From 59b5082bc6c790b82d1e9196995579e84f359a7d Mon Sep 17 00:00:00 2001 From: songjc <969378911@qq.com> Date: Thu, 12 Dec 2024 16:50:33 +0800 Subject: [PATCH 01/12] =?UTF-8?q?=E5=AD=98=E5=82=A8=E7=B3=BB=E7=BB=9F?= =?UTF-8?q?=E6=97=A5=E5=BF=97=E5=88=86=E6=9E=90=E5=A4=84=E7=90=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- common/assets/confs/datamap.config.json | 24 +++ common/pkgs/mq/consts.go | 1 + common/pkgs/mq/datamap/hub.go | 4 + common/pkgs/mq/datamap/object.go | 4 + common/pkgs/mq/datamap/server.go | 72 ++++++++ datamap/internal/cmd/serve.go | 28 ++++ datamap/internal/config/config.go | 26 +++ datamap/internal/db/blockdistribution.go | 1 + datamap/internal/db/config/config.go | 21 +++ datamap/internal/db/db.go | 37 ++++ datamap/internal/db/hub.go | 20 +++ datamap/internal/db/hubrequest.go | 1 + datamap/internal/db/object.go | 1 + datamap/internal/db/storage.go | 1 + datamap/internal/db/storagetransfercount.go | 1 + datamap/internal/models/models.go | 177 ++++++++++++++++++++ datamap/main.go | 16 ++ 17 files changed, 435 insertions(+) create mode 100644 common/assets/confs/datamap.config.json create mode 100644 common/pkgs/mq/datamap/hub.go create mode 100644 common/pkgs/mq/datamap/object.go create mode 100644 common/pkgs/mq/datamap/server.go create mode 100644 datamap/internal/cmd/serve.go create mode 100644 datamap/internal/config/config.go create mode 100644 datamap/internal/db/blockdistribution.go create mode 100644 datamap/internal/db/config/config.go create mode 100644 datamap/internal/db/db.go create mode 100644 datamap/internal/db/hub.go create mode 100644 datamap/internal/db/hubrequest.go create mode 100644 datamap/internal/db/object.go create mode 100644 datamap/internal/db/storage.go create mode 100644 datamap/internal/db/storagetransfercount.go create mode 100644 datamap/internal/models/models.go create mode 100644 datamap/main.go diff --git a/common/assets/confs/datamap.config.json b/common/assets/confs/datamap.config.json new file mode 100644 index 0000000..5d5c206 --- /dev/null +++ b/common/assets/confs/datamap.config.json @@ -0,0 +1,24 @@ +{ + "logger": { + "output": "file", + "outputFileName": "datamap", + "outputDirectory": "log", + "level": "debug" + }, + "db": { + "address": "106.75.6.194:3306", + "account": "root", + "password": "cloudream123456", + "databaseName": "cloudream" + }, + "rabbitMQ": { + "address": "106.75.6.194:5672", + "account": "cloudream", + "password": "123456", + "vhost": "/", + "param": { + "retryNum": 5, + "retryInterval": 5000 + } + } +} \ No newline at end of file diff --git a/common/pkgs/mq/consts.go b/common/pkgs/mq/consts.go index 1f4983a..e7085b0 100644 --- a/common/pkgs/mq/consts.go +++ b/common/pkgs/mq/consts.go @@ -5,6 +5,7 @@ import "fmt" const ( COORDINATOR_QUEUE_NAME = "Coordinator" SCANNER_QUEUE_NAME = "Scanner" + DATAMAP_QUEUE_NAME = "DataMap" ) func MakeAgentQueueName(id int64) string { diff --git a/common/pkgs/mq/datamap/hub.go b/common/pkgs/mq/datamap/hub.go new file mode 100644 index 0000000..71390d5 --- /dev/null +++ b/common/pkgs/mq/datamap/hub.go @@ -0,0 +1,4 @@ +package datamap + +type HubService interface { +} diff --git a/common/pkgs/mq/datamap/object.go b/common/pkgs/mq/datamap/object.go new file mode 100644 index 0000000..fcdccf8 --- /dev/null +++ b/common/pkgs/mq/datamap/object.go @@ -0,0 +1,4 @@ +package datamap + +type ObjectService interface { +} diff --git a/common/pkgs/mq/datamap/server.go b/common/pkgs/mq/datamap/server.go new file mode 100644 index 0000000..9f93bd4 --- /dev/null +++ b/common/pkgs/mq/datamap/server.go @@ -0,0 +1,72 @@ +package datamap + +import ( + "gitlink.org.cn/cloudream/common/pkgs/mq" + "gitlink.org.cn/cloudream/common/utils/sync2" + mymq "gitlink.org.cn/cloudream/storage/common/pkgs/mq" +) + +type Service interface { + HubService + ObjectService +} + +type Server struct { + service Service + rabbitSvr mq.RabbitMQServer +} + +func NewServer(svc Service, cfg *mymq.Config) (*Server, error) { + srv := &Server{ + service: svc, + } + + rabbitSvr, err := mq.NewRabbitMQServer( + cfg.MakeConnectingURL(), + mymq.DATAMAP_QUEUE_NAME, + func(msg *mq.Message) (*mq.Message, error) { + return msgDispatcher.Handle(srv.service, msg) + }, + cfg.Param, + ) + if err != nil { + return nil, err + } + + srv.rabbitSvr = *rabbitSvr + + return srv, nil +} + +func (s *Server) Stop() { + s.rabbitSvr.Close() +} + +func (s *Server) Start(cfg mymq.Config) *sync2.UnboundChannel[mq.RabbitMQServerEvent] { + return s.rabbitSvr.Start() +} + +func (s *Server) OnError(callback func(error)) { + s.rabbitSvr.OnError = callback +} + +var msgDispatcher mq.MessageDispatcher = mq.NewMessageDispatcher() + +// Register 将Service中的一个接口函数作为指定类型消息的处理函数，同时会注册请求和响应的消息类型 +// TODO 需要约束：Service实现了TSvc接口 +func Register[TReq mq.MessageBody, TResp mq.MessageBody](svcFn func(svc Service, msg TReq) (TResp, *mq.CodeMessage)) any { + mq.AddServiceFn(&msgDispatcher, svcFn) + mq.RegisterMessage[TReq]() + mq.RegisterMessage[TResp]() + + return nil +} + +// RegisterNoReply 将Service中的一个*没有返回值的*接口函数作为指定类型消息的处理函数，同时会注册请求和响应的消息类型 +// TODO 需要约束：Service实现了TSvc接口 +func RegisterNoReply[TReq mq.MessageBody](svcFn func(svc Service, msg TReq)) any { + mq.AddNoRespServiceFn(&msgDispatcher, svcFn) + mq.RegisterMessage[TReq]() + + return nil +} diff --git a/datamap/internal/cmd/serve.go b/datamap/internal/cmd/serve.go new file mode 100644 index 0000000..26b6e80 --- /dev/null +++ b/datamap/internal/cmd/serve.go @@ -0,0 +1,28 @@ +package cmd + +import ( + "fmt" + "os" + + "gitlink.org.cn/cloudream/common/pkgs/logger" + "gitlink.org.cn/cloudream/storage/datamap/internal/config" +) + +func serve() { + err := config.Init() + if err != nil { + fmt.Printf("init config failed, err: %s", err.Error()) + os.Exit(1) + } + + err = logger.Init(&config.Cfg().Logger) + if err != nil { + fmt.Printf("init logger failed, err: %s", err.Error()) + os.Exit(1) + } + + // dataSvr, err := datamq.NewServer(mymq.NewService(db2), &config.Cfg().RabbitMQ) + // if err != nil { + // logger.Fatalf("new coordinator server failed, err: %s", err.Error()) + // } +} diff --git a/datamap/internal/config/config.go b/datamap/internal/config/config.go new file mode 100644 index 0000000..c9ab9f2 --- /dev/null +++ b/datamap/internal/config/config.go @@ -0,0 +1,26 @@ +package config + +import ( + "gitlink.org.cn/cloudream/common/pkgs/distlock" + log "gitlink.org.cn/cloudream/common/pkgs/logger" + c "gitlink.org.cn/cloudream/common/utils/config" + db "gitlink.org.cn/cloudream/storage/common/pkgs/db2/config" + stgmq "gitlink.org.cn/cloudream/storage/common/pkgs/mq" +) + +type Config struct { + Logger log.Config `json:"logger"` + DB db.Config `json:"db"` + RabbitMQ stgmq.Config `json:"rabbitMQ"` + DistLock distlock.Config `json:"distlock"` +} + +var cfg Config + +func Init() error { + return c.DefaultLoad("datamap", &cfg) +} + +func Cfg() *Config { + return &cfg +} diff --git a/datamap/internal/db/blockdistribution.go b/datamap/internal/db/blockdistribution.go new file mode 100644 index 0000000..3a49c63 --- /dev/null +++ b/datamap/internal/db/blockdistribution.go @@ -0,0 +1 @@ +package db diff --git a/datamap/internal/db/config/config.go b/datamap/internal/db/config/config.go new file mode 100644 index 0000000..9495b71 --- /dev/null +++ b/datamap/internal/db/config/config.go @@ -0,0 +1,21 @@ +package config + +import "fmt" + +type Config struct { + Address string `json:"address"` + Account string `json:"account"` + Password string `json:"password"` + DatabaseName string `json:"databaseName"` +} + +func (cfg *Config) MakeSourceString() string { + return fmt.Sprintf( + "%s:%s@tcp(%s)/%s?charset=utf8mb4&parseTime=true&loc=%s", + cfg.Account, + cfg.Password, + cfg.Address, + cfg.DatabaseName, + "Asia%2FShanghai", + ) +} diff --git a/datamap/internal/db/db.go b/datamap/internal/db/db.go new file mode 100644 index 0000000..99172d2 --- /dev/null +++ b/datamap/internal/db/db.go @@ -0,0 +1,37 @@ +package db + +import ( + "github.com/sirupsen/logrus" + "gitlink.org.cn/cloudream/storage/datamap/internal/db/config" + "gorm.io/driver/mysql" + "gorm.io/gorm" +) + +type DB struct { + db *gorm.DB +} + +func NewDB(cfg *config.Config) (*DB, error) { + mydb, err := gorm.Open(mysql.Open(cfg.MakeSourceString()), &gorm.Config{}) + if err != nil { + logrus.Fatalf("failed to connect to database: %v", err) + } + + return &DB{ + db: mydb, + }, nil +} + +func (db *DB) DoTx(do func(tx SQLContext) error) error { + return db.db.Transaction(func(tx *gorm.DB) error { + return do(SQLContext{tx}) + }) +} + +type SQLContext struct { + *gorm.DB +} + +func (db *DB) DefCtx() SQLContext { + return SQLContext{db.db} +} diff --git a/datamap/internal/db/hub.go b/datamap/internal/db/hub.go new file mode 100644 index 0000000..b93453f --- /dev/null +++ b/datamap/internal/db/hub.go @@ -0,0 +1,20 @@ +package db + +import ( + "gitlink.org.cn/cloudream/storage/datamap/internal/models" +) + +type HubDB struct { + *DB +} + +func (db *DB) Hub() *HubDB { + return &HubDB{DB: db} +} + +func (*HubDB) GetAllHubs(ctx SQLContext) ([]models.Hub, error) { + var ret []models.Hub + + err := ctx.Table("Hub").Find(&ret).Error + return ret, err +} diff --git a/datamap/internal/db/hubrequest.go b/datamap/internal/db/hubrequest.go new file mode 100644 index 0000000..3a49c63 --- /dev/null +++ b/datamap/internal/db/hubrequest.go @@ -0,0 +1 @@ +package db diff --git a/datamap/internal/db/object.go b/datamap/internal/db/object.go new file mode 100644 index 0000000..3a49c63 --- /dev/null +++ b/datamap/internal/db/object.go @@ -0,0 +1 @@ +package db diff --git a/datamap/internal/db/storage.go b/datamap/internal/db/storage.go new file mode 100644 index 0000000..3a49c63 --- /dev/null +++ b/datamap/internal/db/storage.go @@ -0,0 +1 @@ +package db diff --git a/datamap/internal/db/storagetransfercount.go b/datamap/internal/db/storagetransfercount.go new file mode 100644 index 0000000..3a49c63 --- /dev/null +++ b/datamap/internal/db/storagetransfercount.go @@ -0,0 +1 @@ +package db diff --git a/datamap/internal/models/models.go b/datamap/internal/models/models.go new file mode 100644 index 0000000..167edab --- /dev/null +++ b/datamap/internal/models/models.go @@ -0,0 +1,177 @@ +package models + +import ( + "time" + + cdssdk "gitlink.org.cn/cloudream/common/sdks/storage" +) + +type RequestID int64 + +type BlockID int64 + +type RelationshipID int64 + +type Status int + +const ( + StatusNow Status = iota // =0 表示当前实时状态 + StatusYesterdayAfter // =1 表示前一天调整后的状态 + StatusYesterdayBefore // =2 表示前一天调整前的状态 + StatusTodayBeforeYesterday // =3 表示前两天调整后的状态 +) + +// 节点间关系图 +type HubRelationship struct { + HubID cdssdk.HubID `json:"hubID"` // 节点ID + HubName string `json:"hubName"` // 名称 + HubAddress cdssdk.HubAddressInfo `json:"hubAddress"` // 地址 + Storages []StorageInfo `json:"storages"` // 对应的中心信息 + Requests []Request `json:"requests"` // 与各节点的连接信息（仅记录发出的请求） +} + +type StorageInfo struct { + StorageID cdssdk.StorageID `json:"storageID"` // 中心ID + DataCount int64 `json:"dataCount"` // 总数据量（文件分块数） + NewDataCount int64 `json:"newdataCount"` // 新增数据量（文件分块数） + Timestamp time.Time `json:"timestamp"` // 时间戳 +} + +type Request struct { + SourceHubID cdssdk.HubID `json:"sourceHubID"` // 源节点ID + TargetHubID cdssdk.HubID `json:"targetHubID"` // 目标节点ID + DataTransferCount int64 `json:"dataTransferCount"` // 数据传输量 + RequestCount int64 `json:"requestCount"` // 请求数 + FailedRequestCount int64 `json:"failedRequestCount"` // 失败请求数 + AvgTransferCount int64 `json:"avgTransferCount"` // 平均数据传输量 + MaxTransferCount int64 `json:"maxTransferCount"` // 最大数据传输量 + MinTransferCount int64 `json:"minTransferCount"` // 最小数据传输量 + StartTimestamp time.Time `json:"startTimestamp"` // 起始时间戳 + EndTimestamp time.Time `json:"endTimestamp"` // 结束时间戳 +} + +// 对象块分布结构 +type ObjectDistribution struct { + ObjectID cdssdk.ObjectID `json:"objectID"` // 对象 ID + PackageID cdssdk.PackageID `json:"packageID"` // 包 ID + Path string `json:"path"` // 路径 + Size int64 `json:"size"` // 大小 + FileHash string `json:"fileHash"` // 文件哈希 + States []State `json:"states"` // 各阶段状态信息（只需要传1、2、3阶段） + Relationships []Relationship `json:"relationships"` // 节点间传输量 + Timestamp time.Time `json:"timestamp"` // 请求中的时间戳 +} + +type State struct { + Timestamp time.Time `json:"timestamp"` // 时间戳 + Status string `json:"status"` // 状态 + FaultTolerance string `json:"faultTolerance"` // 容灾度（仅布局调整后） + Redundancy string `json:"redundancy"` // 冗余度（仅布局调整后） + AvgAccessCost float64 `json:"avgAccessCost"` // 平均访问开销（仅布局调整前） + BlockDistributions []BlockDist `json:"blockDistributions"` // 块分布情况 +} + +type BlockDist struct { + StorageID cdssdk.StorageID `json:"storageID"` // 中心ID + Blocks []Block `json:"blocks"` // 该中心的所有块 +} + +type Block struct { + Type string `json:"type"` // 块类型 + Index string `json:"index"` // 块编号 + ID string `json:"id"` // 块ID +} + +type Relationship struct { + Status Status `json:"status"` // 连线左侧的状态 + SourceStorageID string `json:"sourceStorageID"` // 源存储节点 ID + TargetStorageID string `json:"targetStorageID"` // 目标存储节点 ID + DataTransferCount string `json:"dataTransferCount"` // 数据传输量 + Timestamp time.Time `json:"timestamp"` // 变化结束时间戳 +} + +//数据库结构定义 + +type Hub struct { + HubID cdssdk.HubID `gorm:"column:HubID; primaryKey; type:bigint; autoIncrement" json:"hubID"` + Name string `gorm:"column:Name; type:varchar(255); not null" json:"name"` + Address cdssdk.HubAddressInfo `gorm:"column:Address; type:json; serializer:union" json:"address"` +} + +func (Hub) TableName() string { + return "Hub" +} + +type Storage struct { + StorageID cdssdk.StorageID `gorm:"column:StorageID; primaryKey; type:bigint; autoIncrement" json:"storageID"` + HubID cdssdk.HubID `gorm:"column:HubID; type:bigint; not null" json:"hubID"` + DataCount int64 `gorm:"column:DataCount; type:bigint; not null" json:"dataCount"` + NewDataCount int64 `gorm:"column:NewDataCount; type:bigint; not null" json:"newDataCount"` + Timestamp time.Time `gorm:"column:Timestamp; type:datatime; not null" json:"timestamp"` +} + +func (Storage) TableName() string { + return "Storage" +} + +type HubRequest struct { + RequestID int64 `gorm:"column:HubID; primaryKey; type:bigint; autoIncrement" json:"hubID"` + SourceHubID cdssdk.HubID `gorm:"column:SourceHubID; type:bigint; not null" json:"sourceHubID"` + TargetHubID cdssdk.HubID `gorm:"column:TargetHubID; type:bigint; not null" json:"targetHubID"` + DataTransfer int64 `gorm:"column:DataTransfer; type:bigint; not null" json:"dataTransfer"` + RequestCount int64 `gorm:"column:RequestCount; type:bigint; not null" json:"requestCount"` + FailedRequest int64 `gorm:"column:FailedRequest; type:bigint; not null" json:"failedRequest"` + AvgTransferCount int64 `gorm:"column:AvgTransferCount; type:bigint; not null" json:"avgTransferCount"` + MaxTransferCount int64 `gorm:"column:MaxTransferCount; type:bigint; not null" json:"maxTransferCount"` + MinTransferCount int64 `gorm:"column:MinTransferCount; type:bigint; not null" json:"minTransferCount"` + StartTimestamp time.Time `gorm:"column:StartTimestamp; type:datatime; not null" json:"startTimestamp"` + EndTimestamp time.Time `gorm:"column:EndTimestamp; type:datatime; not null" json:"endTimestamp"` +} + +func (HubRequest) TableName() string { + return "HubRequest" +} + +type Object struct { + ObjectID cdssdk.ObjectID `gorm:"column:ObjectID; primaryKey; type:bigint; autoIncrement" json:"objectID"` + PackageID cdssdk.PackageID `gorm:"column:PackageID; type:bigint; not null" json:"packageID"` + Path string `gorm:"column:Path; type:varchar(1024); not null" json:"path"` + Size int64 `gorm:"column:Size; type:bigint; not null" json:"size"` + FileHash string `gorm:"column:FileHash; type:varchar(255); not null" json:"fileHash"` + Status Status `gorm:"column:Status; type:tinyint; not null" json:"status"` + FaultTolerance float64 `gorm:"column:faultTolerance; type:float; not null" json:"faultTolerance"` + Redundancy float64 `gorm:"column:redundancy; type:float; not null" json:"redundancy"` + AvgAccessCost float64 `gorm:"column:avgAccessCost; type:float; not null" json:"avgAccessCost"` + Timestamp time.Time `gorm:"column:Timestamp; type:datatime; not null" json:"timestamp"` +} + +func (Object) TableName() string { + return "Object" +} + +type BlockDistribution struct { + BlockID int64 `gorm:"column:BlockID; primaryKey; type:bigint; autoIncrement" json:"blockID"` + ObjectID cdssdk.ObjectID `gorm:"column:ObjectID; type:bigint; not null" json:"objectID"` + Type string `gorm:"column:Type; type:varchar(1024); not null" json:"type"` + Index int64 `gorm:"column:Index; type:bigint; not null" json:"index"` + StorageID cdssdk.StorageID `gorm:"column:StorageID; type:bigint; not null" json:"storageID"` + Status Status `gorm:"column:Status; type:tinyint; not null" json:"status"` + Timestamp time.Time `gorm:"column:Timestamp; type:datatime; not null" json:"timestamp"` +} + +func (BlockDistribution) TableName() string { + return "BlockDistribution" +} + +type StorageTransferCount struct { + RelationshipID RelationshipID `gorm:"column:RelationshipID; primaryKey; type:bigint; autoIncrement" json:"relationshipID"` + Status Status `gorm:"column:Status; type:tinyint; not null" json:"status"` + SourceStorageID cdssdk.StorageID `gorm:"column:SourceStorageID; type:bigint; not null" json:"sourceStorageID"` + TargetStorageID cdssdk.StorageID `gorm:"column:TargetStorageID; type:bigint; not null" json:"targetStorageID"` + DataTransferCount int64 `gorm:"column:DataTransferCount; type:bigint; not null" json:"dataTransferCount"` + Timestamp time.Time `gorm:"column:Timestamp; type:datatime; not null" json:"timestamp"` +} + +func (StorageTransferCount) TableName() string { + return "StorageTransferCount" +} diff --git a/datamap/main.go b/datamap/main.go new file mode 100644 index 0000000..6d69c12 --- /dev/null +++ b/datamap/main.go @@ -0,0 +1,16 @@ +package main + +import ( + "fmt" + "os" + + "gitlink.org.cn/cloudream/storage/datamap/internal/config" +) + +func main() { + err := config.Init() + if err != nil { + fmt.Printf("init config failed, err: %s", err.Error()) + os.Exit(1) + } +} From 41249846f370607b212abb230cc26964b8c2f553 Mon Sep 17 00:00:00 2001 From: Jake <450705171@qq.com> Date: Tue, 17 Dec 2024 19:29:29 +0800 Subject: [PATCH 02/12] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E5=AD=98=E5=82=A8?= =?UTF-8?q?=E6=97=A5=E5=BF=97=E9=87=87=E9=9B=86=E6=95=B0=E6=8D=AE=E7=BB=93?= =?UTF-8?q?=E6=9E=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- client/internal/services/hub.go | 19 +++ common/globals/pools.go | 5 + common/models/datamap.go | 200 +++++++++++++++++++++++++++++++ common/pkgs/mq/datamap/client.go | 60 ++++++++++ common/pkgs/mq/datamap/hub.go | 52 ++++++++ common/pkgs/mq/datamap/object.go | 31 +++++ datamap/internal/db/hub.go | 9 ++ 7 files changed, 376 insertions(+) create mode 100644 common/models/datamap.go create mode 100644 common/pkgs/mq/datamap/client.go diff --git a/client/internal/services/hub.go b/client/internal/services/hub.go index 9f9b2b5..591306e 100644 --- a/client/internal/services/hub.go +++ b/client/internal/services/hub.go @@ -2,10 +2,12 @@ package services import ( "fmt" + stgmod "gitlink.org.cn/cloudream/storage/common/models" cdssdk "gitlink.org.cn/cloudream/common/sdks/storage" stgglb "gitlink.org.cn/cloudream/storage/common/globals" coormq "gitlink.org.cn/cloudream/storage/common/pkgs/mq/coordinator" + datamapmq "gitlink.org.cn/cloudream/storage/common/pkgs/mq/datamap" ) // HubService 是关于节点操作的服务结构体 @@ -45,3 +47,20 @@ func (svc *HubService) GetHubs(hubIDs []cdssdk.HubID) ([]cdssdk.Hub, error) { // 返回获取到的节点信息 return getResp.Hubs, nil } + +// GetHubStat 根据提供的节点ID列表,获取中心的总数据量 + +func (svc *HubService) GetHubStat(hubID cdssdk.HubID) (stgmod.HubStat, error) { + datamapCli, err := stgglb.DatamapMQPool.Acquire() + if err != nil { + return stgmod.HubStat{}, fmt.Errorf("new coordinator client: %w", err) + } + defer stgglb.DatamapMQPool.Release(datamapCli) + + getResp, err := datamapCli.GetHubStat(datamapmq.NewGetHubStat(hubID)) + if err != nil { + return stgmod.HubStat{}, fmt.Errorf("requesting to coordinator: %w", err) + } + + return getResp.HubStat, nil +} diff --git a/common/globals/pools.go b/common/globals/pools.go index 71eca60..d30a1bc 100644 --- a/common/globals/pools.go +++ b/common/globals/pools.go @@ -5,6 +5,7 @@ import ( stgmq "gitlink.org.cn/cloudream/storage/common/pkgs/mq" agtmq "gitlink.org.cn/cloudream/storage/common/pkgs/mq/agent" coormq "gitlink.org.cn/cloudream/storage/common/pkgs/mq/coordinator" + datamap "gitlink.org.cn/cloudream/storage/common/pkgs/mq/datamap" scmq "gitlink.org.cn/cloudream/storage/common/pkgs/mq/scanner" ) @@ -14,6 +15,8 @@ var CoordinatorMQPool coormq.Pool var ScannerMQPool scmq.Pool +var DatamapMQPool datamap.Pool + // InitMQPool // // @Description: 初始化MQ连接池 @@ -24,6 +27,8 @@ func InitMQPool(cfg *stgmq.Config) { CoordinatorMQPool = coormq.NewPool(cfg) ScannerMQPool = scmq.NewPool(cfg) + + DatamapMQPool = datamap.NewPool(cfg) } var AgentRPCPool *agtrpc.Pool diff --git a/common/models/datamap.go b/common/models/datamap.go new file mode 100644 index 0000000..f7a2492 --- /dev/null +++ b/common/models/datamap.go @@ -0,0 +1,200 @@ +package stgmod + +import "time" + +// HubStat 中心信息 +// 每天一次，各节点统计自身当前的总数据量 +type HubStat struct { + Timestamp time.Time `json:"timestamp"` + Source struct { + Type string `json:"type"` + HubID string `json:"hubID"` + HubName string `json:"hubName"` + } `json:"source"` + Category string `json:"category"` + Body struct { + MasterHubID int `json:"masterHubID"` + MasterHubAddress string `json:"masterHubAddress"` + StorageID int `json:"storageID"` + DataCount int `json:"dataCount"` + } `json:"body"` +} + +// HubTrans 节点传输信息 +// 每天一次，各节点统计自身当天向外部各个节点传输的总数据量 +type HubTrans struct { + Timestamp time.Time `json:"timestamp"` + Source struct { + Type string `json:"type"` + HubID string `json:"hubID"` + HubName string `json:"hubName"` + } `json:"source"` + Category string `json:"category"` + Body struct { + SourceHubID int `json:"sourceHubID"` + TargetHubID int `json:"targetHubID"` + DataTransferCount int `json:"dataTransferCount"` + RequestCount int `json:"requestCount"` + FailedRequestCount int `json:"failedRequestCount"` + AvgTransferCount int `json:"avgTransferCount"` + MaxTransferCount int `json:"maxTransferCount"` + MinTransferCount int `json:"minTransferCount"` + StartTimestamp time.Time `json:"startTimestamp"` + EndTimestamp time.Time `json:"endTimestamp"` + } `json:"body"` +} + +// BlockTransInfo 块传输信息 +/*实时日志，当对象的块信息发生变化时推送，共有4种变化类型： +拷贝 +编解码(一变多、多变一、多变多) +删除 +更新*/ +type BlockTransInfo struct { + Timestamp time.Time `json:"timestamp"` + Source struct { + Type string `json:"type"` + HubID interface{} `json:"hubID"` + HubName interface{} `json:"hubName"` + } `json:"source"` + Category string `json:"category"` + Body struct { + Type string `json:"type"` + ObjectID interface{} `json:"objectID"` + PackageID interface{} `json:"packageID"` + BlockChanges []struct { + Type string `json:"type"` + BlockType interface{} `json:"blockType"` + Index interface{} `json:"index"` + SourceStorageID interface{} `json:"sourceStorageID"` + TargetStorageID interface{} `json:"targetStorageID"` + DataTransferCount interface{} `json:"dataTransferCount"` + Timestamp interface{} `json:"timestamp"` + SourceBlocks []struct { + BlockType interface{} `json:"blockType"` + Index interface{} `json:"index"` + StorageID interface{} `json:"storageID"` + } `json:"sourceBlocks,omitempty"` + TargetBlocks []struct { + BlockType interface{} `json:"blockType"` + Index interface{} `json:"index"` + StorageID interface{} `json:"storageID"` + } `json:"targetBlocks,omitempty"` + DataTransfers []struct { + SourceStorageID interface{} `json:"sourceStorageID"` + TargetStorageID interface{} `json:"targetStorageID"` + DataTransferCount interface{} `json:"dataTransferCount"` + } `json:"dataTransfers,omitempty"` + Blocks []struct { + BlockType interface{} `json:"blockType"` + Index interface{} `json:"index"` + StorageID interface{} `json:"storageID"` + } `json:"blocks,omitempty"` + } `json:"blockChanges"` + } `json:"body"` +} + +// BlockDistribution 块传输信息 +// 每天一次，在调整完成后，将当天调整前后的布局情况一起推送 +type BlockDistribution struct { + Timestamp time.Time `json:"timestamp"` + Source struct { + Type string `json:"type"` + HubID interface{} `json:"hubID"` + HubName interface{} `json:"hubName"` + } `json:"source"` + Category string `json:"category"` + Body struct { + Type string `json:"type"` + ObjectID interface{} `json:"objectID"` + PackageID interface{} `json:"packageID"` + BlockChanges []struct { + Type string `json:"type"` + BlockType interface{} `json:"blockType"` + Index interface{} `json:"index"` + SourceStorageID interface{} `json:"sourceStorageID"` + TargetStorageID interface{} `json:"targetStorageID"` + DataTransferCount interface{} `json:"dataTransferCount"` + Timestamp interface{} `json:"timestamp"` + SourceBlocks []struct { + BlockType interface{} `json:"blockType"` + Index interface{} `json:"index"` + StorageID interface{} `json:"storageID"` + } `json:"sourceBlocks,omitempty"` + TargetBlocks []struct { + BlockType interface{} `json:"blockType"` + Index interface{} `json:"index"` + StorageID interface{} `json:"storageID"` + } `json:"targetBlocks,omitempty"` + DataTransfers []struct { + SourceStorageID interface{} `json:"sourceStorageID"` + TargetStorageID interface{} `json:"targetStorageID"` + DataTransferCount interface{} `json:"dataTransferCount"` + } `json:"dataTransfers,omitempty"` + Blocks []struct { + BlockType interface{} `json:"blockType"` + Index interface{} `json:"index"` + StorageID interface{} `json:"storageID"` + } `json:"blocks,omitempty"` + } `json:"blockChanges"` + } `json:"body"` +} + +// ObjectUpdateInfo Object变化信息 +type ObjectUpdateInfo struct { + Timestamp time.Time `json:"timestamp"` + Source struct { + Type string `json:"type"` + HubID string `json:"hubID"` + HubName string `json:"hubName"` + } `json:"source"` + Category string `json:"category"` + Body struct { + Type string `json:"type"` + ObjectID string `json:"objectID"` + PackageID string `json:"packageID"` + Path string `json:"path"` + Size int `json:"size"` + BlockDistribution []struct { + Type string `json:"type,omitempty"` + Index string `json:"index,omitempty"` + StorageID string `json:"storageID,omitempty"` + } `json:"blockDistribution"` + Timestamp string `json:"timestamp"` + } `json:"body"` +} + +// PackageUpdateInfo package变化信息 +type PackageUpdateInfo struct { + Timestamp time.Time `json:"timestamp"` + Source struct { + Type string `json:"type"` + HubID string `json:"hubID"` + HubName string `json:"hubName"` + } `json:"source"` + Category string `json:"category"` + Body struct { + Type string `json:"type"` + PackageID string `json:"packageID"` + PackageName string `json:"packageName"` + BucketID string `json:"bucketID"` + Timestamp string `json:"timestamp"` + } `json:"body"` +} + +// BucketUpdateInfo bucket变化信息 +type BucketUpdateInfo struct { + Timestamp time.Time `json:"timestamp"` + Source struct { + Type string `json:"type"` + HubID string `json:"hubID"` + HubName string `json:"hubName"` + } `json:"source"` + Category string `json:"category"` + Body struct { + Type string `json:"type"` + BucketID string `json:"bucketID"` + BucketName string `json:"bucketName"` + Timestamp string `json:"timestamp"` + } `json:"body"` +} diff --git a/common/pkgs/mq/datamap/client.go b/common/pkgs/mq/datamap/client.go new file mode 100644 index 0000000..e388e22 --- /dev/null +++ b/common/pkgs/mq/datamap/client.go @@ -0,0 +1,60 @@ +package datamap + +import ( + "sync" + + "gitlink.org.cn/cloudream/common/pkgs/mq" + stgmq "gitlink.org.cn/cloudream/storage/common/pkgs/mq" +) + +type Client struct { + rabbitCli *mq.RabbitMQTransport +} + +func NewClient(cfg *stgmq.Config) (*Client, error) { + rabbitCli, err := mq.NewRabbitMQTransport(cfg.MakeConnectingURL(), stgmq.DATAMAP_QUEUE_NAME, "") + if err != nil { + return nil, err + } + + return &Client{ + rabbitCli: rabbitCli, + }, nil +} + +func (c *Client) Close() { + c.rabbitCli.Close() +} + +type Pool interface { + Acquire() (*Client, error) + Release(cli *Client) +} + +type pool struct { + mqcfg *stgmq.Config + shared *Client + lock sync.Mutex +} + +func NewPool(mqcfg *stgmq.Config) Pool { + return &pool{ + mqcfg: mqcfg, + } +} +func (p *pool) Acquire() (*Client, error) { + p.lock.Lock() + defer p.lock.Unlock() + if p.shared == nil { + var err error + p.shared, err = NewClient(p.mqcfg) + if err != nil { + return nil, err + } + } + + return p.shared, nil +} + +func (p *pool) Release(cli *Client) { +} diff --git a/common/pkgs/mq/datamap/hub.go b/common/pkgs/mq/datamap/hub.go index 71390d5..91d03b5 100644 --- a/common/pkgs/mq/datamap/hub.go +++ b/common/pkgs/mq/datamap/hub.go @@ -1,4 +1,56 @@ package datamap +import ( + "gitlink.org.cn/cloudream/common/pkgs/mq" + cdssdk "gitlink.org.cn/cloudream/common/sdks/storage" + stgmod "gitlink.org.cn/cloudream/storage/common/models" +) + type HubService interface { + GetHubStat(msg *GetHubStat) (*GetHubStatResp, *mq.CodeMessage) + GetHubTrans(msg *GetHubTrans) (*GetHubTransResp, *mq.CodeMessage) +} + +// 查询中心自身当前的总数据量 +var _ = Register(Service.GetHubStat) + +type GetHubStat struct { + mq.MessageBodyBase + HubID cdssdk.HubID `json:"hubID"` +} +type GetHubStatResp struct { + mq.MessageBodyBase + HubStat stgmod.HubStat `json:"hubStat"` +} + +func NewGetHubStat(hubID cdssdk.HubID) *GetHubStat { + return &GetHubStat{ + HubID: hubID, + } +} +func NewGetHubStatResp(hubStat stgmod.HubStat) *GetHubStatResp { + return &GetHubStatResp{ + HubStat: hubStat, + } +} +func (client *Client) GetHubStat(msg *GetHubStat) (*GetHubStatResp, error) { + return mq.Request(Service.GetHubStat, client.rabbitCli, msg) +} + +// 查询中心节点传输的总数据量 + +var _ = Register(Service.GetHubTrans) + +type GetHubTrans struct { + mq.MessageBodyBase + HubID cdssdk.HubID `json:"hubID"` +} + +type GetHubTransResp struct { + mq.MessageBodyBase + HubTrans stgmod.HubTrans `json:"hubTrans"` +} + +func (client *Client) GetHubTrans(msg *GetHubTrans) (*GetHubTransResp, error) { + return mq.Request(Service.GetHubTrans, client.rabbitCli, msg) } diff --git a/common/pkgs/mq/datamap/object.go b/common/pkgs/mq/datamap/object.go index fcdccf8..bdc26b9 100644 --- a/common/pkgs/mq/datamap/object.go +++ b/common/pkgs/mq/datamap/object.go @@ -1,4 +1,35 @@ package datamap +import ( + "gitlink.org.cn/cloudream/common/pkgs/mq" + cdssdk "gitlink.org.cn/cloudream/common/sdks/storage" + stgmod "gitlink.org.cn/cloudream/storage/common/models" +) + type ObjectService interface { + GetBlockTransInfo(msg *GetBlockTransInfo) (*GetBlockTransInfoResp, *mq.CodeMessage) + GetBlockDistribution(msg *GetBlockDistribution) (*GetBlockDistributionResp, *mq.CodeMessage) +} + +type GetBlockTransInfo struct { + mq.MessageBodyBase + //todo 入参待确认 + UserID cdssdk.UserID `json:"userID"` + PackageID cdssdk.PackageID `json:"packageID"` +} + +type GetBlockTransInfoResp struct { + mq.MessageBodyBase + ObjectTransInfo stgmod.BlockTransInfo `json:"objectTransInfo"` +} + +type GetBlockDistribution struct { + mq.MessageBodyBase + //todo 入参待确认 + HubID cdssdk.HubID `json:"hubID"` +} + +type GetBlockDistributionResp struct { + mq.MessageBodyBase + BlockDistribution stgmod.BlockDistribution `json:"blockDistribution"` } diff --git a/datamap/internal/db/hub.go b/datamap/internal/db/hub.go index b93453f..daaef66 100644 --- a/datamap/internal/db/hub.go +++ b/datamap/internal/db/hub.go @@ -1,6 +1,9 @@ package db import ( + "gitlink.org.cn/cloudream/common/pkgs/mq" + stgmod "gitlink.org.cn/cloudream/storage/common/models" + datamapmq "gitlink.org.cn/cloudream/storage/common/pkgs/mq/datamap" "gitlink.org.cn/cloudream/storage/datamap/internal/models" ) @@ -18,3 +21,9 @@ func (*HubDB) GetAllHubs(ctx SQLContext) ([]models.Hub, error) { err := ctx.Table("Hub").Find(&ret).Error return ret, err } + +func (*HubDB) GetHubStat(msg *datamapmq.GetHubStat) (*datamapmq.GetHubStatResp, *mq.CodeMessage) { + //todo 数据库操作 + + return mq.ReplyOK(datamapmq.NewGetHubStatResp(stgmod.HubStat{})) +} From 49b469ae3c9b93a78d8fda21db0ff7596c141b4b Mon Sep 17 00:00:00 2001 From: Jake <450705171@qq.com> Date: Fri, 20 Dec 2024 18:37:31 +0800 Subject: [PATCH 03/12] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E6=95=B0=E6=8D=AE?= =?UTF-8?q?=E7=BB=93=E6=9E=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- client/internal/services/hub.go | 6 ++- common/pkgs/mq/datamap/hub.go | 1 + datamap/internal/db/blockdistribution.go | 41 +++++++++++++++++++++ datamap/internal/db/hub.go | 41 ++++++++++++++++----- datamap/internal/db/hubrequest.go | 23 ++++++++++++ datamap/internal/db/object.go | 35 ++++++++++++++++++ datamap/internal/db/storage.go | 35 ++++++++++++++++++ datamap/internal/db/storagetransfercount.go | 35 ++++++++++++++++++ datamap/internal/mq/hub.go | 22 +++++++++++ datamap/internal/mq/service.go | 15 ++++++++ 10 files changed, 243 insertions(+), 11 deletions(-) create mode 100644 datamap/internal/mq/hub.go create mode 100644 datamap/internal/mq/service.go diff --git a/client/internal/services/hub.go b/client/internal/services/hub.go index 591306e..50d68e7 100644 --- a/client/internal/services/hub.go +++ b/client/internal/services/hub.go @@ -53,14 +53,16 @@ func (svc *HubService) GetHubs(hubIDs []cdssdk.HubID) ([]cdssdk.Hub, error) { func (svc *HubService) GetHubStat(hubID cdssdk.HubID) (stgmod.HubStat, error) { datamapCli, err := stgglb.DatamapMQPool.Acquire() if err != nil { - return stgmod.HubStat{}, fmt.Errorf("new coordinator client: %w", err) + return stgmod.HubStat{}, fmt.Errorf("new datamap client: %w", err) } defer stgglb.DatamapMQPool.Release(datamapCli) getResp, err := datamapCli.GetHubStat(datamapmq.NewGetHubStat(hubID)) if err != nil { - return stgmod.HubStat{}, fmt.Errorf("requesting to coordinator: %w", err) + return stgmod.HubStat{}, fmt.Errorf("requesting to datamap: %w", err) } return getResp.HubStat, nil } + +// diff --git a/common/pkgs/mq/datamap/hub.go b/common/pkgs/mq/datamap/hub.go index 91d03b5..662ef78 100644 --- a/common/pkgs/mq/datamap/hub.go +++ b/common/pkgs/mq/datamap/hub.go @@ -52,5 +52,6 @@ type GetHubTransResp struct { } func (client *Client) GetHubTrans(msg *GetHubTrans) (*GetHubTransResp, error) { + //获取到传输统计之后 return mq.Request(Service.GetHubTrans, client.rabbitCli, msg) } diff --git a/datamap/internal/db/blockdistribution.go b/datamap/internal/db/blockdistribution.go index 3a49c63..cc7d434 100644 --- a/datamap/internal/db/blockdistribution.go +++ b/datamap/internal/db/blockdistribution.go @@ -1 +1,42 @@ package db + +import "gitlink.org.cn/cloudream/storage/datamap/internal/models" + +type BlockDistributionDB struct { + *DB +} + +func (db *DB) BlockDistribution() *BlockDistributionDB { + return &BlockDistributionDB{DB: db} +} + +// GetAllBlockDistribution 查询所有BlockDistribution列表 +func (*HubDB) GetAllBlockDistribution(ctx SQLContext) ([]models.BlockDistribution, error) { + var ret []models.BlockDistribution + + err := ctx.Table("blockdistribution").Find(&ret).Error + return ret, err +} + +// GetBlockDistribution 根据输入的BlockID查询BlockDistribution +func (*HubDB) GetBlockDistribution(ctx SQLContext, BlockID int64) (models.BlockDistribution, error) { + var ret models.BlockDistribution + err := ctx.Table("blockdistribution").Where("BlockID = ?", BlockID).Find(&ret).Error + return ret, err +} + +// CreateBlockDistribution 根据输入的BlockDistribution信息创建BlockDistribution记录 +func (*HubDB) CreateBlockDistribution(ctx SQLContext, blockDistribution models.BlockDistribution) (*models.BlockDistribution, error) { + err := ctx.Table("blockdistribution").Create(&blockDistribution).Error + return &blockDistribution, err +} + +// DeleteBlockDistribution 根据输入的BlockID删除BlockDistribution记录 +func (*HubDB) DeleteBlockDistribution(ctx SQLContext, BlockID int64) error { + return ctx.Table("blockdistribution").Where("BlockID = ?", BlockID).Delete(&models.BlockDistribution{}).Error +} + +// UpdateBlockDistribution 根据输入的BlockDistribution信息更新BlockDistribution记录 +func (*HubDB) UpdateBlockDistribution(ctx SQLContext, blockDistribution models.BlockDistribution) error { + return ctx.Table("blockdistribution").Where("BlockID = ?", blockDistribution.BlockID).Updates(&blockDistribution).Error +} diff --git a/datamap/internal/db/hub.go b/datamap/internal/db/hub.go index daaef66..78f783f 100644 --- a/datamap/internal/db/hub.go +++ b/datamap/internal/db/hub.go @@ -1,9 +1,6 @@ package db import ( - "gitlink.org.cn/cloudream/common/pkgs/mq" - stgmod "gitlink.org.cn/cloudream/storage/common/models" - datamapmq "gitlink.org.cn/cloudream/storage/common/pkgs/mq/datamap" "gitlink.org.cn/cloudream/storage/datamap/internal/models" ) @@ -15,15 +12,41 @@ func (db *DB) Hub() *HubDB { return &HubDB{DB: db} } -func (*HubDB) GetAllHubs(ctx SQLContext) ([]models.Hub, error) { +// GetHubs 获取所有hub列表 +func (*HubDB) GetHubs(ctx SQLContext) ([]models.Hub, error) { var ret []models.Hub - - err := ctx.Table("Hub").Find(&ret).Error + err := ctx.Table("hub").Find(&ret).Error return ret, err } -func (*HubDB) GetHubStat(msg *datamapmq.GetHubStat) (*datamapmq.GetHubStatResp, *mq.CodeMessage) { - //todo 数据库操作 +// GetHub 根据hubId获取该Hub的信息 +func (*HubDB) GetHub(ctx SQLContext, hubID int64) (*models.Hub, error) { + var hub models.Hub + err := ctx.Table("hub").Where("HubID = ?", hubID).Find(&hub).Error + return &hub, err +} + +// DeleteHub 根据hubId删除该Hub +func (*HubDB) DeleteHub(ctx SQLContext, hubID int64) error { + err := ctx.Table("hub").Where("HubID = ?", hubID).Delete(&models.Hub{}).Error + return err +} + +// UpdateHub 根据输入hub信息更新Hub信息 +func (*HubDB) UpdateHub(ctx SQLContext, hub models.Hub) (*models.Hub, error) { + err := ctx.Table("hub").Where("HubID = ?", hub.HubID).Updates(&hub).Error + return &hub, err +} + +// CreateHub 根据输入hub信息创建Hub信息 +func (*HubDB) CreateHub(ctx SQLContext, hub models.Hub) (*models.Hub, error) { + err := ctx.Table("hub").Create(&hub).Error + return &hub, err +} - return mq.ReplyOK(datamapmq.NewGetHubStatResp(stgmod.HubStat{})) +// IsHubExist 根据hubId查询hub是否存在 +func (*HubDB) IsHubExist(ctx SQLContext, hubID int64) (bool, error) { + var count int64 + err := ctx.Table("hub").Where("HubID = ?", hubID).Count(&count).Error + return count > 0, err } diff --git a/datamap/internal/db/hubrequest.go b/datamap/internal/db/hubrequest.go index 3a49c63..f99479b 100644 --- a/datamap/internal/db/hubrequest.go +++ b/datamap/internal/db/hubrequest.go @@ -1 +1,24 @@ package db + +import "gitlink.org.cn/cloudream/storage/datamap/internal/models" + +type HubReqDB struct { + *DB +} + +func (db *DB) HubReq() *HubReqDB { + return &HubReqDB{DB: db} +} + +// GetHubRequest 获取所有hubrequest列表 +func (*HubReqDB) GetHubRequest(ctx SQLContext) ([]models.HubRequest, error) { + var ret []models.HubRequest + err := ctx.Table("hubrequest").Find(&ret).Error + return ret, err +} + +// CreateHubRequest 根据输入的HubRequest信息创建HubRequest信息 +func (*HubReqDB) CreateHubRequest(ctx SQLContext, hubRequest models.HubRequest) (*models.HubRequest, error) { + err := ctx.Table("hubrequest").Create(&hubRequest).Error + return &hubRequest, err +} diff --git a/datamap/internal/db/object.go b/datamap/internal/db/object.go index 3a49c63..1f700eb 100644 --- a/datamap/internal/db/object.go +++ b/datamap/internal/db/object.go @@ -1 +1,36 @@ package db + +import "gitlink.org.cn/cloudream/storage/datamap/internal/models" + +type ObjectDB struct { + *DB +} + +func (db *DB) Object() *ObjectDB { + return &ObjectDB{DB: db} +} + +// GetAllObject 查询所有Object列表 +func (*ObjectDB) GetAllObject(ctx SQLContext) ([]models.Object, error) { + var ret []models.Object + + err := ctx.Table("object").Find(&ret).Error + return ret, err +} + +// GetObject 根据输入的ObjectId查询Object +func (*ObjectDB) GetObject(ctx SQLContext, objectId int64) (models.Object, error) { + var ret models.Object + err := ctx.Table("object").Where("ObjectID = ?", objectId).Find(&ret).Error + return ret, err +} + +// DeleteObject 根据输入的ObjectId删除Object +func (*ObjectDB) DeleteObject(ctx SQLContext, objectId int64) error { + return ctx.Table("object").Where("ObjectID = ?", objectId).Delete(&models.Object{}).Error +} + +// UpdateObject 根据输入的Object信息更新Object +func (*ObjectDB) UpdateObject(ctx SQLContext, object models.Object) error { + return ctx.Table("object").Where("ObjectID = ?", object.ObjectID).Updates(&object).Error +} diff --git a/datamap/internal/db/storage.go b/datamap/internal/db/storage.go index 3a49c63..4076374 100644 --- a/datamap/internal/db/storage.go +++ b/datamap/internal/db/storage.go @@ -1 +1,36 @@ package db + +import "gitlink.org.cn/cloudream/storage/datamap/internal/models" + +type StorageDB struct { + *DB +} + +func (db *DB) Storage() *StorageDB { + return &StorageDB{DB: db} +} + +// GetAllStorage 查询所有Storage列表 +func (*HubDB) GetAllStorage(ctx SQLContext) ([]models.Storage, error) { + var ret []models.Storage + + err := ctx.Table("storage").Find(&ret).Error + return ret, err +} + +// GetStorage 根据输入的StorageId查询Storage +func (*HubDB) GetStorage(ctx SQLContext, storageId int64) (models.Storage, error) { + var ret models.Storage + err := ctx.Table("storage").Where("StorageID = ?", storageId).Find(&ret).Error + return ret, err +} + +// DeleteStorage 根据输入的StorageId删除Storage +func (*HubDB) DeleteStorage(ctx SQLContext, storageId int64) error { + return ctx.Table("storage").Where("StorageID = ?", storageId).Delete(&models.Storage{}).Error +} + +// UpdateStorage 根据输入的Storage信息更新Storage +func (*HubDB) UpdateStorage(ctx SQLContext, storage models.Storage) error { + return ctx.Table("storage").Where("StorageID = ?", storage.StorageID).Updates(&storage).Error +} diff --git a/datamap/internal/db/storagetransfercount.go b/datamap/internal/db/storagetransfercount.go index 3a49c63..f53c389 100644 --- a/datamap/internal/db/storagetransfercount.go +++ b/datamap/internal/db/storagetransfercount.go @@ -1 +1,36 @@ package db + +import "gitlink.org.cn/cloudream/storage/datamap/internal/models" + +type StorageTransferCountDB struct { + *DB +} + +func (db *DB) StorageTransferCount() *StorageTransferCountDB { + return &StorageTransferCountDB{DB: db} +} + +// GetAllStorageTransferCount 查询所有Storage列表 +func (*HubDB) GetAllStorageTransferCount(ctx SQLContext) ([]models.StorageTransferCount, error) { + var ret []models.StorageTransferCount + + err := ctx.Table("storagetransfercount").Find(&ret).Error + return ret, err +} + +// GetStorageTransferCount 根据输入的RelationshipID查询StorageTransferCount +func (*HubDB) GetStorageTransferCount(ctx SQLContext, RelationshipID int64) (models.Storage, error) { + var ret models.Storage + err := ctx.Table("storagetransfercount").Where("RelationshipID = ?", RelationshipID).Find(&ret).Error + return ret, err +} + +// DeleteStorageTransferCount 根据输入的RelationshipID删除StorageTransferCount +func (*HubDB) DeleteStorageTransferCount(ctx SQLContext, RelationshipID int64) error { + return ctx.Table("storagetransfercount").Where("RelationshipID = ?", RelationshipID).Delete(&models.Storage{}).Error +} + +// UpdateStorageTransferCount 根据输入的StorageTransferCount信息更新StorageTransferCount +func (*HubDB) UpdateStorageTransferCount(ctx SQLContext, storageTransferCount models.StorageTransferCount) error { + return ctx.Table("storagetransfercount").Where("RelationshipID = ?", storageTransferCount.RelationshipID).Updates(&storageTransferCount).Error +} diff --git a/datamap/internal/mq/hub.go b/datamap/internal/mq/hub.go new file mode 100644 index 0000000..6b1860d --- /dev/null +++ b/datamap/internal/mq/hub.go @@ -0,0 +1,22 @@ +package mq + +import ( + "gitlink.org.cn/cloudream/common/pkgs/logger" + "gitlink.org.cn/cloudream/common/pkgs/mq" + datamapmq "gitlink.org.cn/cloudream/storage/common/pkgs/mq/datamap" +) + +func (svc *Service) GetHubStat(msg *datamapmq.GetHubStat) (*datamapmq.GetHubStatResp, *mq.CodeMessage) { + logger.WithField("HubID", msg.HubID) + + //从datamapmq队列接收数据到stgmod.HubStat + + //从datamapmq队列接收数据到stgmod.HubStat,然后将该数据和mysql数据库中获取到的models.Storage做对比，用stgmode + return &datamapmq.GetHubStatResp{}, nil +} + +func (svc *Service) GetHubTrans(msg *datamapmq.GetHubTrans) (*datamapmq.GetHubTransResp, *mq.CodeMessage) { + logger.WithField("HubID", msg.HubID) + + return &datamapmq.GetHubTransResp{}, nil +} diff --git a/datamap/internal/mq/service.go b/datamap/internal/mq/service.go new file mode 100644 index 0000000..5e1328e --- /dev/null +++ b/datamap/internal/mq/service.go @@ -0,0 +1,15 @@ +package mq + +import ( + "gorm.io/gorm" +) + +type Service struct { + db *gorm.DB +} + +func NewService(db *gorm.DB) *Service { + return &Service{ + db: db, + } +} From 5f3d43c8f422ae8c5b566441ee8879fe3a3b62df Mon Sep 17 00:00:00 2001 From: Jake <450705171@qq.com> Date: Mon, 23 Dec 2024 20:52:28 +0800 Subject: [PATCH 04/12] =?UTF-8?q?=E8=B0=83=E6=95=B4datamap=E6=A8=A1?= =?UTF-8?q?=E5=9D=97=E6=9E=B6=E6=9E=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- client/internal/services/hub.go | 22 -- common/globals/pools.go | 4 - common/models/datamap.go | 304 +++++++++----------- common/pkgs/mq/datamap/client.go | 60 ---- common/pkgs/mq/datamap/hub.go | 57 ---- common/pkgs/mq/datamap/object.go | 35 --- common/pkgs/mq/datamap/server.go | 72 ----- datamap/internal/cmd/serve.go | 28 -- datamap/internal/config/config.go | 56 +++- datamap/internal/db/blockdistribution.go | 42 --- datamap/internal/db/config/config.go | 21 -- datamap/internal/db/db.go | 42 ++- datamap/internal/db/hub.go | 52 ---- datamap/internal/db/hubrequest.go | 24 -- datamap/internal/db/object.go | 36 --- datamap/internal/db/storage.go | 36 --- datamap/internal/db/storagetransfercount.go | 36 --- datamap/internal/handlers/handlers.go | 30 ++ datamap/internal/models/block.go | 57 ++++ datamap/internal/models/hubs.go | 93 ++++++ datamap/internal/models/hubtrans.go | 53 ++++ datamap/internal/models/models.go | 62 +--- datamap/internal/models/object.go | 70 +++++ datamap/internal/models/repository.go | 43 +++ datamap/internal/mq/hub.go | 22 -- datamap/internal/mq/mq.go | 80 ++++++ datamap/internal/mq/service.go | 15 - datamap/internal/server/server.go | 22 ++ datamap/main.go | 25 +- go.mod | 46 +-- go.sum | 107 ++++--- 31 files changed, 768 insertions(+), 884 deletions(-) delete mode 100644 common/pkgs/mq/datamap/client.go delete mode 100644 common/pkgs/mq/datamap/hub.go delete mode 100644 common/pkgs/mq/datamap/object.go delete mode 100644 common/pkgs/mq/datamap/server.go delete mode 100644 datamap/internal/cmd/serve.go delete mode 100644 datamap/internal/db/blockdistribution.go delete mode 100644 datamap/internal/db/config/config.go delete mode 100644 datamap/internal/db/hub.go delete mode 100644 datamap/internal/db/hubrequest.go delete mode 100644 datamap/internal/db/object.go delete mode 100644 datamap/internal/db/storage.go delete mode 100644 datamap/internal/db/storagetransfercount.go create mode 100644 datamap/internal/handlers/handlers.go create mode 100644 datamap/internal/models/block.go create mode 100644 datamap/internal/models/hubs.go create mode 100644 datamap/internal/models/hubtrans.go create mode 100644 datamap/internal/models/object.go create mode 100644 datamap/internal/models/repository.go delete mode 100644 datamap/internal/mq/hub.go create mode 100644 datamap/internal/mq/mq.go delete mode 100644 datamap/internal/mq/service.go create mode 100644 datamap/internal/server/server.go diff --git a/client/internal/services/hub.go b/client/internal/services/hub.go index 50d68e7..c924b6b 100644 --- a/client/internal/services/hub.go +++ b/client/internal/services/hub.go @@ -2,12 +2,9 @@ package services import ( "fmt" - stgmod "gitlink.org.cn/cloudream/storage/common/models" - cdssdk "gitlink.org.cn/cloudream/common/sdks/storage" stgglb "gitlink.org.cn/cloudream/storage/common/globals" coormq "gitlink.org.cn/cloudream/storage/common/pkgs/mq/coordinator" - datamapmq "gitlink.org.cn/cloudream/storage/common/pkgs/mq/datamap" ) // HubService 是关于节点操作的服务结构体 @@ -47,22 +44,3 @@ func (svc *HubService) GetHubs(hubIDs []cdssdk.HubID) ([]cdssdk.Hub, error) { // 返回获取到的节点信息 return getResp.Hubs, nil } - -// GetHubStat 根据提供的节点ID列表,获取中心的总数据量 - -func (svc *HubService) GetHubStat(hubID cdssdk.HubID) (stgmod.HubStat, error) { - datamapCli, err := stgglb.DatamapMQPool.Acquire() - if err != nil { - return stgmod.HubStat{}, fmt.Errorf("new datamap client: %w", err) - } - defer stgglb.DatamapMQPool.Release(datamapCli) - - getResp, err := datamapCli.GetHubStat(datamapmq.NewGetHubStat(hubID)) - if err != nil { - return stgmod.HubStat{}, fmt.Errorf("requesting to datamap: %w", err) - } - - return getResp.HubStat, nil -} - -// diff --git a/common/globals/pools.go b/common/globals/pools.go index d30a1bc..cb4ab84 100644 --- a/common/globals/pools.go +++ b/common/globals/pools.go @@ -5,7 +5,6 @@ import ( stgmq "gitlink.org.cn/cloudream/storage/common/pkgs/mq" agtmq "gitlink.org.cn/cloudream/storage/common/pkgs/mq/agent" coormq "gitlink.org.cn/cloudream/storage/common/pkgs/mq/coordinator" - datamap "gitlink.org.cn/cloudream/storage/common/pkgs/mq/datamap" scmq "gitlink.org.cn/cloudream/storage/common/pkgs/mq/scanner" ) @@ -15,8 +14,6 @@ var CoordinatorMQPool coormq.Pool var ScannerMQPool scmq.Pool -var DatamapMQPool datamap.Pool - // InitMQPool // // @Description: 初始化MQ连接池 @@ -28,7 +25,6 @@ func InitMQPool(cfg *stgmq.Config) { ScannerMQPool = scmq.NewPool(cfg) - DatamapMQPool = datamap.NewPool(cfg) } var AgentRPCPool *agtrpc.Pool diff --git a/common/models/datamap.go b/common/models/datamap.go index f7a2492..581e8e5 100644 --- a/common/models/datamap.go +++ b/common/models/datamap.go @@ -2,46 +2,46 @@ package stgmod import "time" -// HubStat 中心信息 -// 每天一次，各节点统计自身当前的总数据量 +// HubStat 中心信息 每天一次，各节点统计自身当前的总数据量 + +type Source struct { + Type string `json:"type"` + HubID string `json:"hubID"` + HubName string `json:"hubName"` +} +type HubStatBody struct { + MasterHubID int `json:"masterHubID"` + MasterHubAddress string `json:"masterHubAddress"` + StorageID int `json:"storageID"` + DataCount int `json:"dataCount"` +} type HubStat struct { - Timestamp time.Time `json:"timestamp"` - Source struct { - Type string `json:"type"` - HubID string `json:"hubID"` - HubName string `json:"hubName"` - } `json:"source"` - Category string `json:"category"` - Body struct { - MasterHubID int `json:"masterHubID"` - MasterHubAddress string `json:"masterHubAddress"` - StorageID int `json:"storageID"` - DataCount int `json:"dataCount"` - } `json:"body"` + Timestamp time.Time `json:"timestamp"` + Source Source `json:"source"` + Category string `json:"category"` + Body HubStatBody `json:"body"` } // HubTrans 节点传输信息 // 每天一次，各节点统计自身当天向外部各个节点传输的总数据量 + +type HubTransBody struct { + SourceHubID int64 `json:"sourceHubID"` + TargetHubID int64 `json:"targetHubID"` + DataTransferCount int64 `json:"dataTransferCount"` + RequestCount int64 `json:"requestCount"` + FailedRequestCount int64 `json:"failedRequestCount"` + AvgTransferCount int64 `json:"avgTransferCount"` + MaxTransferCount int64 `json:"maxTransferCount"` + MinTransferCount int64 `json:"minTransferCount"` + StartTimestamp time.Time `json:"startTimestamp"` + EndTimestamp time.Time `json:"endTimestamp"` +} type HubTrans struct { - Timestamp time.Time `json:"timestamp"` - Source struct { - Type string `json:"type"` - HubID string `json:"hubID"` - HubName string `json:"hubName"` - } `json:"source"` - Category string `json:"category"` - Body struct { - SourceHubID int `json:"sourceHubID"` - TargetHubID int `json:"targetHubID"` - DataTransferCount int `json:"dataTransferCount"` - RequestCount int `json:"requestCount"` - FailedRequestCount int `json:"failedRequestCount"` - AvgTransferCount int `json:"avgTransferCount"` - MaxTransferCount int `json:"maxTransferCount"` - MinTransferCount int `json:"minTransferCount"` - StartTimestamp time.Time `json:"startTimestamp"` - EndTimestamp time.Time `json:"endTimestamp"` - } `json:"body"` + Timestamp time.Time `json:"timestamp"` + Source Source `json:"source"` + Category string `json:"category"` + Body HubTransBody `json:"body"` } // BlockTransInfo 块传输信息 @@ -50,151 +50,129 @@ type HubTrans struct { 编解码(一变多、多变一、多变多) 删除 更新*/ +type Block struct { + BlockType string `json:"blockType"` + Index string `json:"index"` + StorageID string `json:"storageID"` +} +type DataTransfer struct { + SourceStorageID string `json:"sourceStorageID"` + TargetStorageID string `json:"targetStorageID"` + DataTransferCount string `json:"dataTransferCount"` +} + +type BlockChange struct { + Type string `json:"type"` + BlockType string `json:"blockType"` + Index string `json:"index"` + SourceStorageID string `json:"sourceStorageID"` + TargetStorageID string `json:"targetStorageID"` + DataTransferCount string `json:"dataTransferCount"` + Timestamp string `json:"timestamp"` + SourceBlocks []Block `json:"sourceBlocks,omitempty"` + TargetBlocks []Block `json:"targetBlocks,omitempty"` + DataTransfers []DataTransfer `json:"dataTransfers,omitempty"` + Blocks []Block `json:"blocks,omitempty"` +} + +type BlockTransInfoBody struct { + Type string `json:"type"` + ObjectID string `json:"objectID"` + PackageID string `json:"packageID"` + BlockChanges []BlockChange `json:"blockChanges"` +} + type BlockTransInfo struct { - Timestamp time.Time `json:"timestamp"` - Source struct { - Type string `json:"type"` - HubID interface{} `json:"hubID"` - HubName interface{} `json:"hubName"` - } `json:"source"` - Category string `json:"category"` - Body struct { - Type string `json:"type"` - ObjectID interface{} `json:"objectID"` - PackageID interface{} `json:"packageID"` - BlockChanges []struct { - Type string `json:"type"` - BlockType interface{} `json:"blockType"` - Index interface{} `json:"index"` - SourceStorageID interface{} `json:"sourceStorageID"` - TargetStorageID interface{} `json:"targetStorageID"` - DataTransferCount interface{} `json:"dataTransferCount"` - Timestamp interface{} `json:"timestamp"` - SourceBlocks []struct { - BlockType interface{} `json:"blockType"` - Index interface{} `json:"index"` - StorageID interface{} `json:"storageID"` - } `json:"sourceBlocks,omitempty"` - TargetBlocks []struct { - BlockType interface{} `json:"blockType"` - Index interface{} `json:"index"` - StorageID interface{} `json:"storageID"` - } `json:"targetBlocks,omitempty"` - DataTransfers []struct { - SourceStorageID interface{} `json:"sourceStorageID"` - TargetStorageID interface{} `json:"targetStorageID"` - DataTransferCount interface{} `json:"dataTransferCount"` - } `json:"dataTransfers,omitempty"` - Blocks []struct { - BlockType interface{} `json:"blockType"` - Index interface{} `json:"index"` - StorageID interface{} `json:"storageID"` - } `json:"blocks,omitempty"` - } `json:"blockChanges"` - } `json:"body"` + Timestamp time.Time `json:"timestamp"` + Source Source `json:"source"` + Category string `json:"category"` + Body BlockTransInfoBody `json:"body"` } // BlockDistribution 块传输信息 // 每天一次，在调整完成后，将当天调整前后的布局情况一起推送 type BlockDistribution struct { - Timestamp time.Time `json:"timestamp"` - Source struct { - Type string `json:"type"` - HubID interface{} `json:"hubID"` - HubName interface{} `json:"hubName"` - } `json:"source"` - Category string `json:"category"` - Body struct { - Type string `json:"type"` - ObjectID interface{} `json:"objectID"` - PackageID interface{} `json:"packageID"` - BlockChanges []struct { - Type string `json:"type"` - BlockType interface{} `json:"blockType"` - Index interface{} `json:"index"` - SourceStorageID interface{} `json:"sourceStorageID"` - TargetStorageID interface{} `json:"targetStorageID"` - DataTransferCount interface{} `json:"dataTransferCount"` - Timestamp interface{} `json:"timestamp"` - SourceBlocks []struct { - BlockType interface{} `json:"blockType"` - Index interface{} `json:"index"` - StorageID interface{} `json:"storageID"` - } `json:"sourceBlocks,omitempty"` - TargetBlocks []struct { - BlockType interface{} `json:"blockType"` - Index interface{} `json:"index"` - StorageID interface{} `json:"storageID"` - } `json:"targetBlocks,omitempty"` - DataTransfers []struct { - SourceStorageID interface{} `json:"sourceStorageID"` - TargetStorageID interface{} `json:"targetStorageID"` - DataTransferCount interface{} `json:"dataTransferCount"` - } `json:"dataTransfers,omitempty"` - Blocks []struct { - BlockType interface{} `json:"blockType"` - Index interface{} `json:"index"` - StorageID interface{} `json:"storageID"` - } `json:"blocks,omitempty"` - } `json:"blockChanges"` - } `json:"body"` + Type string `json:"type"` + Index string `json:"index"` + StorageID string `json:"storageID"` +} + +type Object struct { + ObjectID string `json:"objectID"` + PackageID string `json:"packageID"` + Path string `json:"path"` + Size int64 `json:"size"` + FileHash string `json:"fileHash"` + FaultTolerance string `json:"faultTolerance"` + Redundancy string `json:"redundancy"` + AvgAccessCost string `json:"avgAccessCost"` + BlockDistribution []BlockDistribution `json:"blockDistribution"` + DataTransfers []DataTransfer `json:"dataTransfers"` +} +type BlockDistributionBody struct { + Timestamp time.Time `json:"timestamp"` + Type string `json:"type"` + ObjectID string `json:"objectID"` + PackageID string `json:"packageID"` + SourceBlocks []Block `json:"sourceBlocks,omitempty"` + TargetBlocks []Block `json:"targetBlocks,omitempty"` + DataTransfers []DataTransfer `json:"dataTransfers,omitempty"` + Blocks []Block `json:"blocks,omitempty"` +} + +type BlockDistributionInfo struct { + Timestamp time.Time `json:"timestamp"` + Source Source `json:"source"` + Category string `json:"category"` + Body BlockDistributionBody `json:"body"` } // ObjectUpdateInfo Object变化信息 + +type ObjectUpdateInfoBody struct { + Type string `json:"type"` + ObjectID string `json:"objectID"` + PackageID string `json:"packageID"` + Path string `json:"path"` + Size int `json:"size"` + BlockDistribution []BlockDistribution `json:"blockDistribution"` + Timestamp string `json:"timestamp"` +} type ObjectUpdateInfo struct { - Timestamp time.Time `json:"timestamp"` - Source struct { - Type string `json:"type"` - HubID string `json:"hubID"` - HubName string `json:"hubName"` - } `json:"source"` - Category string `json:"category"` - Body struct { - Type string `json:"type"` - ObjectID string `json:"objectID"` - PackageID string `json:"packageID"` - Path string `json:"path"` - Size int `json:"size"` - BlockDistribution []struct { - Type string `json:"type,omitempty"` - Index string `json:"index,omitempty"` - StorageID string `json:"storageID,omitempty"` - } `json:"blockDistribution"` - Timestamp string `json:"timestamp"` - } `json:"body"` + Timestamp time.Time `json:"timestamp"` + Source Source `json:"source"` + Category string `json:"category"` + Body ObjectUpdateInfoBody `json:"body"` } // PackageUpdateInfo package变化信息 + +type PackageUpdateInfoBody struct { + Type string `json:"type"` + PackageID string `json:"packageID"` + PackageName string `json:"packageName"` + BucketID string `json:"bucketID"` + Timestamp string `json:"timestamp"` +} type PackageUpdateInfo struct { - Timestamp time.Time `json:"timestamp"` - Source struct { - Type string `json:"type"` - HubID string `json:"hubID"` - HubName string `json:"hubName"` - } `json:"source"` - Category string `json:"category"` - Body struct { - Type string `json:"type"` - PackageID string `json:"packageID"` - PackageName string `json:"packageName"` - BucketID string `json:"bucketID"` - Timestamp string `json:"timestamp"` - } `json:"body"` + Timestamp time.Time `json:"timestamp"` + Source Source `json:"source"` + Category string `json:"category"` + Body PackageUpdateInfoBody `json:"body"` } // BucketUpdateInfo bucket变化信息 + +type BucketUpdateInfoBody struct { + Type string `json:"type"` + BucketID string `json:"bucketID"` + BucketName string `json:"bucketName"` + Timestamp string `json:"timestamp"` +} + type BucketUpdateInfo struct { - Timestamp time.Time `json:"timestamp"` - Source struct { - Type string `json:"type"` - HubID string `json:"hubID"` - HubName string `json:"hubName"` - } `json:"source"` - Category string `json:"category"` - Body struct { - Type string `json:"type"` - BucketID string `json:"bucketID"` - BucketName string `json:"bucketName"` - Timestamp string `json:"timestamp"` - } `json:"body"` + Timestamp time.Time `json:"timestamp"` + Source Source `json:"source"` + Category string `json:"category"` + Body BucketUpdateInfoBody `json:"body"` } diff --git a/common/pkgs/mq/datamap/client.go b/common/pkgs/mq/datamap/client.go deleted file mode 100644 index e388e22..0000000 --- a/common/pkgs/mq/datamap/client.go +++ /dev/null @@ -1,60 +0,0 @@ -package datamap - -import ( - "sync" - - "gitlink.org.cn/cloudream/common/pkgs/mq" - stgmq "gitlink.org.cn/cloudream/storage/common/pkgs/mq" -) - -type Client struct { - rabbitCli *mq.RabbitMQTransport -} - -func NewClient(cfg *stgmq.Config) (*Client, error) { - rabbitCli, err := mq.NewRabbitMQTransport(cfg.MakeConnectingURL(), stgmq.DATAMAP_QUEUE_NAME, "") - if err != nil { - return nil, err - } - - return &Client{ - rabbitCli: rabbitCli, - }, nil -} - -func (c *Client) Close() { - c.rabbitCli.Close() -} - -type Pool interface { - Acquire() (*Client, error) - Release(cli *Client) -} - -type pool struct { - mqcfg *stgmq.Config - shared *Client - lock sync.Mutex -} - -func NewPool(mqcfg *stgmq.Config) Pool { - return &pool{ - mqcfg: mqcfg, - } -} -func (p *pool) Acquire() (*Client, error) { - p.lock.Lock() - defer p.lock.Unlock() - if p.shared == nil { - var err error - p.shared, err = NewClient(p.mqcfg) - if err != nil { - return nil, err - } - } - - return p.shared, nil -} - -func (p *pool) Release(cli *Client) { -} diff --git a/common/pkgs/mq/datamap/hub.go b/common/pkgs/mq/datamap/hub.go deleted file mode 100644 index 662ef78..0000000 --- a/common/pkgs/mq/datamap/hub.go +++ /dev/null @@ -1,57 +0,0 @@ -package datamap - -import ( - "gitlink.org.cn/cloudream/common/pkgs/mq" - cdssdk "gitlink.org.cn/cloudream/common/sdks/storage" - stgmod "gitlink.org.cn/cloudream/storage/common/models" -) - -type HubService interface { - GetHubStat(msg *GetHubStat) (*GetHubStatResp, *mq.CodeMessage) - GetHubTrans(msg *GetHubTrans) (*GetHubTransResp, *mq.CodeMessage) -} - -// 查询中心自身当前的总数据量 -var _ = Register(Service.GetHubStat) - -type GetHubStat struct { - mq.MessageBodyBase - HubID cdssdk.HubID `json:"hubID"` -} -type GetHubStatResp struct { - mq.MessageBodyBase - HubStat stgmod.HubStat `json:"hubStat"` -} - -func NewGetHubStat(hubID cdssdk.HubID) *GetHubStat { - return &GetHubStat{ - HubID: hubID, - } -} -func NewGetHubStatResp(hubStat stgmod.HubStat) *GetHubStatResp { - return &GetHubStatResp{ - HubStat: hubStat, - } -} -func (client *Client) GetHubStat(msg *GetHubStat) (*GetHubStatResp, error) { - return mq.Request(Service.GetHubStat, client.rabbitCli, msg) -} - -// 查询中心节点传输的总数据量 - -var _ = Register(Service.GetHubTrans) - -type GetHubTrans struct { - mq.MessageBodyBase - HubID cdssdk.HubID `json:"hubID"` -} - -type GetHubTransResp struct { - mq.MessageBodyBase - HubTrans stgmod.HubTrans `json:"hubTrans"` -} - -func (client *Client) GetHubTrans(msg *GetHubTrans) (*GetHubTransResp, error) { - //获取到传输统计之后 - return mq.Request(Service.GetHubTrans, client.rabbitCli, msg) -} diff --git a/common/pkgs/mq/datamap/object.go b/common/pkgs/mq/datamap/object.go deleted file mode 100644 index bdc26b9..0000000 --- a/common/pkgs/mq/datamap/object.go +++ /dev/null @@ -1,35 +0,0 @@ -package datamap - -import ( - "gitlink.org.cn/cloudream/common/pkgs/mq" - cdssdk "gitlink.org.cn/cloudream/common/sdks/storage" - stgmod "gitlink.org.cn/cloudream/storage/common/models" -) - -type ObjectService interface { - GetBlockTransInfo(msg *GetBlockTransInfo) (*GetBlockTransInfoResp, *mq.CodeMessage) - GetBlockDistribution(msg *GetBlockDistribution) (*GetBlockDistributionResp, *mq.CodeMessage) -} - -type GetBlockTransInfo struct { - mq.MessageBodyBase - //todo 入参待确认 - UserID cdssdk.UserID `json:"userID"` - PackageID cdssdk.PackageID `json:"packageID"` -} - -type GetBlockTransInfoResp struct { - mq.MessageBodyBase - ObjectTransInfo stgmod.BlockTransInfo `json:"objectTransInfo"` -} - -type GetBlockDistribution struct { - mq.MessageBodyBase - //todo 入参待确认 - HubID cdssdk.HubID `json:"hubID"` -} - -type GetBlockDistributionResp struct { - mq.MessageBodyBase - BlockDistribution stgmod.BlockDistribution `json:"blockDistribution"` -} diff --git a/common/pkgs/mq/datamap/server.go b/common/pkgs/mq/datamap/server.go deleted file mode 100644 index 9f93bd4..0000000 --- a/common/pkgs/mq/datamap/server.go +++ /dev/null @@ -1,72 +0,0 @@ -package datamap - -import ( - "gitlink.org.cn/cloudream/common/pkgs/mq" - "gitlink.org.cn/cloudream/common/utils/sync2" - mymq "gitlink.org.cn/cloudream/storage/common/pkgs/mq" -) - -type Service interface { - HubService - ObjectService -} - -type Server struct { - service Service - rabbitSvr mq.RabbitMQServer -} - -func NewServer(svc Service, cfg *mymq.Config) (*Server, error) { - srv := &Server{ - service: svc, - } - - rabbitSvr, err := mq.NewRabbitMQServer( - cfg.MakeConnectingURL(), - mymq.DATAMAP_QUEUE_NAME, - func(msg *mq.Message) (*mq.Message, error) { - return msgDispatcher.Handle(srv.service, msg) - }, - cfg.Param, - ) - if err != nil { - return nil, err - } - - srv.rabbitSvr = *rabbitSvr - - return srv, nil -} - -func (s *Server) Stop() { - s.rabbitSvr.Close() -} - -func (s *Server) Start(cfg mymq.Config) *sync2.UnboundChannel[mq.RabbitMQServerEvent] { - return s.rabbitSvr.Start() -} - -func (s *Server) OnError(callback func(error)) { - s.rabbitSvr.OnError = callback -} - -var msgDispatcher mq.MessageDispatcher = mq.NewMessageDispatcher() - -// Register 将Service中的一个接口函数作为指定类型消息的处理函数，同时会注册请求和响应的消息类型 -// TODO 需要约束：Service实现了TSvc接口 -func Register[TReq mq.MessageBody, TResp mq.MessageBody](svcFn func(svc Service, msg TReq) (TResp, *mq.CodeMessage)) any { - mq.AddServiceFn(&msgDispatcher, svcFn) - mq.RegisterMessage[TReq]() - mq.RegisterMessage[TResp]() - - return nil -} - -// RegisterNoReply 将Service中的一个*没有返回值的*接口函数作为指定类型消息的处理函数，同时会注册请求和响应的消息类型 -// TODO 需要约束：Service实现了TSvc接口 -func RegisterNoReply[TReq mq.MessageBody](svcFn func(svc Service, msg TReq)) any { - mq.AddNoRespServiceFn(&msgDispatcher, svcFn) - mq.RegisterMessage[TReq]() - - return nil -} diff --git a/datamap/internal/cmd/serve.go b/datamap/internal/cmd/serve.go deleted file mode 100644 index 26b6e80..0000000 --- a/datamap/internal/cmd/serve.go +++ /dev/null @@ -1,28 +0,0 @@ -package cmd - -import ( - "fmt" - "os" - - "gitlink.org.cn/cloudream/common/pkgs/logger" - "gitlink.org.cn/cloudream/storage/datamap/internal/config" -) - -func serve() { - err := config.Init() - if err != nil { - fmt.Printf("init config failed, err: %s", err.Error()) - os.Exit(1) - } - - err = logger.Init(&config.Cfg().Logger) - if err != nil { - fmt.Printf("init logger failed, err: %s", err.Error()) - os.Exit(1) - } - - // dataSvr, err := datamq.NewServer(mymq.NewService(db2), &config.Cfg().RabbitMQ) - // if err != nil { - // logger.Fatalf("new coordinator server failed, err: %s", err.Error()) - // } -} diff --git a/datamap/internal/config/config.go b/datamap/internal/config/config.go index c9ab9f2..521ea76 100644 --- a/datamap/internal/config/config.go +++ b/datamap/internal/config/config.go @@ -1,26 +1,54 @@ package config import ( - "gitlink.org.cn/cloudream/common/pkgs/distlock" - log "gitlink.org.cn/cloudream/common/pkgs/logger" - c "gitlink.org.cn/cloudream/common/utils/config" - db "gitlink.org.cn/cloudream/storage/common/pkgs/db2/config" - stgmq "gitlink.org.cn/cloudream/storage/common/pkgs/mq" + "github.com/spf13/viper" ) type Config struct { - Logger log.Config `json:"logger"` - DB db.Config `json:"db"` - RabbitMQ stgmq.Config `json:"rabbitMQ"` - DistLock distlock.Config `json:"distlock"` + Database DatabaseConfig + RabbitMQ RabbitMQConfig + Server ServerConfig } -var cfg Config +type DatabaseConfig struct { + Host string + Port string + User string + Password string + DBName string +} + +type RabbitMQConfig struct { + Host string + Port string + User string + Password string +} -func Init() error { - return c.DefaultLoad("datamap", &cfg) +type ServerConfig struct { + Port string } -func Cfg() *Config { - return &cfg +func LoadConfig() *Config { + viper.SetConfigFile(".env") + viper.ReadInConfig() + + return &Config{ + Database: DatabaseConfig{ + Host: viper.GetString("DB_HOST"), + Port: viper.GetString("DB_PORT"), + User: viper.GetString("DB_USER"), + Password: viper.GetString("DB_PASSWORD"), + DBName: viper.GetString("DB_NAME"), + }, + RabbitMQ: RabbitMQConfig{ + Host: viper.GetString("RABBITMQ_HOST"), + Port: viper.GetString("RABBITMQ_PORT"), + User: viper.GetString("RABBITMQ_USER"), + Password: viper.GetString("RABBITMQ_PASSWORD"), + }, + Server: ServerConfig{ + Port: viper.GetString("SERVER_PORT"), + }, + } } diff --git a/datamap/internal/db/blockdistribution.go b/datamap/internal/db/blockdistribution.go deleted file mode 100644 index cc7d434..0000000 --- a/datamap/internal/db/blockdistribution.go +++ /dev/null @@ -1,42 +0,0 @@ -package db - -import "gitlink.org.cn/cloudream/storage/datamap/internal/models" - -type BlockDistributionDB struct { - *DB -} - -func (db *DB) BlockDistribution() *BlockDistributionDB { - return &BlockDistributionDB{DB: db} -} - -// GetAllBlockDistribution 查询所有BlockDistribution列表 -func (*HubDB) GetAllBlockDistribution(ctx SQLContext) ([]models.BlockDistribution, error) { - var ret []models.BlockDistribution - - err := ctx.Table("blockdistribution").Find(&ret).Error - return ret, err -} - -// GetBlockDistribution 根据输入的BlockID查询BlockDistribution -func (*HubDB) GetBlockDistribution(ctx SQLContext, BlockID int64) (models.BlockDistribution, error) { - var ret models.BlockDistribution - err := ctx.Table("blockdistribution").Where("BlockID = ?", BlockID).Find(&ret).Error - return ret, err -} - -// CreateBlockDistribution 根据输入的BlockDistribution信息创建BlockDistribution记录 -func (*HubDB) CreateBlockDistribution(ctx SQLContext, blockDistribution models.BlockDistribution) (*models.BlockDistribution, error) { - err := ctx.Table("blockdistribution").Create(&blockDistribution).Error - return &blockDistribution, err -} - -// DeleteBlockDistribution 根据输入的BlockID删除BlockDistribution记录 -func (*HubDB) DeleteBlockDistribution(ctx SQLContext, BlockID int64) error { - return ctx.Table("blockdistribution").Where("BlockID = ?", BlockID).Delete(&models.BlockDistribution{}).Error -} - -// UpdateBlockDistribution 根据输入的BlockDistribution信息更新BlockDistribution记录 -func (*HubDB) UpdateBlockDistribution(ctx SQLContext, blockDistribution models.BlockDistribution) error { - return ctx.Table("blockdistribution").Where("BlockID = ?", blockDistribution.BlockID).Updates(&blockDistribution).Error -} diff --git a/datamap/internal/db/config/config.go b/datamap/internal/db/config/config.go deleted file mode 100644 index 9495b71..0000000 --- a/datamap/internal/db/config/config.go +++ /dev/null @@ -1,21 +0,0 @@ -package config - -import "fmt" - -type Config struct { - Address string `json:"address"` - Account string `json:"account"` - Password string `json:"password"` - DatabaseName string `json:"databaseName"` -} - -func (cfg *Config) MakeSourceString() string { - return fmt.Sprintf( - "%s:%s@tcp(%s)/%s?charset=utf8mb4&parseTime=true&loc=%s", - cfg.Account, - cfg.Password, - cfg.Address, - cfg.DatabaseName, - "Asia%2FShanghai", - ) -} diff --git a/datamap/internal/db/db.go b/datamap/internal/db/db.go index 99172d2..d279f64 100644 --- a/datamap/internal/db/db.go +++ b/datamap/internal/db/db.go @@ -1,37 +1,31 @@ package db import ( - "github.com/sirupsen/logrus" - "gitlink.org.cn/cloudream/storage/datamap/internal/db/config" + "fmt" + "gitlink.org.cn/cloudream/storage/datamap/internal/config" + "gitlink.org.cn/cloudream/storage/datamap/internal/models" "gorm.io/driver/mysql" "gorm.io/gorm" ) -type DB struct { - db *gorm.DB -} +// 全局数据库连接实例 +var DB *gorm.DB -func NewDB(cfg *config.Config) (*DB, error) { - mydb, err := gorm.Open(mysql.Open(cfg.MakeSourceString()), &gorm.Config{}) +func InitDB(cfg config.DatabaseConfig) (*gorm.DB, error) { + dsn := fmt.Sprintf("%s:%s@tcp(%s:%s)/%s?charset=utf8mb4&parseTime=True&loc=Local", + cfg.User, cfg.Password, cfg.Host, cfg.Port, cfg.DBName) + db, err := gorm.Open(mysql.Open(dsn), &gorm.Config{}) if err != nil { - logrus.Fatalf("failed to connect to database: %v", err) + return nil, err } - return &DB{ - db: mydb, - }, nil -} - -func (db *DB) DoTx(do func(tx SQLContext) error) error { - return db.db.Transaction(func(tx *gorm.DB) error { - return do(SQLContext{tx}) - }) -} - -type SQLContext struct { - *gorm.DB -} + // 自动迁移表结构 + db.AutoMigrate( + &models.Hub{}, + &models.Storage{}, + &models.HubRequest{}, + &models.BlockDistribution{}, + ) -func (db *DB) DefCtx() SQLContext { - return SQLContext{db.db} + return db, nil } diff --git a/datamap/internal/db/hub.go b/datamap/internal/db/hub.go deleted file mode 100644 index 78f783f..0000000 --- a/datamap/internal/db/hub.go +++ /dev/null @@ -1,52 +0,0 @@ -package db - -import ( - "gitlink.org.cn/cloudream/storage/datamap/internal/models" -) - -type HubDB struct { - *DB -} - -func (db *DB) Hub() *HubDB { - return &HubDB{DB: db} -} - -// GetHubs 获取所有hub列表 -func (*HubDB) GetHubs(ctx SQLContext) ([]models.Hub, error) { - var ret []models.Hub - err := ctx.Table("hub").Find(&ret).Error - return ret, err -} - -// GetHub 根据hubId获取该Hub的信息 -func (*HubDB) GetHub(ctx SQLContext, hubID int64) (*models.Hub, error) { - var hub models.Hub - err := ctx.Table("hub").Where("HubID = ?", hubID).Find(&hub).Error - return &hub, err -} - -// DeleteHub 根据hubId删除该Hub -func (*HubDB) DeleteHub(ctx SQLContext, hubID int64) error { - err := ctx.Table("hub").Where("HubID = ?", hubID).Delete(&models.Hub{}).Error - return err -} - -// UpdateHub 根据输入hub信息更新Hub信息 -func (*HubDB) UpdateHub(ctx SQLContext, hub models.Hub) (*models.Hub, error) { - err := ctx.Table("hub").Where("HubID = ?", hub.HubID).Updates(&hub).Error - return &hub, err -} - -// CreateHub 根据输入hub信息创建Hub信息 -func (*HubDB) CreateHub(ctx SQLContext, hub models.Hub) (*models.Hub, error) { - err := ctx.Table("hub").Create(&hub).Error - return &hub, err -} - -// IsHubExist 根据hubId查询hub是否存在 -func (*HubDB) IsHubExist(ctx SQLContext, hubID int64) (bool, error) { - var count int64 - err := ctx.Table("hub").Where("HubID = ?", hubID).Count(&count).Error - return count > 0, err -} diff --git a/datamap/internal/db/hubrequest.go b/datamap/internal/db/hubrequest.go deleted file mode 100644 index f99479b..0000000 --- a/datamap/internal/db/hubrequest.go +++ /dev/null @@ -1,24 +0,0 @@ -package db - -import "gitlink.org.cn/cloudream/storage/datamap/internal/models" - -type HubReqDB struct { - *DB -} - -func (db *DB) HubReq() *HubReqDB { - return &HubReqDB{DB: db} -} - -// GetHubRequest 获取所有hubrequest列表 -func (*HubReqDB) GetHubRequest(ctx SQLContext) ([]models.HubRequest, error) { - var ret []models.HubRequest - err := ctx.Table("hubrequest").Find(&ret).Error - return ret, err -} - -// CreateHubRequest 根据输入的HubRequest信息创建HubRequest信息 -func (*HubReqDB) CreateHubRequest(ctx SQLContext, hubRequest models.HubRequest) (*models.HubRequest, error) { - err := ctx.Table("hubrequest").Create(&hubRequest).Error - return &hubRequest, err -} diff --git a/datamap/internal/db/object.go b/datamap/internal/db/object.go deleted file mode 100644 index 1f700eb..0000000 --- a/datamap/internal/db/object.go +++ /dev/null @@ -1,36 +0,0 @@ -package db - -import "gitlink.org.cn/cloudream/storage/datamap/internal/models" - -type ObjectDB struct { - *DB -} - -func (db *DB) Object() *ObjectDB { - return &ObjectDB{DB: db} -} - -// GetAllObject 查询所有Object列表 -func (*ObjectDB) GetAllObject(ctx SQLContext) ([]models.Object, error) { - var ret []models.Object - - err := ctx.Table("object").Find(&ret).Error - return ret, err -} - -// GetObject 根据输入的ObjectId查询Object -func (*ObjectDB) GetObject(ctx SQLContext, objectId int64) (models.Object, error) { - var ret models.Object - err := ctx.Table("object").Where("ObjectID = ?", objectId).Find(&ret).Error - return ret, err -} - -// DeleteObject 根据输入的ObjectId删除Object -func (*ObjectDB) DeleteObject(ctx SQLContext, objectId int64) error { - return ctx.Table("object").Where("ObjectID = ?", objectId).Delete(&models.Object{}).Error -} - -// UpdateObject 根据输入的Object信息更新Object -func (*ObjectDB) UpdateObject(ctx SQLContext, object models.Object) error { - return ctx.Table("object").Where("ObjectID = ?", object.ObjectID).Updates(&object).Error -} diff --git a/datamap/internal/db/storage.go b/datamap/internal/db/storage.go deleted file mode 100644 index 4076374..0000000 --- a/datamap/internal/db/storage.go +++ /dev/null @@ -1,36 +0,0 @@ -package db - -import "gitlink.org.cn/cloudream/storage/datamap/internal/models" - -type StorageDB struct { - *DB -} - -func (db *DB) Storage() *StorageDB { - return &StorageDB{DB: db} -} - -// GetAllStorage 查询所有Storage列表 -func (*HubDB) GetAllStorage(ctx SQLContext) ([]models.Storage, error) { - var ret []models.Storage - - err := ctx.Table("storage").Find(&ret).Error - return ret, err -} - -// GetStorage 根据输入的StorageId查询Storage -func (*HubDB) GetStorage(ctx SQLContext, storageId int64) (models.Storage, error) { - var ret models.Storage - err := ctx.Table("storage").Where("StorageID = ?", storageId).Find(&ret).Error - return ret, err -} - -// DeleteStorage 根据输入的StorageId删除Storage -func (*HubDB) DeleteStorage(ctx SQLContext, storageId int64) error { - return ctx.Table("storage").Where("StorageID = ?", storageId).Delete(&models.Storage{}).Error -} - -// UpdateStorage 根据输入的Storage信息更新Storage -func (*HubDB) UpdateStorage(ctx SQLContext, storage models.Storage) error { - return ctx.Table("storage").Where("StorageID = ?", storage.StorageID).Updates(&storage).Error -} diff --git a/datamap/internal/db/storagetransfercount.go b/datamap/internal/db/storagetransfercount.go deleted file mode 100644 index f53c389..0000000 --- a/datamap/internal/db/storagetransfercount.go +++ /dev/null @@ -1,36 +0,0 @@ -package db - -import "gitlink.org.cn/cloudream/storage/datamap/internal/models" - -type StorageTransferCountDB struct { - *DB -} - -func (db *DB) StorageTransferCount() *StorageTransferCountDB { - return &StorageTransferCountDB{DB: db} -} - -// GetAllStorageTransferCount 查询所有Storage列表 -func (*HubDB) GetAllStorageTransferCount(ctx SQLContext) ([]models.StorageTransferCount, error) { - var ret []models.StorageTransferCount - - err := ctx.Table("storagetransfercount").Find(&ret).Error - return ret, err -} - -// GetStorageTransferCount 根据输入的RelationshipID查询StorageTransferCount -func (*HubDB) GetStorageTransferCount(ctx SQLContext, RelationshipID int64) (models.Storage, error) { - var ret models.Storage - err := ctx.Table("storagetransfercount").Where("RelationshipID = ?", RelationshipID).Find(&ret).Error - return ret, err -} - -// DeleteStorageTransferCount 根据输入的RelationshipID删除StorageTransferCount -func (*HubDB) DeleteStorageTransferCount(ctx SQLContext, RelationshipID int64) error { - return ctx.Table("storagetransfercount").Where("RelationshipID = ?", RelationshipID).Delete(&models.Storage{}).Error -} - -// UpdateStorageTransferCount 根据输入的StorageTransferCount信息更新StorageTransferCount -func (*HubDB) UpdateStorageTransferCount(ctx SQLContext, storageTransferCount models.StorageTransferCount) error { - return ctx.Table("storagetransfercount").Where("RelationshipID = ?", storageTransferCount.RelationshipID).Updates(&storageTransferCount).Error -} diff --git a/datamap/internal/handlers/handlers.go b/datamap/internal/handlers/handlers.go new file mode 100644 index 0000000..2b909af --- /dev/null +++ b/datamap/internal/handlers/handlers.go @@ -0,0 +1,30 @@ +package handlers + +import ( + "github.com/gin-gonic/gin" + "gitlink.org.cn/cloudream/storage/datamap/internal/db" + "gitlink.org.cn/cloudream/storage/datamap/internal/models" + "net/http" +) + +// GetStorageData 获取 Storage 表中的全部数据 +func GetStorageData(c *gin.Context) { + repo := models.NewStorageRepository(db.DB) + storages, err := repo.GetAllStorages() + if err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to fetch storage data"}) + return + } + c.JSON(http.StatusOK, storages) +} + +// GetBlockDistributionData 获取 BlockDistribution 表中的全部数据 +func GetBlockDistributionData(c *gin.Context) { + repo := models.NewBlockDistributionRepository(db.DB) + blocks, err := repo.GetAllBlocks() + if err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to fetch block distribution data"}) + return + } + c.JSON(http.StatusOK, blocks) +} diff --git a/datamap/internal/models/block.go b/datamap/internal/models/block.go new file mode 100644 index 0000000..9c15b5c --- /dev/null +++ b/datamap/internal/models/block.go @@ -0,0 +1,57 @@ +package models + +import ( + cdssdk "gitlink.org.cn/cloudream/common/sdks/storage" + stgmod "gitlink.org.cn/cloudream/storage/common/models" + "gorm.io/gorm" + "strconv" + "time" +) + +type BlockDistribution struct { + BlockID int64 `gorm:"column:BlockID; primaryKey; type:bigint; autoIncrement" json:"blockID"` + ObjectID cdssdk.ObjectID `gorm:"column:ObjectID; type:bigint; not null" json:"objectID"` + Type string `gorm:"column:Type; type:varchar(1024); not null" json:"type"` + Index int64 `gorm:"column:Index; type:bigint; not null" json:"index"` + StorageID cdssdk.StorageID `gorm:"column:StorageID; type:bigint; not null" json:"storageID"` + Status Status `gorm:"column:Status; type:tinyint; not null" json:"status"` + Timestamp time.Time `gorm:"column:Timestamp; type:datatime; not null" json:"timestamp"` +} + +type BlockDistributionRepository struct { + repo *GormRepository +} + +func NewBlockDistributionRepository(db *gorm.DB) *BlockDistributionRepository { + return &BlockDistributionRepository{repo: NewGormRepository(db)} +} + +func (r *BlockDistributionRepository) CreateBlockDistribution(block *BlockDistribution) error { + return r.repo.Create(block) +} + +func (r *BlockDistributionRepository) GetAllBlocks() ([]BlockDistribution, error) { + var blocks []BlockDistribution + err := r.repo.GetAll(&blocks) + if err != nil { + return nil, err + } + return blocks, nil +} + +func ProcessBlockDistributionInfo(data stgmod.BlockTransInfo) { + repo := NewBlockDistributionRepository(db) + + for _, change := range data.Body.BlockChanges { + objectID, _ := strconv.ParseInt(data.Body.ObjectID, 10, 64) + index, _ := strconv.ParseInt(change.Index, 10, 64) + targetStorageID, _ := strconv.ParseInt(change.TargetStorageID, 10, 64) + block := &BlockDistribution{ + ObjectID: cdssdk.ObjectID(objectID), + Type: change.Type, + Index: index, + StorageID: cdssdk.StorageID(targetStorageID), + } + repo.CreateBlockDistribution(block) + } +} diff --git a/datamap/internal/models/hubs.go b/datamap/internal/models/hubs.go new file mode 100644 index 0000000..a28cb9b --- /dev/null +++ b/datamap/internal/models/hubs.go @@ -0,0 +1,93 @@ +package models + +import ( + "errors" + cdssdk "gitlink.org.cn/cloudream/common/sdks/storage" + stgmod "gitlink.org.cn/cloudream/storage/common/models" + "gorm.io/gorm" + "log" + "time" +) + +var db *gorm.DB + +func InitDB(gormDB *gorm.DB) { + db = gormDB +} + +type Hub struct { + HubID cdssdk.HubID `gorm:"column:HubID; primaryKey; type:bigint; autoIncrement" json:"hubID"` + Name string `gorm:"column:Name; type:varchar(255); not null" json:"name"` + Address cdssdk.HubAddressInfo `gorm:"column:Address; type:json; serializer:union" json:"address"` +} + +type HubStat struct { + StorageID int `json:"storageID"` + DataCount int `json:"dataCount"` +} + +type Storage struct { + StorageID cdssdk.StorageID `gorm:"column:StorageID; primaryKey; type:bigint; autoIncrement" json:"storageID"` + HubID cdssdk.HubID `gorm:"column:HubID; type:bigint; not null" json:"hubID"` + DataCount int64 `gorm:"column:DataCount; type:bigint; not null" json:"dataCount"` + NewDataCount int64 `gorm:"column:NewDataCount; type:bigint; not null" json:"newDataCount"` + Timestamp time.Time `gorm:"column:Timestamp; type:datatime; not null" json:"timestamp"` +} +type StorageRepository struct { + repo *GormRepository +} + +func NewStorageRepository(db *gorm.DB) *StorageRepository { + return &StorageRepository{repo: NewGormRepository(db)} +} + +func (r *StorageRepository) CreateStorage(storage *Storage) error { + return r.repo.Create(storage) +} + +func (r *StorageRepository) UpdateStorage(storage *Storage) error { + return r.repo.Update(storage) +} + +func (r *StorageRepository) GetStorageByID(id int) (*Storage, error) { + var storage Storage + err := r.repo.GetByID(uint(id), &storage) + if err != nil { + return nil, err + } + return &storage, nil +} + +func (r *StorageRepository) GetAllStorages() ([]Storage, error) { + var storages []Storage + err := r.repo.GetAll(&storages) + if err != nil { + return nil, err + } + return storages, nil +} + +func ProcessHubStat(data stgmod.HubStat) { + repo := NewStorageRepository(db) + + storage, err := repo.GetStorageByID(data.Body.StorageID) + if err != nil { + if errors.Is(err, gorm.ErrRecordNotFound) { + // 插入新记录 + newStorage := &Storage{ + StorageID: cdssdk.StorageID(data.Body.StorageID), + DataCount: int64(data.Body.DataCount), + NewDataCount: 0, + } + repo.CreateStorage(newStorage) + } else { + log.Printf("Error querying storage: %v", err) + } + } else { + // 更新记录 + newDataCount := int64(data.Body.DataCount) - storage.DataCount + storage.DataCount = int64(data.Body.DataCount) + storage.NewDataCount = newDataCount + repo.UpdateStorage(storage) + } +} diff --git a/datamap/internal/models/hubtrans.go b/datamap/internal/models/hubtrans.go new file mode 100644 index 0000000..d209fc8 --- /dev/null +++ b/datamap/internal/models/hubtrans.go @@ -0,0 +1,53 @@ +package models + +import ( + cdssdk "gitlink.org.cn/cloudream/common/sdks/storage" + stgmod "gitlink.org.cn/cloudream/storage/common/models" + "gorm.io/gorm" + "time" +) + +type HubRequest struct { + RequestID int64 `gorm:"column:HubID; primaryKey; type:bigint; autoIncrement" json:"hubID"` + SourceHubID cdssdk.HubID `gorm:"column:SourceHubID; type:bigint; not null" json:"sourceHubID"` + TargetHubID cdssdk.HubID `gorm:"column:TargetHubID; type:bigint; not null" json:"targetHubID"` + DataTransfer int64 `gorm:"column:DataTransfer; type:bigint; not null" json:"dataTransfer"` + RequestCount int64 `gorm:"column:RequestCount; type:bigint; not null" json:"requestCount"` + FailedRequest int64 `gorm:"column:FailedRequest; type:bigint; not null" json:"failedRequest"` + AvgTransferCount int64 `gorm:"column:AvgTransferCount; type:bigint; not null" json:"avgTransferCount"` + MaxTransferCount int64 `gorm:"column:MaxTransferCount; type:bigint; not null" json:"maxTransferCount"` + MinTransferCount int64 `gorm:"column:MinTransferCount; type:bigint; not null" json:"minTransferCount"` + StartTimestamp time.Time `gorm:"column:StartTimestamp; type:datatime; not null" json:"startTimestamp"` + EndTimestamp time.Time `gorm:"column:EndTimestamp; type:datatime; not null" json:"endTimestamp"` +} + +type HubRequestRepository struct { + repo *GormRepository +} + +func NewHubRequestRepository(db *gorm.DB) *HubRequestRepository { + return &HubRequestRepository{repo: NewGormRepository(db)} +} + +func (r *HubRequestRepository) CreateHubRequest(request *HubRequest) error { + return r.repo.Create(request) +} + +func ProcessHubTrans(data stgmod.HubTrans) { + repo := NewHubRequestRepository(db) + + hubRequest := &HubRequest{ + SourceHubID: cdssdk.HubID(data.Body.SourceHubID), + TargetHubID: cdssdk.HubID(data.Body.TargetHubID), + DataTransfer: data.Body.DataTransferCount, + RequestCount: data.Body.RequestCount, + FailedRequest: data.Body.FailedRequestCount, + AvgTransferCount: data.Body.AvgTransferCount, + MaxTransferCount: data.Body.MaxTransferCount, + MinTransferCount: data.Body.MinTransferCount, + StartTimestamp: data.Body.StartTimestamp, + EndTimestamp: data.Body.EndTimestamp, + } + + repo.CreateHubRequest(hubRequest) +} diff --git a/datamap/internal/models/models.go b/datamap/internal/models/models.go index 167edab..2c239e9 100644 --- a/datamap/internal/models/models.go +++ b/datamap/internal/models/models.go @@ -1,9 +1,8 @@ package models import ( - "time" - cdssdk "gitlink.org.cn/cloudream/common/sdks/storage" + "time" ) type RequestID int64 @@ -92,73 +91,14 @@ type Relationship struct { //数据库结构定义 -type Hub struct { - HubID cdssdk.HubID `gorm:"column:HubID; primaryKey; type:bigint; autoIncrement" json:"hubID"` - Name string `gorm:"column:Name; type:varchar(255); not null" json:"name"` - Address cdssdk.HubAddressInfo `gorm:"column:Address; type:json; serializer:union" json:"address"` -} - -func (Hub) TableName() string { - return "Hub" -} - -type Storage struct { - StorageID cdssdk.StorageID `gorm:"column:StorageID; primaryKey; type:bigint; autoIncrement" json:"storageID"` - HubID cdssdk.HubID `gorm:"column:HubID; type:bigint; not null" json:"hubID"` - DataCount int64 `gorm:"column:DataCount; type:bigint; not null" json:"dataCount"` - NewDataCount int64 `gorm:"column:NewDataCount; type:bigint; not null" json:"newDataCount"` - Timestamp time.Time `gorm:"column:Timestamp; type:datatime; not null" json:"timestamp"` -} - -func (Storage) TableName() string { - return "Storage" -} - -type HubRequest struct { - RequestID int64 `gorm:"column:HubID; primaryKey; type:bigint; autoIncrement" json:"hubID"` - SourceHubID cdssdk.HubID `gorm:"column:SourceHubID; type:bigint; not null" json:"sourceHubID"` - TargetHubID cdssdk.HubID `gorm:"column:TargetHubID; type:bigint; not null" json:"targetHubID"` - DataTransfer int64 `gorm:"column:DataTransfer; type:bigint; not null" json:"dataTransfer"` - RequestCount int64 `gorm:"column:RequestCount; type:bigint; not null" json:"requestCount"` - FailedRequest int64 `gorm:"column:FailedRequest; type:bigint; not null" json:"failedRequest"` - AvgTransferCount int64 `gorm:"column:AvgTransferCount; type:bigint; not null" json:"avgTransferCount"` - MaxTransferCount int64 `gorm:"column:MaxTransferCount; type:bigint; not null" json:"maxTransferCount"` - MinTransferCount int64 `gorm:"column:MinTransferCount; type:bigint; not null" json:"minTransferCount"` - StartTimestamp time.Time `gorm:"column:StartTimestamp; type:datatime; not null" json:"startTimestamp"` - EndTimestamp time.Time `gorm:"column:EndTimestamp; type:datatime; not null" json:"endTimestamp"` -} - func (HubRequest) TableName() string { return "HubRequest" } -type Object struct { - ObjectID cdssdk.ObjectID `gorm:"column:ObjectID; primaryKey; type:bigint; autoIncrement" json:"objectID"` - PackageID cdssdk.PackageID `gorm:"column:PackageID; type:bigint; not null" json:"packageID"` - Path string `gorm:"column:Path; type:varchar(1024); not null" json:"path"` - Size int64 `gorm:"column:Size; type:bigint; not null" json:"size"` - FileHash string `gorm:"column:FileHash; type:varchar(255); not null" json:"fileHash"` - Status Status `gorm:"column:Status; type:tinyint; not null" json:"status"` - FaultTolerance float64 `gorm:"column:faultTolerance; type:float; not null" json:"faultTolerance"` - Redundancy float64 `gorm:"column:redundancy; type:float; not null" json:"redundancy"` - AvgAccessCost float64 `gorm:"column:avgAccessCost; type:float; not null" json:"avgAccessCost"` - Timestamp time.Time `gorm:"column:Timestamp; type:datatime; not null" json:"timestamp"` -} - func (Object) TableName() string { return "Object" } -type BlockDistribution struct { - BlockID int64 `gorm:"column:BlockID; primaryKey; type:bigint; autoIncrement" json:"blockID"` - ObjectID cdssdk.ObjectID `gorm:"column:ObjectID; type:bigint; not null" json:"objectID"` - Type string `gorm:"column:Type; type:varchar(1024); not null" json:"type"` - Index int64 `gorm:"column:Index; type:bigint; not null" json:"index"` - StorageID cdssdk.StorageID `gorm:"column:StorageID; type:bigint; not null" json:"storageID"` - Status Status `gorm:"column:Status; type:tinyint; not null" json:"status"` - Timestamp time.Time `gorm:"column:Timestamp; type:datatime; not null" json:"timestamp"` -} - func (BlockDistribution) TableName() string { return "BlockDistribution" } diff --git a/datamap/internal/models/object.go b/datamap/internal/models/object.go new file mode 100644 index 0000000..6fa8747 --- /dev/null +++ b/datamap/internal/models/object.go @@ -0,0 +1,70 @@ +package models + +import ( + cdssdk "gitlink.org.cn/cloudream/common/sdks/storage" + stgmod "gitlink.org.cn/cloudream/storage/common/models" + "gorm.io/gorm" + "log" + "time" +) + +type Object struct { + ObjectID cdssdk.ObjectID `gorm:"column:ObjectID; primaryKey; type:bigint; autoIncrement" json:"objectID"` + PackageID cdssdk.PackageID `gorm:"column:PackageID; type:bigint; not null" json:"packageID"` + Path string `gorm:"column:Path; type:varchar(1024); not null" json:"path"` + Size int64 `gorm:"column:Size; type:bigint; not null" json:"size"` + FileHash string `gorm:"column:FileHash; type:varchar(255); not null" json:"fileHash"` + Status Status `gorm:"column:Status; type:tinyint; not null" json:"status"` + FaultTolerance float64 `gorm:"column:faultTolerance; type:float; not null" json:"faultTolerance"` + Redundancy float64 `gorm:"column:redundancy; type:float; not null" json:"redundancy"` + AvgAccessCost float64 `gorm:"column:avgAccessCost; type:float; not null" json:"avgAccessCost"` + Timestamp time.Time `gorm:"column:Timestamp; type:datatime; not null" json:"timestamp"` +} + +// BlockTransferRepository 块传输记录的 Repository +type BlockTransferRepository struct { + repo *GormRepository +} + +// NewBlockTransferRepository 创建 BlockTransferRepository 实例 +func NewBlockTransferRepository(db *gorm.DB) *BlockTransferRepository { + return &BlockTransferRepository{repo: NewGormRepository(db)} +} + +// CreateBlockTransfer 创建块传输记录 +func (r *BlockTransferRepository) CreateBlockTransfer(transfer *Object) error { + return r.repo.Create(transfer) +} + +// GetAllBlockTransfers 获取所有块传输记录 +func (r *BlockTransferRepository) GetAllBlockTransfers() ([]Object, error) { + var transfers []Object + err := r.repo.GetAll(&transfers) + if err != nil { + return nil, err + } + return transfers, nil +} + +// ProcessBlockTransInfo 处理 BlockTransInfo 数据 +func ProcessBlockTransInfo(data stgmod.Object) { + repo := NewBlockTransferRepository(db) + //TODO 数据待调整 + transfer := &Object{ + ObjectID: 0, + PackageID: 0, + Path: data.Path, + Size: 0, + FileHash: "", + Status: 0, + FaultTolerance: 0, + Redundancy: 0, + AvgAccessCost: 0, + Timestamp: time.Time{}, + } + err := repo.CreateBlockTransfer(transfer) + if err != nil { + log.Printf("Failed to create block transfer record: %v", err) + } + +} diff --git a/datamap/internal/models/repository.go b/datamap/internal/models/repository.go new file mode 100644 index 0000000..4ffce91 --- /dev/null +++ b/datamap/internal/models/repository.go @@ -0,0 +1,43 @@ +package models + +import ( + "gorm.io/gorm" +) + +// Repository 通用接口 +type Repository interface { + Create(value interface{}) error + Update(value interface{}) error + Delete(value interface{}, id uint) error + GetByID(id uint, out interface{}) error + GetAll(out interface{}) error +} + +// GormRepository 基于 GORM 的通用实现 +type GormRepository struct { + db *gorm.DB +} + +func NewGormRepository(db *gorm.DB) *GormRepository { + return &GormRepository{db: db} +} + +func (r *GormRepository) Create(value interface{}) error { + return r.db.Create(value).Error +} + +func (r *GormRepository) Update(value interface{}) error { + return r.db.Save(value).Error +} + +func (r *GormRepository) Delete(value interface{}, id uint) error { + return r.db.Delete(value, id).Error +} + +func (r *GormRepository) GetByID(id uint, out interface{}) error { + return r.db.First(out, id).Error +} + +func (r *GormRepository) GetAll(out interface{}) error { + return r.db.Find(out).Error +} diff --git a/datamap/internal/mq/hub.go b/datamap/internal/mq/hub.go deleted file mode 100644 index 6b1860d..0000000 --- a/datamap/internal/mq/hub.go +++ /dev/null @@ -1,22 +0,0 @@ -package mq - -import ( - "gitlink.org.cn/cloudream/common/pkgs/logger" - "gitlink.org.cn/cloudream/common/pkgs/mq" - datamapmq "gitlink.org.cn/cloudream/storage/common/pkgs/mq/datamap" -) - -func (svc *Service) GetHubStat(msg *datamapmq.GetHubStat) (*datamapmq.GetHubStatResp, *mq.CodeMessage) { - logger.WithField("HubID", msg.HubID) - - //从datamapmq队列接收数据到stgmod.HubStat - - //从datamapmq队列接收数据到stgmod.HubStat,然后将该数据和mysql数据库中获取到的models.Storage做对比，用stgmode - return &datamapmq.GetHubStatResp{}, nil -} - -func (svc *Service) GetHubTrans(msg *datamapmq.GetHubTrans) (*datamapmq.GetHubTransResp, *mq.CodeMessage) { - logger.WithField("HubID", msg.HubID) - - return &datamapmq.GetHubTransResp{}, nil -} diff --git a/datamap/internal/mq/mq.go b/datamap/internal/mq/mq.go new file mode 100644 index 0000000..ba32c6a --- /dev/null +++ b/datamap/internal/mq/mq.go @@ -0,0 +1,80 @@ +package mq + +import ( + "encoding/json" + "fmt" + "github.com/streadway/amqp" + stgmod "gitlink.org.cn/cloudream/storage/common/models" + "gitlink.org.cn/cloudream/storage/datamap/internal/config" + "gitlink.org.cn/cloudream/storage/datamap/internal/models" + "log" +) + +func InitMQ(cfg config.RabbitMQConfig) (*amqp.Connection, error) { + conn, err := amqp.Dial(fmt.Sprintf("amqp://%s:%s@%s:%s/", + cfg.User, cfg.Password, cfg.Host, cfg.Port)) + if err != nil { + return nil, err + } + + // 启动队列监听 + go listenQueues(conn) + + return conn, nil +} + +func listenQueues(conn *amqp.Connection) { + queues := []string{ + "datamap_storageinfo", + "datamap_hubtransfer", + "datamap_blocktransfer", + "datamap_blockdistribution", + "datamap_objectchange", + "datamap_packagechange", + "datamap_bucketchange", + } + + for _, queue := range queues { + go func(q string) { + ch, err := conn.Channel() + if err != nil { + log.Printf("Failed to open channel for queue %s: %v", q, err) + return + } + defer ch.Close() + + msgs, err := ch.Consume(q, "", true, false, false, false, nil) + if err != nil { + log.Printf("Failed to register consumer for queue %s: %v", q, err) + return + } + + for msg := range msgs { + processMessage(q, msg.Body) + } + }(queue) + } +} + +func processMessage(queue string, body []byte) { + switch queue { + case "datamap_storageinfo": + var data stgmod.HubStat + json.Unmarshal(body, &data) + models.ProcessHubStat(data) + case "datamap_hubtransfer": + var data stgmod.HubTrans + json.Unmarshal(body, &data) + models.ProcessHubTrans(data) + case "datamap_blocktransfer": + var data stgmod.Object + json.Unmarshal(body, &data) + models.ProcessBlockTransInfo(data) + case "datamap_blockdistribution": + var data stgmod.BlockTransInfo + json.Unmarshal(body, &data) + models.ProcessBlockDistributionInfo(data) + default: + log.Printf("Unknown queue: %s", queue) + } +} diff --git a/datamap/internal/mq/service.go b/datamap/internal/mq/service.go deleted file mode 100644 index 5e1328e..0000000 --- a/datamap/internal/mq/service.go +++ /dev/null @@ -1,15 +0,0 @@ -package mq - -import ( - "gorm.io/gorm" -) - -type Service struct { - db *gorm.DB -} - -func NewService(db *gorm.DB) *Service { - return &Service{ - db: db, - } -} diff --git a/datamap/internal/server/server.go b/datamap/internal/server/server.go new file mode 100644 index 0000000..d2812c7 --- /dev/null +++ b/datamap/internal/server/server.go @@ -0,0 +1,22 @@ +package server + +import ( + "github.com/gin-gonic/gin" + "github.com/streadway/amqp" + "gitlink.org.cn/cloudream/storage/datamap/internal/handlers" + "gorm.io/gorm" + "log" +) + +func StartServer(db *gorm.DB, mq *amqp.Connection) { + r := gin.Default() + + // 注册HTTP接口 + r.GET("/storage", handlers.GetStorageData) + r.GET("/block-distribution", handlers.GetBlockDistributionData) + + // 启动服务 + if err := r.Run(":8080"); err != nil { + log.Fatalf("Failed to start server: %v", err) + } +} diff --git a/datamap/main.go b/datamap/main.go index 6d69c12..abe8755 100644 --- a/datamap/main.go +++ b/datamap/main.go @@ -1,16 +1,29 @@ package main import ( - "fmt" - "os" - "gitlink.org.cn/cloudream/storage/datamap/internal/config" + "gitlink.org.cn/cloudream/storage/datamap/internal/db" + "gitlink.org.cn/cloudream/storage/datamap/internal/mq" + "gitlink.org.cn/cloudream/storage/datamap/internal/server" + "log" ) func main() { - err := config.Init() + // 加载配置 + cfg := config.LoadConfig() + + // 初始化数据库 + dbConn, err := db.InitDB(cfg.Database) + if err != nil { + log.Fatalf("Failed to initialize database: %v", err) + } + + // 初始化RabbitMQ + mqConn, err := mq.InitMQ(cfg.RabbitMQ) if err != nil { - fmt.Printf("init config failed, err: %s", err.Error()) - os.Exit(1) + log.Fatalf("Failed to initialize RabbitMQ: %v", err) } + + // 启动Gin服务 + server.StartServer(dbConn, mqConn) } diff --git a/go.mod b/go.mod index da28acd..4c44652 100644 --- a/go.mod +++ b/go.mod @@ -22,10 +22,11 @@ require ( github.com/samber/lo v1.38.1 github.com/smartystreets/goconvey v1.8.1 github.com/spf13/cobra v1.8.0 + github.com/spf13/viper v1.19.0 github.com/tencentyun/cos-go-sdk-v5 v0.7.56 gitlink.org.cn/cloudream/common v0.0.0 - google.golang.org/grpc v1.57.0 - google.golang.org/protobuf v1.31.0 + google.golang.org/grpc v1.62.1 + google.golang.org/protobuf v1.33.0 gorm.io/gorm v1.25.7 ) @@ -40,14 +41,25 @@ require ( github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.6 // indirect github.com/aws/smithy-go v1.22.1 // indirect github.com/clbanning/mxj v1.8.4 // indirect + github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/google/go-querystring v1.0.0 // indirect - github.com/google/uuid v1.3.1 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/hashicorp/hcl v1.0.0 // indirect github.com/jinzhu/inflection v1.0.0 // indirect github.com/jinzhu/now v1.1.5 // indirect + github.com/magiconair/properties v1.8.7 // indirect github.com/mozillazg/go-httpheader v0.2.1 // indirect - github.com/stretchr/testify v1.8.4 // indirect + github.com/pelletier/go-toml/v2 v2.2.2 // indirect + github.com/sagikazarmark/locafero v0.4.0 // indirect + github.com/sagikazarmark/slog-shim v0.1.0 // indirect + github.com/sourcegraph/conc v0.3.0 // indirect + github.com/spf13/afero v1.11.0 // indirect + github.com/spf13/cast v1.6.0 // indirect + github.com/subosito/gotenv v1.6.0 // indirect golang.org/x/time v0.7.0 // indirect + gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect ) require ( @@ -60,7 +72,6 @@ require ( github.com/go-playground/validator/v10 v10.8.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/protobuf v1.5.3 // indirect - github.com/google/go-cmp v0.5.9 // indirect github.com/gopherjs/gopherjs v1.17.2 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect @@ -79,23 +90,22 @@ require ( github.com/sirupsen/logrus v1.9.2 github.com/smarty/assertions v1.15.0 // indirect github.com/spf13/pflag v1.0.5 // indirect - github.com/streadway/amqp v1.1.0 // indirect + github.com/streadway/amqp v1.1.0 github.com/ugorji/go/codec v1.2.11 // indirect github.com/zyedidia/generic v1.2.1 // indirect - go.etcd.io/etcd/api/v3 v3.5.9 // indirect - go.etcd.io/etcd/client/pkg/v3 v3.5.9 // indirect - go.etcd.io/etcd/client/v3 v3.5.9 // indirect + go.etcd.io/etcd/api/v3 v3.5.12 // indirect + go.etcd.io/etcd/client/pkg/v3 v3.5.12 // indirect + go.etcd.io/etcd/client/v3 v3.5.12 // indirect go.uber.org/atomic v1.10.0 // indirect go.uber.org/multierr v1.9.0 // indirect go.uber.org/zap v1.24.0 // indirect - golang.org/x/crypto v0.9.0 // indirect - golang.org/x/exp v0.0.0-20230519143937-03e91628a987 // indirect - golang.org/x/net v0.10.0 // indirect - golang.org/x/sync v0.1.0 - golang.org/x/sys v0.8.0 // indirect - golang.org/x/text v0.9.0 // indirect - google.golang.org/genproto v0.0.0-20230526161137-0005af68ea54 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20230525234035-dd9d682886f9 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19 // indirect + golang.org/x/crypto v0.21.0 // indirect + golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect + golang.org/x/net v0.23.0 // indirect + golang.org/x/sync v0.6.0 + golang.org/x/sys v0.18.0 // indirect + golang.org/x/text v0.14.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20240311132316-a219d84964c2 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240314234333-6e1732d8331c // indirect gorm.io/driver/mysql v1.5.7 ) diff --git a/go.sum b/go.sum index 2376f92..2cee4dd 100644 --- a/go.sum +++ b/go.sum @@ -37,8 +37,13 @@ github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8 github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= +github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= +github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= +github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= github.com/gin-gonic/gin v1.7.7 h1:3DoBmSbJbZAWqXJC3SLjAPfutPJJRN1U5pALB7EeTTs= @@ -64,14 +69,14 @@ github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaS github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= -github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4= -github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gopherjs/gopherjs v1.17.2 h1:fQnZVsXk8uxXIStYb0N4bGk7jeyTalG/wsZjQ25dO0g= github.com/gopherjs/gopherjs v1.17.2/go.mod h1:pRRIvn/QzFLrKfvEz3qUuEhtE/zLCWfreZ6J5gM2i+k= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= @@ -81,6 +86,8 @@ github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+l github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= github.com/hashicorp/golang-lru/v2 v2.0.5 h1:wW7h1TG88eUIJ2i69gaE3uNVtEPIagzhGvHgwfx2Vm4= github.com/hashicorp/golang-lru/v2 v2.0.5/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= +github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/huaweicloud/huaweicloud-sdk-go-obs v3.24.9+incompatible h1:XQVXdk+WAJ4fSNB6mMRuYNvFWou7BZs6SZB925hPrnk= github.com/huaweicloud/huaweicloud-sdk-go-obs v3.24.9+incompatible/go.mod h1:l7VUhRbTKCzdOacdT4oWCwATKyvZqUOlOqr0Ous3k4s= github.com/imdario/mergo v0.3.15 h1:M8XP7IuFNsqUx6VPK2P9OSmsYsI/YFaGil0uD21V3dM= @@ -106,12 +113,18 @@ github.com/klauspost/cpuid/v2 v2.2.4 h1:acbojRNwl3o09bUq+yDCtZFc1aiwaAAxtcn8YkZX github.com/klauspost/cpuid/v2 v2.2.4/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= github.com/klauspost/reedsolomon v1.11.8 h1:s8RpUW5TK4hjr+djiOpbZJB4ksx+TdYbRH7vHQpwPOY= github.com/klauspost/reedsolomon v1.11.8/go.mod h1:4bXRN+cVzMdml6ti7qLouuYi32KHJ5MGv0Qd8a47h6A= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY= github.com/leodido/go-urn v1.2.4 h1:XlAE/cm/ms7TE/VMVoduSpNBoyc2dOxHs5MZSwAN63Q= github.com/leodido/go-urn v1.2.4/go.mod h1:7ZrI8mTSeBSHl/UaRyKQW1qZeMgak41ANeCNaVckg+4= github.com/magefile/mage v1.15.0 h1:BvGheCMAsG3bWUDbZ8AyXXpCNwU9u5CB6sM+HNb9HYg= github.com/magefile/mage v1.15.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A= +github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= +github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= @@ -128,14 +141,23 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/mozillazg/go-httpheader v0.2.1 h1:geV7TrjbL8KXSyvghnFm+NyTux/hxwueTSrwhe88TQQ= github.com/mozillazg/go-httpheader v0.2.1/go.mod h1:jJ8xECTlalr6ValeXYdOF8fFUISeBAdw6E61aqQma60= +github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= +github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/profile v1.6.0/go.mod h1:qBsxPvzyUincmltOk6iyRVxHYg4adc0OFOv72ZdLa18= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= +github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= +github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6keLGt6kNQ= +github.com/sagikazarmark/locafero v0.4.0/go.mod h1:Pe1W6UlPYUk/+wc/6KFhbORCfqzgYEpgQ3O5fPuL3H4= +github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE= +github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ= github.com/samber/lo v1.38.1 h1:j2XEAqXKb09Am4ebOg31SpvzUTTs6EN3VfgeLUhPdXM= github.com/samber/lo v1.38.1/go.mod h1:+m/ZKRl6ClXCE2Lgf3MsQlWfh4bn1bz6CXEOxnEXnEA= github.com/sirupsen/logrus v1.9.2 h1:oxx1eChJGI6Uks2ZC4W1zpLlVgqB8ner4EuQwV4Ik1Y= @@ -144,15 +166,24 @@ github.com/smarty/assertions v1.15.0 h1:cR//PqUBUiQRakZWqBiFFQ9wb8emQGDb0HeGdqGB github.com/smarty/assertions v1.15.0/go.mod h1:yABtdzeQs6l1brC900WlRNwj6ZR55d7B+E8C6HtKdec= github.com/smartystreets/goconvey v1.8.1 h1:qGjIddxOk4grTu9JPOU31tVfq3cNdBlNa5sSznIX1xY= github.com/smartystreets/goconvey v1.8.1/go.mod h1:+/u4qLyY6x1jReYOp7GOM2FSt8aP9CzCZL03bI28W60= +github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= +github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0= +github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8= +github.com/spf13/afero v1.11.0/go.mod h1:GH9Y3pIexgf1MTIWtNGyogA5MwRIDXGUr+hbWNoBjkY= +github.com/spf13/cast v1.6.0 h1:GEiTHELF+vaR5dhz3VqZfFSzZjYbgeKDpBxQVS4GYJ0= +github.com/spf13/cast v1.6.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0= github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.19.0 h1:RWq5SEjt8o25SROyN3z2OrDB9l7RPd3lwTWU8EcEdcI= +github.com/spf13/viper v1.19.0/go.mod h1:GQUN9bilAbhU/jgc1bKs99f/suXKeUMct8Adx5+Ntkg= github.com/streadway/amqp v1.1.0 h1:py12iX8XSyI7aN/3dUT8DFIDJazNJsVJdxNVEpnQTZM= github.com/streadway/amqp v1.1.0/go.mod h1:WYSrTEYHOXHd0nwFeUXAe2G2hRnQT+deZJJf88uS9Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= @@ -161,8 +192,11 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.4/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= +github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common v1.0.563/go.mod h1:7sCQWVkxcsR38nffDW057DRGk8mUjK1Ing/EFOK8s8Y= github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/kms v1.0.563/go.mod h1:uom4Nvi9W+Qkom0exYiJ9VWJjXwyxtPYTkKkaLMlfE0= github.com/tencentyun/cos-go-sdk-v5 v0.7.56 h1:fOA3l3XbVN2kTjQKYPvhDms0Fq8zDcinO3boXodFaLw= @@ -175,12 +209,12 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/zyedidia/generic v1.2.1 h1:Zv5KS/N2m0XZZiuLS82qheRG4X1o5gsWreGb0hR7XDc= github.com/zyedidia/generic v1.2.1/go.mod h1:ly2RBz4mnz1yeuVbQA/VFwGjK3mnHGRj1JuoG336Bis= -go.etcd.io/etcd/api/v3 v3.5.9 h1:4wSsluwyTbGGmyjJktOf3wFQoTBIURXHnq9n/G/JQHs= -go.etcd.io/etcd/api/v3 v3.5.9/go.mod h1:uyAal843mC8uUVSLWz6eHa/d971iDGnCRpmKd2Z+X8k= -go.etcd.io/etcd/client/pkg/v3 v3.5.9 h1:oidDC4+YEuSIQbsR94rY9gur91UPL6DnxDCIYd2IGsE= -go.etcd.io/etcd/client/pkg/v3 v3.5.9/go.mod h1:y+CzeSmkMpWN2Jyu1npecjB9BBnABxGM4pN8cGuJeL4= -go.etcd.io/etcd/client/v3 v3.5.9 h1:r5xghnU7CwbUxD/fbUtRyJGaYNfDun8sp/gTr1hew6E= -go.etcd.io/etcd/client/v3 v3.5.9/go.mod h1:i/Eo5LrZ5IKqpbtpPDuaUnDOUv471oDg8cjQaUr2MbA= +go.etcd.io/etcd/api/v3 v3.5.12 h1:W4sw5ZoU2Juc9gBWuLk5U6fHfNVyY1WC5g9uiXZio/c= +go.etcd.io/etcd/api/v3 v3.5.12/go.mod h1:Ot+o0SWSyT6uHhA56al1oCED0JImsRiU9Dc26+C2a+4= +go.etcd.io/etcd/client/pkg/v3 v3.5.12 h1:EYDL6pWwyOsylrQyLp2w+HkQ46ATiOvoEdMarindU2A= +go.etcd.io/etcd/client/pkg/v3 v3.5.12/go.mod h1:seTzl2d9APP8R5Y2hFL3NVlD6qC/dOT+3kvrqPyTas4= +go.etcd.io/etcd/client/v3 v3.5.12 h1:v5lCPXn1pf1Uu3M4laUE2hp/geOTc5uPcYYsNe1lDxg= +go.etcd.io/etcd/client/v3 v3.5.12/go.mod h1:tSbBCakoWmmddL+BKVAJHa9km+O/E+bumDe9mSbPiqw= go.uber.org/atomic v1.10.0 h1:9qC72Qh0+3MqyJbAn8YU5xVq1frD8bn3JtD2oXtafVQ= go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= go.uber.org/goleak v1.1.12 h1:gZAh5/EyT/HQwlpkCy6wTpqfH9H8Lz8zbm3dZh+OyzA= @@ -193,10 +227,10 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.9.0 h1:LF6fAI+IutBocDJ2OT0Q1g8plpYljMZ4+lty+dsqw3g= -golang.org/x/crypto v0.9.0/go.mod h1:yrmDGqONDYtNj3tH8X9dzUun2m2lzPa9ngI6/RUPGR0= -golang.org/x/exp v0.0.0-20230519143937-03e91628a987 h1:3xJIFvzUFbu4ls0BTBYcgbCGhA63eAOEMxIHugyXJqA= -golang.org/x/exp v0.0.0-20230519143937-03e91628a987/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w= +golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= +golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= +golang.org/x/exp v0.0.0-20230905200255-921286631fa9 h1:GoHiUyI/Tp2nVkLI2mCxVkOjsbSXD66ic0XW0js0R9g= +golang.org/x/exp v0.0.0-20230905200255-921286631fa9/go.mod h1:S2oDrQGGwySpoQPVqRShND87VCbxmc6bL1Yd2oYrm6k= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -204,13 +238,13 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M= -golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs= +golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= -golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= +golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -222,15 +256,15 @@ golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU= -golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= +golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= -golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.7.0 h1:ntUhktv3OPE6TgYxXWv9vKvUSJyIFJlyohwbkEwPrKQ= golang.org/x/time v0.7.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -241,20 +275,21 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/genproto v0.0.0-20230526161137-0005af68ea54 h1:9NWlQfY2ePejTmfwUH1OWwmznFa+0kKcHGPDvcPza9M= -google.golang.org/genproto v0.0.0-20230526161137-0005af68ea54/go.mod h1:zqTuNwFlFRsw5zIts5VnzLQxSRqh+CGOTVMlYbY0Eyk= -google.golang.org/genproto/googleapis/api v0.0.0-20230525234035-dd9d682886f9 h1:m8v1xLLLzMe1m5P+gCTF8nJB9epwZQUBERm20Oy1poQ= -google.golang.org/genproto/googleapis/api v0.0.0-20230525234035-dd9d682886f9/go.mod h1:vHYtlOoi6TsQ3Uk2yxR7NI5z8uoV+3pZtR4jmHIkRig= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19 h1:0nDDozoAU19Qb2HwhXadU8OcsiO/09cnTqhUtq2MEOM= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= -google.golang.org/grpc v1.57.0 h1:kfzNeI/klCGD2YPMUlaGNT3pxvYfga7smW3Vth8Zsiw= -google.golang.org/grpc v1.57.0/go.mod h1:Sd+9RMTACXwmub0zcNY2c4arhtrbBYD1AUHI/dt16Mo= +google.golang.org/genproto/googleapis/api v0.0.0-20240311132316-a219d84964c2 h1:rIo7ocm2roD9DcFIX67Ym8icoGCKSARAiPljFhh5suQ= +google.golang.org/genproto/googleapis/api v0.0.0-20240311132316-a219d84964c2/go.mod h1:O1cOfN1Cy6QEYr7VxtjOyP5AdAuR0aJ/MYZaaof623Y= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240314234333-6e1732d8331c h1:lfpJ/2rWPa/kJgxyyXM8PrNnfCzcmxJ265mADgwmvLI= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240314234333-6e1732d8331c/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY= +google.golang.org/grpc v1.62.1 h1:B4n+nfKzOICUXMgyrNd19h/I9oH0L1pizfk1d4zSgTk= +google.golang.org/grpc v1.62.1/go.mod h1:IWTG0VlJLCh1SkC58F7np9ka9mx/WNkjl4PGJaiq+QE= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= -google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= +google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= +gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= From 5fe52ee0689a40e86d848dea71568c0198bdaf60 Mon Sep 17 00:00:00 2001 From: Jake <450705171@qq.com> Date: Wed, 15 Jan 2025 16:41:56 +0800 Subject: [PATCH 05/12] =?UTF-8?q?=E5=AF=B9=E9=BD=90=E5=89=8D=E7=AB=AF?= =?UTF-8?q?=E5=8F=A3=E5=BE=84=20mq=E5=8F=A3=E5=BE=84=E9=83=A8=E5=88=86?= =?UTF-8?q?=E9=80=BB=E8=BE=91=E8=BF=98=E5=BE=85=E4=BF=AE=E5=A4=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- common/models/datamap.go | 222 ++++++++++------- datamap/internal/config/config.go | 2 +- datamap/internal/db/db.go | 25 +- datamap/internal/handlers/handlers.go | 163 +++++++++++-- datamap/internal/models/block.go | 57 ----- datamap/internal/models/blockdistribution.go | 178 ++++++++++++++ datamap/internal/models/blocktransfer.go | 227 ++++++++++++++++++ datamap/internal/models/hub.go | 53 ++++ datamap/internal/models/hubinfo.go | 75 ++++++ datamap/internal/models/hubrequest.go | 84 +++++++ datamap/internal/models/hubtrans.go | 53 ---- datamap/internal/models/models.go | 177 +++++++------- datamap/internal/models/object.go | 65 ++--- .../models/{hubs.go => storageinfo.go} | 50 ++-- datamap/internal/mq/mq.go | 67 ++++-- datamap/internal/server/server.go | 5 +- datamap/main.go | 3 + go.mod | 10 +- go.sum | 41 +++- 19 files changed, 1166 insertions(+), 391 deletions(-) delete mode 100644 datamap/internal/models/block.go create mode 100644 datamap/internal/models/blockdistribution.go create mode 100644 datamap/internal/models/blocktransfer.go create mode 100644 datamap/internal/models/hub.go create mode 100644 datamap/internal/models/hubinfo.go create mode 100644 datamap/internal/models/hubrequest.go delete mode 100644 datamap/internal/models/hubtrans.go rename datamap/internal/models/{hubs.go => storageinfo.go} (66%) diff --git a/common/models/datamap.go b/common/models/datamap.go index 581e8e5..38bf89b 100644 --- a/common/models/datamap.go +++ b/common/models/datamap.go @@ -1,55 +1,118 @@ package stgmod -import "time" +import ( + cdssdk "gitlink.org.cn/cloudream/common/sdks/storage" + "time" +) -// HubStat 中心信息 每天一次，各节点统计自身当前的总数据量 +// HubInfo 节点信息 + +type SourceHub struct { + Type string `json:"type"` +} + +type HubInfoBody struct { + HubID int `json:"hubID"` + HubInfo cdssdk.Hub `json:"hubInfo"` + Type string `json:"type"` +} + +type HubInfo struct { + Timestamp time.Time `json:"timestamp"` + Source SourceHub `json:"source"` + Category string `json:"category"` + Body HubInfoBody `json:"body"` +} + +//StorageInfo 节点信息 + +type SourceStorage struct { + Type string `json:"type"` +} + +type StorageInfoBody struct { + StorageID int64 `json:"storageID"` + StorageInfo cdssdk.Storage `json:"storageInfo"` + Type string `json:"type"` +} + +type StorageInfo struct { + Timestamp time.Time `json:"timestamp"` + Source SourceStorage `json:"source"` + Category string `json:"category"` + Body StorageInfoBody `json:"body"` +} + +// StorageStats 节点信息数据 type Source struct { Type string `json:"type"` HubID string `json:"hubID"` HubName string `json:"hubName"` } -type HubStatBody struct { - MasterHubID int `json:"masterHubID"` - MasterHubAddress string `json:"masterHubAddress"` - StorageID int `json:"storageID"` - DataCount int `json:"dataCount"` + +type StorageStatsBody struct { + StorageID int64 `json:"storageID"` + DataCount int64 `json:"dataCount"` } -type HubStat struct { - Timestamp time.Time `json:"timestamp"` - Source Source `json:"source"` - Category string `json:"category"` - Body HubStatBody `json:"body"` + +type StorageStats struct { + Timestamp time.Time `json:"timestamp"` + Source Source `json:"source"` + Category string `json:"category"` + Body StorageStatsBody `json:"body"` } -// HubTrans 节点传输信息 +// HubTransferStats 节点传输信息 // 每天一次，各节点统计自身当天向外部各个节点传输的总数据量 -type HubTransBody struct { - SourceHubID int64 `json:"sourceHubID"` - TargetHubID int64 `json:"targetHubID"` - DataTransferCount int64 `json:"dataTransferCount"` - RequestCount int64 `json:"requestCount"` - FailedRequestCount int64 `json:"failedRequestCount"` - AvgTransferCount int64 `json:"avgTransferCount"` - MaxTransferCount int64 `json:"maxTransferCount"` - MinTransferCount int64 `json:"minTransferCount"` - StartTimestamp time.Time `json:"startTimestamp"` - EndTimestamp time.Time `json:"endTimestamp"` -} -type HubTrans struct { - Timestamp time.Time `json:"timestamp"` - Source Source `json:"source"` - Category string `json:"category"` - Body HubTransBody `json:"body"` -} - -// BlockTransInfo 块传输信息 +type DataTrans struct { + TotalTransfer int64 `json:"totalTransfer"` + RequestCount int64 `json:"requestCount"` + FailedRequestCount int64 `json:"failedRequestCount"` + AvgTransfer int64 `json:"avgTransfer"` + MaxTransfer int64 `json:"maxTransfer"` + MinTransfer int64 `json:"minTransfer"` +} +type HubTransferStatsBody struct { + SourceHubID int64 `json:"sourceHubID"` + TargetHubID int64 `json:"targetHubID"` + Send DataTrans `json:"send"` + StartTimestamp time.Time `json:"startTimestamp"` + EndTimestamp time.Time `json:"endTimestamp"` +} +type HubTransferStats struct { + Timestamp time.Time `json:"timestamp"` + Source Source `json:"source"` + Category string `json:"category"` + Body HubTransferStatsBody `json:"body"` +} + +//hubStorageTransferStats 节点-中心间传输信息 + +type HubStorageTransferStatsBody struct { + HubID int64 `json:"hubID"` + StorageID int64 `json:"storageID"` + Send DataTrans `json:"send"` + Receive DataTrans `json:"receive"` + StartTimestamp time.Time `json:"startTimestamp"` + EndTimestamp time.Time `json:"endTimestamp"` +} + +type HubStorageTransferStats struct { + Timestamp time.Time `json:"timestamp"` + Source Source `json:"source"` + Category string `json:"category"` + Body HubStorageTransferStatsBody `json:"body"` +} + +// blockTransfer 块传输信息 /*实时日志，当对象的块信息发生变化时推送，共有4种变化类型： 拷贝 编解码(一变多、多变一、多变多) 删除 更新*/ + type Block struct { BlockType string `json:"blockType"` Index string `json:"index"` @@ -68,68 +131,63 @@ type BlockChange struct { SourceStorageID string `json:"sourceStorageID"` TargetStorageID string `json:"targetStorageID"` DataTransferCount string `json:"dataTransferCount"` - Timestamp string `json:"timestamp"` + Timestamp time.Time `json:"timestamp"` SourceBlocks []Block `json:"sourceBlocks,omitempty"` TargetBlocks []Block `json:"targetBlocks,omitempty"` DataTransfers []DataTransfer `json:"dataTransfers,omitempty"` Blocks []Block `json:"blocks,omitempty"` } -type BlockTransInfoBody struct { +type BlockTransferBody struct { Type string `json:"type"` ObjectID string `json:"objectID"` PackageID string `json:"packageID"` BlockChanges []BlockChange `json:"blockChanges"` } -type BlockTransInfo struct { - Timestamp time.Time `json:"timestamp"` - Source Source `json:"source"` - Category string `json:"category"` - Body BlockTransInfoBody `json:"body"` +type BlockTransfer struct { + Timestamp time.Time `json:"timestamp"` + Source Source `json:"source"` + Category string `json:"category"` + Body BlockTransferBody `json:"body"` } // BlockDistribution 块传输信息 // 每天一次，在调整完成后，将当天调整前后的布局情况一起推送 -type BlockDistribution struct { + +type BlockDistributionObjectInfo struct { Type string `json:"type"` Index string `json:"index"` StorageID string `json:"storageID"` } -type Object struct { - ObjectID string `json:"objectID"` - PackageID string `json:"packageID"` - Path string `json:"path"` - Size int64 `json:"size"` - FileHash string `json:"fileHash"` - FaultTolerance string `json:"faultTolerance"` - Redundancy string `json:"redundancy"` - AvgAccessCost string `json:"avgAccessCost"` - BlockDistribution []BlockDistribution `json:"blockDistribution"` - DataTransfers []DataTransfer `json:"dataTransfers"` +type BlockDistributionObject struct { + ObjectID int64 `json:"objectID"` + PackageID int64 `json:"packageID"` + Path string `json:"path"` + Size int64 `json:"size"` + FileHash string `json:"fileHash"` + FaultTolerance string `json:"faultTolerance"` + Redundancy string `json:"redundancy"` + AvgAccessCost string `json:"avgAccessCost"` + BlockDistribution []BlockDistributionObjectInfo `json:"blockDistribution"` + DataTransfers []DataTransfer `json:"dataTransfers"` } type BlockDistributionBody struct { - Timestamp time.Time `json:"timestamp"` - Type string `json:"type"` - ObjectID string `json:"objectID"` - PackageID string `json:"packageID"` - SourceBlocks []Block `json:"sourceBlocks,omitempty"` - TargetBlocks []Block `json:"targetBlocks,omitempty"` - DataTransfers []DataTransfer `json:"dataTransfers,omitempty"` - Blocks []Block `json:"blocks,omitempty"` + Timestamp time.Time `json:"timestamp"` + Object BlockDistributionObject `json:"object,omitempty"` } -type BlockDistributionInfo struct { +type BlockDistribution struct { Timestamp time.Time `json:"timestamp"` Source Source `json:"source"` Category string `json:"category"` Body BlockDistributionBody `json:"body"` } -// ObjectUpdateInfo Object变化信息 +// ObjectChange Object变化信息 -type ObjectUpdateInfoBody struct { +type ObjectChangeBody struct { Type string `json:"type"` ObjectID string `json:"objectID"` PackageID string `json:"packageID"` @@ -138,41 +196,41 @@ type ObjectUpdateInfoBody struct { BlockDistribution []BlockDistribution `json:"blockDistribution"` Timestamp string `json:"timestamp"` } -type ObjectUpdateInfo struct { - Timestamp time.Time `json:"timestamp"` - Source Source `json:"source"` - Category string `json:"category"` - Body ObjectUpdateInfoBody `json:"body"` +type ObjectChange struct { + Timestamp time.Time `json:"timestamp"` + Source Source `json:"source"` + Category string `json:"category"` + Body ObjectChangeBody `json:"body"` } -// PackageUpdateInfo package变化信息 +// PackageChange package变化信息 -type PackageUpdateInfoBody struct { +type PackageChangeBody struct { Type string `json:"type"` PackageID string `json:"packageID"` PackageName string `json:"packageName"` BucketID string `json:"bucketID"` Timestamp string `json:"timestamp"` } -type PackageUpdateInfo struct { - Timestamp time.Time `json:"timestamp"` - Source Source `json:"source"` - Category string `json:"category"` - Body PackageUpdateInfoBody `json:"body"` +type PackageChange struct { + Timestamp time.Time `json:"timestamp"` + Source Source `json:"source"` + Category string `json:"category"` + Body PackageChangeBody `json:"body"` } -// BucketUpdateInfo bucket变化信息 +// BucketChange bucket变化信息 -type BucketUpdateInfoBody struct { +type BucketChangeBody struct { Type string `json:"type"` BucketID string `json:"bucketID"` BucketName string `json:"bucketName"` Timestamp string `json:"timestamp"` } -type BucketUpdateInfo struct { - Timestamp time.Time `json:"timestamp"` - Source Source `json:"source"` - Category string `json:"category"` - Body BucketUpdateInfoBody `json:"body"` +type BucketChange struct { + Timestamp time.Time `json:"timestamp"` + Source Source `json:"source"` + Category string `json:"category"` + Body BucketChangeBody `json:"body"` } diff --git a/datamap/internal/config/config.go b/datamap/internal/config/config.go index 521ea76..069d877 100644 --- a/datamap/internal/config/config.go +++ b/datamap/internal/config/config.go @@ -30,7 +30,7 @@ type ServerConfig struct { } func LoadConfig() *Config { - viper.SetConfigFile(".env") + viper.SetConfigFile("C:\\Users\\Administrator\\workspace\\workspace\\storage\\datamap\\.env") viper.ReadInConfig() return &Config{ diff --git a/datamap/internal/db/db.go b/datamap/internal/db/db.go index d279f64..4f2d1d5 100644 --- a/datamap/internal/db/db.go +++ b/datamap/internal/db/db.go @@ -3,29 +3,30 @@ package db import ( "fmt" "gitlink.org.cn/cloudream/storage/datamap/internal/config" - "gitlink.org.cn/cloudream/storage/datamap/internal/models" "gorm.io/driver/mysql" "gorm.io/gorm" + "gorm.io/gorm/schema" ) -// 全局数据库连接实例 -var DB *gorm.DB - func InitDB(cfg config.DatabaseConfig) (*gorm.DB, error) { dsn := fmt.Sprintf("%s:%s@tcp(%s:%s)/%s?charset=utf8mb4&parseTime=True&loc=Local", cfg.User, cfg.Password, cfg.Host, cfg.Port, cfg.DBName) - db, err := gorm.Open(mysql.Open(dsn), &gorm.Config{}) + db, err := gorm.Open(mysql.Open(dsn), &gorm.Config{ + NamingStrategy: schema.NamingStrategy{ + SingularTable: true, // 禁用自动复数化表名 + }, + }) if err != nil { return nil, err } - // 自动迁移表结构 - db.AutoMigrate( - &models.Hub{}, - &models.Storage{}, - &models.HubRequest{}, - &models.BlockDistribution{}, - ) + //// 自动迁移表结构 + //db.AutoMigrate( + // &models.Hub{}, + // &models.Storage{}, + // &models.HubRequest{}, + // &models.BlockDistribution{}, + //) return db, nil } diff --git a/datamap/internal/handlers/handlers.go b/datamap/internal/handlers/handlers.go index 2b909af..0fa46ba 100644 --- a/datamap/internal/handlers/handlers.go +++ b/datamap/internal/handlers/handlers.go @@ -2,29 +2,160 @@ package handlers import ( "github.com/gin-gonic/gin" - "gitlink.org.cn/cloudream/storage/datamap/internal/db" "gitlink.org.cn/cloudream/storage/datamap/internal/models" + "gorm.io/gorm" "net/http" + "strconv" ) -// GetStorageData 获取 Storage 表中的全部数据 -func GetStorageData(c *gin.Context) { - repo := models.NewStorageRepository(db.DB) - storages, err := repo.GetAllStorages() - if err != nil { - c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to fetch storage data"}) - return +// DB 全局数据库连接实例 +var DB *gorm.DB + +// SetDB 设置数据库连接实例 +func SetDB(db *gorm.DB) { + DB = db +} + +// GetHubInfo 获取 节点交互数据 +func GetHubInfo(c *gin.Context) { + + repoHub := models.NewHubRepository(DB) + repoStorage := models.NewStorageRepository(DB) + repoHubReq := models.NewHubRequestRepository(DB) + + nodes := make([]models.Node, 0) + edges := make([]models.Edge, 0) + + //添加所有节点信息 + hubs, _ := repoHub.GetAllHubs() + storages, _ := repoStorage.GetAllStorages() + for _, hub := range hubs { + node := models.Node{ + ID: int64(hub.HubID), + NodeType: "hub", + Name: hub.Name, + Address: hub.Address, + } + nodes = append(nodes, node) + } + for _, storage := range storages { + node := models.Node{ + ID: int64(storage.StorageID), + NodeType: "storage", + Name: storage.StorageName, + DataCount: storage.DataCount, + NewDataCount: storage.NewDataCount, + Timestamp: storage.Timestamp, + } + nodes = append(nodes, node) + } + + // 添加所有边信息 + hubReqs, _ := repoHubReq.GetAllHubRequests() + for _, hubReq := range hubReqs { + edge := models.Edge{ + SourceType: hubReq.SourceType, + SourceID: int64(hubReq.SourceID), + TargetType: hubReq.TargetType, + TargetID: int64(hubReq.TargetID), + DataTransferCount: hubReq.DataTransferCount, + RequestCount: hubReq.RequestCount, + FailedRequestCount: hubReq.FailedRequestCount, + AvgTransferCount: hubReq.AvgTransferCount, + MaxTransferCount: hubReq.MaxTransferCount, + MinTransferCount: hubReq.MinTransferCount, + StartTimestamp: hubReq.StartTimestamp, + EndTimestamp: hubReq.EndTimestamp, + } + edges = append(edges, edge) + } + hubRelationship := models.HubRelationship{ + Nodes: nodes, + Edges: edges, } - c.JSON(http.StatusOK, storages) + c.JSON(http.StatusOK, hubRelationship) } -// GetBlockDistributionData 获取 BlockDistribution 表中的全部数据 -func GetBlockDistributionData(c *gin.Context) { - repo := models.NewBlockDistributionRepository(db.DB) - blocks, err := repo.GetAllBlocks() - if err != nil { - c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to fetch block distribution data"}) +func containsCombo(combos []models.Combo, targetID string, targetComboType string) bool { + for _, combo := range combos { + if combo.ID == targetID && combo.ComboType == targetComboType { + return true + } + } + return false +} + +// GetDataTransfer 数据对象的节点间传输量 +func GetDataTransfer(c *gin.Context) { + + repoObject := models.NewObjectRepository(DB) + repoBlockDistribution := models.NewBlockDistributionRepository(DB) + repoStorageTrans := models.NewStorageTransferCountRepository(DB) + + //首先判断object是否存在 + objectIDStr := c.Param("objectID") + objectID, _ := strconv.ParseInt(objectIDStr, 10, 64) + object, _ := repoObject.GetObjectByID(objectID) + if object == nil { + c.JSON(http.StatusOK, []interface{}{}) return } - c.JSON(http.StatusOK, blocks) + + nodes := make([]models.DistNode, 0) + combos := make([]models.Combo, 0) + edges := make([]models.DistEdge, 0) + + //根据ObjectID查询出在所有storage中存储的块或副本 + blocks, _ := repoBlockDistribution.GetBlockDistributionByObjectID(objectID) + for _, block := range blocks { + //nodes --------- block + //添加node信息 + node := models.DistNode{ + //block id + ID: strconv.FormatInt(block.BlockID, 10), + //storage id + ComboID: strconv.FormatInt(block.StorageID, 10), + //block index + Label: strconv.FormatInt(block.Index, 10), + //block type + NodeType: block.Type, + } + nodes = append(nodes, node) + + //combos ------- state or storage + //添加storage combo信息 + if !containsCombo(combos, strconv.FormatInt(block.StorageID, 10), "storage") { + combo := models.Combo{ + ID: strconv.FormatInt(block.StorageID, 10), + Label: "存储中心" + strconv.FormatInt(block.StorageID, 10), + ParentId: strconv.Itoa(block.Status), + ComboType: "storage", + } + combos = append(combos, combo) + } + //添加state combo信息 + if !containsCombo(combos, strconv.Itoa(block.Status), "state") { + combo := models.Combo{ + ID: strconv.Itoa(block.Status), + Label: "存储状态" + string(block.Status), + ComboType: "state", + } + combos = append(combos, combo) + } + } + //edges data trans between storage and storage + relations, _ := repoStorageTrans.GetStorageTransferCountByObjectID(objectID) + for _, relation := range relations { + edge := models.DistEdge{ + Source: strconv.FormatInt(relation.SourceStorageID, 10), + Target: strconv.FormatInt(relation.TargetStorageID, 10), + } + edges = append(edges, edge) + } + result := models.ObjectDistribution{ + Nodes: nodes, + Combos: combos, + Edges: edges, + } + c.JSON(http.StatusOK, result) } diff --git a/datamap/internal/models/block.go b/datamap/internal/models/block.go deleted file mode 100644 index 9c15b5c..0000000 --- a/datamap/internal/models/block.go +++ /dev/null @@ -1,57 +0,0 @@ -package models - -import ( - cdssdk "gitlink.org.cn/cloudream/common/sdks/storage" - stgmod "gitlink.org.cn/cloudream/storage/common/models" - "gorm.io/gorm" - "strconv" - "time" -) - -type BlockDistribution struct { - BlockID int64 `gorm:"column:BlockID; primaryKey; type:bigint; autoIncrement" json:"blockID"` - ObjectID cdssdk.ObjectID `gorm:"column:ObjectID; type:bigint; not null" json:"objectID"` - Type string `gorm:"column:Type; type:varchar(1024); not null" json:"type"` - Index int64 `gorm:"column:Index; type:bigint; not null" json:"index"` - StorageID cdssdk.StorageID `gorm:"column:StorageID; type:bigint; not null" json:"storageID"` - Status Status `gorm:"column:Status; type:tinyint; not null" json:"status"` - Timestamp time.Time `gorm:"column:Timestamp; type:datatime; not null" json:"timestamp"` -} - -type BlockDistributionRepository struct { - repo *GormRepository -} - -func NewBlockDistributionRepository(db *gorm.DB) *BlockDistributionRepository { - return &BlockDistributionRepository{repo: NewGormRepository(db)} -} - -func (r *BlockDistributionRepository) CreateBlockDistribution(block *BlockDistribution) error { - return r.repo.Create(block) -} - -func (r *BlockDistributionRepository) GetAllBlocks() ([]BlockDistribution, error) { - var blocks []BlockDistribution - err := r.repo.GetAll(&blocks) - if err != nil { - return nil, err - } - return blocks, nil -} - -func ProcessBlockDistributionInfo(data stgmod.BlockTransInfo) { - repo := NewBlockDistributionRepository(db) - - for _, change := range data.Body.BlockChanges { - objectID, _ := strconv.ParseInt(data.Body.ObjectID, 10, 64) - index, _ := strconv.ParseInt(change.Index, 10, 64) - targetStorageID, _ := strconv.ParseInt(change.TargetStorageID, 10, 64) - block := &BlockDistribution{ - ObjectID: cdssdk.ObjectID(objectID), - Type: change.Type, - Index: index, - StorageID: cdssdk.StorageID(targetStorageID), - } - repo.CreateBlockDistribution(block) - } -} diff --git a/datamap/internal/models/blockdistribution.go b/datamap/internal/models/blockdistribution.go new file mode 100644 index 0000000..a3e6e18 --- /dev/null +++ b/datamap/internal/models/blockdistribution.go @@ -0,0 +1,178 @@ +package models + +import ( + "errors" + cdssdk "gitlink.org.cn/cloudream/common/sdks/storage" + stgmod "gitlink.org.cn/cloudream/storage/common/models" + "gorm.io/gorm" + "log" + "strconv" + "time" +) + +type BlockDistribution struct { + BlockID int64 `gorm:"column:BlockID; primaryKey; type:bigint; autoIncrement" json:"blockID"` + ObjectID int64 `gorm:"column:ObjectID; type:bigint; not null" json:"objectID"` + Type string `gorm:"column:Type; type:varchar(1024); not null" json:"type"` + Index int64 `gorm:"column:Index; type:bigint; not null" json:"index"` + StorageID int64 `gorm:"column:StorageID; type:bigint; not null" json:"storageID"` + Status int `gorm:"column:Status; type:tinyint; not null" json:"status"` + Timestamp time.Time `gorm:"column:Timestamp; type:datatime; not null" json:"timestamp"` +} + +func (BlockDistribution) TableName() string { + return "blockdistribution" +} + +type BlockDistributionRepository struct { + repo *GormRepository +} + +func NewBlockDistributionRepository(db *gorm.DB) *BlockDistributionRepository { + return &BlockDistributionRepository{repo: NewGormRepository(db)} +} + +func (r *BlockDistributionRepository) CreateBlockDistribution(block *BlockDistribution) error { + return r.repo.Create(block) +} + +func (r *BlockDistributionRepository) UpdateBlockDistribution(block *BlockDistribution) error { + return r.repo.Update(block) +} + +func (r *BlockDistributionRepository) GetAllBlocks() ([]BlockDistribution, error) { + var blocks []BlockDistribution + err := r.repo.GetAll(&blocks) + if err != nil { + return nil, err + } + return blocks, nil +} + +func (r *BlockDistributionRepository) GetBlockDistributionByObjectID(objectID int64) ([]BlockDistribution, error) { + var blocks []BlockDistribution + query := "SELECT * FROM blockdistribution WHERE ObjectID = ?" + err := r.repo.db.Raw(query, objectID).Scan(&blocks).Error + if errors.Is(err, gorm.ErrRecordNotFound) { + return []BlockDistribution{}, errors.New("block not found") + } + return blocks, nil +} + +func (r *BlockDistributionRepository) GetStorageIDsByObjectID(objectID int64) ([]int64, error) { + var storageIDs []int64 + query := "SELECT distinct storageID FROM blockdistribution WHERE ObjectID = ?" + // 通过 ObjectID 查询 + err := r.repo.db.Raw(query, objectID).Scan(&storageIDs).Error + if errors.Is(err, gorm.ErrRecordNotFound) { + return []int64{}, errors.New("block not found") + } + return storageIDs, nil +} + +func (r *BlockDistributionRepository) GetBlockDistributionByIndex(objectID int64, index int64, storageID int64) (BlockDistribution, error) { + var block BlockDistribution + query := "SELECT * FROM blockdistribution WHERE ObjectID = ? AND `Index` = ? AND StorageID = ?" + // 通过 ObjectID 和 Index 联合查询 + err := r.repo.db.Exec(query, objectID, index, storageID).First(&block).Error + if errors.Is(err, gorm.ErrRecordNotFound) { + return BlockDistribution{}, errors.New("block not found") + } + return block, nil +} + +// DeleteBlockDistribution 删除 BlockDistribution 记录 (根据 ObjectID 和 Index) + +func (r *BlockDistributionRepository) DeleteBlockDistribution(objectID int64, index int64, storageID int64) error { + query := "DELETE FROM blockdistribution WHERE ObjectID = ? AND `Index` = ? AND StorageID = ?" + + return r.repo.db.Exec(query, objectID, index, storageID).Error +} + +// ProcessBlockDistribution mq推送各节点统计自身当前的总数据量时的处理逻辑 + +func ProcessBlockDistribution(data stgmod.BlockDistribution) { + repoObject := NewObjectRepository(DB) + repoBlock := NewBlockDistributionRepository(DB) + repoStorage := NewStorageTransferCountRepository(DB) + //更新object表中的状态 + object, err := repoObject.GetObjectByID(data.Body.Object.ObjectID) + faultTolerance, _ := strconv.ParseFloat(data.Body.Object.FaultTolerance, 64) + redundancy, _ := strconv.ParseFloat(data.Body.Object.Redundancy, 64) + avgAccessCost, _ := strconv.ParseFloat(data.Body.Object.AvgAccessCost, 64) + if errors.Is(err, gorm.ErrRecordNotFound) { + err := repoObject.CreateObject(&Object{ + ObjectID: cdssdk.ObjectID(data.Body.Object.ObjectID), + PackageID: cdssdk.PackageID(data.Body.Object.PackageID), + Path: data.Body.Object.Path, + Size: data.Body.Object.Size, + FileHash: data.Body.Object.FileHash, + Status: StatusYesterdayAfter, + FaultTolerance: faultTolerance, + Redundancy: redundancy, + AvgAccessCost: avgAccessCost, + Timestamp: time.Now(), + }) + if err != nil { + log.Printf("Error create object: %v", err) + } + } else { + object.Status = StatusYesterdayAfter + err = repoObject.UpdateObject(object) + if err != nil { + log.Printf("Error update object: %v", err) + } + } + + //更新block表中的状态 + for _, blockDistribution := range data.Body.Object.BlockDistribution { + blockIndex, _ := strconv.ParseInt(blockDistribution.Index, 10, 64) + blockStorageID, _ := strconv.ParseInt(blockDistribution.StorageID, 10, 64) + blockDist, err := repoBlock.GetBlockDistributionByIndex(data.Body.Object.ObjectID, blockIndex, blockStorageID) + if errors.Is(err, gorm.ErrRecordNotFound) { + err := repoBlock.CreateBlockDistribution(&BlockDistribution{ + BlockID: blockDist.BlockID, + ObjectID: blockDist.ObjectID, + Type: blockDistribution.Type, + Index: blockIndex, + StorageID: blockStorageID, + Status: StatusYesterdayAfter, + Timestamp: time.Now(), + }) + if err != nil { + log.Printf("Error create BlockDistribution: %v", err) + } + } else { + err := repoBlock.UpdateBlockDistribution(&BlockDistribution{ + BlockID: blockDist.BlockID, + ObjectID: blockDist.ObjectID, + Type: blockDistribution.Type, + Index: blockIndex, + StorageID: blockStorageID, + Status: StatusYesterdayAfter, + Timestamp: time.Now(), + }) + if err != nil { + log.Printf("Error update BlockDistribution: %v", err) + } + } + } + //在storageTransferCount表中添加记录 + for _, dataTransfer := range data.Body.Object.DataTransfers { + sourceStorageID, _ := strconv.ParseInt(dataTransfer.SourceStorageID, 10, 64) + targetStorageID, _ := strconv.ParseInt(dataTransfer.TargetStorageID, 10, 64) + dataTransferCount, _ := strconv.ParseInt(dataTransfer.DataTransferCount, 10, 64) + + err := repoStorage.CreateStorageTransferCount(&StorageTransferCount{ + ObjectID: data.Body.Object.ObjectID, + Status: StatusTodayBeforeYesterday, + SourceStorageID: sourceStorageID, + TargetStorageID: targetStorageID, + DataTransferCount: dataTransferCount, + Timestamp: time.Now(), + }) + if err != nil { + log.Printf("Error create StorageTransferCount : %v", err) + } + } +} diff --git a/datamap/internal/models/blocktransfer.go b/datamap/internal/models/blocktransfer.go new file mode 100644 index 0000000..0bcd351 --- /dev/null +++ b/datamap/internal/models/blocktransfer.go @@ -0,0 +1,227 @@ +package models + +import ( + "errors" + cdssdk "gitlink.org.cn/cloudream/common/sdks/storage" + stgmod "gitlink.org.cn/cloudream/storage/common/models" + "gorm.io/gorm" + "log" + "strconv" + "time" +) + +type StorageTransferCount struct { + RelationshipID int64 `gorm:"column:RelationshipID; primaryKey; type:bigint; autoIncrement" json:"relationshipID"` + ObjectID int64 `gorm:"column:ObjectID; type:bigint; not null" json:"objectID"` + Status int64 `gorm:"column:Status; type:bigint; not null" json:"status"` // 连线左侧的状态 + SourceStorageID int64 `gorm:"column:SourceStorageID; type:bigint; not null" json:"sourceStorageID"` // 源存储节点 ID + TargetStorageID int64 `gorm:"column:TargetStorageID; type:bigint; not null" json:"targetStorageID"` // 目标存储节点 ID + DataTransferCount int64 `gorm:"column:DataTransferCount; type:bigint; not null" json:"dataTransferCount"` // 数据传输量 + Timestamp time.Time `gorm:"column:Timestamp; type:datatime; not null" json:"timestamp"` // 变化结束时间戳 +} + +func (StorageTransferCount) TableName() string { + return "storagetransfercount" +} + +type StorageTransferCountRepository struct { + repo *GormRepository +} + +func NewStorageTransferCountRepository(db *gorm.DB) *StorageTransferCountRepository { + return &StorageTransferCountRepository{repo: NewGormRepository(db)} +} + +func (r *StorageTransferCountRepository) CreateStorageTransferCount(storageTransferCount *StorageTransferCount) error { + return r.repo.Create(storageTransferCount) +} + +func (r *StorageTransferCountRepository) UpdateStorageTransferCount(storageTransferCount *StorageTransferCount) error { + return r.repo.Update(storageTransferCount) +} + +func (r *StorageTransferCountRepository) GetStorageTransferCountByID(id int) (*StorageTransferCount, error) { + var storageTransferCount StorageTransferCount + err := r.repo.GetByID(uint(id), &storageTransferCount) + if err != nil { + return nil, err + } + return &storageTransferCount, nil +} + +func (r *StorageTransferCountRepository) GetStorageTransferCountByObjectID(objectID int64) ([]StorageTransferCount, error) { + var storageTransferCounts []StorageTransferCount + query := "SELECT * FROM storagetransfercount WHERE ObjectID = ?" + err := r.repo.db.Raw(query, objectID).Scan(&storageTransferCounts).Error + if err != nil { + return nil, err + } + return storageTransferCounts, nil +} + +func (r *StorageTransferCountRepository) GetAllStorageTransferCounts() ([]StorageTransferCount, error) { + var storageTransferCounts []StorageTransferCount + err := r.repo.GetAll(&storageTransferCounts) + if err != nil { + return nil, err + } + return storageTransferCounts, nil +} + +// ProcessBlockTransfer 处理块传输信息的实时日志，当对象的块信息发生变化时推送触发数据库刷新 + +func ProcessBlockTransfer(data stgmod.BlockTransfer) { + + repoDist := NewBlockDistributionRepository(DB) + repoStorage := NewStorageRepository(DB) + repoStorageTrans := NewStorageTransferCountRepository(DB) + repoObject := NewObjectRepository(DB) + + for _, change := range data.Body.BlockChanges { + + objectID, _ := strconv.ParseInt(data.Body.ObjectID, 10, 64) + object, _ := repoObject.GetObjectByID(objectID) + index, _ := strconv.ParseInt(change.Index, 10, 64) + sourceStorageID, _ := strconv.ParseInt(change.SourceStorageID, 10, 64) + targetStorageID, _ := strconv.ParseInt(change.TargetStorageID, 10, 64) + newDataCount, _ := strconv.ParseInt(change.DataTransferCount, 10, 64) + + switch change.Type { + case "0": //拷贝 + //查询出存储在数据库中的BlockDistribution信息 + blockSource, errSource := repoDist.GetBlockDistributionByIndex(objectID, index, sourceStorageID) + //没有记录就将source和target的信息都保存到库中 + if errors.Is(errSource, gorm.ErrRecordNotFound) { + err := repoDist.CreateBlockDistribution(&BlockDistribution{ + ObjectID: objectID, + Type: change.BlockType, + Index: index, + StorageID: sourceStorageID, + Status: StatusNow, + Timestamp: time.Now(), + }) + if err != nil { + log.Printf("Error create source blockdistribution: %v", err) + } + } else { + //有数据则新增一条storageID为targetStorageID的记录,同时更新状态 + err := repoDist.CreateBlockDistribution(&BlockDistribution{ + ObjectID: blockSource.ObjectID, + Type: change.BlockType, + Index: index, + StorageID: targetStorageID, + Status: StatusNow, + Timestamp: time.Now(), + }) + if err != nil { + log.Printf("Error update blockdistribution: %v", err) + } + //复制完成之后增加的dataCount要加到targetStorage的记录中 + storageOld, err := repoStorage.GetStorageByID(targetStorageID) + if errors.Is(err, gorm.ErrRecordNotFound) { + err = repoStorage.CreateStorage(&Storage{ + StorageID: cdssdk.StorageID(targetStorageID), + DataCount: newDataCount, + Timestamp: time.Now(), + }) + if err != nil { + log.Printf("Error increase datacount in targetstorage: %v", err) + } + } else { + err = repoStorage.UpdateStorage(&Storage{ + StorageID: cdssdk.StorageID(targetStorageID), + DataCount: storageOld.DataCount + newDataCount, + Timestamp: time.Now(), + }) + if err != nil { + log.Printf("Error increase datacount in targetstorage: %v", err) + } + } + + } + //新增记录到storageTransferCount表中 + err := repoStorageTrans.CreateStorageTransferCount(&StorageTransferCount{ + ObjectID: objectID, + Status: int64(blockSource.Status), + SourceStorageID: sourceStorageID, + TargetStorageID: targetStorageID, + DataTransferCount: newDataCount, + Timestamp: time.Now(), + }) + if err != nil { + log.Printf("Error create StorageTransferCount : %v", err) + } + case "1": //编解码 + //删除所有的sourceBlock + for _, sourceBlock := range change.SourceBlocks { + sourceBlockIndex, _ := strconv.ParseInt(sourceBlock.Index, 10, 64) + err := repoDist.DeleteBlockDistribution(objectID, sourceBlockIndex, sourceStorageID) + if err != nil { + log.Printf("Error delete blockdistribution: %v", err) + } + } + //插入所有的targetBlock + for _, targetBlock := range change.TargetBlocks { + storageID, _ := strconv.ParseInt(targetBlock.StorageID, 10, 64) + err := repoDist.CreateBlockDistribution(&BlockDistribution{ + ObjectID: objectID, + Type: targetBlock.BlockType, + Index: index, + //直接保存到目标中心 + StorageID: storageID, + Status: StatusNow, + Timestamp: time.Now(), + }) + if err != nil { + log.Printf("Error create blockdistribution: %v", err) + } + } + //新增记录到storageTransferCount表中 + err := repoStorageTrans.CreateStorageTransferCount(&StorageTransferCount{ + ObjectID: objectID, + Status: int64(object.Status), + SourceStorageID: sourceStorageID, + TargetStorageID: targetStorageID, + DataTransferCount: newDataCount, + Timestamp: time.Now(), + }) + if err != nil { + log.Printf("Error create StorageTransferCount : %v", err) + } + + case "2": //删除 + for _, block := range change.Blocks { + storageID, _ := strconv.ParseInt(block.StorageID, 10, 64) + changeIndex, _ := strconv.ParseInt(block.Index, 10, 64) + err := repoDist.DeleteBlockDistribution(objectID, changeIndex, storageID) + if err != nil { + log.Printf("Error delete blockdistribution: %v", err) + } + } + + case "3": //更新 + for _, blockUpdate := range change.Blocks { + //查询出存储在数据库中的BlockDistribution信息 + blockIndex, _ := strconv.ParseInt(blockUpdate.Index, 10, 64) + blockOld, err := repoDist.GetBlockDistributionByIndex(objectID, blockIndex, sourceStorageID) + newStorageID, _ := strconv.ParseInt(blockUpdate.StorageID, 10, 64) + err = repoDist.UpdateBlockDistribution(&BlockDistribution{ + BlockID: blockOld.BlockID, + ObjectID: blockOld.ObjectID, + Type: blockUpdate.BlockType, + Index: blockIndex, + StorageID: newStorageID, + Status: StatusNow, + Timestamp: time.Now(), + }) + if err != nil { + log.Printf("Error delete blockdistribution: %v", err) + } + } + + default: + break + } + } + +} diff --git a/datamap/internal/models/hub.go b/datamap/internal/models/hub.go new file mode 100644 index 0000000..fdb7795 --- /dev/null +++ b/datamap/internal/models/hub.go @@ -0,0 +1,53 @@ +package models + +import ( + cdssdk "gitlink.org.cn/cloudream/common/sdks/storage" + "gorm.io/datatypes" + "gorm.io/gorm" +) + +type Hub struct { + HubID cdssdk.HubID `gorm:"column:HubID; primaryKey; type:bigint; autoIncrement" json:"hubID"` + Name string `gorm:"column:Name; type:varchar(255); not null" json:"name"` + Address datatypes.JSON `gorm:"column:Address; type:json; " json:"address"` +} + +func (Hub) TableName() string { return "hub" } + +type HubRepository struct { + repo *GormRepository +} + +func NewHubRepository(db *gorm.DB) *HubRepository { + return &HubRepository{repo: NewGormRepository(db)} +} + +func (r *HubRepository) CreateHub(hub *Hub) error { + return r.repo.Create(hub) +} + +func (r *HubRepository) UpdateHub(hub *Hub) error { + return r.repo.Update(hub) +} + +func (r *HubRepository) DeleteHub(hub *Hub) error { + return r.repo.Delete(hub, uint(hub.HubID)) +} + +func (r *HubRepository) GetHubByID(id int) (*Hub, error) { + var hub Hub + err := r.repo.GetByID(uint(id), &hub) + if err != nil { + return nil, err + } + return &hub, nil +} + +func (r *HubRepository) GetAllHubs() ([]Hub, error) { + var hubs []Hub + err := r.repo.GetAll(&hubs) + if err != nil { + return nil, err + } + return hubs, nil +} diff --git a/datamap/internal/models/hubinfo.go b/datamap/internal/models/hubinfo.go new file mode 100644 index 0000000..81b1369 --- /dev/null +++ b/datamap/internal/models/hubinfo.go @@ -0,0 +1,75 @@ +package models + +import ( + "encoding/json" + cdssdk "gitlink.org.cn/cloudream/common/sdks/storage" + stgmod "gitlink.org.cn/cloudream/storage/common/models" + "gorm.io/datatypes" +) + +// LocalHub 本地结构体，嵌入cdssdk.Hub +type LocalHub struct { + cdssdk.Hub +} + +type ConcreteHubType struct { + Address string +} + +func (c ConcreteHubType) GetStorageType() string { + return c.Address +} + +func (c ConcreteHubType) String() string { + return c.Address +} + +func (s *LocalHub) UnmarshalJSON(data []byte) error { + // 定义一个临时结构体来解析 JSON + type Alias LocalHub + aux := &struct { + Address string `json:"address"` + *Alias + }{ + Alias: (*Alias)(s), + } + + if err := json.Unmarshal(data, &aux); err != nil { + return err + } + + s.Address = ConcreteHubType{Address: aux.Address} + return nil +} + +func ProcessHubInfo(data stgmod.HubInfo) { + repo := NewHubRepository(DB) + jsonData, _ := json.Marshal(data.Body.HubInfo.Address) + HubInfo := &Hub{ + HubID: cdssdk.HubID(data.Body.HubID), + Name: data.Body.HubInfo.Name, + Address: datatypes.JSON(jsonData), + } + + //先判断传输数据的类型 + switch data.Body.Type { + case "add": + err := repo.CreateHub(HubInfo) + if err != nil { + return + } + case "update": + err := repo.UpdateHub(HubInfo) + if err != nil { + return + } + case "delete": + err := repo.DeleteHub(HubInfo) + if err != nil { + return + } + default: + return + } + +} diff --git a/datamap/internal/models/hubrequest.go b/datamap/internal/models/hubrequest.go new file mode 100644 index 0000000..47a77c8 --- /dev/null +++ b/datamap/internal/models/hubrequest.go @@ -0,0 +1,84 @@ +package models + +import ( + cdssdk "gitlink.org.cn/cloudream/common/sdks/storage" + stgmod "gitlink.org.cn/cloudream/storage/common/models" + "gorm.io/gorm" + "log" + "time" +) + +type HubRequest struct { + RequestID int64 `gorm:"column:RequestID; primaryKey; type:bigint; autoIncrement" json:"RequestID"` + SourceType string `gorm:"column:SourceType; type:varchar(255); not null" json:"sourceType"` + SourceID cdssdk.HubID `gorm:"column:SourceHubID; type:bigint; not null" json:"sourceID"` + TargetType string `gorm:"column:TargetType; type:varchar(255); not null" json:"targetType"` + TargetID cdssdk.HubID `gorm:"column:TargetHubID; type:bigint; not null" json:"targetID"` + DataTransferCount int64 `gorm:"column:DataTransferCount; type:bigint; not null" json:"dataTransferCount"` + RequestCount int64 `gorm:"column:RequestCount; type:bigint; not null" json:"requestCount"` + FailedRequestCount int64 `gorm:"column:FailedRequestCount; type:bigint; not null" json:"failedRequestCount"` + AvgTransferCount int64 `gorm:"column:AvgTransferCount; type:bigint; not null" json:"avgTransferCount"` + MaxTransferCount int64 `gorm:"column:MaxTransferCount; type:bigint; not null" json:"maxTransferCount"` + MinTransferCount int64 `gorm:"column:MinTransferCount; type:bigint; not null" json:"minTransferCount"` + StartTimestamp time.Time `gorm:"column:StartTimestamp; type:datatime; not null" json:"startTimestamp"` + EndTimestamp time.Time `gorm:"column:EndTimestamp; type:datatime; not null" json:"endTimestamp"` +} + +func (HubRequest) TableName() string { return "hubrequest" } + +type HubRequestRepository struct { + repo *GormRepository +} + +func NewHubRequestRepository(db *gorm.DB) *HubRequestRepository { + return &HubRequestRepository{repo: NewGormRepository(db)} +} + +func (r *HubRequestRepository) CreateHubRequest(request *HubRequest) error { + return r.repo.Create(request) +} + +func (r *HubRequestRepository) GetHubRequestByHubID(hubId int64) ([]HubRequest, error) { + var hubRequests []HubRequest + query := "SELECT * FROM hubrequest WHERE SourceHubID = ?" + err := r.repo.db.Raw(query, hubId).Scan(&hubRequests).Error + if err != nil { + return nil, err + } + return hubRequests, nil +} + +func (r *HubRequestRepository) GetAllHubRequests() ([]HubRequest, error) { + var hubRequests []HubRequest + err := r.repo.GetAll(&hubRequests) + if err != nil { + return nil, err + } + return hubRequests, nil +} + +// ProcessHubTransfer mq推送各节点统计自身当天向外部各个节点传输的总数据量时的处理逻辑 +func ProcessHubTransfer(data stgmod.HubTransferStats) { + repo := NewHubRequestRepository(DB) + + //todo 加字段 + hubRequest := &HubRequest{ + + SourceID: cdssdk.HubID(data.Body.SourceHubID), + TargetID: cdssdk.HubID(data.Body.TargetHubID), + DataTransferCount: data.Body.Send.TotalTransfer, + RequestCount: data.Body.Send.RequestCount, + FailedRequestCount: data.Body.Send.FailedRequestCount, + AvgTransferCount: data.Body.Send.AvgTransfer, + MaxTransferCount: data.Body.Send.MaxTransfer, + MinTransferCount: data.Body.Send.MinTransfer, + StartTimestamp: data.Body.StartTimestamp, + EndTimestamp: data.Body.EndTimestamp, + } + + err := repo.CreateHubRequest(hubRequest) + if err != nil { + log.Printf("Error update hubrequest: %v", err) + + } +} diff --git a/datamap/internal/models/hubtrans.go b/datamap/internal/models/hubtrans.go deleted file mode 100644 index d209fc8..0000000 --- a/datamap/internal/models/hubtrans.go +++ /dev/null @@ -1,53 +0,0 @@ -package models - -import ( - cdssdk "gitlink.org.cn/cloudream/common/sdks/storage" - stgmod "gitlink.org.cn/cloudream/storage/common/models" - "gorm.io/gorm" - "time" -) - -type HubRequest struct { - RequestID int64 `gorm:"column:HubID; primaryKey; type:bigint; autoIncrement" json:"hubID"` - SourceHubID cdssdk.HubID `gorm:"column:SourceHubID; type:bigint; not null" json:"sourceHubID"` - TargetHubID cdssdk.HubID `gorm:"column:TargetHubID; type:bigint; not null" json:"targetHubID"` - DataTransfer int64 `gorm:"column:DataTransfer; type:bigint; not null" json:"dataTransfer"` - RequestCount int64 `gorm:"column:RequestCount; type:bigint; not null" json:"requestCount"` - FailedRequest int64 `gorm:"column:FailedRequest; type:bigint; not null" json:"failedRequest"` - AvgTransferCount int64 `gorm:"column:AvgTransferCount; type:bigint; not null" json:"avgTransferCount"` - MaxTransferCount int64 `gorm:"column:MaxTransferCount; type:bigint; not null" json:"maxTransferCount"` - MinTransferCount int64 `gorm:"column:MinTransferCount; type:bigint; not null" json:"minTransferCount"` - StartTimestamp time.Time `gorm:"column:StartTimestamp; type:datatime; not null" json:"startTimestamp"` - EndTimestamp time.Time `gorm:"column:EndTimestamp; type:datatime; not null" json:"endTimestamp"` -} - -type HubRequestRepository struct { - repo *GormRepository -} - -func NewHubRequestRepository(db *gorm.DB) *HubRequestRepository { - return &HubRequestRepository{repo: NewGormRepository(db)} -} - -func (r *HubRequestRepository) CreateHubRequest(request *HubRequest) error { - return r.repo.Create(request) -} - -func ProcessHubTrans(data stgmod.HubTrans) { - repo := NewHubRequestRepository(db) - - hubRequest := &HubRequest{ - SourceHubID: cdssdk.HubID(data.Body.SourceHubID), - TargetHubID: cdssdk.HubID(data.Body.TargetHubID), - DataTransfer: data.Body.DataTransferCount, - RequestCount: data.Body.RequestCount, - FailedRequest: data.Body.FailedRequestCount, - AvgTransferCount: data.Body.AvgTransferCount, - MaxTransferCount: data.Body.MaxTransferCount, - MinTransferCount: data.Body.MinTransferCount, - StartTimestamp: data.Body.StartTimestamp, - EndTimestamp: data.Body.EndTimestamp, - } - - repo.CreateHubRequest(hubRequest) -} diff --git a/datamap/internal/models/models.go b/datamap/internal/models/models.go index 2c239e9..b3b1161 100644 --- a/datamap/internal/models/models.go +++ b/datamap/internal/models/models.go @@ -2,116 +2,123 @@ package models import ( cdssdk "gitlink.org.cn/cloudream/common/sdks/storage" + "gorm.io/gorm" "time" ) -type RequestID int64 +var DB *gorm.DB -type BlockID int64 +// InitDB 初始化数据库连接 +func InitDB(db *gorm.DB) { + DB = db +} +type RequestID int64 +type BlockID int64 type RelationshipID int64 - type Status int const ( - StatusNow Status = iota // =0 表示当前实时状态 - StatusYesterdayAfter // =1 表示前一天调整后的状态 - StatusYesterdayBefore // =2 表示前一天调整前的状态 - StatusTodayBeforeYesterday // =3 表示前两天调整后的状态 + StatusNow = 0 //表示当前实时状态 + StatusYesterdayAfter = 1 // 表示前一天调整后的状态 + StatusYesterdayBefore = 2 // 表示前一天调整前的状态 + StatusTodayBeforeYesterday = 3 // 表示前两天调整后的状态 ) // 节点间关系图 + type HubRelationship struct { - HubID cdssdk.HubID `json:"hubID"` // 节点ID - HubName string `json:"hubName"` // 名称 - HubAddress cdssdk.HubAddressInfo `json:"hubAddress"` // 地址 - Storages []StorageInfo `json:"storages"` // 对应的中心信息 - Requests []Request `json:"requests"` // 与各节点的连接信息（仅记录发出的请求） + Nodes []Node `json:"nodes"` // 节点列表 + Edges []Edge `json:"edges"` // 名称 } -type StorageInfo struct { - StorageID cdssdk.StorageID `json:"storageID"` // 中心ID - DataCount int64 `json:"dataCount"` // 总数据量（文件分块数） - NewDataCount int64 `json:"newdataCount"` // 新增数据量（文件分块数） - Timestamp time.Time `json:"timestamp"` // 时间戳 +type Node struct { + ID int64 `json:"id"` // 节点/中心ID + NodeType string `json:"nodeType"` //节点类型 storage/hub + Name string `json:"name"` // 节点/中心名称 + Address cdssdk.HubAddressInfo `json:"address"` // 地址 + DataCount int64 `json:"dataCount"` // 总数据量（文件分块数） + NewDataCount int64 `json:"newdataCount"` // 新增数据量（文件分块数） + Timestamp time.Time `json:"timestamp"` // 时间戳 } -type Request struct { - SourceHubID cdssdk.HubID `json:"sourceHubID"` // 源节点ID - TargetHubID cdssdk.HubID `json:"targetHubID"` // 目标节点ID - DataTransferCount int64 `json:"dataTransferCount"` // 数据传输量 - RequestCount int64 `json:"requestCount"` // 请求数 - FailedRequestCount int64 `json:"failedRequestCount"` // 失败请求数 - AvgTransferCount int64 `json:"avgTransferCount"` // 平均数据传输量 - MaxTransferCount int64 `json:"maxTransferCount"` // 最大数据传输量 - MinTransferCount int64 `json:"minTransferCount"` // 最小数据传输量 - StartTimestamp time.Time `json:"startTimestamp"` // 起始时间戳 - EndTimestamp time.Time `json:"endTimestamp"` // 结束时间戳 +type Edge struct { + SourceType string `json:"sourceType"` // 源节点类型 + SourceID int64 `json:"sourceID"` // 源节点ID + TargetType string `json:"targetType"` // 目标节点类型 + TargetID int64 `json:"targetID"` // 目标节点ID + DataTransferCount int64 `json:"dataTransferCount"` // 数据传输量 + RequestCount int64 `json:"requestCount"` // 请求数 + FailedRequestCount int64 `json:"failedRequestCount"` // 失败请求数 + AvgTransferCount int64 `json:"avgTransferCount"` // 平均数据传输量 + MaxTransferCount int64 `json:"maxTransferCount"` // 最大数据传输量 + MinTransferCount int64 `json:"minTransferCount"` // 最小数据传输量 + StartTimestamp time.Time `json:"startTimestamp"` // 起始时间戳 + EndTimestamp time.Time `json:"endTimestamp"` // 结束时间戳 } -// 对象块分布结构 +// 对象分布 type ObjectDistribution struct { - ObjectID cdssdk.ObjectID `json:"objectID"` // 对象 ID - PackageID cdssdk.PackageID `json:"packageID"` // 包 ID - Path string `json:"path"` // 路径 - Size int64 `json:"size"` // 大小 - FileHash string `json:"fileHash"` // 文件哈希 - States []State `json:"states"` // 各阶段状态信息（只需要传1、2、3阶段） - Relationships []Relationship `json:"relationships"` // 节点间传输量 - Timestamp time.Time `json:"timestamp"` // 请求中的时间戳 + Nodes []DistNode `json:"nodes"` + Combos []Combo `json:"combos"` + Edges []DistEdge `json:"edges"` } -type State struct { - Timestamp time.Time `json:"timestamp"` // 时间戳 - Status string `json:"status"` // 状态 - FaultTolerance string `json:"faultTolerance"` // 容灾度（仅布局调整后） - Redundancy string `json:"redundancy"` // 冗余度（仅布局调整后） - AvgAccessCost float64 `json:"avgAccessCost"` // 平均访问开销（仅布局调整前） - BlockDistributions []BlockDist `json:"blockDistributions"` // 块分布情况 +type DistNode struct { + ID string `json:"id"` + ComboID string `json:"comboID"` + Label string `json:"label"` + NodeType string `json:"nodeType"` } -type BlockDist struct { - StorageID cdssdk.StorageID `json:"storageID"` // 中心ID - Blocks []Block `json:"blocks"` // 该中心的所有块 +type Combo struct { + ID string `json:"id"` + Label string `json:"label"` + ParentId string `json:"parentId"` + ComboType string `json:"comboType"` } -type Block struct { - Type string `json:"type"` // 块类型 - Index string `json:"index"` // 块编号 - ID string `json:"id"` // 块ID +type DistEdge struct { + Source string `json:"source"` + Target string `json:"target"` } -type Relationship struct { - Status Status `json:"status"` // 连线左侧的状态 - SourceStorageID string `json:"sourceStorageID"` // 源存储节点 ID - TargetStorageID string `json:"targetStorageID"` // 目标存储节点 ID - DataTransferCount string `json:"dataTransferCount"` // 数据传输量 - Timestamp time.Time `json:"timestamp"` // 变化结束时间戳 -} - -//数据库结构定义 - -func (HubRequest) TableName() string { - return "HubRequest" -} - -func (Object) TableName() string { - return "Object" -} - -func (BlockDistribution) TableName() string { - return "BlockDistribution" -} - -type StorageTransferCount struct { - RelationshipID RelationshipID `gorm:"column:RelationshipID; primaryKey; type:bigint; autoIncrement" json:"relationshipID"` - Status Status `gorm:"column:Status; type:tinyint; not null" json:"status"` - SourceStorageID cdssdk.StorageID `gorm:"column:SourceStorageID; type:bigint; not null" json:"sourceStorageID"` - TargetStorageID cdssdk.StorageID `gorm:"column:TargetStorageID; type:bigint; not null" json:"targetStorageID"` - DataTransferCount int64 `gorm:"column:DataTransferCount; type:bigint; not null" json:"dataTransferCount"` - Timestamp time.Time `gorm:"column:Timestamp; type:datatime; not null" json:"timestamp"` -} - -func (StorageTransferCount) TableName() string { - return "StorageTransferCount" -} +// 对象块分布结构 +//type ObjectDistribution struct { +// ObjectID cdssdk.ObjectID `json:"objectID"` // 对象 ID +// PackageID cdssdk.PackageID `json:"packageID"` // 包 ID +// Path string `json:"path"` // 路径 +// Size int64 `json:"size"` // 大小 +// FileHash string `json:"fileHash"` // 文件哈希 +// States []State `json:"states"` // 各阶段状态信息（只需要传1、2、3阶段） +// Relationships []Relationship `json:"relationships"` // 节点间传输量 +// Timestamp time.Time `json:"timestamp"` // 请求中的时间戳 +//} +// +//type State struct { +// Timestamp time.Time `json:"timestamp"` // 时间戳 +// Status string `json:"status"` // 状态 +// FaultTolerance string `json:"faultTolerance"` // 容灾度（仅布局调整后） +// Redundancy string `json:"redundancy"` // 冗余度（仅布局调整后） +// AvgAccessCost float64 `json:"avgAccessCost"` // 平均访问开销（仅布局调整前） +// BlockDistributions []BlockDist `json:"blockDistributions"` // 块分布情况 +//} +// +//type BlockDist struct { +// StorageID cdssdk.StorageID `json:"storageID"` // 中心ID +// Blocks []Block `json:"blocks"` // 该中心的所有块 +//} +// +//type Block struct { +// Type string `json:"type"` // 块类型 +// Index string `json:"index"` // 块编号 +// ID string `json:"id"` // 块ID +//} +// +//type Relationship struct { +// Status Status `json:"status"` // 连线左侧的状态 +// SourceStorageID string `json:"sourceStorageID"` // 源存储节点 ID +// TargetStorageID string `json:"targetStorageID"` // 目标存储节点 ID +// DataTransferCount string `json:"dataTransferCount"` // 数据传输量 +// Timestamp time.Time `json:"timestamp"` // 变化结束时间戳 +//} diff --git a/datamap/internal/models/object.go b/datamap/internal/models/object.go index 6fa8747..b2c8855 100644 --- a/datamap/internal/models/object.go +++ b/datamap/internal/models/object.go @@ -4,7 +4,6 @@ import ( cdssdk "gitlink.org.cn/cloudream/common/sdks/storage" stgmod "gitlink.org.cn/cloudream/storage/common/models" "gorm.io/gorm" - "log" "time" ) @@ -21,50 +20,52 @@ type Object struct { Timestamp time.Time `gorm:"column:Timestamp; type:datatime; not null" json:"timestamp"` } -// BlockTransferRepository 块传输记录的 Repository -type BlockTransferRepository struct { +func (Object) TableName() string { + return "object" +} + +// ObjectRepository 块传输记录的 Repository +type ObjectRepository struct { repo *GormRepository } -// NewBlockTransferRepository 创建 BlockTransferRepository 实例 -func NewBlockTransferRepository(db *gorm.DB) *BlockTransferRepository { - return &BlockTransferRepository{repo: NewGormRepository(db)} +// NewObjectRepository 创建 ObjectRepository 实例 +func NewObjectRepository(db *gorm.DB) *ObjectRepository { + return &ObjectRepository{repo: NewGormRepository(db)} } -// CreateBlockTransfer 创建块传输记录 -func (r *BlockTransferRepository) CreateBlockTransfer(transfer *Object) error { - return r.repo.Create(transfer) +// CreateObject 创建块传输记录 +func (r *ObjectRepository) CreateObject(object *Object) error { + return r.repo.Create(object) } -// GetAllBlockTransfers 获取所有块传输记录 -func (r *BlockTransferRepository) GetAllBlockTransfers() ([]Object, error) { - var transfers []Object - err := r.repo.GetAll(&transfers) +// GetObjectByID 查询单个Object +func (r *ObjectRepository) GetObjectByID(objectID int64) (*Object, error) { + var object Object + query := "SELECT * FROM object WHERE ObjectID = ?" + err := r.repo.db.Raw(query, objectID).First(&object).Error if err != nil { return nil, err } - return transfers, nil + return &object, nil } -// ProcessBlockTransInfo 处理 BlockTransInfo 数据 -func ProcessBlockTransInfo(data stgmod.Object) { - repo := NewBlockTransferRepository(db) - //TODO 数据待调整 - transfer := &Object{ - ObjectID: 0, - PackageID: 0, - Path: data.Path, - Size: 0, - FileHash: "", - Status: 0, - FaultTolerance: 0, - Redundancy: 0, - AvgAccessCost: 0, - Timestamp: time.Time{}, - } - err := repo.CreateBlockTransfer(transfer) +// UpdateObject 更新块传输记录 +func (r *ObjectRepository) UpdateObject(object *Object) error { + return r.repo.Update(object) +} + +// GetAllObjects 获取所有块传输记录 +func (r *ObjectRepository) GetAllObjects() ([]Object, error) { + var objects []Object + err := r.repo.GetAll(&objects) if err != nil { - log.Printf("Failed to create block transfer record: %v", err) + return nil, err } + return objects, nil +} + +// ProcessObject 处理 Object 数据 +func ProcessObject(data stgmod.ObjectChange) { } diff --git a/datamap/internal/models/hubs.go b/datamap/internal/models/storageinfo.go similarity index 66% rename from datamap/internal/models/hubs.go rename to datamap/internal/models/storageinfo.go index a28cb9b..9f9dbeb 100644 --- a/datamap/internal/models/hubs.go +++ b/datamap/internal/models/storageinfo.go @@ -9,30 +9,17 @@ import ( "time" ) -var db *gorm.DB - -func InitDB(gormDB *gorm.DB) { - db = gormDB -} - -type Hub struct { - HubID cdssdk.HubID `gorm:"column:HubID; primaryKey; type:bigint; autoIncrement" json:"hubID"` - Name string `gorm:"column:Name; type:varchar(255); not null" json:"name"` - Address cdssdk.HubAddressInfo `gorm:"column:Address; type:json; serializer:union" json:"address"` -} - -type HubStat struct { - StorageID int `json:"storageID"` - DataCount int `json:"dataCount"` -} - type Storage struct { StorageID cdssdk.StorageID `gorm:"column:StorageID; primaryKey; type:bigint; autoIncrement" json:"storageID"` + StorageName string `gorm:"column:StorageName; type:varchar(1024); not null" json:"storageName"` HubID cdssdk.HubID `gorm:"column:HubID; type:bigint; not null" json:"hubID"` DataCount int64 `gorm:"column:DataCount; type:bigint; not null" json:"dataCount"` NewDataCount int64 `gorm:"column:NewDataCount; type:bigint; not null" json:"newDataCount"` Timestamp time.Time `gorm:"column:Timestamp; type:datatime; not null" json:"timestamp"` } + +func (Storage) TableName() string { return "storage" } + type StorageRepository struct { repo *GormRepository } @@ -49,7 +36,7 @@ func (r *StorageRepository) UpdateStorage(storage *Storage) error { return r.repo.Update(storage) } -func (r *StorageRepository) GetStorageByID(id int) (*Storage, error) { +func (r *StorageRepository) GetStorageByID(id int64) (*Storage, error) { var storage Storage err := r.repo.GetByID(uint(id), &storage) if err != nil { @@ -58,6 +45,16 @@ func (r *StorageRepository) GetStorageByID(id int) (*Storage, error) { return &storage, nil } +func (r *StorageRepository) GetStorageByHubID(hubId int64) (*Storage, error) { + var storage Storage + query := "SELECT * FROM storage WHERE HubID = ?" + err := r.repo.db.Raw(query, hubId).Scan(&storage).Error + if err != nil { + return nil, err + } + return &storage, nil +} + func (r *StorageRepository) GetAllStorages() ([]Storage, error) { var storages []Storage err := r.repo.GetAll(&storages) @@ -67,8 +64,10 @@ func (r *StorageRepository) GetAllStorages() ([]Storage, error) { return storages, nil } -func ProcessHubStat(data stgmod.HubStat) { - repo := NewStorageRepository(db) +//ProcessHubStat mq推送各节点统计自身当前的总数据量时的处理逻辑 + +func ProcessStorageInfo(data stgmod.StorageStats) { + repo := NewStorageRepository(DB) storage, err := repo.GetStorageByID(data.Body.StorageID) if err != nil { @@ -76,7 +75,7 @@ func ProcessHubStat(data stgmod.HubStat) { // 插入新记录 newStorage := &Storage{ StorageID: cdssdk.StorageID(data.Body.StorageID), - DataCount: int64(data.Body.DataCount), + DataCount: data.Body.DataCount, NewDataCount: 0, } repo.CreateStorage(newStorage) @@ -85,9 +84,12 @@ func ProcessHubStat(data stgmod.HubStat) { } } else { // 更新记录 - newDataCount := int64(data.Body.DataCount) - storage.DataCount - storage.DataCount = int64(data.Body.DataCount) + newDataCount := data.Body.DataCount - storage.DataCount + storage.DataCount = data.Body.DataCount storage.NewDataCount = newDataCount - repo.UpdateStorage(storage) + err := repo.UpdateStorage(storage) + if err != nil { + log.Printf("Error update storage: %v", err) + } } } diff --git a/datamap/internal/mq/mq.go b/datamap/internal/mq/mq.go index ba32c6a..53240fd 100644 --- a/datamap/internal/mq/mq.go +++ b/datamap/internal/mq/mq.go @@ -29,9 +29,6 @@ func listenQueues(conn *amqp.Connection) { "datamap_hubtransfer", "datamap_blocktransfer", "datamap_blockdistribution", - "datamap_objectchange", - "datamap_packagechange", - "datamap_bucketchange", } for _, queue := range queues { @@ -58,22 +55,60 @@ func listenQueues(conn *amqp.Connection) { func processMessage(queue string, body []byte) { switch queue { + case "datamap_hubinfo": + var data stgmod.HubInfo + + if err := json.Unmarshal(body, &data); err != nil { + log.Printf("Failed to unmarshal HubInfo: %v, body: %s", err, body) + return + } + models.ProcessHubInfo(data) case "datamap_storageinfo": - var data stgmod.HubStat - json.Unmarshal(body, &data) - models.ProcessHubStat(data) - case "datamap_hubtransfer": - var data stgmod.HubTrans - json.Unmarshal(body, &data) - models.ProcessHubTrans(data) + var data stgmod.StorageInfo + if err := json.Unmarshal(body, &data); err != nil { + log.Printf("Failed to unmarshal StorageInfo: %v, body: %s", err, body) + return + } + //models.ProcessStorageInfo(data) + case "datamap_storagestats": + var data stgmod.StorageStats + if err := json.Unmarshal(body, &data); err != nil { + log.Printf("Failed to unmarshal StorageStats: %v, body: %s", err, body) + return + } + //models.ProcessStorageInfo(data) + case "datamap_hubtransferstats": + var data stgmod.HubTransferStats + err := json.Unmarshal(body, &data) + if err != nil { + log.Printf("Failed to unmarshal HubTransferStats: %v, body: %s", err, body) + return + } + models.ProcessHubTransfer(data) + case "datamap_hubstoragetransferstats": + var data stgmod.HubStorageTransferStats + err := json.Unmarshal(body, &data) + if err != nil { + log.Printf("Failed to unmarshal HubStorageTransferStats: %v, body: %s", err, body) + return + } + //models.ProcessHubTransfer(data) case "datamap_blocktransfer": - var data stgmod.Object - json.Unmarshal(body, &data) - models.ProcessBlockTransInfo(data) + var data stgmod.BlockTransfer + err := json.Unmarshal(body, &data) + if err != nil { + log.Printf("Failed to unmarshal BlockTransfer: %v, body: %s", err, body) + return + } + models.ProcessBlockTransfer(data) case "datamap_blockdistribution": - var data stgmod.BlockTransInfo - json.Unmarshal(body, &data) - models.ProcessBlockDistributionInfo(data) + var data stgmod.BlockDistribution + err := json.Unmarshal(body, &data) + if err != nil { + log.Printf("Failed to unmarshal BlockDistribution: %v, body: %s", err, body) + return + } + models.ProcessBlockDistribution(data) default: log.Printf("Unknown queue: %s", queue) } diff --git a/datamap/internal/server/server.go b/datamap/internal/server/server.go index d2812c7..f3233cb 100644 --- a/datamap/internal/server/server.go +++ b/datamap/internal/server/server.go @@ -11,9 +11,10 @@ import ( func StartServer(db *gorm.DB, mq *amqp.Connection) { r := gin.Default() + handlers.SetDB(db) // 注册HTTP接口 - r.GET("/storage", handlers.GetStorageData) - r.GET("/block-distribution", handlers.GetBlockDistributionData) + r.GET("/hubInfo", handlers.GetHubInfo) + r.GET("/dataTransfer/:objectID", handlers.GetDataTransfer) // 启动服务 if err := r.Run(":8080"); err != nil { diff --git a/datamap/main.go b/datamap/main.go index abe8755..da0ce32 100644 --- a/datamap/main.go +++ b/datamap/main.go @@ -3,6 +3,7 @@ package main import ( "gitlink.org.cn/cloudream/storage/datamap/internal/config" "gitlink.org.cn/cloudream/storage/datamap/internal/db" + "gitlink.org.cn/cloudream/storage/datamap/internal/models" "gitlink.org.cn/cloudream/storage/datamap/internal/mq" "gitlink.org.cn/cloudream/storage/datamap/internal/server" "log" @@ -18,6 +19,8 @@ func main() { log.Fatalf("Failed to initialize database: %v", err) } + models.InitDB(dbConn) + // 初始化RabbitMQ mqConn, err := mq.InitMQ(cfg.RabbitMQ) if err != nil { diff --git a/go.mod b/go.mod index 4c44652..4e4e0fd 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/aws/aws-sdk-go-v2/credentials v1.17.47 github.com/aws/aws-sdk-go-v2/service/s3 v1.71.0 github.com/gin-gonic/gin v1.7.7 - github.com/go-sql-driver/mysql v1.7.1 + github.com/go-sql-driver/mysql v1.8.1 github.com/hashicorp/golang-lru/v2 v2.0.5 github.com/huaweicloud/huaweicloud-sdk-go-obs v3.24.9+incompatible github.com/inhies/go-bytesize v0.0.0-20220417184213-4913239db9cf @@ -27,10 +27,12 @@ require ( gitlink.org.cn/cloudream/common v0.0.0 google.golang.org/grpc v1.62.1 google.golang.org/protobuf v1.33.0 - gorm.io/gorm v1.25.7 + gorm.io/datatypes v1.2.5 + gorm.io/gorm v1.25.11 ) require ( + filippo.io/edwards25519 v1.1.0 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.7 // indirect github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.25 // indirect github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.25 // indirect @@ -99,11 +101,11 @@ require ( go.uber.org/atomic v1.10.0 // indirect go.uber.org/multierr v1.9.0 // indirect go.uber.org/zap v1.24.0 // indirect - golang.org/x/crypto v0.21.0 // indirect + golang.org/x/crypto v0.22.0 // indirect golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect golang.org/x/net v0.23.0 // indirect golang.org/x/sync v0.6.0 - golang.org/x/sys v0.18.0 // indirect + golang.org/x/sys v0.19.0 // indirect golang.org/x/text v0.14.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20240311132316-a219d84964c2 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240314234333-6e1732d8331c // indirect diff --git a/go.sum b/go.sum index 2cee4dd..ac97be9 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,5 @@ +filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA= +filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= github.com/QcloudApi/qcloud_sign_golang v0.0.0-20141224014652-e4130a326409/go.mod h1:1pk82RBxDY/JZnPQrtqHlUFfCctgdorsd9M06fMynOM= github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible h1:8psS8a+wKfiLt1iVDX79F7Y6wUM49Lcha2FMXt4UM8g= github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8= @@ -58,12 +60,16 @@ github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn github.com/go-playground/validator/v10 v10.8.0 h1:1kAa0fCrnpv+QYdkdcRzrRM7AyYs5o8+jZdJCz9xj6k= github.com/go-playground/validator/v10 v10.8.0/go.mod h1:9JhgTzTaE31GZDpH/HSvHiRJrJ3iKAgqqH0Bl/Ocjdk= github.com/go-sql-driver/mysql v1.7.0/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI= -github.com/go-sql-driver/mysql v1.7.1 h1:lUIinVbN1DY0xBg0eMOzmmtGoHwWBbvnWubQUrtU8EI= -github.com/go-sql-driver/mysql v1.7.1/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI= +github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpvNJ1Y= +github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang-jwt/jwt/v5 v5.2.1/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= +github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9 h1:au07oEsX2xN0ktxqI+Sida1w446QrXBRJ0nee3SNZlA= +github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0= +github.com/golang-sql/sqlexp v0.1.0 h1:ZCD6MBpcuOVfGVqsEmY5/4FtYiKz6tSyUv9LPEDei6A= +github.com/golang-sql/sqlexp v0.1.0/go.mod h1:J4ad9Vo8ZCWQ2GMrC4UCQy1JpCbwU9m3EOqtpKwwwHI= github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= @@ -96,6 +102,14 @@ github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/inhies/go-bytesize v0.0.0-20220417184213-4913239db9cf h1:FtEj8sfIcaaBfAKrE1Cwb61YDtYq9JxChK1c7AKce7s= github.com/inhies/go-bytesize v0.0.0-20220417184213-4913239db9cf/go.mod h1:yrqSXGoD/4EKfF26AOGzscPOgTTJcyAwM2rpixWT+t4= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgservicefile v0.0.0-20231201235250-de7065d80cb9 h1:L0QtFUgDarD7Fpv9jeVMgy/+Ec0mtnmYuImjTz6dtDA= +github.com/jackc/pgservicefile v0.0.0-20231201235250-de7065d80cb9/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgx/v5 v5.5.5 h1:amBjrZVmksIdNjxGW/IiIMzxMKZFelXbUoPNb+8sjQw= +github.com/jackc/pgx/v5 v5.5.5/go.mod h1:ez9gk+OAat140fv9ErkZDYFWmXLfV+++K0uAOiwgm1A= +github.com/jackc/puddle/v2 v2.2.1 h1:RhxXJtFG022u4ibrCSMSiu5aOq1i77R3OHKNJj77OAk= +github.com/jackc/puddle/v2 v2.2.1/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/jedib0t/go-pretty/v6 v6.4.7 h1:lwiTJr1DEkAgzljsUsORmWsVn5MQjt1BPJdPCtJ6KXE= github.com/jedib0t/go-pretty/v6 v6.4.7/go.mod h1:Ndk3ase2CkQbXLLNf5QDHoYb6J9WtVfmHZu9n8rk2xs= github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= @@ -130,6 +144,10 @@ github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APP github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-runewidth v0.0.13 h1:lTGmDsbAYt5DmK6OnoV7EuIF1wEIFAcxld6ypU4OSgU= github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= +github.com/mattn/go-sqlite3 v1.14.15 h1:vfoHhTN1af61xCRSWzFIWzx2YskyMTwHLrExkBOjvxI= +github.com/mattn/go-sqlite3 v1.14.15/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= +github.com/microsoft/go-mssqldb v1.7.2 h1:CHkFJiObW7ItKTJfHo1QX7QBBD1iV+mn1eOyRP3b/PA= +github.com/microsoft/go-mssqldb v1.7.2/go.mod h1:kOvZKUdrhhFQmxLZqbwUV0rHkNkZpthMITIb2Ko1IoA= github.com/mitchellh/mapstructure v1.4.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= @@ -227,8 +245,8 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= -golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= +golang.org/x/crypto v0.22.0 h1:g1v0xeRhjcugydODzvb3mEM9SQ0HGp9s/nh3COQ/C30= +golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M= golang.org/x/exp v0.0.0-20230905200255-921286631fa9 h1:GoHiUyI/Tp2nVkLI2mCxVkOjsbSXD66ic0XW0js0R9g= golang.org/x/exp v0.0.0-20230905200255-921286631fa9/go.mod h1:S2oDrQGGwySpoQPVqRShND87VCbxmc6bL1Yd2oYrm6k= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= @@ -256,8 +274,8 @@ golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= -golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o= +golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= @@ -297,7 +315,16 @@ gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gorm.io/datatypes v1.2.5 h1:9UogU3jkydFVW1bIVVeoYsTpLRgwDVW3rHfJG6/Ek9I= +gorm.io/datatypes v1.2.5/go.mod h1:I5FUdlKpLb5PMqeMQhm30CQ6jXP8Rj89xkTeCSAaAD4= gorm.io/driver/mysql v1.5.7 h1:MndhOPYOfEp2rHKgkZIhJ16eVUIRf2HmzgoPmh7FCWo= gorm.io/driver/mysql v1.5.7/go.mod h1:sEtPWMiqiN1N1cMXoXmBbd8C6/l+TESwriotuRRpkDM= -gorm.io/gorm v1.25.7 h1:VsD6acwRjz2zFxGO50gPO6AkNs7KKnvfzUjHQhZDz/A= +gorm.io/driver/postgres v1.5.0 h1:u2FXTy14l45qc3UeCJ7QaAXZmZfDDv0YrthvmRq1l0U= +gorm.io/driver/postgres v1.5.0/go.mod h1:FUZXzO+5Uqg5zzwzv4KK49R8lvGIyscBOqYrtI1Ce9A= +gorm.io/driver/sqlite v1.4.3 h1:HBBcZSDnWi5BW3B3rwvVTc510KGkBkexlOg0QrmLUuU= +gorm.io/driver/sqlite v1.4.3/go.mod h1:0Aq3iPO+v9ZKbcdiz8gLWRw5VOPcBOPUQJFLq5e2ecI= +gorm.io/driver/sqlserver v1.5.4 h1:xA+Y1KDNspv79q43bPyjDMUgHoYHLhXYmdFcYPobg8g= +gorm.io/driver/sqlserver v1.5.4/go.mod h1:+frZ/qYmuna11zHPlh5oc2O6ZA/lS88Keb0XSH1Zh/g= gorm.io/gorm v1.25.7/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8= +gorm.io/gorm v1.25.11 h1:/Wfyg1B/je1hnDx3sMkX+gAlxrlZpn6X0BXRlwXlvHg= +gorm.io/gorm v1.25.11/go.mod h1:xh7N7RHfYlNc5EmcI/El95gXusucDrQnHXe0+CgWcLQ= From 8869f8e6e994fb657641e7cdf6864a991dd3fb0a Mon Sep 17 00:00:00 2001 From: Jake <450705171@qq.com> Date: Thu, 16 Jan 2025 11:13:30 +0800 Subject: [PATCH 06/12] =?UTF-8?q?=E9=83=A8=E5=88=86=E5=AD=97=E6=AE=B5?= =?UTF-8?q?=E8=B0=83=E6=95=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- datamap/internal/handlers/handlers.go | 35 ++++++++++++------ datamap/internal/models/hubinfo.go | 1 - datamap/internal/models/models.go | 6 ++-- datamap/internal/models/storageStats.go | 30 ++++++++++++++++ datamap/internal/models/storageinfo.go | 47 +++++++++++++------------ datamap/internal/mq/mq.go | 24 +++++++------ 6 files changed, 94 insertions(+), 49 deletions(-) create mode 100644 datamap/internal/models/storageStats.go diff --git a/datamap/internal/handlers/handlers.go b/datamap/internal/handlers/handlers.go index 0fa46ba..66abf46 100644 --- a/datamap/internal/handlers/handlers.go +++ b/datamap/internal/handlers/handlers.go @@ -31,7 +31,7 @@ func GetHubInfo(c *gin.Context) { storages, _ := repoStorage.GetAllStorages() for _, hub := range hubs { node := models.Node{ - ID: int64(hub.HubID), + ID: "hub" + strconv.FormatInt(int64(hub.HubID), 10), NodeType: "hub", Name: hub.Name, Address: hub.Address, @@ -40,7 +40,7 @@ func GetHubInfo(c *gin.Context) { } for _, storage := range storages { node := models.Node{ - ID: int64(storage.StorageID), + ID: "storage" + strconv.FormatInt(int64(storage.StorageID), 10), NodeType: "storage", Name: storage.StorageName, DataCount: storage.DataCount, @@ -55,9 +55,9 @@ func GetHubInfo(c *gin.Context) { for _, hubReq := range hubReqs { edge := models.Edge{ SourceType: hubReq.SourceType, - SourceID: int64(hubReq.SourceID), + SourceID: hubReq.SourceType + strconv.FormatInt(int64(hubReq.SourceID), 10), TargetType: hubReq.TargetType, - TargetID: int64(hubReq.TargetID), + TargetID: hubReq.TargetType + strconv.FormatInt(int64(hubReq.TargetID), 10), DataTransferCount: hubReq.DataTransferCount, RequestCount: hubReq.RequestCount, FailedRequestCount: hubReq.FailedRequestCount, @@ -116,7 +116,7 @@ func GetDataTransfer(c *gin.Context) { //storage id ComboID: strconv.FormatInt(block.StorageID, 10), //block index - Label: strconv.FormatInt(block.Index, 10), + Label: block.Type + strconv.FormatInt(block.Index, 10), //block type NodeType: block.Type, } @@ -126,7 +126,7 @@ func GetDataTransfer(c *gin.Context) { //添加storage combo信息 if !containsCombo(combos, strconv.FormatInt(block.StorageID, 10), "storage") { combo := models.Combo{ - ID: strconv.FormatInt(block.StorageID, 10), + ID: "storage" + strconv.FormatInt(block.StorageID, 10), Label: "存储中心" + strconv.FormatInt(block.StorageID, 10), ParentId: strconv.Itoa(block.Status), ComboType: "storage", @@ -134,10 +134,23 @@ func GetDataTransfer(c *gin.Context) { combos = append(combos, combo) } //添加state combo信息 - if !containsCombo(combos, strconv.Itoa(block.Status), "state") { + if !containsCombo(combos, "state"+strconv.Itoa(block.Status), "state") { + var statusStr string + switch block.Status { + case 0: + statusStr = "实时情况" + case 1: + statusStr = block.Timestamp.Format("2006-01-02") + "布局调整后" + case 2: + statusStr = block.Timestamp.Format("2006-01-02") + "布局调整前" + case 3: + statusStr = block.Timestamp.Format("2006-01-02") + "布局调整后" + default: + statusStr = "未知状态" + } combo := models.Combo{ - ID: strconv.Itoa(block.Status), - Label: "存储状态" + string(block.Status), + ID: "state" + strconv.Itoa(block.Status), + Label: statusStr, ComboType: "state", } combos = append(combos, combo) @@ -147,8 +160,8 @@ func GetDataTransfer(c *gin.Context) { relations, _ := repoStorageTrans.GetStorageTransferCountByObjectID(objectID) for _, relation := range relations { edge := models.DistEdge{ - Source: strconv.FormatInt(relation.SourceStorageID, 10), - Target: strconv.FormatInt(relation.TargetStorageID, 10), + Source: "storage" + strconv.FormatInt(relation.SourceStorageID, 10), + Target: "storage" + strconv.FormatInt(relation.TargetStorageID, 10), } edges = append(edges, edge) } diff --git a/datamap/internal/models/hubinfo.go b/datamap/internal/models/hubinfo.go index 81b1369..e515846 100644 --- a/datamap/internal/models/hubinfo.go +++ b/datamap/internal/models/hubinfo.go @@ -71,5 +71,4 @@ func ProcessHubInfo(data stgmod.HubInfo) { default: return } - } diff --git a/datamap/internal/models/models.go b/datamap/internal/models/models.go index b3b1161..85fb10a 100644 --- a/datamap/internal/models/models.go +++ b/datamap/internal/models/models.go @@ -33,7 +33,7 @@ type HubRelationship struct { } type Node struct { - ID int64 `json:"id"` // 节点/中心ID + ID string `json:"id"` // 节点/中心ID NodeType string `json:"nodeType"` //节点类型 storage/hub Name string `json:"name"` // 节点/中心名称 Address cdssdk.HubAddressInfo `json:"address"` // 地址 @@ -44,9 +44,9 @@ type Node struct { type Edge struct { SourceType string `json:"sourceType"` // 源节点类型 - SourceID int64 `json:"sourceID"` // 源节点ID + SourceID string `json:"source"` // 源节点ID TargetType string `json:"targetType"` // 目标节点类型 - TargetID int64 `json:"targetID"` // 目标节点ID + TargetID string `json:"target"` // 目标节点ID DataTransferCount int64 `json:"dataTransferCount"` // 数据传输量 RequestCount int64 `json:"requestCount"` // 请求数 FailedRequestCount int64 `json:"failedRequestCount"` // 失败请求数 diff --git a/datamap/internal/models/storageStats.go b/datamap/internal/models/storageStats.go new file mode 100644 index 0000000..d7e6928 --- /dev/null +++ b/datamap/internal/models/storageStats.go @@ -0,0 +1,30 @@ +package models + +// +//func ProcessStorageInfo(data stgmod.StorageInfo) { +// repo := NewStorageRepository(DB) +// +// storage, err := repo.GetStorageByID(data.Body.StorageID) +// if err != nil { +// if errors.Is(err, gorm.ErrRecordNotFound) { +// // 插入新记录 +// newStorage := &Storage{ +// StorageID: cdssdk.StorageID(data.Body.StorageID), +// DataCount: data.Body.DataCount, +// NewDataCount: 0, +// } +// repo.CreateStorage(newStorage) +// } else { +// log.Printf("Error querying storage: %v", err) +// } +// } else { +// // 更新记录 +// newDataCount := data.Body.DataCount - storage.DataCount +// storage.DataCount = data.Body.DataCount +// storage.NewDataCount = newDataCount +// err := repo.UpdateStorage(storage) +// if err != nil { +// log.Printf("Error update storage: %v", err) +// } +// } +//} diff --git a/datamap/internal/models/storageinfo.go b/datamap/internal/models/storageinfo.go index 9f9dbeb..74be392 100644 --- a/datamap/internal/models/storageinfo.go +++ b/datamap/internal/models/storageinfo.go @@ -1,11 +1,9 @@ package models import ( - "errors" cdssdk "gitlink.org.cn/cloudream/common/sdks/storage" stgmod "gitlink.org.cn/cloudream/storage/common/models" "gorm.io/gorm" - "log" "time" ) @@ -35,6 +33,9 @@ func (r *StorageRepository) CreateStorage(storage *Storage) error { func (r *StorageRepository) UpdateStorage(storage *Storage) error { return r.repo.Update(storage) } +func (r *StorageRepository) DeleteStorage(storage *Storage) error { + return r.repo.Delete(storage, uint(storage.StorageID)) +} func (r *StorageRepository) GetStorageByID(id int64) (*Storage, error) { var storage Storage @@ -64,32 +65,32 @@ func (r *StorageRepository) GetAllStorages() ([]Storage, error) { return storages, nil } -//ProcessHubStat mq推送各节点统计自身当前的总数据量时的处理逻辑 - -func ProcessStorageInfo(data stgmod.StorageStats) { +func ProcessStorageInfo(data stgmod.StorageInfo) { repo := NewStorageRepository(DB) + storage := &Storage{ + StorageID: cdssdk.StorageID(data.Body.StorageID), + StorageName: data.Body.StorageInfo.Name, + HubID: data.Body.StorageInfo.MasterHub, + Timestamp: data.Timestamp, + } - storage, err := repo.GetStorageByID(data.Body.StorageID) - if err != nil { - if errors.Is(err, gorm.ErrRecordNotFound) { - // 插入新记录 - newStorage := &Storage{ - StorageID: cdssdk.StorageID(data.Body.StorageID), - DataCount: data.Body.DataCount, - NewDataCount: 0, - } - repo.CreateStorage(newStorage) - } else { - log.Printf("Error querying storage: %v", err) + switch data.Body.Type { + case "add": + err := repo.CreateStorage(storage) + if err != nil { + return } - } else { - // 更新记录 - newDataCount := data.Body.DataCount - storage.DataCount - storage.DataCount = data.Body.DataCount - storage.NewDataCount = newDataCount + case "update": err := repo.UpdateStorage(storage) if err != nil { - log.Printf("Error update storage: %v", err) + return + } + case "delete": + err := repo.DeleteStorage(storage) + if err != nil { + return } + default: + return } } diff --git a/datamap/internal/mq/mq.go b/datamap/internal/mq/mq.go index 53240fd..7b3e18c 100644 --- a/datamap/internal/mq/mq.go +++ b/datamap/internal/mq/mq.go @@ -1,8 +1,8 @@ package mq import ( - "encoding/json" "fmt" + jsoniter "github.com/json-iterator/go" "github.com/streadway/amqp" stgmod "gitlink.org.cn/cloudream/storage/common/models" "gitlink.org.cn/cloudream/storage/datamap/internal/config" @@ -25,8 +25,11 @@ func InitMQ(cfg config.RabbitMQConfig) (*amqp.Connection, error) { func listenQueues(conn *amqp.Connection) { queues := []string{ + "datamap_hubinfo", "datamap_storageinfo", - "datamap_hubtransfer", + "datamap_storagestats", + "datamap_hubtransferstats", + "datamap_hubstoragetransferstats", "datamap_blocktransfer", "datamap_blockdistribution", } @@ -57,29 +60,28 @@ func processMessage(queue string, body []byte) { switch queue { case "datamap_hubinfo": var data stgmod.HubInfo - - if err := json.Unmarshal(body, &data); err != nil { + if err := jsoniter.Unmarshal(body, &data); err != nil { log.Printf("Failed to unmarshal HubInfo: %v, body: %s", err, body) return } models.ProcessHubInfo(data) case "datamap_storageinfo": var data stgmod.StorageInfo - if err := json.Unmarshal(body, &data); err != nil { + if err := jsoniter.Unmarshal(body, &data); err != nil { log.Printf("Failed to unmarshal StorageInfo: %v, body: %s", err, body) return } - //models.ProcessStorageInfo(data) + models.ProcessStorageInfo(data) case "datamap_storagestats": var data stgmod.StorageStats - if err := json.Unmarshal(body, &data); err != nil { + if err := jsoniter.Unmarshal(body, &data); err != nil { log.Printf("Failed to unmarshal StorageStats: %v, body: %s", err, body) return } //models.ProcessStorageInfo(data) case "datamap_hubtransferstats": var data stgmod.HubTransferStats - err := json.Unmarshal(body, &data) + err := jsoniter.Unmarshal(body, &data) if err != nil { log.Printf("Failed to unmarshal HubTransferStats: %v, body: %s", err, body) return @@ -87,7 +89,7 @@ func processMessage(queue string, body []byte) { models.ProcessHubTransfer(data) case "datamap_hubstoragetransferstats": var data stgmod.HubStorageTransferStats - err := json.Unmarshal(body, &data) + err := jsoniter.Unmarshal(body, &data) if err != nil { log.Printf("Failed to unmarshal HubStorageTransferStats: %v, body: %s", err, body) return @@ -95,7 +97,7 @@ func processMessage(queue string, body []byte) { //models.ProcessHubTransfer(data) case "datamap_blocktransfer": var data stgmod.BlockTransfer - err := json.Unmarshal(body, &data) + err := jsoniter.Unmarshal(body, &data) if err != nil { log.Printf("Failed to unmarshal BlockTransfer: %v, body: %s", err, body) return @@ -103,7 +105,7 @@ func processMessage(queue string, body []byte) { models.ProcessBlockTransfer(data) case "datamap_blockdistribution": var data stgmod.BlockDistribution - err := json.Unmarshal(body, &data) + err := jsoniter.Unmarshal(body, &data) if err != nil { log.Printf("Failed to unmarshal BlockDistribution: %v, body: %s", err, body) return From b2c65d7eded625ba64fe42445a533cc5a01b092f Mon Sep 17 00:00:00 2001 From: Jake <450705171@qq.com> Date: Fri, 17 Jan 2025 10:22:31 +0800 Subject: [PATCH 07/12] =?UTF-8?q?=E9=83=A8=E5=88=86=E5=AD=97=E6=AE=B5?= =?UTF-8?q?=E8=B0=83=E6=95=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- common/models/datamap.go | 10 ++-- datamap/.env | 15 ++++++ datamap/internal/handlers/handlers.go | 6 +-- datamap/internal/models/hubrequest.go | 55 +++++++++++++++++++-- datamap/internal/models/models.go | 2 +- datamap/internal/models/storageStats.go | 63 ++++++++++++++----------- datamap/internal/mq/mq.go | 4 +- 7 files changed, 112 insertions(+), 43 deletions(-) create mode 100644 datamap/.env diff --git a/common/models/datamap.go b/common/models/datamap.go index 38bf89b..54c1f3a 100644 --- a/common/models/datamap.go +++ b/common/models/datamap.go @@ -194,7 +194,7 @@ type ObjectChangeBody struct { Path string `json:"path"` Size int `json:"size"` BlockDistribution []BlockDistribution `json:"blockDistribution"` - Timestamp string `json:"timestamp"` + Timestamp time.Time `json:"timestamp"` } type ObjectChange struct { Timestamp time.Time `json:"timestamp"` @@ -222,10 +222,10 @@ type PackageChange struct { // BucketChange bucket变化信息 type BucketChangeBody struct { - Type string `json:"type"` - BucketID string `json:"bucketID"` - BucketName string `json:"bucketName"` - Timestamp string `json:"timestamp"` + Type string `json:"type"` + BucketID string `json:"bucketID"` + BucketName string `json:"bucketName"` + Timestamp time.Time `json:"timestamp"` } type BucketChange struct { diff --git a/datamap/.env b/datamap/.env new file mode 100644 index 0000000..12a1727 --- /dev/null +++ b/datamap/.env @@ -0,0 +1,15 @@ +# 数据库配置 +DB_HOST=175.178.223.172 +DB_PORT=3306 +DB_USER=root +DB_PASSWORD= +DB_NAME=storage_datamap + +# RabbitMQ 配置 +RABBITMQ_HOST=175.178.223.172 +RABBITMQ_PORT=5672 +RABBITMQ_USER=guest +RABBITMQ_PASSWORD= + +# 服务配置 +SERVER_PORT=8080 \ No newline at end of file diff --git a/datamap/internal/handlers/handlers.go b/datamap/internal/handlers/handlers.go index 66abf46..b206d0a 100644 --- a/datamap/internal/handlers/handlers.go +++ b/datamap/internal/handlers/handlers.go @@ -114,7 +114,7 @@ func GetDataTransfer(c *gin.Context) { //block id ID: strconv.FormatInt(block.BlockID, 10), //storage id - ComboID: strconv.FormatInt(block.StorageID, 10), + ComboID: "storage" + strconv.FormatInt(block.StorageID, 10), //block index Label: block.Type + strconv.FormatInt(block.Index, 10), //block type @@ -124,11 +124,11 @@ func GetDataTransfer(c *gin.Context) { //combos ------- state or storage //添加storage combo信息 - if !containsCombo(combos, strconv.FormatInt(block.StorageID, 10), "storage") { + if !containsCombo(combos, "storage"+strconv.FormatInt(block.StorageID, 10), "storage") { combo := models.Combo{ ID: "storage" + strconv.FormatInt(block.StorageID, 10), Label: "存储中心" + strconv.FormatInt(block.StorageID, 10), - ParentId: strconv.Itoa(block.Status), + ParentId: "state" + strconv.Itoa(block.Status), ComboType: "storage", } combos = append(combos, combo) diff --git a/datamap/internal/models/hubrequest.go b/datamap/internal/models/hubrequest.go index 47a77c8..cc9abda 100644 --- a/datamap/internal/models/hubrequest.go +++ b/datamap/internal/models/hubrequest.go @@ -11,9 +11,9 @@ import ( type HubRequest struct { RequestID int64 `gorm:"column:RequestID; primaryKey; type:bigint; autoIncrement" json:"RequestID"` SourceType string `gorm:"column:SourceType; type:varchar(255); not null" json:"sourceType"` - SourceID cdssdk.HubID `gorm:"column:SourceHubID; type:bigint; not null" json:"sourceID"` + SourceID cdssdk.HubID `gorm:"column:SourceID; type:bigint; not null" json:"sourceID"` TargetType string `gorm:"column:TargetType; type:varchar(255); not null" json:"targetType"` - TargetID cdssdk.HubID `gorm:"column:TargetHubID; type:bigint; not null" json:"targetID"` + TargetID cdssdk.HubID `gorm:"column:TargetID; type:bigint; not null" json:"targetID"` DataTransferCount int64 `gorm:"column:DataTransferCount; type:bigint; not null" json:"dataTransferCount"` RequestCount int64 `gorm:"column:RequestCount; type:bigint; not null" json:"requestCount"` FailedRequestCount int64 `gorm:"column:FailedRequestCount; type:bigint; not null" json:"failedRequestCount"` @@ -61,10 +61,10 @@ func (r *HubRequestRepository) GetAllHubRequests() ([]HubRequest, error) { func ProcessHubTransfer(data stgmod.HubTransferStats) { repo := NewHubRequestRepository(DB) - //todo 加字段 hubRequest := &HubRequest{ - + SourceType: "hub", SourceID: cdssdk.HubID(data.Body.SourceHubID), + TargetType: "hub", TargetID: cdssdk.HubID(data.Body.TargetHubID), DataTransferCount: data.Body.Send.TotalTransfer, RequestCount: data.Body.Send.RequestCount, @@ -82,3 +82,50 @@ func ProcessHubTransfer(data stgmod.HubTransferStats) { } } + +// ProcessHubStorageTransfer 节点中心之间数据传输处理 +func ProcessHubStorageTransfer(data stgmod.HubStorageTransferStats) { + repo := NewHubRequestRepository(DB) + + hubRequestSend := &HubRequest{ + SourceType: "hub", + SourceID: cdssdk.HubID(data.Body.HubID), + TargetType: "storage", + TargetID: cdssdk.HubID(data.Body.StorageID), + DataTransferCount: data.Body.Send.TotalTransfer, + RequestCount: data.Body.Send.RequestCount, + FailedRequestCount: data.Body.Send.FailedRequestCount, + AvgTransferCount: data.Body.Send.AvgTransfer, + MaxTransferCount: data.Body.Send.MaxTransfer, + MinTransferCount: data.Body.Send.MinTransfer, + StartTimestamp: data.Body.StartTimestamp, + EndTimestamp: data.Body.EndTimestamp, + } + + err := repo.CreateHubRequest(hubRequestSend) + if err != nil { + log.Printf("Error update hubrequest: %v", err) + + } + + hubRequestReceive := &HubRequest{ + SourceType: "storage", + SourceID: cdssdk.HubID(data.Body.StorageID), + TargetType: "hub", + TargetID: cdssdk.HubID(data.Body.HubID), + DataTransferCount: data.Body.Receive.TotalTransfer, + RequestCount: data.Body.Receive.RequestCount, + FailedRequestCount: data.Body.Receive.FailedRequestCount, + AvgTransferCount: data.Body.Receive.AvgTransfer, + MaxTransferCount: data.Body.Receive.MaxTransfer, + MinTransferCount: data.Body.Receive.MinTransfer, + StartTimestamp: data.Body.StartTimestamp, + EndTimestamp: data.Body.EndTimestamp, + } + + err = repo.CreateHubRequest(hubRequestReceive) + if err != nil { + log.Printf("Error update hubrequest: %v", err) + + } +} diff --git a/datamap/internal/models/models.go b/datamap/internal/models/models.go index 85fb10a..3c2bb51 100644 --- a/datamap/internal/models/models.go +++ b/datamap/internal/models/models.go @@ -66,7 +66,7 @@ type ObjectDistribution struct { type DistNode struct { ID string `json:"id"` - ComboID string `json:"comboID"` + ComboID string `json:"comboId"` Label string `json:"label"` NodeType string `json:"nodeType"` } diff --git a/datamap/internal/models/storageStats.go b/datamap/internal/models/storageStats.go index d7e6928..f26c553 100644 --- a/datamap/internal/models/storageStats.go +++ b/datamap/internal/models/storageStats.go @@ -1,30 +1,37 @@ package models -// -//func ProcessStorageInfo(data stgmod.StorageInfo) { -// repo := NewStorageRepository(DB) -// -// storage, err := repo.GetStorageByID(data.Body.StorageID) -// if err != nil { -// if errors.Is(err, gorm.ErrRecordNotFound) { -// // 插入新记录 -// newStorage := &Storage{ -// StorageID: cdssdk.StorageID(data.Body.StorageID), -// DataCount: data.Body.DataCount, -// NewDataCount: 0, -// } -// repo.CreateStorage(newStorage) -// } else { -// log.Printf("Error querying storage: %v", err) -// } -// } else { -// // 更新记录 -// newDataCount := data.Body.DataCount - storage.DataCount -// storage.DataCount = data.Body.DataCount -// storage.NewDataCount = newDataCount -// err := repo.UpdateStorage(storage) -// if err != nil { -// log.Printf("Error update storage: %v", err) -// } -// } -//} +import ( + "errors" + cdssdk "gitlink.org.cn/cloudream/common/sdks/storage" + stgmod "gitlink.org.cn/cloudream/storage/common/models" + "gorm.io/gorm" + "log" +) + +func ProcessStorageStats(data stgmod.StorageStats) { + repo := NewStorageRepository(DB) + + storage, err := repo.GetStorageByID(data.Body.StorageID) + if err != nil { + if errors.Is(err, gorm.ErrRecordNotFound) { + // 插入新记录 + newStorage := &Storage{ + StorageID: cdssdk.StorageID(data.Body.StorageID), + DataCount: data.Body.DataCount, + NewDataCount: 0, + } + repo.CreateStorage(newStorage) + } else { + log.Printf("Error querying storage: %v", err) + } + } else { + // 更新记录 + newDataCount := data.Body.DataCount - storage.DataCount + storage.DataCount = data.Body.DataCount + storage.NewDataCount = newDataCount + err := repo.UpdateStorage(storage) + if err != nil { + log.Printf("Error update storage: %v", err) + } + } +} diff --git a/datamap/internal/mq/mq.go b/datamap/internal/mq/mq.go index 7b3e18c..07c9d9b 100644 --- a/datamap/internal/mq/mq.go +++ b/datamap/internal/mq/mq.go @@ -78,7 +78,7 @@ func processMessage(queue string, body []byte) { log.Printf("Failed to unmarshal StorageStats: %v, body: %s", err, body) return } - //models.ProcessStorageInfo(data) + models.ProcessStorageStats(data) case "datamap_hubtransferstats": var data stgmod.HubTransferStats err := jsoniter.Unmarshal(body, &data) @@ -94,7 +94,7 @@ func processMessage(queue string, body []byte) { log.Printf("Failed to unmarshal HubStorageTransferStats: %v, body: %s", err, body) return } - //models.ProcessHubTransfer(data) + models.ProcessHubStorageTransfer(data) case "datamap_blocktransfer": var data stgmod.BlockTransfer err := jsoniter.Unmarshal(body, &data) From e60c76f2de53e9fa58edce5264d8b94f098d1e95 Mon Sep 17 00:00:00 2001 From: Jake <450705171@qq.com> Date: Thu, 23 Jan 2025 19:05:52 +0800 Subject: [PATCH 08/12] =?UTF-8?q?=E8=B0=83=E6=95=B4=E6=95=B0=E6=8D=AE?= =?UTF-8?q?=E7=BB=93=E6=9E=84=E5=AE=9A=E4=B9=89=E4=BB=A5=E5=8F=8Amq?= =?UTF-8?q?=E6=95=B0=E6=8D=AE=E5=A4=84=E7=90=86=E6=96=B9=E5=BC=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- common/models/datamap.go | 285 ++++++++----------- common/pkgs/sysevent/sysevent.go | 14 +- datamap/internal/models/blockdistribution.go | 170 ++++++----- datamap/internal/models/blocktransfer.go | 284 +++++++++--------- datamap/internal/models/hub.go | 7 +- datamap/internal/models/hubinfo.go | 69 +++-- datamap/internal/models/hubrequest.go | 150 ++++++---- datamap/internal/models/object.go | 18 +- datamap/internal/models/storageStats.go | 54 ++-- datamap/internal/models/storageinfo.go | 63 ++-- 10 files changed, 585 insertions(+), 529 deletions(-) diff --git a/common/models/datamap.go b/common/models/datamap.go index 54c1f3a..41fae27 100644 --- a/common/models/datamap.go +++ b/common/models/datamap.go @@ -1,71 +1,67 @@ package stgmod import ( + "gitlink.org.cn/cloudream/common/pkgs/types" cdssdk "gitlink.org.cn/cloudream/common/sdks/storage" + "gitlink.org.cn/cloudream/common/utils/serder" "time" ) -// HubInfo 节点信息 - -type SourceHub struct { - Type string `json:"type"` -} - -type HubInfoBody struct { - HubID int `json:"hubID"` - HubInfo cdssdk.Hub `json:"hubInfo"` - Type string `json:"type"` +type Source interface { } -type HubInfo struct { - Timestamp time.Time `json:"timestamp"` - Source SourceHub `json:"source"` - Category string `json:"category"` - Body HubInfoBody `json:"body"` +type Body interface { } -//StorageInfo 节点信息 - -type SourceStorage struct { - Type string `json:"type"` -} +var _ = serder.UseTypeUnionInternallyTagged(types.Ref(types.NewTypeUnion[Source]( + (*SourceCoordinator)(nil), + (*SourceScanner)(nil), +)), "type") -type StorageInfoBody struct { - StorageID int64 `json:"storageID"` - StorageInfo cdssdk.Storage `json:"storageInfo"` - Type string `json:"type"` +type SourceCoordinator struct { + serder.Metadata `union:"SourceCoordinator"` + Type string `json:"type"` } -type StorageInfo struct { - Timestamp time.Time `json:"timestamp"` - Source SourceStorage `json:"source"` - Category string `json:"category"` - Body StorageInfoBody `json:"body"` +type SourceScanner struct { + serder.Metadata `union:"SourceScanner"` + Type string `json:"type"` + HubID cdssdk.HubID `json:"hubID"` + HubName string `json:"hubName"` } -// StorageStats 节点信息数据 +var _ = serder.UseTypeUnionInternallyTagged(types.Ref(types.NewTypeUnion[Body]( + (*BodyHubInfo)(nil), + (*BodyStorageInfo)(nil), + (*BodyStorageStats)(nil), + (*BodyHubTransferStats)(nil), + (*BodyHubStorageTransferStats)(nil), + (*BodyBlockTransfer)(nil), + (*BodyBlockDistribution)(nil), + (*BodyObjectChange)(nil), + (*BodyPackageChange)(nil), + (*BodyBucketChange)(nil), +)), "type") -type Source struct { - Type string `json:"type"` - HubID string `json:"hubID"` - HubName string `json:"hubName"` +type BodyHubInfo struct { + serder.Metadata `union:"BodyHubInfo"` + HubID cdssdk.HubID `json:"hubID"` + HubInfo cdssdk.Hub `json:"hubInfo"` + Type string `json:"type"` } -type StorageStatsBody struct { - StorageID int64 `json:"storageID"` - DataCount int64 `json:"dataCount"` +type BodyStorageInfo struct { + serder.Metadata `union:"BodyStorageInfo"` + StorageID cdssdk.StorageID `json:"storageID"` + StorageInfo cdssdk.Storage `json:"storageInfo"` + Type string `json:"type"` } - -type StorageStats struct { - Timestamp time.Time `json:"timestamp"` - Source Source `json:"source"` - Category string `json:"category"` - Body StorageStatsBody `json:"body"` +type BodyStorageStats struct { + serder.Metadata `union:"BodyStorageStats"` + StorageID cdssdk.StorageID `json:"storageID"` + DataCount int64 `json:"dataCount"` } -// HubTransferStats 节点传输信息 -// 每天一次，各节点统计自身当天向外部各个节点传输的总数据量 - type DataTrans struct { TotalTransfer int64 `json:"totalTransfer"` RequestCount int64 `json:"requestCount"` @@ -74,87 +70,58 @@ type DataTrans struct { MaxTransfer int64 `json:"maxTransfer"` MinTransfer int64 `json:"minTransfer"` } -type HubTransferStatsBody struct { - SourceHubID int64 `json:"sourceHubID"` - TargetHubID int64 `json:"targetHubID"` - Send DataTrans `json:"send"` - StartTimestamp time.Time `json:"startTimestamp"` - EndTimestamp time.Time `json:"endTimestamp"` -} -type HubTransferStats struct { - Timestamp time.Time `json:"timestamp"` - Source Source `json:"source"` - Category string `json:"category"` - Body HubTransferStatsBody `json:"body"` -} -//hubStorageTransferStats 节点-中心间传输信息 - -type HubStorageTransferStatsBody struct { - HubID int64 `json:"hubID"` - StorageID int64 `json:"storageID"` - Send DataTrans `json:"send"` - Receive DataTrans `json:"receive"` - StartTimestamp time.Time `json:"startTimestamp"` - EndTimestamp time.Time `json:"endTimestamp"` +type BodyHubTransferStats struct { + serder.Metadata `union:"BodyHubTransferStats"` + SourceHubID cdssdk.HubID `json:"sourceHubID"` + TargetHubID cdssdk.HubID `json:"targetHubID"` + Send DataTrans `json:"send"` + StartTimestamp time.Time `json:"startTimestamp"` + EndTimestamp time.Time `json:"endTimestamp"` } -type HubStorageTransferStats struct { - Timestamp time.Time `json:"timestamp"` - Source Source `json:"source"` - Category string `json:"category"` - Body HubStorageTransferStatsBody `json:"body"` +type BodyHubStorageTransferStats struct { + serder.Metadata `union:"BodyHubStorageTransferStats"` + HubID cdssdk.HubID `json:"hubID"` + StorageID cdssdk.StorageID `json:"storageID"` + Send DataTrans `json:"send"` + Receive DataTrans `json:"receive"` + StartTimestamp time.Time `json:"startTimestamp"` + EndTimestamp time.Time `json:"endTimestamp"` } -// blockTransfer 块传输信息 -/*实时日志，当对象的块信息发生变化时推送，共有4种变化类型： -拷贝 -编解码(一变多、多变一、多变多) -删除 -更新*/ - type Block struct { - BlockType string `json:"blockType"` - Index string `json:"index"` - StorageID string `json:"storageID"` + BlockType string `json:"blockType"` + Index string `json:"index"` + StorageID cdssdk.StorageID `json:"storageID"` } type DataTransfer struct { - SourceStorageID string `json:"sourceStorageID"` - TargetStorageID string `json:"targetStorageID"` - DataTransferCount string `json:"dataTransferCount"` + SourceStorageID cdssdk.StorageID `json:"sourceStorageID"` + TargetStorageID cdssdk.StorageID `json:"targetStorageID"` + DataTransferCount string `json:"dataTransferCount"` } type BlockChange struct { - Type string `json:"type"` - BlockType string `json:"blockType"` - Index string `json:"index"` - SourceStorageID string `json:"sourceStorageID"` - TargetStorageID string `json:"targetStorageID"` - DataTransferCount string `json:"dataTransferCount"` - Timestamp time.Time `json:"timestamp"` - SourceBlocks []Block `json:"sourceBlocks,omitempty"` - TargetBlocks []Block `json:"targetBlocks,omitempty"` - DataTransfers []DataTransfer `json:"dataTransfers,omitempty"` - Blocks []Block `json:"blocks,omitempty"` -} - -type BlockTransferBody struct { - Type string `json:"type"` - ObjectID string `json:"objectID"` - PackageID string `json:"packageID"` - BlockChanges []BlockChange `json:"blockChanges"` -} - -type BlockTransfer struct { - Timestamp time.Time `json:"timestamp"` - Source Source `json:"source"` - Category string `json:"category"` - Body BlockTransferBody `json:"body"` + Type string `json:"type"` + BlockType string `json:"blockType"` + Index string `json:"index"` + SourceStorageID cdssdk.StorageID `json:"sourceStorageID"` + TargetStorageID cdssdk.StorageID `json:"targetStorageID"` + DataTransferCount string `json:"dataTransferCount"` + Timestamp time.Time `json:"timestamp"` + SourceBlocks []Block `json:"sourceBlocks,omitempty"` + TargetBlocks []Block `json:"targetBlocks,omitempty"` + DataTransfers []DataTransfer `json:"dataTransfers,omitempty"` + Blocks []Block `json:"blocks,omitempty"` +} +type BodyBlockTransfer struct { + serder.Metadata `union:"BodyBlockTransfer"` + Type string `json:"type"` + ObjectID cdssdk.ObjectID `json:"objectID"` + PackageID cdssdk.PackageID `json:"packageID"` + BlockChanges []BlockChange `json:"blockChanges"` } -// BlockDistribution 块传输信息 -// 每天一次，在调整完成后，将当天调整前后的布局情况一起推送 - type BlockDistributionObjectInfo struct { Type string `json:"type"` Index string `json:"index"` @@ -162,8 +129,8 @@ type BlockDistributionObjectInfo struct { } type BlockDistributionObject struct { - ObjectID int64 `json:"objectID"` - PackageID int64 `json:"packageID"` + ObjectID cdssdk.ObjectID `json:"objectID"` + PackageID cdssdk.PackageID `json:"packageID"` Path string `json:"path"` Size int64 `json:"size"` FileHash string `json:"fileHash"` @@ -173,64 +140,34 @@ type BlockDistributionObject struct { BlockDistribution []BlockDistributionObjectInfo `json:"blockDistribution"` DataTransfers []DataTransfer `json:"dataTransfers"` } -type BlockDistributionBody struct { - Timestamp time.Time `json:"timestamp"` - Object BlockDistributionObject `json:"object,omitempty"` -} - -type BlockDistribution struct { - Timestamp time.Time `json:"timestamp"` - Source Source `json:"source"` - Category string `json:"category"` - Body BlockDistributionBody `json:"body"` -} - -// ObjectChange Object变化信息 - -type ObjectChangeBody struct { - Type string `json:"type"` - ObjectID string `json:"objectID"` - PackageID string `json:"packageID"` - Path string `json:"path"` - Size int `json:"size"` - BlockDistribution []BlockDistribution `json:"blockDistribution"` - Timestamp time.Time `json:"timestamp"` -} -type ObjectChange struct { - Timestamp time.Time `json:"timestamp"` - Source Source `json:"source"` - Category string `json:"category"` - Body ObjectChangeBody `json:"body"` -} -// PackageChange package变化信息 - -type PackageChangeBody struct { - Type string `json:"type"` - PackageID string `json:"packageID"` - PackageName string `json:"packageName"` - BucketID string `json:"bucketID"` - Timestamp string `json:"timestamp"` -} -type PackageChange struct { - Timestamp time.Time `json:"timestamp"` - Source Source `json:"source"` - Category string `json:"category"` - Body PackageChangeBody `json:"body"` -} - -// BucketChange bucket变化信息 - -type BucketChangeBody struct { - Type string `json:"type"` - BucketID string `json:"bucketID"` - BucketName string `json:"bucketName"` - Timestamp time.Time `json:"timestamp"` +type BodyBlockDistribution struct { + serder.Metadata `union:"BodyBlockDistribution"` + Timestamp time.Time `json:"timestamp"` + Object BlockDistributionObject `json:"object,omitempty"` } - -type BucketChange struct { - Timestamp time.Time `json:"timestamp"` - Source Source `json:"source"` - Category string `json:"category"` - Body BucketChangeBody `json:"body"` +type BodyObjectChange struct { + serder.Metadata `union:"BodyObjectChange"` + Type string `json:"type"` + ObjectID cdssdk.ObjectID `json:"objectID"` + PackageID cdssdk.PackageID `json:"packageID"` + Path string `json:"path"` + Size int `json:"size"` + BlockDistribution []BlockDistributionObjectInfo `json:"blockDistribution"` + Timestamp time.Time `json:"timestamp"` +} +type BodyPackageChange struct { + serder.Metadata `union:"BodyPackageChange"` + Type string `json:"type"` + PackageID cdssdk.PackageID `json:"packageID"` + PackageName string `json:"packageName"` + BucketID cdssdk.BucketID `json:"bucketID"` + Timestamp time.Time `json:"timestamp"` +} +type BodyBucketChange struct { + serder.Metadata `union:"BodyBucketChange"` + Type string `json:"type"` + BucketID cdssdk.BucketID `json:"bucketID"` + BucketName string `json:"bucketName"` + Timestamp time.Time `json:"timestamp"` } diff --git a/common/pkgs/sysevent/sysevent.go b/common/pkgs/sysevent/sysevent.go index 082b93e..41a1f8f 100644 --- a/common/pkgs/sysevent/sysevent.go +++ b/common/pkgs/sysevent/sysevent.go @@ -1,9 +1,17 @@ package sysevent +import ( + stgmod "gitlink.org.cn/cloudream/storage/common/models" + "time" +) + const ( SysEventQueueName = "SysEventQueue" ) -type SysEvent = any // TODO 换成具体的类型 - -type Source = any // TODO 换成具体的类型 +type SysEvent struct { + Timestamp time.Time `json:"timestamp"` + Source stgmod.Source `json:"source"` + Category string `json:"category"` + Body stgmod.Body `json:"body"` +} diff --git a/datamap/internal/models/blockdistribution.go b/datamap/internal/models/blockdistribution.go index a3e6e18..cdf4005 100644 --- a/datamap/internal/models/blockdistribution.go +++ b/datamap/internal/models/blockdistribution.go @@ -2,8 +2,9 @@ package models import ( "errors" - cdssdk "gitlink.org.cn/cloudream/common/sdks/storage" + "fmt" stgmod "gitlink.org.cn/cloudream/storage/common/models" + "gitlink.org.cn/cloudream/storage/common/pkgs/sysevent" "gorm.io/gorm" "log" "strconv" @@ -89,90 +90,103 @@ func (r *BlockDistributionRepository) DeleteBlockDistribution(objectID int64, in return r.repo.db.Exec(query, objectID, index, storageID).Error } -// ProcessBlockDistribution mq推送各节点统计自身当前的总数据量时的处理逻辑 +type BlockDistributionWatcher struct { + Name string +} -func ProcessBlockDistribution(data stgmod.BlockDistribution) { +func (w *BlockDistributionWatcher) OnEvent(event sysevent.SysEvent) { repoObject := NewObjectRepository(DB) repoBlock := NewBlockDistributionRepository(DB) repoStorage := NewStorageTransferCountRepository(DB) - //更新object表中的状态 - object, err := repoObject.GetObjectByID(data.Body.Object.ObjectID) - faultTolerance, _ := strconv.ParseFloat(data.Body.Object.FaultTolerance, 64) - redundancy, _ := strconv.ParseFloat(data.Body.Object.Redundancy, 64) - avgAccessCost, _ := strconv.ParseFloat(data.Body.Object.AvgAccessCost, 64) - if errors.Is(err, gorm.ErrRecordNotFound) { - err := repoObject.CreateObject(&Object{ - ObjectID: cdssdk.ObjectID(data.Body.Object.ObjectID), - PackageID: cdssdk.PackageID(data.Body.Object.PackageID), - Path: data.Body.Object.Path, - Size: data.Body.Object.Size, - FileHash: data.Body.Object.FileHash, - Status: StatusYesterdayAfter, - FaultTolerance: faultTolerance, - Redundancy: redundancy, - AvgAccessCost: avgAccessCost, - Timestamp: time.Now(), - }) - if err != nil { - log.Printf("Error create object: %v", err) - } - } else { - object.Status = StatusYesterdayAfter - err = repoObject.UpdateObject(object) - if err != nil { - log.Printf("Error update object: %v", err) - } - } - //更新block表中的状态 - for _, blockDistribution := range data.Body.Object.BlockDistribution { - blockIndex, _ := strconv.ParseInt(blockDistribution.Index, 10, 64) - blockStorageID, _ := strconv.ParseInt(blockDistribution.StorageID, 10, 64) - blockDist, err := repoBlock.GetBlockDistributionByIndex(data.Body.Object.ObjectID, blockIndex, blockStorageID) - if errors.Is(err, gorm.ErrRecordNotFound) { - err := repoBlock.CreateBlockDistribution(&BlockDistribution{ - BlockID: blockDist.BlockID, - ObjectID: blockDist.ObjectID, - Type: blockDistribution.Type, - Index: blockIndex, - StorageID: blockStorageID, - Status: StatusYesterdayAfter, - Timestamp: time.Now(), - }) - if err != nil { - log.Printf("Error create BlockDistribution: %v", err) + if event.Category == "blockDistribution" { + if blockDistribution, ok := event.Body.(*stgmod.BodyBlockDistribution); ok { + + //更新object表中的状态 + + object, err := repoObject.GetObjectByID(int64(blockDistribution.Object.ObjectID)) + faultTolerance, _ := strconv.ParseFloat(blockDistribution.Object.FaultTolerance, 64) + redundancy, _ := strconv.ParseFloat(blockDistribution.Object.Redundancy, 64) + avgAccessCost, _ := strconv.ParseFloat(blockDistribution.Object.AvgAccessCost, 64) + if errors.Is(err, gorm.ErrRecordNotFound) { + err := repoObject.CreateObject(&Object{ + ObjectID: blockDistribution.Object.ObjectID, + PackageID: blockDistribution.Object.PackageID, + Path: blockDistribution.Object.Path, + Size: blockDistribution.Object.Size, + FileHash: blockDistribution.Object.FileHash, + Status: StatusYesterdayAfter, + FaultTolerance: faultTolerance, + Redundancy: redundancy, + AvgAccessCost: avgAccessCost, + Timestamp: time.Now(), + }) + if err != nil { + log.Printf("Error create object: %v", err) + } + } else { + object.Status = StatusYesterdayAfter + err = repoObject.UpdateObject(object) + if err != nil { + log.Printf("Error update object: %v", err) + } } - } else { - err := repoBlock.UpdateBlockDistribution(&BlockDistribution{ - BlockID: blockDist.BlockID, - ObjectID: blockDist.ObjectID, - Type: blockDistribution.Type, - Index: blockIndex, - StorageID: blockStorageID, - Status: StatusYesterdayAfter, - Timestamp: time.Now(), - }) - if err != nil { - log.Printf("Error update BlockDistribution: %v", err) + + //更新block表中的状态 + for _, blockDist := range blockDistribution.Object.BlockDistribution { + blockIndex, _ := strconv.ParseInt(blockDist.Index, 10, 64) + blockStorageID, _ := strconv.ParseInt(blockDist.StorageID, 10, 64) + blockDist, err := repoBlock.GetBlockDistributionByIndex(int64(blockDistribution.Object.ObjectID), blockIndex, blockStorageID) + if errors.Is(err, gorm.ErrRecordNotFound) { + err := repoBlock.CreateBlockDistribution(&BlockDistribution{ + BlockID: blockDist.BlockID, + ObjectID: blockDist.ObjectID, + Type: blockDist.Type, + Index: blockIndex, + StorageID: blockStorageID, + Status: StatusYesterdayAfter, + Timestamp: time.Now(), + }) + if err != nil { + log.Printf("Error create BlockDistribution: %v", err) + } + } else { + err := repoBlock.UpdateBlockDistribution(&BlockDistribution{ + BlockID: blockDist.BlockID, + ObjectID: blockDist.ObjectID, + Type: blockDist.Type, + Index: blockIndex, + StorageID: blockStorageID, + Status: StatusYesterdayAfter, + Timestamp: time.Now(), + }) + if err != nil { + log.Printf("Error update BlockDistribution: %v", err) + } + } } + //在storageTransferCount表中添加记录 + for _, dataTransfer := range blockDistribution.Object.DataTransfers { + sourceStorageID, _ := strconv.ParseInt(string(dataTransfer.SourceStorageID), 10, 64) + targetStorageID, _ := strconv.ParseInt(string(dataTransfer.TargetStorageID), 10, 64) + dataTransferCount, _ := strconv.ParseInt(dataTransfer.DataTransferCount, 10, 64) + + err := repoStorage.CreateStorageTransferCount(&StorageTransferCount{ + ObjectID: int64(blockDistribution.Object.ObjectID), + Status: StatusTodayBeforeYesterday, + SourceStorageID: sourceStorageID, + TargetStorageID: targetStorageID, + DataTransferCount: dataTransferCount, + Timestamp: time.Now(), + }) + if err != nil { + log.Printf("Error create StorageTransferCount : %v", err) + } + } + } else { + fmt.Printf("Watcher %s: Unexpected Body type, expected *BodyStorageInfo, got %T\n", w.Name, event.Body) } - } - //在storageTransferCount表中添加记录 - for _, dataTransfer := range data.Body.Object.DataTransfers { - sourceStorageID, _ := strconv.ParseInt(dataTransfer.SourceStorageID, 10, 64) - targetStorageID, _ := strconv.ParseInt(dataTransfer.TargetStorageID, 10, 64) - dataTransferCount, _ := strconv.ParseInt(dataTransfer.DataTransferCount, 10, 64) - - err := repoStorage.CreateStorageTransferCount(&StorageTransferCount{ - ObjectID: data.Body.Object.ObjectID, - Status: StatusTodayBeforeYesterday, - SourceStorageID: sourceStorageID, - TargetStorageID: targetStorageID, - DataTransferCount: dataTransferCount, - Timestamp: time.Now(), - }) - if err != nil { - log.Printf("Error create StorageTransferCount : %v", err) - } + } else { + fmt.Printf("Watcher %s received an event with category %s\n", w.Name, event.Category) } } diff --git a/datamap/internal/models/blocktransfer.go b/datamap/internal/models/blocktransfer.go index 0bcd351..98e210d 100644 --- a/datamap/internal/models/blocktransfer.go +++ b/datamap/internal/models/blocktransfer.go @@ -2,8 +2,10 @@ package models import ( "errors" + "fmt" cdssdk "gitlink.org.cn/cloudream/common/sdks/storage" stgmod "gitlink.org.cn/cloudream/storage/common/models" + "gitlink.org.cn/cloudream/storage/common/pkgs/sysevent" "gorm.io/gorm" "log" "strconv" @@ -68,160 +70,170 @@ func (r *StorageTransferCountRepository) GetAllStorageTransferCounts() ([]Storag return storageTransferCounts, nil } -// ProcessBlockTransfer 处理块传输信息的实时日志，当对象的块信息发生变化时推送触发数据库刷新 +type BlockTransferWatcher struct { + Name string +} -func ProcessBlockTransfer(data stgmod.BlockTransfer) { +func (w *BlockTransferWatcher) OnEvent(event sysevent.SysEvent) { repoDist := NewBlockDistributionRepository(DB) repoStorage := NewStorageRepository(DB) repoStorageTrans := NewStorageTransferCountRepository(DB) repoObject := NewObjectRepository(DB) - for _, change := range data.Body.BlockChanges { - - objectID, _ := strconv.ParseInt(data.Body.ObjectID, 10, 64) - object, _ := repoObject.GetObjectByID(objectID) - index, _ := strconv.ParseInt(change.Index, 10, 64) - sourceStorageID, _ := strconv.ParseInt(change.SourceStorageID, 10, 64) - targetStorageID, _ := strconv.ParseInt(change.TargetStorageID, 10, 64) - newDataCount, _ := strconv.ParseInt(change.DataTransferCount, 10, 64) - - switch change.Type { - case "0": //拷贝 - //查询出存储在数据库中的BlockDistribution信息 - blockSource, errSource := repoDist.GetBlockDistributionByIndex(objectID, index, sourceStorageID) - //没有记录就将source和target的信息都保存到库中 - if errors.Is(errSource, gorm.ErrRecordNotFound) { - err := repoDist.CreateBlockDistribution(&BlockDistribution{ - ObjectID: objectID, - Type: change.BlockType, - Index: index, - StorageID: sourceStorageID, - Status: StatusNow, - Timestamp: time.Now(), - }) - if err != nil { - log.Printf("Error create source blockdistribution: %v", err) - } - } else { - //有数据则新增一条storageID为targetStorageID的记录,同时更新状态 - err := repoDist.CreateBlockDistribution(&BlockDistribution{ - ObjectID: blockSource.ObjectID, - Type: change.BlockType, - Index: index, - StorageID: targetStorageID, - Status: StatusNow, - Timestamp: time.Now(), - }) - if err != nil { - log.Printf("Error update blockdistribution: %v", err) - } - //复制完成之后增加的dataCount要加到targetStorage的记录中 - storageOld, err := repoStorage.GetStorageByID(targetStorageID) - if errors.Is(err, gorm.ErrRecordNotFound) { - err = repoStorage.CreateStorage(&Storage{ - StorageID: cdssdk.StorageID(targetStorageID), - DataCount: newDataCount, - Timestamp: time.Now(), + if event.Category == "blockTransfer" { + if blockTransfer, ok := event.Body.(*stgmod.BodyBlockTransfer); ok { + + for _, change := range blockTransfer.BlockChanges { + + objectID, _ := strconv.ParseInt(string(blockTransfer.ObjectID), 10, 64) + object, _ := repoObject.GetObjectByID(objectID) + index, _ := strconv.ParseInt(change.Index, 10, 64) + sourceStorageID, _ := strconv.ParseInt(string(change.SourceStorageID), 10, 64) + targetStorageID, _ := strconv.ParseInt(string(change.TargetStorageID), 10, 64) + newDataCount, _ := strconv.ParseInt(change.DataTransferCount, 10, 64) + + switch change.Type { + case "0": //拷贝 + //查询出存储在数据库中的BlockDistribution信息 + blockSource, errSource := repoDist.GetBlockDistributionByIndex(objectID, index, sourceStorageID) + //没有记录就将source和target的信息都保存到库中 + if errors.Is(errSource, gorm.ErrRecordNotFound) { + err := repoDist.CreateBlockDistribution(&BlockDistribution{ + ObjectID: objectID, + Type: change.BlockType, + Index: index, + StorageID: sourceStorageID, + Status: StatusNow, + Timestamp: time.Now(), + }) + if err != nil { + log.Printf("Error create source blockdistribution: %v", err) + } + } else { + //有数据则新增一条storageID为targetStorageID的记录,同时更新状态 + err := repoDist.CreateBlockDistribution(&BlockDistribution{ + ObjectID: blockSource.ObjectID, + Type: change.BlockType, + Index: index, + StorageID: targetStorageID, + Status: StatusNow, + Timestamp: time.Now(), + }) + if err != nil { + log.Printf("Error update blockdistribution: %v", err) + } + //复制完成之后增加的dataCount要加到targetStorage的记录中 + storageOld, err := repoStorage.GetStorageByID(targetStorageID) + if errors.Is(err, gorm.ErrRecordNotFound) { + err = repoStorage.CreateStorage(&Storage{ + StorageID: cdssdk.StorageID(targetStorageID), + DataCount: newDataCount, + Timestamp: time.Now(), + }) + if err != nil { + log.Printf("Error increase datacount in targetstorage: %v", err) + } + } else { + err = repoStorage.UpdateStorage(&Storage{ + StorageID: cdssdk.StorageID(targetStorageID), + DataCount: storageOld.DataCount + newDataCount, + Timestamp: time.Now(), + }) + if err != nil { + log.Printf("Error increase datacount in targetstorage: %v", err) + } + } + + } + //新增记录到storageTransferCount表中 + err := repoStorageTrans.CreateStorageTransferCount(&StorageTransferCount{ + ObjectID: objectID, + Status: int64(blockSource.Status), + SourceStorageID: sourceStorageID, + TargetStorageID: targetStorageID, + DataTransferCount: newDataCount, + Timestamp: time.Now(), }) if err != nil { - log.Printf("Error increase datacount in targetstorage: %v", err) + log.Printf("Error create StorageTransferCount : %v", err) + } + case "1": //编解码 + //删除所有的sourceBlock + for _, sourceBlock := range change.SourceBlocks { + sourceBlockIndex, _ := strconv.ParseInt(sourceBlock.Index, 10, 64) + err := repoDist.DeleteBlockDistribution(objectID, sourceBlockIndex, sourceStorageID) + if err != nil { + log.Printf("Error delete blockdistribution: %v", err) + } } - } else { - err = repoStorage.UpdateStorage(&Storage{ - StorageID: cdssdk.StorageID(targetStorageID), - DataCount: storageOld.DataCount + newDataCount, - Timestamp: time.Now(), + //插入所有的targetBlock + for _, targetBlock := range change.TargetBlocks { + storageID, _ := strconv.ParseInt(string(targetBlock.StorageID), 10, 64) + err := repoDist.CreateBlockDistribution(&BlockDistribution{ + ObjectID: objectID, + Type: targetBlock.BlockType, + Index: index, + //直接保存到目标中心 + StorageID: storageID, + Status: StatusNow, + Timestamp: time.Now(), + }) + if err != nil { + log.Printf("Error create blockdistribution: %v", err) + } + } + //新增记录到storageTransferCount表中 + err := repoStorageTrans.CreateStorageTransferCount(&StorageTransferCount{ + ObjectID: objectID, + Status: int64(object.Status), + SourceStorageID: sourceStorageID, + TargetStorageID: targetStorageID, + DataTransferCount: newDataCount, + Timestamp: time.Now(), }) if err != nil { - log.Printf("Error increase datacount in targetstorage: %v", err) + log.Printf("Error create StorageTransferCount : %v", err) } - } - } - //新增记录到storageTransferCount表中 - err := repoStorageTrans.CreateStorageTransferCount(&StorageTransferCount{ - ObjectID: objectID, - Status: int64(blockSource.Status), - SourceStorageID: sourceStorageID, - TargetStorageID: targetStorageID, - DataTransferCount: newDataCount, - Timestamp: time.Now(), - }) - if err != nil { - log.Printf("Error create StorageTransferCount : %v", err) - } - case "1": //编解码 - //删除所有的sourceBlock - for _, sourceBlock := range change.SourceBlocks { - sourceBlockIndex, _ := strconv.ParseInt(sourceBlock.Index, 10, 64) - err := repoDist.DeleteBlockDistribution(objectID, sourceBlockIndex, sourceStorageID) - if err != nil { - log.Printf("Error delete blockdistribution: %v", err) - } - } - //插入所有的targetBlock - for _, targetBlock := range change.TargetBlocks { - storageID, _ := strconv.ParseInt(targetBlock.StorageID, 10, 64) - err := repoDist.CreateBlockDistribution(&BlockDistribution{ - ObjectID: objectID, - Type: targetBlock.BlockType, - Index: index, - //直接保存到目标中心 - StorageID: storageID, - Status: StatusNow, - Timestamp: time.Now(), - }) - if err != nil { - log.Printf("Error create blockdistribution: %v", err) - } - } - //新增记录到storageTransferCount表中 - err := repoStorageTrans.CreateStorageTransferCount(&StorageTransferCount{ - ObjectID: objectID, - Status: int64(object.Status), - SourceStorageID: sourceStorageID, - TargetStorageID: targetStorageID, - DataTransferCount: newDataCount, - Timestamp: time.Now(), - }) - if err != nil { - log.Printf("Error create StorageTransferCount : %v", err) - } + case "2": //删除 + for _, block := range change.Blocks { + storageID, _ := strconv.ParseInt(string(block.StorageID), 10, 64) + changeIndex, _ := strconv.ParseInt(block.Index, 10, 64) + err := repoDist.DeleteBlockDistribution(objectID, changeIndex, storageID) + if err != nil { + log.Printf("Error delete blockdistribution: %v", err) + } + } - case "2": //删除 - for _, block := range change.Blocks { - storageID, _ := strconv.ParseInt(block.StorageID, 10, 64) - changeIndex, _ := strconv.ParseInt(block.Index, 10, 64) - err := repoDist.DeleteBlockDistribution(objectID, changeIndex, storageID) - if err != nil { - log.Printf("Error delete blockdistribution: %v", err) - } - } + case "3": //更新 + for _, blockUpdate := range change.Blocks { + //查询出存储在数据库中的BlockDistribution信息 + blockIndex, _ := strconv.ParseInt(blockUpdate.Index, 10, 64) + blockOld, err := repoDist.GetBlockDistributionByIndex(objectID, blockIndex, sourceStorageID) + newStorageID, _ := strconv.ParseInt(string(blockUpdate.StorageID), 10, 64) + err = repoDist.UpdateBlockDistribution(&BlockDistribution{ + BlockID: blockOld.BlockID, + ObjectID: blockOld.ObjectID, + Type: blockUpdate.BlockType, + Index: blockIndex, + StorageID: newStorageID, + Status: StatusNow, + Timestamp: time.Now(), + }) + if err != nil { + log.Printf("Error delete blockdistribution: %v", err) + } + } - case "3": //更新 - for _, blockUpdate := range change.Blocks { - //查询出存储在数据库中的BlockDistribution信息 - blockIndex, _ := strconv.ParseInt(blockUpdate.Index, 10, 64) - blockOld, err := repoDist.GetBlockDistributionByIndex(objectID, blockIndex, sourceStorageID) - newStorageID, _ := strconv.ParseInt(blockUpdate.StorageID, 10, 64) - err = repoDist.UpdateBlockDistribution(&BlockDistribution{ - BlockID: blockOld.BlockID, - ObjectID: blockOld.ObjectID, - Type: blockUpdate.BlockType, - Index: blockIndex, - StorageID: newStorageID, - Status: StatusNow, - Timestamp: time.Now(), - }) - if err != nil { - log.Printf("Error delete blockdistribution: %v", err) + default: + break } } - - default: - break + } else { + fmt.Printf("Watcher %s: Unexpected Body type, expected *BodyStorageInfo, got %T\n", w.Name, event.Body) } + } else { + fmt.Printf("Watcher %s received an event with category %s\n", w.Name, event.Category) } - } diff --git a/datamap/internal/models/hub.go b/datamap/internal/models/hub.go index fdb7795..327c22b 100644 --- a/datamap/internal/models/hub.go +++ b/datamap/internal/models/hub.go @@ -2,14 +2,13 @@ package models import ( cdssdk "gitlink.org.cn/cloudream/common/sdks/storage" - "gorm.io/datatypes" "gorm.io/gorm" ) type Hub struct { - HubID cdssdk.HubID `gorm:"column:HubID; primaryKey; type:bigint; autoIncrement" json:"hubID"` - Name string `gorm:"column:Name; type:varchar(255); not null" json:"name"` - Address datatypes.JSON `gorm:"column:Address; type:json; " json:"address"` + HubID cdssdk.HubID `gorm:"column:HubID; primaryKey; type:bigint; autoIncrement" json:"hubID"` + Name string `gorm:"column:Name; type:varchar(255); not null" json:"name"` + Address cdssdk.HubAddressInfo `gorm:"column:Address; type:json; " json:"address"` } func (Hub) TableName() string { return "hub" } diff --git a/datamap/internal/models/hubinfo.go b/datamap/internal/models/hubinfo.go index e515846..bd133bb 100644 --- a/datamap/internal/models/hubinfo.go +++ b/datamap/internal/models/hubinfo.go @@ -2,9 +2,10 @@ package models import ( "encoding/json" + "fmt" cdssdk "gitlink.org.cn/cloudream/common/sdks/storage" stgmod "gitlink.org.cn/cloudream/storage/common/models" - "gorm.io/datatypes" + "gitlink.org.cn/cloudream/storage/common/pkgs/sysevent" ) // LocalHub 本地结构体，嵌入cdssdk.Hub @@ -42,33 +43,49 @@ func (s *LocalHub) UnmarshalJSON(data []byte) error { return nil } -func ProcessHubInfo(data stgmod.HubInfo) { +// 实现 Watcher 接口的结构体 +type HubInfoWatcher struct { + Name string +} + +// 实现 OnEvent 方法 +func (w *HubInfoWatcher) OnEvent(event sysevent.SysEvent) { repo := NewHubRepository(DB) - jsonData, _ := json.Marshal(data.Body.HubInfo.Address) - HubInfo := &Hub{ - HubID: cdssdk.HubID(data.Body.HubID), - Name: data.Body.HubInfo.Name, - Address: datatypes.JSON(jsonData), - } - //先判断传输数据的类型 - switch data.Body.Type { - case "add": - err := repo.CreateHub(HubInfo) - if err != nil { - return - } - case "update": - err := repo.UpdateHub(HubInfo) - if err != nil { - return - } - case "delete": - err := repo.DeleteHub(HubInfo) - if err != nil { - return + if event.Category == "hubInfo" { + if hubInfo, ok := event.Body.(*stgmod.BodyHubInfo); ok { + + hub := &Hub{ + HubID: hubInfo.HubID, + Name: hubInfo.HubInfo.Name, + Address: hubInfo.HubInfo.Address, + } + //先判断传输数据的类型 + switch hubInfo.Type { + case "add": + err := repo.CreateHub(hub) + if err != nil { + return + } + case "update": + err := repo.UpdateHub(hub) + if err != nil { + return + } + case "delete": + err := repo.DeleteHub(hub) + if err != nil { + return + } + default: + return + } + } else { + // 如果 Body 不是我们期望的类型，打印错误信息 + fmt.Printf("Watcher %s: Unexpected Body type, expected *BodyHubInfo, got %T\n", w.Name, event.Body) } - default: - return + } else { + // 如果事件的 Category 不是 hubInfo，打印默认信息 + fmt.Printf("Watcher %s received an event with category %s\n", w.Name, event.Category) } } diff --git a/datamap/internal/models/hubrequest.go b/datamap/internal/models/hubrequest.go index cc9abda..766c085 100644 --- a/datamap/internal/models/hubrequest.go +++ b/datamap/internal/models/hubrequest.go @@ -1,14 +1,17 @@ package models import ( + "fmt" cdssdk "gitlink.org.cn/cloudream/common/sdks/storage" stgmod "gitlink.org.cn/cloudream/storage/common/models" + "gitlink.org.cn/cloudream/storage/common/pkgs/sysevent" "gorm.io/gorm" "log" "time" ) type HubRequest struct { + //todo source和target类型的区分 RequestID int64 `gorm:"column:RequestID; primaryKey; type:bigint; autoIncrement" json:"RequestID"` SourceType string `gorm:"column:SourceType; type:varchar(255); not null" json:"sourceType"` SourceID cdssdk.HubID `gorm:"column:SourceID; type:bigint; not null" json:"sourceID"` @@ -57,75 +60,98 @@ func (r *HubRequestRepository) GetAllHubRequests() ([]HubRequest, error) { return hubRequests, nil } -// ProcessHubTransfer mq推送各节点统计自身当天向外部各个节点传输的总数据量时的处理逻辑 -func ProcessHubTransfer(data stgmod.HubTransferStats) { - repo := NewHubRequestRepository(DB) - - hubRequest := &HubRequest{ - SourceType: "hub", - SourceID: cdssdk.HubID(data.Body.SourceHubID), - TargetType: "hub", - TargetID: cdssdk.HubID(data.Body.TargetHubID), - DataTransferCount: data.Body.Send.TotalTransfer, - RequestCount: data.Body.Send.RequestCount, - FailedRequestCount: data.Body.Send.FailedRequestCount, - AvgTransferCount: data.Body.Send.AvgTransfer, - MaxTransferCount: data.Body.Send.MaxTransfer, - MinTransferCount: data.Body.Send.MinTransfer, - StartTimestamp: data.Body.StartTimestamp, - EndTimestamp: data.Body.EndTimestamp, - } - - err := repo.CreateHubRequest(hubRequest) - if err != nil { - log.Printf("Error update hubrequest: %v", err) - - } +type HubTransferStatsWatcher struct { + Name string } -// ProcessHubStorageTransfer 节点中心之间数据传输处理 -func ProcessHubStorageTransfer(data stgmod.HubStorageTransferStats) { +func (w *HubTransferStatsWatcher) OnEvent(event sysevent.SysEvent) { repo := NewHubRequestRepository(DB) - hubRequestSend := &HubRequest{ - SourceType: "hub", - SourceID: cdssdk.HubID(data.Body.HubID), - TargetType: "storage", - TargetID: cdssdk.HubID(data.Body.StorageID), - DataTransferCount: data.Body.Send.TotalTransfer, - RequestCount: data.Body.Send.RequestCount, - FailedRequestCount: data.Body.Send.FailedRequestCount, - AvgTransferCount: data.Body.Send.AvgTransfer, - MaxTransferCount: data.Body.Send.MaxTransfer, - MinTransferCount: data.Body.Send.MinTransfer, - StartTimestamp: data.Body.StartTimestamp, - EndTimestamp: data.Body.EndTimestamp, - } - - err := repo.CreateHubRequest(hubRequestSend) - if err != nil { - log.Printf("Error update hubrequest: %v", err) - + if event.Category == "hubTransferStats" { + if hubTransferStats, ok := event.Body.(*stgmod.BodyHubTransferStats); ok { + hubRequest := &HubRequest{ + SourceType: "hub", + SourceID: hubTransferStats.SourceHubID, + TargetType: "hub", + TargetID: hubTransferStats.TargetHubID, + DataTransferCount: hubTransferStats.Send.TotalTransfer, + RequestCount: hubTransferStats.Send.RequestCount, + FailedRequestCount: hubTransferStats.Send.FailedRequestCount, + AvgTransferCount: hubTransferStats.Send.AvgTransfer, + MaxTransferCount: hubTransferStats.Send.MaxTransfer, + MinTransferCount: hubTransferStats.Send.MinTransfer, + StartTimestamp: hubTransferStats.StartTimestamp, + EndTimestamp: hubTransferStats.EndTimestamp, + } + + err := repo.CreateHubRequest(hubRequest) + if err != nil { + log.Printf("Error update hubrequest: %v", err) + + } + } else { + fmt.Printf("Watcher %s: Unexpected Body type, expected *BodyStorageInfo, got %T\n", w.Name, event.Body) + } + } else { + fmt.Printf("Watcher %s received an event with category %s\n", w.Name, event.Category) } +} - hubRequestReceive := &HubRequest{ - SourceType: "storage", - SourceID: cdssdk.HubID(data.Body.StorageID), - TargetType: "hub", - TargetID: cdssdk.HubID(data.Body.HubID), - DataTransferCount: data.Body.Receive.TotalTransfer, - RequestCount: data.Body.Receive.RequestCount, - FailedRequestCount: data.Body.Receive.FailedRequestCount, - AvgTransferCount: data.Body.Receive.AvgTransfer, - MaxTransferCount: data.Body.Receive.MaxTransfer, - MinTransferCount: data.Body.Receive.MinTransfer, - StartTimestamp: data.Body.StartTimestamp, - EndTimestamp: data.Body.EndTimestamp, - } +type HubStorageTransferStatsWatcher struct { + Name string +} - err = repo.CreateHubRequest(hubRequestReceive) - if err != nil { - log.Printf("Error update hubrequest: %v", err) +func (w *HubStorageTransferStatsWatcher) OnEvent(event sysevent.SysEvent) { + repo := NewHubRequestRepository(DB) + if event.Category == "hubStorageTransferStats" { + if hubStorageTransferStats, ok := event.Body.(*stgmod.BodyHubStorageTransferStats); ok { + + hubRequestSend := &HubRequest{ + SourceType: "hub", + SourceID: hubStorageTransferStats.HubID, + TargetType: "storage", + TargetID: cdssdk.HubID(hubStorageTransferStats.StorageID), + DataTransferCount: hubStorageTransferStats.Send.TotalTransfer, + RequestCount: hubStorageTransferStats.Send.RequestCount, + FailedRequestCount: hubStorageTransferStats.Send.FailedRequestCount, + AvgTransferCount: hubStorageTransferStats.Send.AvgTransfer, + MaxTransferCount: hubStorageTransferStats.Send.MaxTransfer, + MinTransferCount: hubStorageTransferStats.Send.MinTransfer, + StartTimestamp: hubStorageTransferStats.StartTimestamp, + EndTimestamp: hubStorageTransferStats.EndTimestamp, + } + + err := repo.CreateHubRequest(hubRequestSend) + if err != nil { + log.Printf("Error update hubrequest: %v", err) + + } + + hubRequestReceive := &HubRequest{ + SourceType: "storage", + SourceID: cdssdk.HubID(hubStorageTransferStats.StorageID), + TargetType: "hub", + TargetID: hubStorageTransferStats.HubID, + DataTransferCount: hubStorageTransferStats.Receive.TotalTransfer, + RequestCount: hubStorageTransferStats.Receive.RequestCount, + FailedRequestCount: hubStorageTransferStats.Receive.FailedRequestCount, + AvgTransferCount: hubStorageTransferStats.Receive.AvgTransfer, + MaxTransferCount: hubStorageTransferStats.Receive.MaxTransfer, + MinTransferCount: hubStorageTransferStats.Receive.MinTransfer, + StartTimestamp: hubStorageTransferStats.StartTimestamp, + EndTimestamp: hubStorageTransferStats.EndTimestamp, + } + + err = repo.CreateHubRequest(hubRequestReceive) + if err != nil { + log.Printf("Error update hubrequest: %v", err) + + } + } else { + fmt.Printf("Watcher %s: Unexpected Body type, expected *BodyStorageInfo, got %T\n", w.Name, event.Body) + } + } else { + fmt.Printf("Watcher %s received an event with category %s\n", w.Name, event.Category) } } diff --git a/datamap/internal/models/object.go b/datamap/internal/models/object.go index b2c8855..f83375d 100644 --- a/datamap/internal/models/object.go +++ b/datamap/internal/models/object.go @@ -1,8 +1,10 @@ package models import ( + "fmt" cdssdk "gitlink.org.cn/cloudream/common/sdks/storage" stgmod "gitlink.org.cn/cloudream/storage/common/models" + "gitlink.org.cn/cloudream/storage/common/pkgs/sysevent" "gorm.io/gorm" "time" ) @@ -65,7 +67,19 @@ func (r *ObjectRepository) GetAllObjects() ([]Object, error) { return objects, nil } -// ProcessObject 处理 Object 数据 -func ProcessObject(data stgmod.ObjectChange) { +type ObjectWatcher struct { + Name string +} + +func (w *ObjectWatcher) OnEvent(event sysevent.SysEvent) { + if event.Category == "objectChange" { + if _, ok := event.Body.(*stgmod.BodyObjectChange); ok { + + } else { + fmt.Printf("Watcher %s: Unexpected Body type, expected *ObjectInfo, got %T\n", w.Name, event.Body) + } + } else { + fmt.Printf("Watcher %s received an event with category %s\n", w.Name, event.Category) + } } diff --git a/datamap/internal/models/storageStats.go b/datamap/internal/models/storageStats.go index f26c553..2020946 100644 --- a/datamap/internal/models/storageStats.go +++ b/datamap/internal/models/storageStats.go @@ -2,36 +2,50 @@ package models import ( "errors" - cdssdk "gitlink.org.cn/cloudream/common/sdks/storage" + "fmt" stgmod "gitlink.org.cn/cloudream/storage/common/models" + "gitlink.org.cn/cloudream/storage/common/pkgs/sysevent" "gorm.io/gorm" "log" ) -func ProcessStorageStats(data stgmod.StorageStats) { +type StorageStatsWatcher struct { + Name string +} + +func (w *StorageStatsWatcher) OnEvent(event sysevent.SysEvent) { repo := NewStorageRepository(DB) - storage, err := repo.GetStorageByID(data.Body.StorageID) - if err != nil { - if errors.Is(err, gorm.ErrRecordNotFound) { - // 插入新记录 - newStorage := &Storage{ - StorageID: cdssdk.StorageID(data.Body.StorageID), - DataCount: data.Body.DataCount, - NewDataCount: 0, + if event.Category == "storageStats" { + if storageStats, ok := event.Body.(*stgmod.BodyStorageStats); ok { + + storage, err := repo.GetStorageByID(int64(storageStats.StorageID)) + if err != nil { + if errors.Is(err, gorm.ErrRecordNotFound) { + // 插入新记录 + newStorage := &Storage{ + StorageID: storageStats.StorageID, + DataCount: storageStats.DataCount, + NewDataCount: 0, + } + repo.CreateStorage(newStorage) + } else { + log.Printf("Error querying storage: %v", err) + } + } else { + // 更新记录 + newDataCount := storageStats.DataCount - storage.DataCount + storage.DataCount = storageStats.DataCount + storage.NewDataCount = newDataCount + err := repo.UpdateStorage(storage) + if err != nil { + log.Printf("Error update storage: %v", err) + } } - repo.CreateStorage(newStorage) } else { - log.Printf("Error querying storage: %v", err) + fmt.Printf("Watcher %s: Unexpected Body type, expected *BodyStorageInfo, got %T\n", w.Name, event.Body) } } else { - // 更新记录 - newDataCount := data.Body.DataCount - storage.DataCount - storage.DataCount = data.Body.DataCount - storage.NewDataCount = newDataCount - err := repo.UpdateStorage(storage) - if err != nil { - log.Printf("Error update storage: %v", err) - } + fmt.Printf("Watcher %s received an event with category %s\n", w.Name, event.Category) } } diff --git a/datamap/internal/models/storageinfo.go b/datamap/internal/models/storageinfo.go index 74be392..64f47f6 100644 --- a/datamap/internal/models/storageinfo.go +++ b/datamap/internal/models/storageinfo.go @@ -1,8 +1,10 @@ package models import ( + "fmt" cdssdk "gitlink.org.cn/cloudream/common/sdks/storage" stgmod "gitlink.org.cn/cloudream/storage/common/models" + "gitlink.org.cn/cloudream/storage/common/pkgs/sysevent" "gorm.io/gorm" "time" ) @@ -65,32 +67,45 @@ func (r *StorageRepository) GetAllStorages() ([]Storage, error) { return storages, nil } -func ProcessStorageInfo(data stgmod.StorageInfo) { +type StorageInfoWatcher struct { + Name string +} + +func (w *StorageInfoWatcher) OnEvent(event sysevent.SysEvent) { repo := NewStorageRepository(DB) - storage := &Storage{ - StorageID: cdssdk.StorageID(data.Body.StorageID), - StorageName: data.Body.StorageInfo.Name, - HubID: data.Body.StorageInfo.MasterHub, - Timestamp: data.Timestamp, - } - switch data.Body.Type { - case "add": - err := repo.CreateStorage(storage) - if err != nil { - return - } - case "update": - err := repo.UpdateStorage(storage) - if err != nil { - return - } - case "delete": - err := repo.DeleteStorage(storage) - if err != nil { - return + if event.Category == "storageInfo" { + if storageInfo, ok := event.Body.(*stgmod.BodyStorageInfo); ok { + storage := &Storage{ + StorageID: storageInfo.StorageID, + StorageName: storageInfo.StorageInfo.Name, + HubID: storageInfo.StorageInfo.MasterHub, + Timestamp: time.Now(), + } + + switch storageInfo.Type { + case "add": + err := repo.CreateStorage(storage) + if err != nil { + return + } + case "update": + err := repo.UpdateStorage(storage) + if err != nil { + return + } + case "delete": + err := repo.DeleteStorage(storage) + if err != nil { + return + } + default: + return + } + } else { + fmt.Printf("Watcher %s: Unexpected Body type, expected *BodyStorageInfo, got %T\n", w.Name, event.Body) } - default: - return + } else { + fmt.Printf("Watcher %s received an event with category %s\n", w.Name, event.Category) } } From bafc6fe38536a2003af544a6381ea8c3d036d4fb Mon Sep 17 00:00:00 2001 From: Sydonian <794346190@qq.com> Date: Fri, 24 Jan 2025 17:45:59 +0800 Subject: [PATCH 09/12] =?UTF-8?q?=E8=B0=83=E6=95=B4=E6=95=B0=E6=8D=AE?= =?UTF-8?q?=E7=BB=93=E6=9E=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- common/models/datamap.go | 356 +++++++++++++++++++++++++------ common/pkgs/sysevent/sysevent.go | 10 +- 2 files changed, 292 insertions(+), 74 deletions(-) diff --git a/common/models/datamap.go b/common/models/datamap.go index 41fae27..a363c44 100644 --- a/common/models/datamap.go +++ b/common/models/datamap.go @@ -1,78 +1,174 @@ package stgmod import ( + "time" + "gitlink.org.cn/cloudream/common/pkgs/types" cdssdk "gitlink.org.cn/cloudream/common/sdks/storage" "gitlink.org.cn/cloudream/common/utils/serder" - "time" ) -type Source interface { +// 系统事件 +type SysEvent struct { + Timestamp time.Time `json:"timestamp"` + Source SysEventSource `json:"source"` + Category string `json:"category"` + Body SysEventBody `json:"body"` } -type Body interface { +// 事件源 +type SysEventSource interface { + GetSourceType() string } -var _ = serder.UseTypeUnionInternallyTagged(types.Ref(types.NewTypeUnion[Source]( +var _ = serder.UseTypeUnionInternallyTagged(types.Ref(types.NewTypeUnion[SysEventSource]( (*SourceCoordinator)(nil), (*SourceScanner)(nil), + (*SourceHub)(nil), )), "type") type SourceCoordinator struct { - serder.Metadata `union:"SourceCoordinator"` + serder.Metadata `union:"Coordinator"` Type string `json:"type"` } +func (s *SourceCoordinator) GetSourceType() string { + return "Coordinator" +} + type SourceScanner struct { - serder.Metadata `union:"SourceScanner"` + serder.Metadata `union:"Scanner"` + Type string `json:"type"` +} + +func (s *SourceScanner) GetSourceType() string { + return "Scanner" +} + +type SourceHub struct { + serder.Metadata `union:"Hub"` Type string `json:"type"` HubID cdssdk.HubID `json:"hubID"` HubName string `json:"hubName"` } -var _ = serder.UseTypeUnionInternallyTagged(types.Ref(types.NewTypeUnion[Body]( - (*BodyHubInfo)(nil), - (*BodyStorageInfo)(nil), +func (s *SourceHub) GetSourceType() string { + return "Hub" +} + +// 事件体 +type SysEventBody interface { + GetBodyType() string +} + +var _ = serder.UseTypeUnionInternallyTagged(types.Ref(types.NewTypeUnion[SysEventBody]( + (*BodyNewHub)(nil), + (*BodyHubUpdated)(nil), + (*BodyHubDeleted)(nil), + + (*BodyNewStorage)(nil), + (*BodyStorageUpdated)(nil), + (*BodyStorageDeleted)(nil), + (*BodyStorageStats)(nil), (*BodyHubTransferStats)(nil), (*BodyHubStorageTransferStats)(nil), (*BodyBlockTransfer)(nil), (*BodyBlockDistribution)(nil), - (*BodyObjectChange)(nil), - (*BodyPackageChange)(nil), - (*BodyBucketChange)(nil), + + (*BodyNewObject)(nil), + (*BodyObjectUpdated)(nil), + (*BodyObjectDeleted)(nil), + + (*BodyNewPackage)(nil), + (*BodyPackageDeleted)(nil), + + (*BodyNewBucket)(nil), + (*BodyBucketDeleted)(nil), )), "type") -type BodyHubInfo struct { - serder.Metadata `union:"BodyHubInfo"` - HubID cdssdk.HubID `json:"hubID"` - HubInfo cdssdk.Hub `json:"hubInfo"` +// 新增Hub的事件 +type BodyNewHub struct { + serder.Metadata `union:"NewHub"` + Type string `json:"type"` + Info cdssdk.Hub `json:"info"` +} + +func (b *BodyNewHub) GetBodyType() string { + return "NewHub" +} + +// Hub信息更新的事件 +type BodyHubUpdated struct { + serder.Metadata `union:"HubUpdated"` + Type string `json:"type"` + Info cdssdk.Hub `json:"info"` +} + +func (b *BodyHubUpdated) GetBodyType() string { + return "HubUpdated" +} + +// Hub删除的事件 +type BodyHubDeleted struct { + serder.Metadata `union:"HubDeleted"` Type string `json:"type"` + HubID cdssdk.HubID `json:"hubID"` } -type BodyStorageInfo struct { - serder.Metadata `union:"BodyStorageInfo"` - StorageID cdssdk.StorageID `json:"storageID"` - StorageInfo cdssdk.Storage `json:"storageInfo"` +func (b *BodyHubDeleted) GetBodyType() string { + return "HubDeleted" +} + +// 新增Storage的事件 +type BodyNewStorage struct { + serder.Metadata `union:"NewStorage"` + Info cdssdk.Storage `json:"info"` + Type string `json:"type"` +} + +func (b *BodyNewStorage) GetBodyType() string { + return "NewStorage" +} + +// Storage信息更新的事件 +type BodyStorageUpdated struct { + serder.Metadata `union:"StorageUpdated"` + Type string `json:"type"` + Info cdssdk.Storage `json:"info"` +} + +func (b *BodyStorageUpdated) GetBodyType() string { + return "StorageUpdated" +} + +// Storage删除的事件 +type BodyStorageDeleted struct { + serder.Metadata `union:"StorageDeleted"` Type string `json:"type"` + StorageID cdssdk.StorageID `json:"storageID"` +} + +func (b *BodyStorageDeleted) GetBodyType() string { + return "StorageDeleted" } + +// Storage统计信息的事件 type BodyStorageStats struct { - serder.Metadata `union:"BodyStorageStats"` + serder.Metadata `union:"StorageStats"` + Type string `json:"type"` StorageID cdssdk.StorageID `json:"storageID"` DataCount int64 `json:"dataCount"` } -type DataTrans struct { - TotalTransfer int64 `json:"totalTransfer"` - RequestCount int64 `json:"requestCount"` - FailedRequestCount int64 `json:"failedRequestCount"` - AvgTransfer int64 `json:"avgTransfer"` - MaxTransfer int64 `json:"maxTransfer"` - MinTransfer int64 `json:"minTransfer"` +func (b *BodyStorageStats) GetBodyType() string { + return "StorageStats" } +// Hub数据传输统计信息的事件 type BodyHubTransferStats struct { - serder.Metadata `union:"BodyHubTransferStats"` + serder.Metadata `union:"HubTransferStats"` + Type string `json:"type"` SourceHubID cdssdk.HubID `json:"sourceHubID"` TargetHubID cdssdk.HubID `json:"targetHubID"` Send DataTrans `json:"send"` @@ -80,8 +176,23 @@ type BodyHubTransferStats struct { EndTimestamp time.Time `json:"endTimestamp"` } +func (b *BodyHubTransferStats) GetBodyType() string { + return "HubTransferStats" +} + +type DataTrans struct { + TotalTransfer int64 `json:"totalTransfer"` + RequestCount int64 `json:"requestCount"` + FailedRequestCount int64 `json:"failedRequestCount"` + AvgTransfer int64 `json:"avgTransfer"` + MaxTransfer int64 `json:"maxTransfer"` + MinTransfer int64 `json:"minTransfer"` +} + +// Hub和Storage数据传输统计信息的事件 type BodyHubStorageTransferStats struct { - serder.Metadata `union:"BodyHubStorageTransferStats"` + serder.Metadata `union:"HubStorageTransferStats"` + Type string `json:"type"` HubID cdssdk.HubID `json:"hubID"` StorageID cdssdk.StorageID `json:"storageID"` Send DataTrans `json:"send"` @@ -90,6 +201,35 @@ type BodyHubStorageTransferStats struct { EndTimestamp time.Time `json:"endTimestamp"` } +func (b *BodyHubStorageTransferStats) GetBodyType() string { + return "HubStorageTransferStats" +} + +// 块传输的事件 +type BodyBlockTransfer struct { + serder.Metadata `union:"BlockTransfer"` + Type string `json:"type"` + ObjectID cdssdk.ObjectID `json:"objectID"` + PackageID cdssdk.PackageID `json:"packageID"` + BlockChanges []BlockChange `json:"blockChanges"` +} + +func (b *BodyBlockTransfer) GetBodyType() string { + return "BlockTransfer" +} + +// 块变化类型 +type BlockChange interface { + GetBlockChangeType() string +} + +var _ = serder.UseTypeUnionInternallyTagged(types.Ref(types.NewTypeUnion[BlockChange]( + (*BlockChangeClone)(nil), + (*BlockChangeDeleted)(nil), + (*BlockChangeEnDecode)(nil), + // (*BlockChangeUpdated)(nil), +)), "type") + type Block struct { BlockType string `json:"blockType"` Index string `json:"index"` @@ -101,34 +241,58 @@ type DataTransfer struct { DataTransferCount string `json:"dataTransferCount"` } -type BlockChange struct { +type BlockChangeClone struct { + serder.Metadata `union:"BlockChangeClone"` Type string `json:"type"` BlockType string `json:"blockType"` Index string `json:"index"` SourceStorageID cdssdk.StorageID `json:"sourceStorageID"` TargetStorageID cdssdk.StorageID `json:"targetStorageID"` DataTransferCount string `json:"dataTransferCount"` - Timestamp time.Time `json:"timestamp"` - SourceBlocks []Block `json:"sourceBlocks,omitempty"` - TargetBlocks []Block `json:"targetBlocks,omitempty"` - DataTransfers []DataTransfer `json:"dataTransfers,omitempty"` - Blocks []Block `json:"blocks,omitempty"` } -type BodyBlockTransfer struct { - serder.Metadata `union:"BodyBlockTransfer"` - Type string `json:"type"` - ObjectID cdssdk.ObjectID `json:"objectID"` - PackageID cdssdk.PackageID `json:"packageID"` - BlockChanges []BlockChange `json:"blockChanges"` + +func (b *BlockChangeClone) GetBlockChangeType() string { + return "Clone" } -type BlockDistributionObjectInfo struct { - Type string `json:"type"` - Index string `json:"index"` - StorageID string `json:"storageID"` +type BlockChangeDeleted struct { + serder.Metadata `union:"BlockChangeDeleted"` + Type string `json:"type"` + Index string `json:"index"` + StorageID string `json:"storageID"` +} + +func (b *BlockChangeDeleted) GetBlockChangeType() string { + return "Deleted" +} + +type BlockChangeEnDecode struct { + serder.Metadata `union:"BlockChangeEnDecode"` + Type string `json:"type"` + SourceBlocks []Block `json:"sourceBlocks,omitempty"` + TargetBlocks []Block `json:"targetBlocks,omitempty"` + DataTransfers []DataTransfer `json:"dataTransfers,omitempty"` +} + +func (b *BlockChangeEnDecode) GetBlockChangeType() string { + return "EnDecode" } -type BlockDistributionObject struct { +// TODO 块更新应该是说对象被重新上传，此时事件内应该包含全部对象块的信息，因此应该使用ObjectUpdated事件 +// type BlockChangeUpdated struct { +// serder.Metadata `union:"BlockChangeUpdated"` +// Type string `json:"type"` +// Blocks []Block `json:"blocks"` +// } + +// func (b *BlockChangeUpdated) GetBlockChangeType() string { +// return "Updated" +// } + +// 块分布的事件 +type BodyBlockDistribution struct { + serder.Metadata `union:"BlockDistribution"` + Type string `json:"type"` ObjectID cdssdk.ObjectID `json:"objectID"` PackageID cdssdk.PackageID `json:"packageID"` Path string `json:"path"` @@ -141,33 +305,91 @@ type BlockDistributionObject struct { DataTransfers []DataTransfer `json:"dataTransfers"` } -type BodyBlockDistribution struct { - serder.Metadata `union:"BodyBlockDistribution"` - Timestamp time.Time `json:"timestamp"` - Object BlockDistributionObject `json:"object,omitempty"` +func (b *BodyBlockDistribution) GetBodyType() string { + return "BlockDistribution" } -type BodyObjectChange struct { - serder.Metadata `union:"BodyObjectChange"` + +type BlockDistributionObjectInfo struct { + Type string `json:"type"` + Index string `json:"index"` + StorageID string `json:"storageID"` +} + +// 新增Object的事件 +type BodyNewObject struct { + serder.Metadata `union:"NewObject"` Type string `json:"type"` - ObjectID cdssdk.ObjectID `json:"objectID"` - PackageID cdssdk.PackageID `json:"packageID"` - Path string `json:"path"` - Size int `json:"size"` + Info cdssdk.Object `json:"info"` BlockDistribution []BlockDistributionObjectInfo `json:"blockDistribution"` - Timestamp time.Time `json:"timestamp"` } -type BodyPackageChange struct { - serder.Metadata `union:"BodyPackageChange"` + +func (b *BodyNewObject) GetBodyType() string { + return "NewObject" +} + +// Object更新的事件 +type BodyObjectUpdated struct { + serder.Metadata `union:"ObjectUpdated"` + Type string `json:"type"` + Info cdssdk.Object `json:"info"` + BlockDistribution []BlockDistributionObjectInfo `json:"blockDistribution"` +} + +func (b *BodyObjectUpdated) GetBodyType() string { + return "ObjectUpdated" +} + +// Object删除的事件 +type BodyObjectDeleted struct { + serder.Metadata `union:"ObjectDeleted"` + Type string `json:"type"` + ObjectID cdssdk.ObjectID `json:"objectID"` +} + +func (b *BodyObjectDeleted) GetBodyType() string { + return "ObjectDeleted" +} + +// 新增Package的事件 +type BodyNewPackage struct { + serder.Metadata `union:"NewPackage"` + Type string `json:"type"` + Info cdssdk.Package `json:"info"` +} + +func (b *BodyNewPackage) GetBodyType() string { + return "NewPackage" +} + +// Package删除的事件 +type BodyPackageDeleted struct { + serder.Metadata `union:"PackageDeleted"` Type string `json:"type"` PackageID cdssdk.PackageID `json:"packageID"` - PackageName string `json:"packageName"` - BucketID cdssdk.BucketID `json:"bucketID"` - Timestamp time.Time `json:"timestamp"` } -type BodyBucketChange struct { - serder.Metadata `union:"BodyBucketChange"` + +func (b *BodyPackageDeleted) GetBodyType() string { + return "PackageDeleted" +} + +// 新增Bucket的事件 +type BodyNewBucket struct { + serder.Metadata `union:"NewBucket"` + Type string `json:"type"` + Info cdssdk.Bucket `json:"info"` +} + +func (b *BodyNewBucket) GetBodyType() string { + return "NewBucket" +} + +// Bucket删除的事件 +type BodyBucketDeleted struct { + serder.Metadata `union:"BucketDeleted"` Type string `json:"type"` BucketID cdssdk.BucketID `json:"bucketID"` - BucketName string `json:"bucketName"` - Timestamp time.Time `json:"timestamp"` +} + +func (b *BodyBucketDeleted) GetBodyType() string { + return "BucketDeleted" } diff --git a/common/pkgs/sysevent/sysevent.go b/common/pkgs/sysevent/sysevent.go index 41a1f8f..93c2a62 100644 --- a/common/pkgs/sysevent/sysevent.go +++ b/common/pkgs/sysevent/sysevent.go @@ -2,16 +2,12 @@ package sysevent import ( stgmod "gitlink.org.cn/cloudream/storage/common/models" - "time" ) const ( SysEventQueueName = "SysEventQueue" ) -type SysEvent struct { - Timestamp time.Time `json:"timestamp"` - Source stgmod.Source `json:"source"` - Category string `json:"category"` - Body stgmod.Body `json:"body"` -} +type SysEvent = stgmod.SysEvent + +type Source = stgmod.SysEventSource From e898591a00d079fb9f27c2b2c7b0532f4e46a5f5 Mon Sep 17 00:00:00 2001 From: Sydonian <794346190@qq.com> Date: Fri, 24 Jan 2025 17:46:24 +0800 Subject: [PATCH 10/12] =?UTF-8?q?=E8=B0=83=E6=95=B4=E5=A4=84=E7=90=86?= =?UTF-8?q?=E4=BA=8B=E4=BB=B6=E7=9A=84=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- datamap/internal/models/blockdistribution.go | 173 ++++++----- datamap/internal/models/blocktransfer.go | 291 +++++++++---------- datamap/internal/models/hubinfo.go | 62 ++-- datamap/internal/models/object.go | 5 +- datamap/internal/models/storageinfo.go | 65 ++--- 5 files changed, 290 insertions(+), 306 deletions(-) diff --git a/datamap/internal/models/blockdistribution.go b/datamap/internal/models/blockdistribution.go index cdf4005..12e1dc7 100644 --- a/datamap/internal/models/blockdistribution.go +++ b/datamap/internal/models/blockdistribution.go @@ -2,13 +2,13 @@ package models import ( "errors" - "fmt" - stgmod "gitlink.org.cn/cloudream/storage/common/models" - "gitlink.org.cn/cloudream/storage/common/pkgs/sysevent" - "gorm.io/gorm" "log" "strconv" "time" + + stgmod "gitlink.org.cn/cloudream/storage/common/models" + "gitlink.org.cn/cloudream/storage/common/pkgs/sysevent" + "gorm.io/gorm" ) type BlockDistribution struct { @@ -95,98 +95,93 @@ type BlockDistributionWatcher struct { } func (w *BlockDistributionWatcher) OnEvent(event sysevent.SysEvent) { + body, ok := event.Body.(*stgmod.BodyBlockDistribution) + if !ok { + return + } + repoObject := NewObjectRepository(DB) repoBlock := NewBlockDistributionRepository(DB) repoStorage := NewStorageTransferCountRepository(DB) - if event.Category == "blockDistribution" { - if blockDistribution, ok := event.Body.(*stgmod.BodyBlockDistribution); ok { - - //更新object表中的状态 - - object, err := repoObject.GetObjectByID(int64(blockDistribution.Object.ObjectID)) - faultTolerance, _ := strconv.ParseFloat(blockDistribution.Object.FaultTolerance, 64) - redundancy, _ := strconv.ParseFloat(blockDistribution.Object.Redundancy, 64) - avgAccessCost, _ := strconv.ParseFloat(blockDistribution.Object.AvgAccessCost, 64) - if errors.Is(err, gorm.ErrRecordNotFound) { - err := repoObject.CreateObject(&Object{ - ObjectID: blockDistribution.Object.ObjectID, - PackageID: blockDistribution.Object.PackageID, - Path: blockDistribution.Object.Path, - Size: blockDistribution.Object.Size, - FileHash: blockDistribution.Object.FileHash, - Status: StatusYesterdayAfter, - FaultTolerance: faultTolerance, - Redundancy: redundancy, - AvgAccessCost: avgAccessCost, - Timestamp: time.Now(), - }) - if err != nil { - log.Printf("Error create object: %v", err) - } - } else { - object.Status = StatusYesterdayAfter - err = repoObject.UpdateObject(object) - if err != nil { - log.Printf("Error update object: %v", err) - } - } + //更新object表中的状态 + object, err := repoObject.GetObjectByID(int64(body.ObjectID)) + faultTolerance, _ := strconv.ParseFloat(body.FaultTolerance, 64) + redundancy, _ := strconv.ParseFloat(body.Redundancy, 64) + avgAccessCost, _ := strconv.ParseFloat(body.AvgAccessCost, 64) + if errors.Is(err, gorm.ErrRecordNotFound) { + err := repoObject.CreateObject(&Object{ + ObjectID: body.ObjectID, + PackageID: body.PackageID, + Path: body.Path, + Size: body.Size, + FileHash: body.FileHash, + Status: StatusYesterdayAfter, + FaultTolerance: faultTolerance, + Redundancy: redundancy, + AvgAccessCost: avgAccessCost, + Timestamp: time.Now(), + }) + if err != nil { + log.Printf("Error create object: %v", err) + } + } else { + object.Status = StatusYesterdayAfter + err = repoObject.UpdateObject(object) + if err != nil { + log.Printf("Error update object: %v", err) + } + } - //更新block表中的状态 - for _, blockDist := range blockDistribution.Object.BlockDistribution { - blockIndex, _ := strconv.ParseInt(blockDist.Index, 10, 64) - blockStorageID, _ := strconv.ParseInt(blockDist.StorageID, 10, 64) - blockDist, err := repoBlock.GetBlockDistributionByIndex(int64(blockDistribution.Object.ObjectID), blockIndex, blockStorageID) - if errors.Is(err, gorm.ErrRecordNotFound) { - err := repoBlock.CreateBlockDistribution(&BlockDistribution{ - BlockID: blockDist.BlockID, - ObjectID: blockDist.ObjectID, - Type: blockDist.Type, - Index: blockIndex, - StorageID: blockStorageID, - Status: StatusYesterdayAfter, - Timestamp: time.Now(), - }) - if err != nil { - log.Printf("Error create BlockDistribution: %v", err) - } - } else { - err := repoBlock.UpdateBlockDistribution(&BlockDistribution{ - BlockID: blockDist.BlockID, - ObjectID: blockDist.ObjectID, - Type: blockDist.Type, - Index: blockIndex, - StorageID: blockStorageID, - Status: StatusYesterdayAfter, - Timestamp: time.Now(), - }) - if err != nil { - log.Printf("Error update BlockDistribution: %v", err) - } - } - } - //在storageTransferCount表中添加记录 - for _, dataTransfer := range blockDistribution.Object.DataTransfers { - sourceStorageID, _ := strconv.ParseInt(string(dataTransfer.SourceStorageID), 10, 64) - targetStorageID, _ := strconv.ParseInt(string(dataTransfer.TargetStorageID), 10, 64) - dataTransferCount, _ := strconv.ParseInt(dataTransfer.DataTransferCount, 10, 64) - - err := repoStorage.CreateStorageTransferCount(&StorageTransferCount{ - ObjectID: int64(blockDistribution.Object.ObjectID), - Status: StatusTodayBeforeYesterday, - SourceStorageID: sourceStorageID, - TargetStorageID: targetStorageID, - DataTransferCount: dataTransferCount, - Timestamp: time.Now(), - }) - if err != nil { - log.Printf("Error create StorageTransferCount : %v", err) - } + //更新block表中的状态 + for _, blockDist := range body.BlockDistribution { + blockIndex, _ := strconv.ParseInt(blockDist.Index, 10, 64) + blockStorageID, _ := strconv.ParseInt(blockDist.StorageID, 10, 64) + blockDist, err := repoBlock.GetBlockDistributionByIndex(int64(body.ObjectID), blockIndex, blockStorageID) + if errors.Is(err, gorm.ErrRecordNotFound) { + err := repoBlock.CreateBlockDistribution(&BlockDistribution{ + BlockID: blockDist.BlockID, + ObjectID: blockDist.ObjectID, + Type: blockDist.Type, + Index: blockIndex, + StorageID: blockStorageID, + Status: StatusYesterdayAfter, + Timestamp: time.Now(), + }) + if err != nil { + log.Printf("Error create BlockDistribution: %v", err) } } else { - fmt.Printf("Watcher %s: Unexpected Body type, expected *BodyStorageInfo, got %T\n", w.Name, event.Body) + err := repoBlock.UpdateBlockDistribution(&BlockDistribution{ + BlockID: blockDist.BlockID, + ObjectID: blockDist.ObjectID, + Type: blockDist.Type, + Index: blockIndex, + StorageID: blockStorageID, + Status: StatusYesterdayAfter, + Timestamp: time.Now(), + }) + if err != nil { + log.Printf("Error update BlockDistribution: %v", err) + } + } + } + //在storageTransferCount表中添加记录 + for _, dataTransfer := range body.DataTransfers { + sourceStorageID, _ := strconv.ParseInt(string(dataTransfer.SourceStorageID), 10, 64) + targetStorageID, _ := strconv.ParseInt(string(dataTransfer.TargetStorageID), 10, 64) + dataTransferCount, _ := strconv.ParseInt(dataTransfer.DataTransferCount, 10, 64) + + err := repoStorage.CreateStorageTransferCount(&StorageTransferCount{ + ObjectID: int64(body.ObjectID), + Status: StatusTodayBeforeYesterday, + SourceStorageID: sourceStorageID, + TargetStorageID: targetStorageID, + DataTransferCount: dataTransferCount, + Timestamp: time.Now(), + }) + if err != nil { + log.Printf("Error create StorageTransferCount : %v", err) } - } else { - fmt.Printf("Watcher %s received an event with category %s\n", w.Name, event.Category) } } diff --git a/datamap/internal/models/blocktransfer.go b/datamap/internal/models/blocktransfer.go index 98e210d..6b40071 100644 --- a/datamap/internal/models/blocktransfer.go +++ b/datamap/internal/models/blocktransfer.go @@ -2,14 +2,14 @@ package models import ( "errors" - "fmt" + "log" + "strconv" + "time" + cdssdk "gitlink.org.cn/cloudream/common/sdks/storage" stgmod "gitlink.org.cn/cloudream/storage/common/models" "gitlink.org.cn/cloudream/storage/common/pkgs/sysevent" "gorm.io/gorm" - "log" - "strconv" - "time" ) type StorageTransferCount struct { @@ -75,165 +75,160 @@ type BlockTransferWatcher struct { } func (w *BlockTransferWatcher) OnEvent(event sysevent.SysEvent) { + body, ok := event.Body.(*stgmod.BodyBlockTransfer) + if !ok { + return + } repoDist := NewBlockDistributionRepository(DB) repoStorage := NewStorageRepository(DB) repoStorageTrans := NewStorageTransferCountRepository(DB) repoObject := NewObjectRepository(DB) - if event.Category == "blockTransfer" { - if blockTransfer, ok := event.Body.(*stgmod.BodyBlockTransfer); ok { - - for _, change := range blockTransfer.BlockChanges { - - objectID, _ := strconv.ParseInt(string(blockTransfer.ObjectID), 10, 64) - object, _ := repoObject.GetObjectByID(objectID) - index, _ := strconv.ParseInt(change.Index, 10, 64) - sourceStorageID, _ := strconv.ParseInt(string(change.SourceStorageID), 10, 64) - targetStorageID, _ := strconv.ParseInt(string(change.TargetStorageID), 10, 64) - newDataCount, _ := strconv.ParseInt(change.DataTransferCount, 10, 64) - - switch change.Type { - case "0": //拷贝 - //查询出存储在数据库中的BlockDistribution信息 - blockSource, errSource := repoDist.GetBlockDistributionByIndex(objectID, index, sourceStorageID) - //没有记录就将source和target的信息都保存到库中 - if errors.Is(errSource, gorm.ErrRecordNotFound) { - err := repoDist.CreateBlockDistribution(&BlockDistribution{ - ObjectID: objectID, - Type: change.BlockType, - Index: index, - StorageID: sourceStorageID, - Status: StatusNow, - Timestamp: time.Now(), - }) - if err != nil { - log.Printf("Error create source blockdistribution: %v", err) - } - } else { - //有数据则新增一条storageID为targetStorageID的记录,同时更新状态 - err := repoDist.CreateBlockDistribution(&BlockDistribution{ - ObjectID: blockSource.ObjectID, - Type: change.BlockType, - Index: index, - StorageID: targetStorageID, - Status: StatusNow, - Timestamp: time.Now(), - }) - if err != nil { - log.Printf("Error update blockdistribution: %v", err) - } - //复制完成之后增加的dataCount要加到targetStorage的记录中 - storageOld, err := repoStorage.GetStorageByID(targetStorageID) - if errors.Is(err, gorm.ErrRecordNotFound) { - err = repoStorage.CreateStorage(&Storage{ - StorageID: cdssdk.StorageID(targetStorageID), - DataCount: newDataCount, - Timestamp: time.Now(), - }) - if err != nil { - log.Printf("Error increase datacount in targetstorage: %v", err) - } - } else { - err = repoStorage.UpdateStorage(&Storage{ - StorageID: cdssdk.StorageID(targetStorageID), - DataCount: storageOld.DataCount + newDataCount, - Timestamp: time.Now(), - }) - if err != nil { - log.Printf("Error increase datacount in targetstorage: %v", err) - } - } - - } - //新增记录到storageTransferCount表中 - err := repoStorageTrans.CreateStorageTransferCount(&StorageTransferCount{ - ObjectID: objectID, - Status: int64(blockSource.Status), - SourceStorageID: sourceStorageID, - TargetStorageID: targetStorageID, - DataTransferCount: newDataCount, - Timestamp: time.Now(), + for _, change := range body.BlockChanges { + + objectID, _ := strconv.ParseInt(string(body.ObjectID), 10, 64) + object, _ := repoObject.GetObjectByID(objectID) + // index, _ := strconv.ParseInt(change.Index, 10, 64) + // sourceStorageID, _ := strconv.ParseInt(string(change.SourceStorageID), 10, 64) + // targetStorageID, _ := strconv.ParseInt(string(change.TargetStorageID), 10, 64) + // newDataCount, _ := strconv.ParseInt(change.DataTransferCount, 10, 64) + + switch change := change.(type) { + case *stgmod.BlockChangeClone: //拷贝 + // TODO 从change中获取index, sourceStorageID, targetStorageID, newDataCount，下同 + + //查询出存储在数据库中的BlockDistribution信息 + blockSource, errSource := repoDist.GetBlockDistributionByIndex(objectID, index, sourceStorageID) + //没有记录就将source和target的信息都保存到库中 + if errors.Is(errSource, gorm.ErrRecordNotFound) { + err := repoDist.CreateBlockDistribution(&BlockDistribution{ + ObjectID: objectID, + Type: change.BlockType, + Index: index, + StorageID: sourceStorageID, + Status: StatusNow, + Timestamp: time.Now(), + }) + if err != nil { + log.Printf("Error create source blockdistribution: %v", err) + } + } else { + //有数据则新增一条storageID为targetStorageID的记录,同时更新状态 + err := repoDist.CreateBlockDistribution(&BlockDistribution{ + ObjectID: blockSource.ObjectID, + Type: change.BlockType, + Index: index, + StorageID: targetStorageID, + Status: StatusNow, + Timestamp: time.Now(), + }) + if err != nil { + log.Printf("Error update blockdistribution: %v", err) + } + //复制完成之后增加的dataCount要加到targetStorage的记录中 + storageOld, err := repoStorage.GetStorageByID(targetStorageID) + if errors.Is(err, gorm.ErrRecordNotFound) { + err = repoStorage.CreateStorage(&Storage{ + StorageID: cdssdk.StorageID(targetStorageID), + DataCount: newDataCount, + Timestamp: time.Now(), }) if err != nil { - log.Printf("Error create StorageTransferCount : %v", err) - } - case "1": //编解码 - //删除所有的sourceBlock - for _, sourceBlock := range change.SourceBlocks { - sourceBlockIndex, _ := strconv.ParseInt(sourceBlock.Index, 10, 64) - err := repoDist.DeleteBlockDistribution(objectID, sourceBlockIndex, sourceStorageID) - if err != nil { - log.Printf("Error delete blockdistribution: %v", err) - } - } - //插入所有的targetBlock - for _, targetBlock := range change.TargetBlocks { - storageID, _ := strconv.ParseInt(string(targetBlock.StorageID), 10, 64) - err := repoDist.CreateBlockDistribution(&BlockDistribution{ - ObjectID: objectID, - Type: targetBlock.BlockType, - Index: index, - //直接保存到目标中心 - StorageID: storageID, - Status: StatusNow, - Timestamp: time.Now(), - }) - if err != nil { - log.Printf("Error create blockdistribution: %v", err) - } + log.Printf("Error increase datacount in targetstorage: %v", err) } - //新增记录到storageTransferCount表中 - err := repoStorageTrans.CreateStorageTransferCount(&StorageTransferCount{ - ObjectID: objectID, - Status: int64(object.Status), - SourceStorageID: sourceStorageID, - TargetStorageID: targetStorageID, - DataTransferCount: newDataCount, - Timestamp: time.Now(), + } else { + err = repoStorage.UpdateStorage(&Storage{ + StorageID: cdssdk.StorageID(targetStorageID), + DataCount: storageOld.DataCount + newDataCount, + Timestamp: time.Now(), }) if err != nil { - log.Printf("Error create StorageTransferCount : %v", err) - } - - case "2": //删除 - for _, block := range change.Blocks { - storageID, _ := strconv.ParseInt(string(block.StorageID), 10, 64) - changeIndex, _ := strconv.ParseInt(block.Index, 10, 64) - err := repoDist.DeleteBlockDistribution(objectID, changeIndex, storageID) - if err != nil { - log.Printf("Error delete blockdistribution: %v", err) - } - } - - case "3": //更新 - for _, blockUpdate := range change.Blocks { - //查询出存储在数据库中的BlockDistribution信息 - blockIndex, _ := strconv.ParseInt(blockUpdate.Index, 10, 64) - blockOld, err := repoDist.GetBlockDistributionByIndex(objectID, blockIndex, sourceStorageID) - newStorageID, _ := strconv.ParseInt(string(blockUpdate.StorageID), 10, 64) - err = repoDist.UpdateBlockDistribution(&BlockDistribution{ - BlockID: blockOld.BlockID, - ObjectID: blockOld.ObjectID, - Type: blockUpdate.BlockType, - Index: blockIndex, - StorageID: newStorageID, - Status: StatusNow, - Timestamp: time.Now(), - }) - if err != nil { - log.Printf("Error delete blockdistribution: %v", err) - } + log.Printf("Error increase datacount in targetstorage: %v", err) } + } - default: - break + } + //新增记录到storageTransferCount表中 + err := repoStorageTrans.CreateStorageTransferCount(&StorageTransferCount{ + ObjectID: objectID, + Status: int64(blockSource.Status), + SourceStorageID: sourceStorageID, + TargetStorageID: targetStorageID, + DataTransferCount: newDataCount, + Timestamp: time.Now(), + }) + if err != nil { + log.Printf("Error create StorageTransferCount : %v", err) + } + case *stgmod.BlockChangeEnDecode: //编解码 + //删除所有的sourceBlock + for _, sourceBlock := range change.SourceBlocks { + sourceBlockIndex, _ := strconv.ParseInt(sourceBlock.Index, 10, 64) + err := repoDist.DeleteBlockDistribution(objectID, sourceBlockIndex, sourceStorageID) + if err != nil { + log.Printf("Error delete blockdistribution: %v", err) } } - } else { - fmt.Printf("Watcher %s: Unexpected Body type, expected *BodyStorageInfo, got %T\n", w.Name, event.Body) + //插入所有的targetBlock + for _, targetBlock := range change.TargetBlocks { + storageID, _ := strconv.ParseInt(string(targetBlock.StorageID), 10, 64) + err := repoDist.CreateBlockDistribution(&BlockDistribution{ + ObjectID: objectID, + Type: targetBlock.BlockType, + Index: index, + //直接保存到目标中心 + StorageID: storageID, + Status: StatusNow, + Timestamp: time.Now(), + }) + if err != nil { + log.Printf("Error create blockdistribution: %v", err) + } + } + //新增记录到storageTransferCount表中 + err := repoStorageTrans.CreateStorageTransferCount(&StorageTransferCount{ + ObjectID: objectID, + Status: int64(object.Status), + SourceStorageID: sourceStorageID, + TargetStorageID: targetStorageID, + DataTransferCount: newDataCount, + Timestamp: time.Now(), + }) + if err != nil { + log.Printf("Error create StorageTransferCount : %v", err) + } + + case *stgmod.BlockChangeDeleted: //删除 + storageID, _ := strconv.ParseInt(string(change.StorageID), 10, 64) + changeIndex, _ := strconv.ParseInt(change.Index, 10, 64) + err := repoDist.DeleteBlockDistribution(objectID, changeIndex, storageID) + if err != nil { + log.Printf("Error delete blockdistribution: %v", err) + } + + // case *stgmod.BlockChangeUpdated: //更新 + // for _, blockUpdate := range change.Blocks { + // //查询出存储在数据库中的BlockDistribution信息 + // blockIndex, _ := strconv.ParseInt(blockUpdate.Index, 10, 64) + // blockOld, err := repoDist.GetBlockDistributionByIndex(objectID, blockIndex, sourceStorageID) + // newStorageID, _ := strconv.ParseInt(string(blockUpdate.StorageID), 10, 64) + // err = repoDist.UpdateBlockDistribution(&BlockDistribution{ + // BlockID: blockOld.BlockID, + // ObjectID: blockOld.ObjectID, + // Type: blockUpdate.BlockType, + // Index: blockIndex, + // StorageID: newStorageID, + // Status: StatusNow, + // Timestamp: time.Now(), + // }) + // if err != nil { + // log.Printf("Error delete blockdistribution: %v", err) + // } + // } + + default: + break } - } else { - fmt.Printf("Watcher %s received an event with category %s\n", w.Name, event.Category) } } diff --git a/datamap/internal/models/hubinfo.go b/datamap/internal/models/hubinfo.go index bd133bb..423c1e6 100644 --- a/datamap/internal/models/hubinfo.go +++ b/datamap/internal/models/hubinfo.go @@ -2,7 +2,7 @@ package models import ( "encoding/json" - "fmt" + cdssdk "gitlink.org.cn/cloudream/common/sdks/storage" stgmod "gitlink.org.cn/cloudream/storage/common/models" "gitlink.org.cn/cloudream/storage/common/pkgs/sysevent" @@ -50,42 +50,36 @@ type HubInfoWatcher struct { // 实现 OnEvent 方法 func (w *HubInfoWatcher) OnEvent(event sysevent.SysEvent) { + repo := NewHubRepository(DB) - if event.Category == "hubInfo" { - if hubInfo, ok := event.Body.(*stgmod.BodyHubInfo); ok { + switch body := event.Body.(type) { + case *stgmod.BodyNewHub: + err := repo.CreateHub(&Hub{ + HubID: body.Info.HubID, + Name: body.Info.Name, + Address: body.Info.Address, + }) + if err != nil { + return + } + + case *stgmod.BodyHubUpdated: + err := repo.UpdateHub(&Hub{ + HubID: body.Info.HubID, + Name: body.Info.Name, + Address: body.Info.Address, + }) + if err != nil { + return + } - hub := &Hub{ - HubID: hubInfo.HubID, - Name: hubInfo.HubInfo.Name, - Address: hubInfo.HubInfo.Address, - } - //先判断传输数据的类型 - switch hubInfo.Type { - case "add": - err := repo.CreateHub(hub) - if err != nil { - return - } - case "update": - err := repo.UpdateHub(hub) - if err != nil { - return - } - case "delete": - err := repo.DeleteHub(hub) - if err != nil { - return - } - default: - return - } - } else { - // 如果 Body 不是我们期望的类型，打印错误信息 - fmt.Printf("Watcher %s: Unexpected Body type, expected *BodyHubInfo, got %T\n", w.Name, event.Body) + case *stgmod.BodyHubDeleted: + err := repo.DeleteHub(&Hub{ + HubID: body.HubID, + }) + if err != nil { + return } - } else { - // 如果事件的 Category 不是 hubInfo，打印默认信息 - fmt.Printf("Watcher %s received an event with category %s\n", w.Name, event.Category) } } diff --git a/datamap/internal/models/object.go b/datamap/internal/models/object.go index f83375d..3cb2373 100644 --- a/datamap/internal/models/object.go +++ b/datamap/internal/models/object.go @@ -2,11 +2,12 @@ package models import ( "fmt" + "time" + cdssdk "gitlink.org.cn/cloudream/common/sdks/storage" stgmod "gitlink.org.cn/cloudream/storage/common/models" "gitlink.org.cn/cloudream/storage/common/pkgs/sysevent" "gorm.io/gorm" - "time" ) type Object struct { @@ -74,7 +75,7 @@ type ObjectWatcher struct { func (w *ObjectWatcher) OnEvent(event sysevent.SysEvent) { if event.Category == "objectChange" { - if _, ok := event.Body.(*stgmod.BodyObjectChange); ok { + if _, ok := event.Body.(*stgmod.BodyNewObject); ok { } else { fmt.Printf("Watcher %s: Unexpected Body type, expected *ObjectInfo, got %T\n", w.Name, event.Body) diff --git a/datamap/internal/models/storageinfo.go b/datamap/internal/models/storageinfo.go index 64f47f6..7d13510 100644 --- a/datamap/internal/models/storageinfo.go +++ b/datamap/internal/models/storageinfo.go @@ -1,12 +1,12 @@ package models import ( - "fmt" + "time" + cdssdk "gitlink.org.cn/cloudream/common/sdks/storage" stgmod "gitlink.org.cn/cloudream/storage/common/models" "gitlink.org.cn/cloudream/storage/common/pkgs/sysevent" "gorm.io/gorm" - "time" ) type Storage struct { @@ -74,38 +74,37 @@ type StorageInfoWatcher struct { func (w *StorageInfoWatcher) OnEvent(event sysevent.SysEvent) { repo := NewStorageRepository(DB) - if event.Category == "storageInfo" { - if storageInfo, ok := event.Body.(*stgmod.BodyStorageInfo); ok { - storage := &Storage{ - StorageID: storageInfo.StorageID, - StorageName: storageInfo.StorageInfo.Name, - HubID: storageInfo.StorageInfo.MasterHub, - Timestamp: time.Now(), - } + switch body := event.Body.(type) { + case *stgmod.BodyNewStorage: + storage := &Storage{ + StorageID: body.Info.StorageID, + StorageName: body.Info.Name, + HubID: body.Info.MasterHub, + Timestamp: time.Now(), + } + err := repo.CreateStorage(storage) + if err != nil { + return + } - switch storageInfo.Type { - case "add": - err := repo.CreateStorage(storage) - if err != nil { - return - } - case "update": - err := repo.UpdateStorage(storage) - if err != nil { - return - } - case "delete": - err := repo.DeleteStorage(storage) - if err != nil { - return - } - default: - return - } - } else { - fmt.Printf("Watcher %s: Unexpected Body type, expected *BodyStorageInfo, got %T\n", w.Name, event.Body) + case *stgmod.BodyStorageUpdated: + storage := &Storage{ + StorageID: body.Info.StorageID, + StorageName: body.Info.Name, + HubID: body.Info.MasterHub, + Timestamp: time.Now(), + } + err := repo.UpdateStorage(storage) + if err != nil { + return + } + case *stgmod.BodyStorageDeleted: + storage := &Storage{ + StorageID: body.StorageID, + } + err := repo.DeleteStorage(storage) + if err != nil { + return } - } else { - fmt.Printf("Watcher %s received an event with category %s\n", w.Name, event.Category) } } From 6cf8240002b90b97cae6f5cec7601f9441e9bcd8 Mon Sep 17 00:00:00 2001 From: Sydonian <794346190@qq.com> Date: Mon, 10 Feb 2025 17:31:18 +0800 Subject: [PATCH 11/12] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E6=94=B6=E9=9B=86Hub?= =?UTF-8?q?=E4=BC=A0=E8=BE=93=E6=95=B0=E6=8D=AE=E9=87=8F=E7=9A=84=E5=8A=9F?= =?UTF-8?q?=E8=83=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- agent/internal/cmd/serve.go | 77 +++++++++++++ agent/internal/tickevent/report_hub_stats.go | 58 ++++++++++ .../tickevent/report_storage_stats.go | 23 ++++ common/globals/stats.go | 5 + common/models/datamap.go | 1 - common/pkgs/ioswitch2/agent_worker.go | 21 +++- common/pkgs/ioswitch2/http_hub_worker.go | 22 +++- .../servicestats/hub_strorage_transfer.go | 103 ++++++++++++++++++ common/pkgs/servicestats/hub_transfter.go | 103 ++++++++++++++++++ common/pkgs/servicestats/service_stats.go | 37 +++++++ common/pkgs/storage/agtpool/pool.go | 12 ++ common/pkgs/storage/local/shard_store.go | 17 ++- common/pkgs/storage/s3/shard_store.go | 17 ++- common/pkgs/storage/types/shard_store.go | 2 +- common/pkgs/sysevent/config.go | 11 ++ common/pkgs/sysevent/publisher.go | 13 ++- go.mod | 7 +- go.sum | 25 +++-- 18 files changed, 525 insertions(+), 29 deletions(-) create mode 100644 agent/internal/tickevent/report_hub_stats.go create mode 100644 agent/internal/tickevent/report_storage_stats.go create mode 100644 common/globals/stats.go create mode 100644 common/pkgs/servicestats/hub_strorage_transfer.go create mode 100644 common/pkgs/servicestats/hub_transfter.go create mode 100644 common/pkgs/servicestats/service_stats.go diff --git a/agent/internal/cmd/serve.go b/agent/internal/cmd/serve.go index ca6b9cf..c2f21de 100644 --- a/agent/internal/cmd/serve.go +++ b/agent/internal/cmd/serve.go @@ -1,12 +1,15 @@ package cmd import ( + "context" "fmt" "net" "os" "time" + "github.com/go-co-op/gocron/v2" "gitlink.org.cn/cloudream/storage/agent/internal/http" + "gitlink.org.cn/cloudream/storage/agent/internal/tickevent" "gitlink.org.cn/cloudream/common/pkgs/ioswitch/exec" "gitlink.org.cn/cloudream/common/pkgs/logger" @@ -15,6 +18,7 @@ import ( "gitlink.org.cn/cloudream/storage/agent/internal/config" "gitlink.org.cn/cloudream/storage/agent/internal/task" stgglb "gitlink.org.cn/cloudream/storage/common/globals" + stgmod "gitlink.org.cn/cloudream/storage/common/models" "gitlink.org.cn/cloudream/storage/common/pkgs/accessstat" "gitlink.org.cn/cloudream/storage/common/pkgs/connectivity" "gitlink.org.cn/cloudream/storage/common/pkgs/distlock" @@ -23,6 +27,7 @@ import ( agtrpc "gitlink.org.cn/cloudream/storage/common/pkgs/grpc/agent" "gitlink.org.cn/cloudream/storage/common/pkgs/metacache" "gitlink.org.cn/cloudream/storage/common/pkgs/storage/agtpool" + "gitlink.org.cn/cloudream/storage/common/pkgs/sysevent" "gitlink.org.cn/cloudream/storage/common/pkgs/uploader" "google.golang.org/grpc" @@ -141,6 +146,21 @@ func serve(configPath string) { // 初始化任务管理器 taskMgr := task.NewManager(distlock, &conCol, &dlder, acStat, stgAgts, uploader) + // 初始化系统事件发布器 + evtPub, err := sysevent.NewPublisher(sysevent.ConfigFromMQConfig(config.Cfg().RabbitMQ), &stgmod.SourceHub{ + HubID: hubCfg.Hub.HubID, + HubName: hubCfg.Hub.Name, + }) + if err != nil { + logger.Errorf("new sysevent publisher: %v", err) + os.Exit(1) + } + go servePublisher(evtPub) + + // 初始化定时任务执行器 + sch := setupTickTask(stgAgts, evtPub) + defer sch.Shutdown() + // 启动命令服务器 // TODO 需要设计AgentID持久化机制 agtSvr, err := agtmq.NewServer(cmdsvc.NewService(&taskMgr, stgAgts), config.Cfg().ID, config.Cfg().RabbitMQ) @@ -185,6 +205,63 @@ func downloadHubConfig() coormq.GetHubConfigResp { return *cfgResp } +func servePublisher(evtPub *sysevent.Publisher) { + logger.Info("start serving sysevent publisher") + + ch := evtPub.Start() + +loop: + for { + val, err := ch.Receive().Wait(context.Background()) + if err != nil { + logger.Errorf("sysevent publisher stopped with error: %s", err.Error()) + break + } + + switch val := val.(type) { + case sysevent.PublishError: + logger.Errorf("publishing event: %v", val) + + case sysevent.PublisherExited: + if val.Err != nil { + logger.Errorf("publisher exited with error: %v", val.Err) + } else { + logger.Info("publisher exited") + } + break loop + + case sysevent.OtherError: + logger.Errorf("sysevent: %v", val) + } + } + logger.Info("sysevent publisher stopped") + + // TODO 仅简单结束了程序 + os.Exit(1) +} + +func setupTickTask(agtPool *agtpool.AgentPool, evtPub *sysevent.Publisher) gocron.Scheduler { + sch, err := gocron.NewScheduler() + if err != nil { + logger.Errorf("new cron scheduler: %s", err.Error()) + os.Exit(1) + } + + sch.NewJob(gocron.DailyJob(1, gocron.NewAtTimes( + gocron.NewAtTime(0, 0, 0), + )), gocron.NewTask(tickevent.ReportStorageStats, agtPool, evtPub)) + + sch.NewJob(gocron.DailyJob(1, gocron.NewAtTimes( + gocron.NewAtTime(0, 0, 1), + )), gocron.NewTask(tickevent.ReportHubTransferStats, evtPub)) + + sch.NewJob(gocron.DailyJob(1, gocron.NewAtTimes( + gocron.NewAtTime(0, 0, 2), + )), gocron.NewTask(tickevent.ReportHubStorageTransferStats, evtPub)) + + return sch +} + func serveAgentServer(server *agtmq.Server) { logger.Info("start serving command server") diff --git a/agent/internal/tickevent/report_hub_stats.go b/agent/internal/tickevent/report_hub_stats.go new file mode 100644 index 0000000..7c3c6d1 --- /dev/null +++ b/agent/internal/tickevent/report_hub_stats.go @@ -0,0 +1,58 @@ +package tickevent + +import ( + stgglb "gitlink.org.cn/cloudream/storage/common/globals" + stgmod "gitlink.org.cn/cloudream/storage/common/models" + "gitlink.org.cn/cloudream/storage/common/pkgs/storage/agtpool" + "gitlink.org.cn/cloudream/storage/common/pkgs/sysevent" +) + +func ReportHubTransferStats(evtPub *sysevent.Publisher) { + if stgglb.Stats.HubTransfer == nil { + return + } + + data := stgglb.Stats.HubTransfer.DumpData() + endTime := stgglb.Stats.HubTransfer.Reset() + + for hubID, entry := range data.Entries { + evtPub.Publish(&stgmod.BodyHubTransferStats{ + SourceHubID: *stgglb.Local.HubID, + TargetHubID: hubID, + Send: stgmod.DataTrans{ + TotalTransfer: entry.OutputBytes, + RequestCount: entry.TotalOutput, + FailedRequestCount: entry.TotalInput - entry.SuccessInput, + AvgTransfer: entry.OutputBytes / entry.TotalOutput, + MinTransfer: entry.MinOutputBytes, + MaxTransfer: entry.MaxOutputBytes, + }, + StartTimestamp: data.StartTime, + EndTimestamp: endTime, + }) + } +} + +func ReportHubStorageTransferStats(stgAgts *agtpool.AgentPool, evtPub *sysevent.Publisher) { + if stgglb.Stats.HubStorageTransfer == nil { + return + } + + data := stgglb.Stats.HubStorageTransfer.DumpData() + endTime := stgglb.Stats.HubStorageTransfer.Reset() + + for storageID, stg := range data.Entries { + evtPub.Publish(&stgmod.BodyHubStorageTransferStats{ + HubID: *stgglb.Local.HubID, + StorageID: storageID, + Send: stgmod.DataTrans{ + TotalTransfer: stg.OutputBytes, + RequestCount: stg.TotalOutput, + FailedRequestCount: stg.TotalInput - stg.SuccessInput, + AvgTransfer: stg.OutputBytes / stg.TotalOutput, + }, + StartTimestamp: data.StartTime, + EndTimestamp: endTime, + }) + } +} diff --git a/agent/internal/tickevent/report_storage_stats.go b/agent/internal/tickevent/report_storage_stats.go new file mode 100644 index 0000000..600acf6 --- /dev/null +++ b/agent/internal/tickevent/report_storage_stats.go @@ -0,0 +1,23 @@ +package tickevent + +import ( + stgmod "gitlink.org.cn/cloudream/storage/common/models" + "gitlink.org.cn/cloudream/storage/common/pkgs/storage/agtpool" + "gitlink.org.cn/cloudream/storage/common/pkgs/sysevent" +) + +func ReportStorageStats(agtPool *agtpool.AgentPool, evtPub *sysevent.Publisher) { + stgs := agtPool.GetAllAgents() + for _, stg := range stgs { + shard, err := stg.GetShardStore() + if err != nil { + continue + } + + stats := shard.Stats() + evtPub.Publish(&stgmod.BodyStorageStats{ + StorageID: stg.Info().Storage.StorageID, + DataCount: stats.FileCount, + }) + } +} diff --git a/common/globals/stats.go b/common/globals/stats.go new file mode 100644 index 0000000..87ed604 --- /dev/null +++ b/common/globals/stats.go @@ -0,0 +1,5 @@ +package stgglb + +import "gitlink.org.cn/cloudream/storage/common/pkgs/servicestats" + +var Stats servicestats.StatsHost diff --git a/common/models/datamap.go b/common/models/datamap.go index a363c44..c61b4ac 100644 --- a/common/models/datamap.go +++ b/common/models/datamap.go @@ -12,7 +12,6 @@ import ( type SysEvent struct { Timestamp time.Time `json:"timestamp"` Source SysEventSource `json:"source"` - Category string `json:"category"` Body SysEventBody `json:"body"` } diff --git a/common/pkgs/ioswitch2/agent_worker.go b/common/pkgs/ioswitch2/agent_worker.go index 393819d..757cb86 100644 --- a/common/pkgs/ioswitch2/agent_worker.go +++ b/common/pkgs/ioswitch2/agent_worker.go @@ -7,6 +7,7 @@ import ( "gitlink.org.cn/cloudream/common/pkgs/ioswitch/exec" "gitlink.org.cn/cloudream/common/pkgs/types" cdssdk "gitlink.org.cn/cloudream/common/sdks/storage" + "gitlink.org.cn/cloudream/common/utils/io2" "gitlink.org.cn/cloudream/common/utils/serder" stgglb "gitlink.org.cn/cloudream/storage/common/globals" agtrpc "gitlink.org.cn/cloudream/storage/common/pkgs/grpc/agent" @@ -45,20 +46,34 @@ func (w *AgentWorker) Equals(worker exec.WorkerInfo) bool { } type AgentWorkerClient struct { - cli *agtrpc.PoolClient + hubID cdssdk.HubID + cli *agtrpc.PoolClient } func (c *AgentWorkerClient) ExecutePlan(ctx context.Context, plan exec.Plan) error { return c.cli.ExecuteIOPlan(ctx, plan) } func (c *AgentWorkerClient) SendStream(ctx context.Context, planID exec.PlanID, id exec.VarID, stream io.ReadCloser) error { - return c.cli.SendStream(ctx, planID, id, stream) + return c.cli.SendStream(ctx, planID, id, io2.CounterCloser(stream, func(cnt int64, err error) { + if stgglb.Stats.HubTransfer != nil { + stgglb.Stats.HubTransfer.RecordOutput(c.hubID, cnt, err == nil || err == io.EOF) + } + })) } func (c *AgentWorkerClient) SendVar(ctx context.Context, planID exec.PlanID, id exec.VarID, value exec.VarValue) error { return c.cli.SendVar(ctx, planID, id, value) } func (c *AgentWorkerClient) GetStream(ctx context.Context, planID exec.PlanID, streamID exec.VarID, signalID exec.VarID, signal exec.VarValue) (io.ReadCloser, error) { - return c.cli.GetStream(ctx, planID, streamID, signalID, signal) + str, err := c.cli.GetStream(ctx, planID, streamID, signalID, signal) + if err != nil { + return nil, err + } + + return io2.CounterCloser(str, func(cnt int64, err error) { + if stgglb.Stats.HubTransfer != nil { + stgglb.Stats.HubTransfer.RecordInput(c.hubID, cnt, err == nil || err == io.EOF) + } + }), nil } func (c *AgentWorkerClient) GetVar(ctx context.Context, planID exec.PlanID, varID exec.VarID, signalID exec.VarID, signal exec.VarValue) (exec.VarValue, error) { return c.cli.GetVar(ctx, planID, varID, signalID, signal) diff --git a/common/pkgs/ioswitch2/http_hub_worker.go b/common/pkgs/ioswitch2/http_hub_worker.go index fe4a25d..f145f25 100644 --- a/common/pkgs/ioswitch2/http_hub_worker.go +++ b/common/pkgs/ioswitch2/http_hub_worker.go @@ -8,6 +8,8 @@ import ( "gitlink.org.cn/cloudream/common/pkgs/ioswitch/exec" cdssdk "gitlink.org.cn/cloudream/common/sdks/storage" "gitlink.org.cn/cloudream/common/sdks/storage/cdsapi" + "gitlink.org.cn/cloudream/common/utils/io2" + stgglb "gitlink.org.cn/cloudream/storage/common/globals" ) type HttpHubWorker struct { @@ -44,7 +46,8 @@ func (w *HttpHubWorker) Equals(worker exec.WorkerInfo) bool { } type HttpHubWorkerClient struct { - cli *cdsapi.Client + hubID cdssdk.HubID + cli *cdsapi.Client } func (c *HttpHubWorkerClient) ExecutePlan(ctx context.Context, plan exec.Plan) error { @@ -58,7 +61,11 @@ func (c *HttpHubWorkerClient) SendStream(ctx context.Context, planID exec.PlanID PlanID: planID, VarID: id, }, - Stream: stream, + Stream: io2.CounterCloser(stream, func(cnt int64, err error) { + if stgglb.Stats.HubTransfer != nil { + stgglb.Stats.HubTransfer.RecordOutput(c.hubID, cnt, err == nil || err == io.EOF) + } + }), }) } func (c *HttpHubWorkerClient) SendVar(ctx context.Context, planID exec.PlanID, id exec.VarID, value exec.VarValue) error { @@ -69,12 +76,21 @@ func (c *HttpHubWorkerClient) SendVar(ctx context.Context, planID exec.PlanID, i }) } func (c *HttpHubWorkerClient) GetStream(ctx context.Context, planID exec.PlanID, streamID exec.VarID, signalID exec.VarID, signal exec.VarValue) (io.ReadCloser, error) { - return c.cli.GetStream(cdsapi.GetStreamReq{ + str, err := c.cli.GetStream(cdsapi.GetStreamReq{ PlanID: planID, VarID: streamID, SignalID: signalID, Signal: signal, }) + if err != nil { + return nil, err + } + + return io2.CounterCloser(str, func(cnt int64, err error) { + if stgglb.Stats.HubTransfer != nil { + stgglb.Stats.HubTransfer.RecordInput(c.hubID, cnt, err == nil || err == io.EOF) + } + }), nil } func (c *HttpHubWorkerClient) GetVar(ctx context.Context, planID exec.PlanID, varID exec.VarID, signalID exec.VarID, signal exec.VarValue) (exec.VarValue, error) { resp, err := c.cli.GetVar(cdsapi.GetVarReq{ diff --git a/common/pkgs/servicestats/hub_strorage_transfer.go b/common/pkgs/servicestats/hub_strorage_transfer.go new file mode 100644 index 0000000..4f86b71 --- /dev/null +++ b/common/pkgs/servicestats/hub_strorage_transfer.go @@ -0,0 +1,103 @@ +package servicestats + +import ( + "math" + "sync" + "time" + + cdssdk "gitlink.org.cn/cloudream/common/sdks/storage" + "gitlink.org.cn/cloudream/common/utils/math2" +) + +type HubStorageTransferStats struct { + data HubStorageTransferStatsData + fromHubID cdssdk.HubID + lock *sync.Mutex +} + +type HubStorageTransferStatsData struct { + Entries map[cdssdk.StorageID]*HubStorageTransferStatsEntry + StartTime time.Time +} + +type HubStorageTransferStatsEntry struct { + DestStorageID cdssdk.StorageID + + OutputBytes int64 + MaxOutputBytes int64 + MinOutputBytes int64 + TotalOutput int64 + SuccessOutput int64 + + InputBytes int64 + MaxInputBytes int64 + MinInputBytes int64 + TotalInput int64 + SuccessInput int64 +} + +func (s *HubStorageTransferStats) RecordUpload(dstStorageID cdssdk.StorageID, transferBytes int64, isSuccess bool) { + s.lock.Lock() + defer s.lock.Unlock() + + e := s.data.Entries[dstStorageID] + if e == nil { + e = &HubStorageTransferStatsEntry{ + DestStorageID: dstStorageID, + MinInputBytes: math.MaxInt64, + MinOutputBytes: math.MaxInt64, + } + s.data.Entries[dstStorageID] = e + } + e.OutputBytes += transferBytes + e.MaxOutputBytes = math2.Max(e.MaxOutputBytes, transferBytes) + e.MinOutputBytes = math2.Min(e.MinOutputBytes, transferBytes) + if isSuccess { + e.SuccessOutput++ + } + e.TotalOutput++ +} + +func (s *HubStorageTransferStats) RecordDownload(dstStorageID cdssdk.StorageID, transferBytes int64, isSuccess bool) { + s.lock.Lock() + defer s.lock.Unlock() + + e := s.data.Entries[dstStorageID] + if e == nil { + e = &HubStorageTransferStatsEntry{ + DestStorageID: dstStorageID, + MinInputBytes: math.MaxInt64, + MinOutputBytes: math.MaxInt64, + } + s.data.Entries[dstStorageID] = e + } + e.InputBytes += transferBytes + e.MaxInputBytes = math2.Max(e.MaxInputBytes, transferBytes) + e.MinInputBytes = math2.Min(e.MinInputBytes, transferBytes) + if isSuccess { + e.SuccessInput++ + } +} + +func (s *HubStorageTransferStats) Reset() time.Time { + s.lock.Lock() + defer s.lock.Unlock() + + s.data.Entries = make(map[cdssdk.StorageID]*HubStorageTransferStatsEntry) + s.data.StartTime = time.Now() + return s.data.StartTime +} + +func (s *HubStorageTransferStats) DumpData() HubStorageTransferStatsData { + s.lock.Lock() + defer s.lock.Unlock() + + data := s.data + data.Entries = make(map[cdssdk.StorageID]*HubStorageTransferStatsEntry) + for k, v := range s.data.Entries { + v2 := *v + data.Entries[k] = &v2 + } + + return data +} diff --git a/common/pkgs/servicestats/hub_transfter.go b/common/pkgs/servicestats/hub_transfter.go new file mode 100644 index 0000000..9eeec51 --- /dev/null +++ b/common/pkgs/servicestats/hub_transfter.go @@ -0,0 +1,103 @@ +package servicestats + +import ( + "math" + "sync" + "time" + + cdssdk "gitlink.org.cn/cloudream/common/sdks/storage" + "gitlink.org.cn/cloudream/common/utils/math2" +) + +type HubTransferStats struct { + data HubTransferStatsData + fromHubID cdssdk.HubID + lock *sync.Mutex +} + +type HubTransferStatsData struct { + Entries map[cdssdk.HubID]*HubTransferStatsEntry + StartTime time.Time +} + +type HubTransferStatsEntry struct { + DestHubID cdssdk.HubID + + OutputBytes int64 + MaxOutputBytes int64 + MinOutputBytes int64 + TotalOutput int64 + SuccessOutput int64 + + InputBytes int64 + MaxInputBytes int64 + MinInputBytes int64 + TotalInput int64 + SuccessInput int64 +} + +func (s *HubTransferStats) RecordOutput(dstHubID cdssdk.HubID, transferBytes int64, isSuccess bool) { + s.lock.Lock() + defer s.lock.Unlock() + + e := s.data.Entries[dstHubID] + if e == nil { + e = &HubTransferStatsEntry{ + DestHubID: dstHubID, + MinInputBytes: math.MaxInt64, + MinOutputBytes: math.MaxInt64, + } + s.data.Entries[dstHubID] = e + } + e.OutputBytes += transferBytes + e.MaxOutputBytes = math2.Max(e.MaxOutputBytes, transferBytes) + e.MinOutputBytes = math2.Min(e.MinOutputBytes, transferBytes) + if isSuccess { + e.SuccessOutput++ + } + e.TotalOutput++ +} + +func (s *HubTransferStats) RecordInput(dstHubID cdssdk.HubID, transferBytes int64, isSuccess bool) { + s.lock.Lock() + defer s.lock.Unlock() + + e := s.data.Entries[dstHubID] + if e == nil { + e = &HubTransferStatsEntry{ + DestHubID: dstHubID, + MinInputBytes: math.MaxInt64, + MinOutputBytes: math.MaxInt64, + } + s.data.Entries[dstHubID] = e + } + e.InputBytes += transferBytes + e.MaxInputBytes = math2.Max(e.MaxInputBytes, transferBytes) + e.MinInputBytes = math2.Min(e.MinInputBytes, transferBytes) + if isSuccess { + e.SuccessInput++ + } + e.TotalInput++ +} + +func (s *HubTransferStats) Reset() time.Time { + s.lock.Lock() + defer s.lock.Unlock() + + s.data.StartTime = time.Now() + s.data.Entries = make(map[cdssdk.HubID]*HubTransferStatsEntry) + return s.data.StartTime +} + +func (s *HubTransferStats) DumpData() HubTransferStatsData { + s.lock.Lock() + defer s.lock.Unlock() + + data := s.data + data.Entries = make(map[cdssdk.HubID]*HubTransferStatsEntry) + for k, v := range s.data.Entries { + v2 := *v + data.Entries[k] = &v2 + } + return data +} diff --git a/common/pkgs/servicestats/service_stats.go b/common/pkgs/servicestats/service_stats.go new file mode 100644 index 0000000..51c8333 --- /dev/null +++ b/common/pkgs/servicestats/service_stats.go @@ -0,0 +1,37 @@ +package servicestats + +import ( + "sync" + "time" + + cdssdk "gitlink.org.cn/cloudream/common/sdks/storage" +) + +type StatsHost struct { + // 统计Hub间的传输数据，仅包含当前Hub主动发送或接收的数量。 + HubTransfer *HubTransferStats + // 统计Hub与存储系统间的传输数据，仅包含当前Hub主动发送或接收的数量。 + HubStorageTransfer *HubStorageTransferStats +} + +func (h *StatsHost) SetupHubTransfer(fromHubID cdssdk.HubID) { + h.HubTransfer = &HubTransferStats{ + fromHubID: fromHubID, + lock: &sync.Mutex{}, + data: HubTransferStatsData{ + StartTime: time.Now(), + Entries: make(map[cdssdk.HubID]*HubTransferStatsEntry), + }, + } +} + +func (h *StatsHost) SetupHubStorageTransfer(fromHubID cdssdk.HubID) { + h.HubStorageTransfer = &HubStorageTransferStats{ + fromHubID: fromHubID, + lock: &sync.Mutex{}, + data: HubStorageTransferStatsData{ + StartTime: time.Now(), + Entries: make(map[cdssdk.StorageID]*HubStorageTransferStatsEntry), + }, + } +} diff --git a/common/pkgs/storage/agtpool/pool.go b/common/pkgs/storage/agtpool/pool.go index 0f93632..69ed71e 100644 --- a/common/pkgs/storage/agtpool/pool.go +++ b/common/pkgs/storage/agtpool/pool.go @@ -74,6 +74,18 @@ func (m *AgentPool) GetAgent(stgID cdssdk.StorageID) (types.StorageAgent, error) return stg.Agent, nil } +func (m *AgentPool) GetAllAgents() []types.StorageAgent { + m.lock.Lock() + defer m.lock.Unlock() + + agents := make([]types.StorageAgent, 0, len(m.storages)) + for _, stg := range m.storages { + agents = append(agents, stg.Agent) + } + + return agents +} + // 查找指定Storage的ShardStore组件 func (m *AgentPool) GetShardStore(stgID cdssdk.StorageID) (types.ShardStore, error) { m.lock.Lock() diff --git a/common/pkgs/storage/local/shard_store.go b/common/pkgs/storage/local/shard_store.go index da6b00e..88d8208 100644 --- a/common/pkgs/storage/local/shard_store.go +++ b/common/pkgs/storage/local/shard_store.go @@ -14,6 +14,7 @@ import ( "gitlink.org.cn/cloudream/common/pkgs/logger" cdssdk "gitlink.org.cn/cloudream/common/sdks/storage" "gitlink.org.cn/cloudream/common/utils/io2" + stgglb "gitlink.org.cn/cloudream/storage/common/globals" "gitlink.org.cn/cloudream/storage/common/pkgs/storage/types" ) @@ -133,7 +134,11 @@ func (s *ShardStore) Create(stream io.Reader) (types.FileInfo, error) { return types.FileInfo{}, err } - size, hash, err := s.writeTempFile(file, stream) + counter := io2.Counter(stream) + size, hash, err := s.writeTempFile(file, counter) + if stgglb.Stats.HubStorageTransfer != nil { + stgglb.Stats.HubStorageTransfer.RecordUpload(s.agt.Detail.Storage.StorageID, counter.Count(), err == nil) + } if err != nil { // Name是文件完整路径 s.onCreateFailed(file.Name()) @@ -272,11 +277,17 @@ func (s *ShardStore) Open(opt types.OpenOption) (io.ReadCloser, error) { } } + var ret io.ReadCloser = file + if opt.Length >= 0 { - return io2.Length(file, opt.Length), nil + ret = io2.Length(ret, opt.Length) } - return file, nil + return io2.CounterCloser(ret, func(cnt int64, err error) { + if stgglb.Stats.HubStorageTransfer != nil { + stgglb.Stats.HubStorageTransfer.RecordDownload(s.agt.Detail.Storage.StorageID, cnt, err == nil || err == io.EOF) + } + }), nil } func (s *ShardStore) Info(hash cdssdk.FileHash) (types.FileInfo, error) { diff --git a/common/pkgs/storage/s3/shard_store.go b/common/pkgs/storage/s3/shard_store.go index b53d956..e2e0a5e 100644 --- a/common/pkgs/storage/s3/shard_store.go +++ b/common/pkgs/storage/s3/shard_store.go @@ -17,6 +17,7 @@ import ( cdssdk "gitlink.org.cn/cloudream/common/sdks/storage" "gitlink.org.cn/cloudream/common/utils/io2" "gitlink.org.cn/cloudream/common/utils/os2" + stgglb "gitlink.org.cn/cloudream/storage/common/globals" stgmod "gitlink.org.cn/cloudream/storage/common/models" "gitlink.org.cn/cloudream/storage/common/pkgs/storage/types" ) @@ -178,7 +179,7 @@ func (s *ShardStore) createWithAwsSha256(stream io.Reader) (types.FileInfo, erro key, fileName := s.createTempFile() - counter := io2.NewCounter(stream) + counter := io2.Counter(stream) resp, err := s.cli.PutObject(context.TODO(), &s3.PutObjectInput{ Bucket: aws.String(s.Bucket), @@ -186,6 +187,9 @@ func (s *ShardStore) createWithAwsSha256(stream io.Reader) (types.FileInfo, erro Body: counter, ChecksumAlgorithm: s3types.ChecksumAlgorithmSha256, }) + if stgglb.Stats.HubStorageTransfer != nil { + stgglb.Stats.HubStorageTransfer.RecordUpload(s.Detail.Storage.StorageID, counter.Count(), err == nil) + } if err != nil { log.Warnf("uploading file %v: %v", key, err) @@ -218,13 +222,16 @@ func (s *ShardStore) createWithCalcSha256(stream io.Reader) (types.FileInfo, err key, fileName := s.createTempFile() hashStr := io2.NewReadHasher(sha256.New(), stream) - counter := io2.NewCounter(hashStr) + counter := io2.Counter(hashStr) _, err := s.cli.PutObject(context.TODO(), &s3.PutObjectInput{ Bucket: aws.String(s.Bucket), Key: aws.String(key), Body: counter, }) + if stgglb.Stats.HubStorageTransfer != nil { + stgglb.Stats.HubStorageTransfer.RecordUpload(s.Detail.Storage.StorageID, counter.Count(), err == nil) + } if err != nil { log.Warnf("uploading file %v: %v", key, err) @@ -320,7 +327,11 @@ func (s *ShardStore) Open(opt types.OpenOption) (io.ReadCloser, error) { return nil, err } - return resp.Body, nil + return io2.CounterCloser(resp.Body, func(cnt int64, err error) { + if stgglb.Stats.HubStorageTransfer != nil { + stgglb.Stats.HubStorageTransfer.RecordDownload(s.Detail.Storage.StorageID, cnt, err == nil || err == io.EOF) + } + }), nil } func (s *ShardStore) Info(hash cdssdk.FileHash) (types.FileInfo, error) { diff --git a/common/pkgs/storage/types/shard_store.go b/common/pkgs/storage/types/shard_store.go index 12a5f30..be6e3be 100644 --- a/common/pkgs/storage/types/shard_store.go +++ b/common/pkgs/storage/types/shard_store.go @@ -55,7 +55,7 @@ type Stats struct { // 存储服务状态，如果状态正常，此值应该是StatusOK Status Status // 文件总数 - FileCount int + FileCount int64 // 存储空间总大小 TotalSize int64 // 已使用的存储空间大小，可以超过存储空间总大小 diff --git a/common/pkgs/sysevent/config.go b/common/pkgs/sysevent/config.go index dfe1bf5..9c5b96b 100644 --- a/common/pkgs/sysevent/config.go +++ b/common/pkgs/sysevent/config.go @@ -1,8 +1,19 @@ package sysevent +import "gitlink.org.cn/cloudream/common/pkgs/mq" + type Config struct { Address string `json:"address"` Account string `json:"account"` Password string `json:"password"` VHost string `json:"vhost"` } + +func ConfigFromMQConfig(mqCfg mq.Config) Config { + return Config{ + Address: mqCfg.Address, + Account: mqCfg.Account, + Password: mqCfg.Password, + VHost: mqCfg.VHost, + } +} diff --git a/common/pkgs/sysevent/publisher.go b/common/pkgs/sysevent/publisher.go index 5b42c93..dfb482b 100644 --- a/common/pkgs/sysevent/publisher.go +++ b/common/pkgs/sysevent/publisher.go @@ -2,10 +2,12 @@ package sysevent import ( "fmt" + "time" "github.com/streadway/amqp" "gitlink.org.cn/cloudream/common/pkgs/async" "gitlink.org.cn/cloudream/common/utils/serder" + stgmod "gitlink.org.cn/cloudream/storage/common/models" ) type PublisherEvent interface{} @@ -107,10 +109,13 @@ func (p *Publisher) Start() *async.UnboundChannel[PublisherEvent] { return ch } -// Publish 发布事件，自动补齐时间戳和源信息 -func (p *Publisher) Publish(evt SysEvent) { - // TODO 补齐时间戳和源信息 - p.eventChan.Send(evt) +// Publish 发布事件，会自动补齐必要信息 +func (p *Publisher) Publish(eventBody stgmod.SysEventBody) { + p.eventChan.Send(stgmod.SysEvent{ + Timestamp: time.Now(), + Source: p.thisSource, + Body: eventBody, + }) } // PublishRaw 完全原样发布事件，不补齐任何信息 diff --git a/go.mod b/go.mod index 8c50e1c..9dcf692 100644 --- a/go.mod +++ b/go.mod @@ -11,6 +11,7 @@ require ( github.com/aws/aws-sdk-go-v2/credentials v1.17.47 github.com/aws/aws-sdk-go-v2/service/s3 v1.71.0 github.com/gin-gonic/gin v1.7.7 + github.com/go-co-op/gocron/v2 v2.15.0 github.com/go-sql-driver/mysql v1.8.1 github.com/hashicorp/golang-lru/v2 v2.0.5 github.com/huaweicloud/huaweicloud-sdk-go-obs v3.24.9+incompatible @@ -24,7 +25,7 @@ require ( github.com/spf13/cobra v1.8.0 github.com/spf13/viper v1.19.0 gitlink.org.cn/cloudream/common v0.0.0 - golang.org/x/sync v0.6.0 + golang.org/x/sync v0.7.0 google.golang.org/grpc v1.62.1 google.golang.org/protobuf v1.33.0 gorm.io/gorm v1.25.11 @@ -46,8 +47,10 @@ require ( github.com/hashicorp/hcl v1.0.0 // indirect github.com/jinzhu/inflection v1.0.0 // indirect github.com/jinzhu/now v1.1.5 // indirect + github.com/jonboulle/clockwork v0.4.0 // indirect github.com/magiconair/properties v1.8.7 // indirect github.com/pelletier/go-toml/v2 v2.2.2 // indirect + github.com/robfig/cron/v3 v3.0.1 // indirect github.com/sagikazarmark/locafero v0.4.0 // indirect github.com/sagikazarmark/slog-shim v0.1.0 // indirect github.com/sourcegraph/conc v0.3.0 // indirect @@ -57,7 +60,7 @@ require ( github.com/tjfoc/gmsm v1.4.1 // indirect go.mongodb.org/mongo-driver v1.12.0 // indirect golang.org/x/crypto v0.23.0 // indirect - golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect + golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8 // indirect golang.org/x/net v0.23.0 // indirect golang.org/x/sys v0.20.0 // indirect golang.org/x/text v0.15.0 // indirect diff --git a/go.sum b/go.sum index 916f9b3..4262e4a 100644 --- a/go.sum +++ b/go.sum @@ -53,6 +53,8 @@ github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= github.com/gin-gonic/gin v1.7.7 h1:3DoBmSbJbZAWqXJC3SLjAPfutPJJRN1U5pALB7EeTTs= github.com/gin-gonic/gin v1.7.7/go.mod h1:axIBovoeJpVj8S3BwE0uPMTeReE4+AfFtqpqaZ1qq1U= +github.com/go-co-op/gocron/v2 v2.15.0 h1:Kpvo71VSihE+RImmpA+3ta5CcMhoRzMGw4dJawrj4zo= +github.com/go-co-op/gocron/v2 v2.15.0/go.mod h1:ZF70ZwEqz0OO4RBXE1sNxnANy/zvwLcattWEFsqpKig= github.com/go-playground/assert/v2 v2.0.1 h1:MsBgLAaY856+nPRTKrp3/OZK38U/wa0CcBYNjji3q3A= github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= github.com/go-playground/locales v0.13.0 h1:HyWk6mgj5qFqCT5fjGBuRArbVDfE4hi8+e8ceBS/t7Q= @@ -121,6 +123,8 @@ github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= +github.com/jonboulle/clockwork v0.4.0 h1:p4Cf1aMWXnXAUh8lVfewRBx1zaTSYKrKMF2g3ST4RZ4= +github.com/jonboulle/clockwork v0.4.0/go.mod h1:xgRqUGwRcjKCO1vbZUEtSLrqKoPSsUpK7fnezOII0kc= github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= @@ -170,6 +174,8 @@ github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= +github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= +github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= @@ -212,8 +218,9 @@ github.com/stretchr/testify v1.7.4/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= -github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= github.com/tjfoc/gmsm v1.4.1 h1:aMe1GlZb+0bLjn+cKTPEvvn9oUEBlJitaZiiBwsbgho= @@ -241,8 +248,8 @@ go.mongodb.org/mongo-driver v1.12.0 h1:aPx33jmn/rQuJXPQLZQ8NtfPQG8CaqgLThFtqRb0P go.mongodb.org/mongo-driver v1.12.0/go.mod h1:AZkxhPnFJUoH7kZlFkVKucV20K387miPfm7oimrSmK0= go.uber.org/atomic v1.10.0 h1:9qC72Qh0+3MqyJbAn8YU5xVq1frD8bn3JtD2oXtafVQ= go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= -go.uber.org/goleak v1.1.12 h1:gZAh5/EyT/HQwlpkCy6wTpqfH9H8Lz8zbm3dZh+OyzA= -go.uber.org/goleak v1.1.12/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI= go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTVQ= go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60= @@ -258,8 +265,8 @@ golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDf golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI= golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20230905200255-921286631fa9 h1:GoHiUyI/Tp2nVkLI2mCxVkOjsbSXD66ic0XW0js0R9g= -golang.org/x/exp v0.0.0-20230905200255-921286631fa9/go.mod h1:S2oDrQGGwySpoQPVqRShND87VCbxmc6bL1Yd2oYrm6k= +golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8 h1:yixxcjnhBmY0nkL253HFVIm0JsFHwrHdT3Yh6szTnfY= +golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8/go.mod h1:jj3sYF3dwk5D+ghuXyeI3r5MFf+NT2An6/9dOA95KSI= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= @@ -292,8 +299,8 @@ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= -golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -370,8 +377,8 @@ google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQ google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= From b954e4e38128da1d03247d097cd3c6c08a9272dd Mon Sep 17 00:00:00 2001 From: Sydonian <794346190@qq.com> Date: Thu, 13 Feb 2025 14:56:33 +0800 Subject: [PATCH 12/12] =?UTF-8?q?=E5=88=9D=E6=AD=A5=E5=AE=8C=E6=88=90?= =?UTF-8?q?=E7=B3=BB=E7=BB=9F=E4=BA=8B=E4=BB=B6=E7=9A=84=E4=BA=A7=E7=94=9F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- agent/internal/cmd/serve.go | 9 + agent/internal/tickevent/report_hub_stats.go | 7 +- client/internal/cmdline/sysevent.go | 60 ++++ client/internal/services/object.go | 2 + common/models/datamap.go | 221 ++++++++++--- common/pkgs/db2/object.go | 82 +---- common/pkgs/ioswitch2/agent_worker.go | 2 +- common/pkgs/ioswitch2/http_hub_worker.go | 2 +- common/pkgs/mq/coordinator/object.go | 7 +- common/pkgs/mq/coordinator/package.go | 8 +- common/pkgs/sysevent/watcher.go | 14 + common/pkgs/uploader/create_load.go | 2 +- common/pkgs/uploader/update.go | 2 +- coordinator/internal/cmd/serve.go | 48 ++- coordinator/internal/mq/bucket.go | 9 + coordinator/internal/mq/object.go | 146 ++++++++- coordinator/internal/mq/package.go | 51 ++- coordinator/internal/mq/service.go | 9 +- coordinator/internal/mq/utils.go | 23 ++ datamap/internal/models/blockdistribution.go | 2 +- datamap/internal/models/object.go | 2 +- .../event/check_package_redundancy.go | 294 ++++++++++++++++-- scanner/internal/event/clean_pinned.go | 192 +++++++++++- scanner/internal/event/event.go | 5 +- scanner/main.go | 48 ++- 25 files changed, 1033 insertions(+), 214 deletions(-) create mode 100644 client/internal/cmdline/sysevent.go create mode 100644 coordinator/internal/mq/utils.go diff --git a/agent/internal/cmd/serve.go b/agent/internal/cmd/serve.go index c2f21de..654db1a 100644 --- a/agent/internal/cmd/serve.go +++ b/agent/internal/cmd/serve.go @@ -55,6 +55,8 @@ func serve(configPath string) { stgglb.InitLocal(&config.Cfg().Local) stgglb.InitMQPool(config.Cfg().RabbitMQ) stgglb.InitAgentRPCPool(&agtrpc.PoolConfig{}) + stgglb.Stats.SetupHubStorageTransfer(*config.Cfg().Local.HubID) + stgglb.Stats.SetupHubTransfer(*config.Cfg().Local.HubID) // 获取Hub配置 hubCfg := downloadHubConfig() @@ -159,6 +161,7 @@ func serve(configPath string) { // 初始化定时任务执行器 sch := setupTickTask(stgAgts, evtPub) + sch.Start() defer sch.Shutdown() // 启动命令服务器 @@ -259,6 +262,12 @@ func setupTickTask(agtPool *agtpool.AgentPool, evtPub *sysevent.Publisher) gocro gocron.NewAtTime(0, 0, 2), )), gocron.NewTask(tickevent.ReportHubStorageTransferStats, evtPub)) + // sch.NewJob(gocron.DurationJob(time.Minute), gocron.NewTask(tickevent.ReportStorageStats, agtPool, evtPub)) + + // sch.NewJob(gocron.DurationJob(time.Minute), gocron.NewTask(tickevent.ReportHubTransferStats, evtPub)) + + // sch.NewJob(gocron.DurationJob(time.Minute), gocron.NewTask(tickevent.ReportHubStorageTransferStats, agtPool, evtPub)) + return sch } diff --git a/agent/internal/tickevent/report_hub_stats.go b/agent/internal/tickevent/report_hub_stats.go index 7c3c6d1..4c744f3 100644 --- a/agent/internal/tickevent/report_hub_stats.go +++ b/agent/internal/tickevent/report_hub_stats.go @@ -1,6 +1,7 @@ package tickevent import ( + "gitlink.org.cn/cloudream/common/utils/math2" stgglb "gitlink.org.cn/cloudream/storage/common/globals" stgmod "gitlink.org.cn/cloudream/storage/common/models" "gitlink.org.cn/cloudream/storage/common/pkgs/storage/agtpool" @@ -23,7 +24,7 @@ func ReportHubTransferStats(evtPub *sysevent.Publisher) { TotalTransfer: entry.OutputBytes, RequestCount: entry.TotalOutput, FailedRequestCount: entry.TotalInput - entry.SuccessInput, - AvgTransfer: entry.OutputBytes / entry.TotalOutput, + AvgTransfer: math2.DivOrDefault(entry.OutputBytes, entry.TotalOutput, 0), MinTransfer: entry.MinOutputBytes, MaxTransfer: entry.MaxOutputBytes, }, @@ -49,7 +50,9 @@ func ReportHubStorageTransferStats(stgAgts *agtpool.AgentPool, evtPub *sysevent. TotalTransfer: stg.OutputBytes, RequestCount: stg.TotalOutput, FailedRequestCount: stg.TotalInput - stg.SuccessInput, - AvgTransfer: stg.OutputBytes / stg.TotalOutput, + AvgTransfer: math2.DivOrDefault(stg.OutputBytes, stg.TotalOutput, 0), + MinTransfer: stg.MinOutputBytes, + MaxTransfer: stg.MaxOutputBytes, }, StartTimestamp: data.StartTime, EndTimestamp: endTime, diff --git a/client/internal/cmdline/sysevent.go b/client/internal/cmdline/sysevent.go new file mode 100644 index 0000000..a0bd0bb --- /dev/null +++ b/client/internal/cmdline/sysevent.go @@ -0,0 +1,60 @@ +package cmdline + +import ( + "context" + "fmt" + + "github.com/spf13/cobra" + "gitlink.org.cn/cloudream/storage/client/internal/config" + "gitlink.org.cn/cloudream/storage/common/pkgs/sysevent" +) + +func init() { + cmd := &cobra.Command{ + Use: "sysevent", + } + + rootCmd.AddCommand(cmd) + + cmd.AddCommand(&cobra.Command{ + Use: "watch", + Short: "Watch system events", + Run: func(cmd *cobra.Command, args []string) { + watchSysEvent(GetCmdCtx(cmd)) + }, + }) +} + +func watchSysEvent(cmdCtx *CommandContext) { + host, err := sysevent.NewWatcherHost(sysevent.ConfigFromMQConfig(config.Cfg().RabbitMQ)) + if err != nil { + fmt.Println(err) + return + } + + ch := host.Start() + host.AddWatcherFn(func(event sysevent.SysEvent) { + fmt.Println(event.String()) + }) + for { + e, err := ch.Receive().Wait(context.Background()) + if err != nil { + fmt.Println(err) + return + } + + switch e := e.(type) { + case sysevent.PublishError: + fmt.Printf("Publish error: %v\n", e.Err) + + case sysevent.PublisherExited: + if e.Err != nil { + fmt.Printf("Publisher exited with error: %v\n", e.Err) + } + return + + case sysevent.OtherError: + fmt.Printf("Other error: %v\n", e.Err) + } + } +} diff --git a/client/internal/services/object.go b/client/internal/services/object.go index b155ece..4c2478d 100644 --- a/client/internal/services/object.go +++ b/client/internal/services/object.go @@ -64,6 +64,8 @@ func (svc *ObjectService) UpdateInfo(userID cdssdk.UserID, updatings []cdsapi.Up return nil, fmt.Errorf("requsting to coodinator: %w", err) } + // TODO 考虑产生Update事件 + return resp.Successes, nil } diff --git a/common/models/datamap.go b/common/models/datamap.go index c61b4ac..5b3b47a 100644 --- a/common/models/datamap.go +++ b/common/models/datamap.go @@ -1,6 +1,7 @@ package stgmod import ( + "fmt" "time" "gitlink.org.cn/cloudream/common/pkgs/types" @@ -15,6 +16,10 @@ type SysEvent struct { Body SysEventBody `json:"body"` } +func (e *SysEvent) String() string { + return fmt.Sprintf("%v [%v] %+v", e.Timestamp.Format("2006-01-02 15:04:05"), e.Source, e.Body) +} + // 事件源 type SysEventSource interface { GetSourceType() string @@ -35,6 +40,14 @@ func (s *SourceCoordinator) GetSourceType() string { return "Coordinator" } +func (s *SourceCoordinator) OnUnionSerializing() { + s.Type = s.GetSourceType() +} + +func (s *SourceCoordinator) String() string { + return "Coordinator" +} + type SourceScanner struct { serder.Metadata `union:"Scanner"` Type string `json:"type"` @@ -44,6 +57,14 @@ func (s *SourceScanner) GetSourceType() string { return "Scanner" } +func (s *SourceScanner) OnUnionSerializing() { + s.Type = s.GetSourceType() +} + +func (s *SourceScanner) String() string { + return "Scanner" +} + type SourceHub struct { serder.Metadata `union:"Hub"` Type string `json:"type"` @@ -55,6 +76,14 @@ func (s *SourceHub) GetSourceType() string { return "Hub" } +func (s *SourceHub) OnUnionSerializing() { + s.Type = s.GetSourceType() +} + +func (s *SourceHub) String() string { + return fmt.Sprintf("Hub(%d, %s)", s.HubID, s.HubName) +} + // 事件体 type SysEventBody interface { GetBodyType() string @@ -75,11 +104,12 @@ var _ = serder.UseTypeUnionInternallyTagged(types.Ref(types.NewTypeUnion[SysEven (*BodyBlockTransfer)(nil), (*BodyBlockDistribution)(nil), - (*BodyNewObject)(nil), - (*BodyObjectUpdated)(nil), + (*BodyNewOrUpdateObject)(nil), + (*BodyObjectInfoUpdated)(nil), (*BodyObjectDeleted)(nil), (*BodyNewPackage)(nil), + (*BodyPackageCloned)(nil), (*BodyPackageDeleted)(nil), (*BodyNewBucket)(nil), @@ -97,6 +127,10 @@ func (b *BodyNewHub) GetBodyType() string { return "NewHub" } +func (b *BodyNewHub) OnUnionSerializing() { + b.Type = b.GetBodyType() +} + // Hub信息更新的事件 type BodyHubUpdated struct { serder.Metadata `union:"HubUpdated"` @@ -108,6 +142,10 @@ func (b *BodyHubUpdated) GetBodyType() string { return "HubUpdated" } +func (b *BodyHubUpdated) OnUnionSerializing() { + b.Type = b.GetBodyType() +} + // Hub删除的事件 type BodyHubDeleted struct { serder.Metadata `union:"HubDeleted"` @@ -119,6 +157,10 @@ func (b *BodyHubDeleted) GetBodyType() string { return "HubDeleted" } +func (b *BodyHubDeleted) OnUnionSerializing() { + b.Type = b.GetBodyType() +} + // 新增Storage的事件 type BodyNewStorage struct { serder.Metadata `union:"NewStorage"` @@ -130,6 +172,10 @@ func (b *BodyNewStorage) GetBodyType() string { return "NewStorage" } +func (b *BodyNewStorage) OnUnionSerializing() { + b.Type = b.GetBodyType() +} + // Storage信息更新的事件 type BodyStorageUpdated struct { serder.Metadata `union:"StorageUpdated"` @@ -141,6 +187,10 @@ func (b *BodyStorageUpdated) GetBodyType() string { return "StorageUpdated" } +func (b *BodyStorageUpdated) OnUnionSerializing() { + b.Type = b.GetBodyType() +} + // Storage删除的事件 type BodyStorageDeleted struct { serder.Metadata `union:"StorageDeleted"` @@ -152,6 +202,10 @@ func (b *BodyStorageDeleted) GetBodyType() string { return "StorageDeleted" } +func (b *BodyStorageDeleted) OnUnionSerializing() { + b.Type = b.GetBodyType() +} + // Storage统计信息的事件 type BodyStorageStats struct { serder.Metadata `union:"StorageStats"` @@ -164,6 +218,10 @@ func (b *BodyStorageStats) GetBodyType() string { return "StorageStats" } +func (b *BodyStorageStats) OnUnionSerializing() { + b.Type = b.GetBodyType() +} + // Hub数据传输统计信息的事件 type BodyHubTransferStats struct { serder.Metadata `union:"HubTransferStats"` @@ -179,6 +237,10 @@ func (b *BodyHubTransferStats) GetBodyType() string { return "HubTransferStats" } +func (b *BodyHubTransferStats) OnUnionSerializing() { + b.Type = b.GetBodyType() +} + type DataTrans struct { TotalTransfer int64 `json:"totalTransfer"` RequestCount int64 `json:"requestCount"` @@ -204,6 +266,10 @@ func (b *BodyHubStorageTransferStats) GetBodyType() string { return "HubStorageTransferStats" } +func (b *BodyHubStorageTransferStats) OnUnionSerializing() { + b.Type = b.GetBodyType() +} + // 块传输的事件 type BodyBlockTransfer struct { serder.Metadata `union:"BlockTransfer"` @@ -217,6 +283,10 @@ func (b *BodyBlockTransfer) GetBodyType() string { return "BlockTransfer" } +func (b *BodyBlockTransfer) OnUnionSerializing() { + b.Type = b.GetBodyType() +} + // 块变化类型 type BlockChange interface { GetBlockChangeType() string @@ -226,47 +296,60 @@ var _ = serder.UseTypeUnionInternallyTagged(types.Ref(types.NewTypeUnion[BlockCh (*BlockChangeClone)(nil), (*BlockChangeDeleted)(nil), (*BlockChangeEnDecode)(nil), - // (*BlockChangeUpdated)(nil), )), "type") +const ( + BlockTypeRaw = "Raw" + BlockTypeEC = "EC" + BlockTypeSegment = "Segment" +) + type Block struct { BlockType string `json:"blockType"` - Index string `json:"index"` + Index int `json:"index"` StorageID cdssdk.StorageID `json:"storageID"` } type DataTransfer struct { - SourceStorageID cdssdk.StorageID `json:"sourceStorageID"` - TargetStorageID cdssdk.StorageID `json:"targetStorageID"` - DataTransferCount string `json:"dataTransferCount"` + SourceStorageID cdssdk.StorageID `json:"sourceStorageID"` + TargetStorageID cdssdk.StorageID `json:"targetStorageID"` + TransferBytes int64 `json:"transferBytes"` } type BlockChangeClone struct { - serder.Metadata `union:"BlockChangeClone"` - Type string `json:"type"` - BlockType string `json:"blockType"` - Index string `json:"index"` - SourceStorageID cdssdk.StorageID `json:"sourceStorageID"` - TargetStorageID cdssdk.StorageID `json:"targetStorageID"` - DataTransferCount string `json:"dataTransferCount"` + serder.Metadata `union:"Clone"` + Type string `json:"type"` + BlockType string `json:"blockType"` + Index int `json:"index"` + SourceStorageID cdssdk.StorageID `json:"sourceStorageID"` + TargetStorageID cdssdk.StorageID `json:"targetStorageID"` + TransferBytes int64 `json:"transferBytes"` } func (b *BlockChangeClone) GetBlockChangeType() string { return "Clone" } +func (b *BlockChangeClone) OnUnionSerializing() { + b.Type = b.GetBlockChangeType() +} + type BlockChangeDeleted struct { - serder.Metadata `union:"BlockChangeDeleted"` - Type string `json:"type"` - Index string `json:"index"` - StorageID string `json:"storageID"` + serder.Metadata `union:"Deleted"` + Type string `json:"type"` + Index int `json:"index"` + StorageID cdssdk.StorageID `json:"storageID"` } func (b *BlockChangeDeleted) GetBlockChangeType() string { return "Deleted" } +func (b *BlockChangeDeleted) OnUnionSerializing() { + b.Type = b.GetBlockChangeType() +} + type BlockChangeEnDecode struct { - serder.Metadata `union:"BlockChangeEnDecode"` + serder.Metadata `union:"EnDecode"` Type string `json:"type"` SourceBlocks []Block `json:"sourceBlocks,omitempty"` TargetBlocks []Block `json:"targetBlocks,omitempty"` @@ -277,16 +360,9 @@ func (b *BlockChangeEnDecode) GetBlockChangeType() string { return "EnDecode" } -// TODO 块更新应该是说对象被重新上传，此时事件内应该包含全部对象块的信息，因此应该使用ObjectUpdated事件 -// type BlockChangeUpdated struct { -// serder.Metadata `union:"BlockChangeUpdated"` -// Type string `json:"type"` -// Blocks []Block `json:"blocks"` -// } - -// func (b *BlockChangeUpdated) GetBlockChangeType() string { -// return "Updated" -// } +func (b *BlockChangeEnDecode) OnUnionSerializing() { + b.Type = b.GetBlockChangeType() +} // 块分布的事件 type BodyBlockDistribution struct { @@ -296,10 +372,10 @@ type BodyBlockDistribution struct { PackageID cdssdk.PackageID `json:"packageID"` Path string `json:"path"` Size int64 `json:"size"` - FileHash string `json:"fileHash"` - FaultTolerance string `json:"faultTolerance"` - Redundancy string `json:"redundancy"` - AvgAccessCost string `json:"avgAccessCost"` + FileHash cdssdk.FileHash `json:"fileHash"` + FaultTolerance float64 `json:"faultTolerance"` + Redundancy float64 `json:"redundancy"` + AvgAccessCost float64 `json:"avgAccessCost"` BlockDistribution []BlockDistributionObjectInfo `json:"blockDistribution"` DataTransfers []DataTransfer `json:"dataTransfers"` } @@ -308,34 +384,45 @@ func (b *BodyBlockDistribution) GetBodyType() string { return "BlockDistribution" } +func (b *BodyBlockDistribution) OnUnionSerializing() { + b.Type = b.GetBodyType() +} + type BlockDistributionObjectInfo struct { - Type string `json:"type"` - Index string `json:"index"` - StorageID string `json:"storageID"` + BlockType string `json:"type"` + Index int `json:"index"` + StorageID cdssdk.StorageID `json:"storageID"` } -// 新增Object的事件 -type BodyNewObject struct { - serder.Metadata `union:"NewObject"` +// 新增或者重新上传Object的事件 +type BodyNewOrUpdateObject struct { + serder.Metadata `union:"NewOrUpdateObject"` Type string `json:"type"` Info cdssdk.Object `json:"info"` BlockDistribution []BlockDistributionObjectInfo `json:"blockDistribution"` } -func (b *BodyNewObject) GetBodyType() string { - return "NewObject" +func (b *BodyNewOrUpdateObject) GetBodyType() string { + return "NewOrUpdateObject" } -// Object更新的事件 -type BodyObjectUpdated struct { - serder.Metadata `union:"ObjectUpdated"` - Type string `json:"type"` - Info cdssdk.Object `json:"info"` - BlockDistribution []BlockDistributionObjectInfo `json:"blockDistribution"` +func (b *BodyNewOrUpdateObject) OnUnionSerializing() { + b.Type = b.GetBodyType() +} + +// Object的基本信息更新的事件 +type BodyObjectInfoUpdated struct { + serder.Metadata `union:"ObjectInfoUpdated"` + Type string `json:"type"` + Object cdssdk.Object `json:"object"` } -func (b *BodyObjectUpdated) GetBodyType() string { - return "ObjectUpdated" +func (b *BodyObjectInfoUpdated) GetBodyType() string { + return "ObjectInfoUpdated" +} + +func (b *BodyObjectInfoUpdated) OnUnionSerializing() { + b.Type = b.GetBodyType() } // Object删除的事件 @@ -349,6 +436,10 @@ func (b *BodyObjectDeleted) GetBodyType() string { return "ObjectDeleted" } +func (b *BodyObjectDeleted) OnUnionSerializing() { + b.Type = b.GetBodyType() +} + // 新增Package的事件 type BodyNewPackage struct { serder.Metadata `union:"NewPackage"` @@ -360,6 +451,26 @@ func (b *BodyNewPackage) GetBodyType() string { return "NewPackage" } +func (b *BodyNewPackage) OnUnionSerializing() { + b.Type = b.GetBodyType() +} + +// Package克隆的事件 +type BodyPackageCloned struct { + serder.Metadata `union:"PackageCloned"` + Type string `json:"type"` + SourcePackageID cdssdk.PackageID `json:"sourcePackageID"` + NewPackage cdssdk.Package `json:"newPackage"` +} + +func (b *BodyPackageCloned) GetBodyType() string { + return "PackageCloned" +} + +func (b *BodyPackageCloned) OnUnionSerializing() { + b.Type = b.GetBodyType() +} + // Package删除的事件 type BodyPackageDeleted struct { serder.Metadata `union:"PackageDeleted"` @@ -371,6 +482,10 @@ func (b *BodyPackageDeleted) GetBodyType() string { return "PackageDeleted" } +func (b *BodyPackageDeleted) OnUnionSerializing() { + b.Type = b.GetBodyType() +} + // 新增Bucket的事件 type BodyNewBucket struct { serder.Metadata `union:"NewBucket"` @@ -382,6 +497,10 @@ func (b *BodyNewBucket) GetBodyType() string { return "NewBucket" } +func (b *BodyNewBucket) OnUnionSerializing() { + b.Type = b.GetBodyType() +} + // Bucket删除的事件 type BodyBucketDeleted struct { serder.Metadata `union:"BucketDeleted"` @@ -392,3 +511,7 @@ type BodyBucketDeleted struct { func (b *BodyBucketDeleted) GetBodyType() string { return "BucketDeleted" } + +func (b *BodyBucketDeleted) OnUnionSerializing() { + b.Type = b.GetBodyType() +} diff --git a/common/pkgs/db2/object.go b/common/pkgs/db2/object.go index 00a7069..6d86e30 100644 --- a/common/pkgs/db2/object.go +++ b/common/pkgs/db2/object.go @@ -298,12 +298,12 @@ func (db *ObjectDB) BatchAdd(ctx SQLContext, packageID cdssdk.PackageID, adds [] if len(affectedObjIDs) > 0 { // 批量删除 ObjectBlock - if err := ctx.Table("ObjectBlock").Where("ObjectID IN ?", affectedObjIDs).Delete(&stgmod.ObjectBlock{}).Error; err != nil { + if err := db.ObjectBlock().BatchDeleteByObjectID(ctx, affectedObjIDs); err != nil { return nil, fmt.Errorf("batch delete object blocks: %w", err) } // 批量删除 PinnedObject - if err := ctx.Table("PinnedObject").Where("ObjectID IN ?", affectedObjIDs).Delete(&cdssdk.PinnedObject{}).Error; err != nil { + if err := db.PinnedObject().BatchDeleteByObjectID(ctx, affectedObjIDs); err != nil { return nil, fmt.Errorf("batch delete pinned objects: %w", err) } } @@ -343,84 +343,6 @@ func (db *ObjectDB) BatchAdd(ctx SQLContext, packageID cdssdk.PackageID, adds [] return affectedObjs, nil } -func (db *ObjectDB) BatchUpdateRedundancy(ctx SQLContext, objs []coormq.UpdatingObjectRedundancy) error { - if len(objs) == 0 { - return nil - } - - nowTime := time.Now() - objIDs := make([]cdssdk.ObjectID, 0, len(objs)) - dummyObjs := make([]cdssdk.Object, 0, len(objs)) - for _, obj := range objs { - objIDs = append(objIDs, obj.ObjectID) - dummyObjs = append(dummyObjs, cdssdk.Object{ - ObjectID: obj.ObjectID, - Redundancy: obj.Redundancy, - CreateTime: nowTime, // 实际不会更新，只因为不能是0值 - UpdateTime: nowTime, - }) - } - - err := db.Object().BatchUpdateColumns(ctx, dummyObjs, []string{"Redundancy", "UpdateTime"}) - if err != nil { - return fmt.Errorf("batch update object redundancy: %w", err) - } - - // 删除原本所有的编码块记录，重新添加 - err = db.ObjectBlock().BatchDeleteByObjectID(ctx, objIDs) - if err != nil { - return fmt.Errorf("batch delete object blocks: %w", err) - } - - // 删除原本Pin住的Object。暂不考虑FileHash没有变化的情况 - err = db.PinnedObject().BatchDeleteByObjectID(ctx, objIDs) - if err != nil { - return fmt.Errorf("batch delete pinned object: %w", err) - } - - blocks := make([]stgmod.ObjectBlock, 0, len(objs)) - for _, obj := range objs { - blocks = append(blocks, obj.Blocks...) - } - err = db.ObjectBlock().BatchCreate(ctx, blocks) - if err != nil { - return fmt.Errorf("batch create object blocks: %w", err) - } - - caches := make([]model.Cache, 0, len(objs)) - for _, obj := range objs { - for _, blk := range obj.Blocks { - caches = append(caches, model.Cache{ - FileHash: blk.FileHash, - StorageID: blk.StorageID, - CreateTime: nowTime, - Priority: 0, - }) - } - } - err = db.Cache().BatchCreate(ctx, caches) - if err != nil { - return fmt.Errorf("batch create object caches: %w", err) - } - - pinneds := make([]cdssdk.PinnedObject, 0, len(objs)) - for _, obj := range objs { - for _, p := range obj.PinnedAt { - pinneds = append(pinneds, cdssdk.PinnedObject{ - ObjectID: obj.ObjectID, - StorageID: p, - CreateTime: nowTime, - }) - } - } - err = db.PinnedObject().BatchTryCreate(ctx, pinneds) - if err != nil { - return fmt.Errorf("batch create pinned objects: %w", err) - } - - return nil -} - func (db *ObjectDB) BatchDelete(ctx SQLContext, ids []cdssdk.ObjectID) error { if len(ids) == 0 { return nil diff --git a/common/pkgs/ioswitch2/agent_worker.go b/common/pkgs/ioswitch2/agent_worker.go index 757cb86..f2bcdc4 100644 --- a/common/pkgs/ioswitch2/agent_worker.go +++ b/common/pkgs/ioswitch2/agent_worker.go @@ -29,7 +29,7 @@ func (w *AgentWorker) NewClient() (exec.WorkerClient, error) { return nil, err } - return &AgentWorkerClient{cli: cli}, nil + return &AgentWorkerClient{hubID: w.Hub.HubID, cli: cli}, nil } func (w *AgentWorker) String() string { diff --git a/common/pkgs/ioswitch2/http_hub_worker.go b/common/pkgs/ioswitch2/http_hub_worker.go index f145f25..c1db4f9 100644 --- a/common/pkgs/ioswitch2/http_hub_worker.go +++ b/common/pkgs/ioswitch2/http_hub_worker.go @@ -29,7 +29,7 @@ func (w *HttpHubWorker) NewClient() (exec.WorkerClient, error) { return nil, err } - return &HttpHubWorkerClient{cli: cli}, nil + return &HttpHubWorkerClient{hubID: w.Hub.HubID, cli: cli}, nil } func (w *HttpHubWorker) String() string { diff --git a/common/pkgs/mq/coordinator/object.go b/common/pkgs/mq/coordinator/object.go index c90712d..ad63e6e 100644 --- a/common/pkgs/mq/coordinator/object.go +++ b/common/pkgs/mq/coordinator/object.go @@ -272,6 +272,7 @@ type DeleteObjects struct { type DeleteObjectsResp struct { mq.MessageBodyBase + Successes []cdssdk.ObjectID `json:"successes"` } func ReqDeleteObjects(userID cdssdk.UserID, objectIDs []cdssdk.ObjectID) *DeleteObjects { @@ -280,8 +281,10 @@ func ReqDeleteObjects(userID cdssdk.UserID, objectIDs []cdssdk.ObjectID) *Delete ObjectIDs: objectIDs, } } -func RespDeleteObjects() *DeleteObjectsResp { - return &DeleteObjectsResp{} +func RespDeleteObjects(sucs []cdssdk.ObjectID) *DeleteObjectsResp { + return &DeleteObjectsResp{ + Successes: sucs, + } } func (client *Client) DeleteObjects(msg *DeleteObjects) (*DeleteObjectsResp, error) { return mq.Request(Service.DeleteObjects, client.rabbitCli, msg) diff --git a/common/pkgs/mq/coordinator/package.go b/common/pkgs/mq/coordinator/package.go index e077a7b..1bbc0b3 100644 --- a/common/pkgs/mq/coordinator/package.go +++ b/common/pkgs/mq/coordinator/package.go @@ -120,9 +120,8 @@ var _ = Register(Service.UpdatePackage) type UpdatePackage struct { mq.MessageBodyBase - PackageID cdssdk.PackageID `json:"packageID"` - Adds []AddObjectEntry `json:"adds"` - Deletes []cdssdk.ObjectID `json:"deletes"` + PackageID cdssdk.PackageID `json:"packageID"` + Adds []AddObjectEntry `json:"adds"` } type UpdatePackageResp struct { mq.MessageBodyBase @@ -136,11 +135,10 @@ type AddObjectEntry struct { StorageIDs []cdssdk.StorageID `json:"storageIDs"` } -func NewUpdatePackage(packageID cdssdk.PackageID, adds []AddObjectEntry, deletes []cdssdk.ObjectID) *UpdatePackage { +func NewUpdatePackage(packageID cdssdk.PackageID, adds []AddObjectEntry) *UpdatePackage { return &UpdatePackage{ PackageID: packageID, Adds: adds, - Deletes: deletes, } } func NewUpdatePackageResp(added []cdssdk.Object) *UpdatePackageResp { diff --git a/common/pkgs/sysevent/watcher.go b/common/pkgs/sysevent/watcher.go index 6ffbd72..c93cca5 100644 --- a/common/pkgs/sysevent/watcher.go +++ b/common/pkgs/sysevent/watcher.go @@ -113,9 +113,23 @@ func (w *WatcherHost) AddWatcher(watcher Watcher) { w.watchers = append(w.watchers, watcher) } +func (w *WatcherHost) AddWatcherFn(fn func(event SysEvent)) Watcher { + watcher := &fnWatcher{fn: fn} + w.AddWatcher(watcher) + return watcher +} + func (w *WatcherHost) RemoveWatcher(watcher Watcher) { w.lock.Lock() defer w.lock.Unlock() w.watchers = lo2.Remove(w.watchers, watcher) } + +type fnWatcher struct { + fn func(event SysEvent) +} + +func (w *fnWatcher) OnEvent(event SysEvent) { + w.fn(event) +} diff --git a/common/pkgs/uploader/create_load.go b/common/pkgs/uploader/create_load.go index 359ea27..3827c16 100644 --- a/common/pkgs/uploader/create_load.go +++ b/common/pkgs/uploader/create_load.go @@ -96,7 +96,7 @@ func (u *CreateLoadUploader) Commit() (CreateLoadResult, error) { } defer stgglb.CoordinatorMQPool.Release(coorCli) - updateResp, err := coorCli.UpdatePackage(coormq.NewUpdatePackage(u.pkg.PackageID, u.successes, nil)) + updateResp, err := coorCli.UpdatePackage(coormq.NewUpdatePackage(u.pkg.PackageID, u.successes)) if err != nil { return CreateLoadResult{}, fmt.Errorf("updating package: %w", err) } diff --git a/common/pkgs/uploader/update.go b/common/pkgs/uploader/update.go index 596db35..127b156 100644 --- a/common/pkgs/uploader/update.go +++ b/common/pkgs/uploader/update.go @@ -100,7 +100,7 @@ func (w *UpdateUploader) Commit() (UpdateResult, error) { } defer stgglb.CoordinatorMQPool.Release(coorCli) - updateResp, err := coorCli.UpdatePackage(coormq.NewUpdatePackage(w.pkgID, w.successes, nil)) + updateResp, err := coorCli.UpdatePackage(coormq.NewUpdatePackage(w.pkgID, w.successes)) if err != nil { return UpdateResult{}, fmt.Errorf("updating package: %w", err) } diff --git a/coordinator/internal/cmd/serve.go b/coordinator/internal/cmd/serve.go index 59057d5..664dfdf 100644 --- a/coordinator/internal/cmd/serve.go +++ b/coordinator/internal/cmd/serve.go @@ -1,13 +1,16 @@ package cmd import ( + "context" "fmt" "os" "gitlink.org.cn/cloudream/common/pkgs/logger" "gitlink.org.cn/cloudream/common/pkgs/mq" + stgmod "gitlink.org.cn/cloudream/storage/common/models" "gitlink.org.cn/cloudream/storage/common/pkgs/db2" coormq "gitlink.org.cn/cloudream/storage/common/pkgs/mq/coordinator" + "gitlink.org.cn/cloudream/storage/common/pkgs/sysevent" "gitlink.org.cn/cloudream/storage/coordinator/internal/config" mymq "gitlink.org.cn/cloudream/storage/coordinator/internal/mq" ) @@ -30,7 +33,15 @@ func serve(configPath string) { logger.Fatalf("new db2 failed, err: %s", err.Error()) } - coorSvr, err := coormq.NewServer(mymq.NewService(db2), config.Cfg().RabbitMQ) + // 初始化系统事件发布器 + evtPub, err := sysevent.NewPublisher(sysevent.ConfigFromMQConfig(config.Cfg().RabbitMQ), &stgmod.SourceCoordinator{}) + if err != nil { + logger.Errorf("new sysevent publisher: %v", err) + os.Exit(1) + } + go servePublisher(evtPub) + + coorSvr, err := coormq.NewServer(mymq.NewService(db2, evtPub), config.Cfg().RabbitMQ) if err != nil { logger.Fatalf("new coordinator server failed, err: %s", err.Error()) } @@ -46,6 +57,41 @@ func serve(configPath string) { <-forever } +func servePublisher(evtPub *sysevent.Publisher) { + logger.Info("start serving sysevent publisher") + + ch := evtPub.Start() + +loop: + for { + val, err := ch.Receive().Wait(context.Background()) + if err != nil { + logger.Errorf("sysevent publisher stopped with error: %s", err.Error()) + break + } + + switch val := val.(type) { + case sysevent.PublishError: + logger.Errorf("publishing event: %v", val) + + case sysevent.PublisherExited: + if val.Err != nil { + logger.Errorf("publisher exited with error: %v", val.Err) + } else { + logger.Info("publisher exited") + } + break loop + + case sysevent.OtherError: + logger.Errorf("sysevent: %v", val) + } + } + logger.Info("sysevent publisher stopped") + + // TODO 仅简单结束了程序 + os.Exit(1) +} + func serveCoorServer(server *coormq.Server, cfg mq.Config) { logger.Info("start serving command server") diff --git a/coordinator/internal/mq/bucket.go b/coordinator/internal/mq/bucket.go index ccaf031..3d79d0c 100644 --- a/coordinator/internal/mq/bucket.go +++ b/coordinator/internal/mq/bucket.go @@ -4,6 +4,7 @@ import ( "errors" "fmt" + stgmod "gitlink.org.cn/cloudream/storage/common/models" "gitlink.org.cn/cloudream/storage/common/pkgs/db2" "gorm.io/gorm" @@ -93,6 +94,10 @@ func (svc *Service) CreateBucket(msg *coormq.CreateBucket) (*coormq.CreateBucket return nil, mq.Failed(errorcode.OperationFailed, err.Error()) } + svc.evtPub.Publish(&stgmod.BodyNewBucket{ + Info: bucket, + }) + return mq.ReplyOK(coormq.NewCreateBucketResp(bucket)) } @@ -133,5 +138,9 @@ func (svc *Service) DeleteBucket(msg *coormq.DeleteBucket) (*coormq.DeleteBucket return nil, mq.Failed(errorcode.OperationFailed, "delete bucket failed") } + svc.evtPub.Publish(&stgmod.BodyBucketDeleted{ + BucketID: msg.BucketID, + }) + return mq.ReplyOK(coormq.NewDeleteBucketResp()) } diff --git a/coordinator/internal/mq/object.go b/coordinator/internal/mq/object.go index 4431e16..1c2e256 100644 --- a/coordinator/internal/mq/object.go +++ b/coordinator/internal/mq/object.go @@ -3,8 +3,10 @@ package mq import ( "errors" "fmt" + "time" "gitlink.org.cn/cloudream/storage/common/pkgs/db2" + "gitlink.org.cn/cloudream/storage/common/pkgs/db2/model" "gorm.io/gorm" "github.com/samber/lo" @@ -192,8 +194,95 @@ func (svc *Service) GetObjectDetails(msg *coormq.GetObjectDetails) (*coormq.GetO } func (svc *Service) UpdateObjectRedundancy(msg *coormq.UpdateObjectRedundancy) (*coormq.UpdateObjectRedundancyResp, *mq.CodeMessage) { - err := svc.db2.DoTx(func(tx db2.SQLContext) error { - return svc.db2.Object().BatchUpdateRedundancy(tx, msg.Updatings) + err := svc.db2.DoTx(func(ctx db2.SQLContext) error { + db := svc.db2 + objs := msg.Updatings + + nowTime := time.Now() + objIDs := make([]cdssdk.ObjectID, 0, len(objs)) + for _, obj := range objs { + objIDs = append(objIDs, obj.ObjectID) + } + + avaiIDs, err := db.Object().BatchTestObjectID(ctx, objIDs) + if err != nil { + return fmt.Errorf("batch test object id: %w", err) + } + + // 过滤掉已经不存在的对象。 + // 注意，objIDs没有被过滤，因为后续逻辑不过滤也不会出错 + objs = lo.Filter(objs, func(obj coormq.UpdatingObjectRedundancy, _ int) bool { + return avaiIDs[obj.ObjectID] + }) + + dummyObjs := make([]cdssdk.Object, 0, len(objs)) + for _, obj := range objs { + dummyObjs = append(dummyObjs, cdssdk.Object{ + ObjectID: obj.ObjectID, + Redundancy: obj.Redundancy, + CreateTime: nowTime, // 实际不会更新，只因为不能是0值 + UpdateTime: nowTime, + }) + } + + err = db.Object().BatchUpdateColumns(ctx, dummyObjs, []string{"Redundancy", "UpdateTime"}) + if err != nil { + return fmt.Errorf("batch update object redundancy: %w", err) + } + + // 删除原本所有的编码块记录，重新添加 + err = db.ObjectBlock().BatchDeleteByObjectID(ctx, objIDs) + if err != nil { + return fmt.Errorf("batch delete object blocks: %w", err) + } + + // 删除原本Pin住的Object。暂不考虑FileHash没有变化的情况 + err = db.PinnedObject().BatchDeleteByObjectID(ctx, objIDs) + if err != nil { + return fmt.Errorf("batch delete pinned object: %w", err) + } + + blocks := make([]stgmod.ObjectBlock, 0, len(objs)) + for _, obj := range objs { + blocks = append(blocks, obj.Blocks...) + } + err = db.ObjectBlock().BatchCreate(ctx, blocks) + if err != nil { + return fmt.Errorf("batch create object blocks: %w", err) + } + + caches := make([]model.Cache, 0, len(objs)) + for _, obj := range objs { + for _, blk := range obj.Blocks { + caches = append(caches, model.Cache{ + FileHash: blk.FileHash, + StorageID: blk.StorageID, + CreateTime: nowTime, + Priority: 0, + }) + } + } + err = db.Cache().BatchCreate(ctx, caches) + if err != nil { + return fmt.Errorf("batch create object caches: %w", err) + } + + pinneds := make([]cdssdk.PinnedObject, 0, len(objs)) + for _, obj := range objs { + for _, p := range obj.PinnedAt { + pinneds = append(pinneds, cdssdk.PinnedObject{ + ObjectID: obj.ObjectID, + StorageID: p, + CreateTime: nowTime, + }) + } + } + err = db.PinnedObject().BatchTryCreate(ctx, pinneds) + if err != nil { + return fmt.Errorf("batch create pinned objects: %w", err) + } + + return nil }) if err != nil { logger.Warnf("batch updating redundancy: %s", err.Error()) @@ -275,6 +364,8 @@ func pickByObjectIDs[T any](objs []T, objIDs []cdssdk.ObjectID, getID func(T) cd func (svc *Service) MoveObjects(msg *coormq.MoveObjects) (*coormq.MoveObjectsResp, *mq.CodeMessage) { var sucs []cdssdk.ObjectID + var evt []*stgmod.BodyObjectInfoUpdated + err := svc.db2.DoTx(func(tx db2.SQLContext) error { msg.Movings = sort2.Sort(msg.Movings, func(o1, o2 cdsapi.MovingObject) int { return sort2.Cmp(o1.ObjectID, o2.ObjectID) @@ -336,6 +427,11 @@ func (svc *Service) MoveObjects(msg *coormq.MoveObjects) (*coormq.MoveObjectsRes } sucs = lo.Map(newObjs, func(obj cdssdk.Object, _ int) cdssdk.ObjectID { return obj.ObjectID }) + evt = lo.Map(newObjs, func(obj cdssdk.Object, _ int) *stgmod.BodyObjectInfoUpdated { + return &stgmod.BodyObjectInfoUpdated{ + Object: obj, + } + }) return nil }) if err != nil { @@ -343,6 +439,10 @@ func (svc *Service) MoveObjects(msg *coormq.MoveObjects) (*coormq.MoveObjectsRes return nil, mq.Failed(errorcode.OperationFailed, "move objects failed") } + for _, e := range evt { + svc.evtPub.Publish(e) + } + return mq.ReplyOK(coormq.RespMoveObjects(sucs)) } @@ -453,8 +553,15 @@ func (svc *Service) checkPathChangedObjects(tx db2.SQLContext, userID cdssdk.Use } func (svc *Service) DeleteObjects(msg *coormq.DeleteObjects) (*coormq.DeleteObjectsResp, *mq.CodeMessage) { + var sucs []cdssdk.ObjectID err := svc.db2.DoTx(func(tx db2.SQLContext) error { - err := svc.db2.Object().BatchDelete(tx, msg.ObjectIDs) + avaiIDs, err := svc.db2.Object().BatchTestObjectID(tx, msg.ObjectIDs) + if err != nil { + return fmt.Errorf("batch testing object id: %w", err) + } + sucs = lo.Keys(avaiIDs) + + err = svc.db2.Object().BatchDelete(tx, msg.ObjectIDs) if err != nil { return fmt.Errorf("batch deleting objects: %w", err) } @@ -481,7 +588,13 @@ func (svc *Service) DeleteObjects(msg *coormq.DeleteObjects) (*coormq.DeleteObje return nil, mq.Failed(errorcode.OperationFailed, "batch delete objects failed") } - return mq.ReplyOK(coormq.RespDeleteObjects()) + for _, objID := range sucs { + svc.evtPub.Publish(&stgmod.BodyObjectDeleted{ + ObjectID: objID, + }) + } + + return mq.ReplyOK(coormq.RespDeleteObjects(sucs)) } func (svc *Service) CloneObjects(msg *coormq.CloneObjects) (*coormq.CloneObjectsResp, *mq.CodeMessage) { @@ -494,6 +607,8 @@ func (svc *Service) CloneObjects(msg *coormq.CloneObjects) (*coormq.CloneObjects Clonings map[string]CloningObject } + var evt []*stgmod.BodyNewOrUpdateObject + // TODO 要检查用户是否有Object、Package的权限 clonings := make(map[cdssdk.PackageID]*PackageClonings) for i, cloning := range msg.Clonings { @@ -600,6 +715,25 @@ func (svc *Service) CloneObjects(msg *coormq.CloneObjects) (*coormq.CloneObjects for i, cloning := range avaiClonings { ret[cloning.OrgIndex] = &newObjs[i] } + + for i, cloning := range avaiClonings { + var evtBlks []stgmod.BlockDistributionObjectInfo + blkType := getBlockTypeFromRed(newObjs[i].Redundancy) + + oldBlks := avaiDetailsMap[cloning.Cloning.ObjectID].Blocks + for _, blk := range oldBlks { + evtBlks = append(evtBlks, stgmod.BlockDistributionObjectInfo{ + BlockType: blkType, + Index: blk.Index, + StorageID: blk.StorageID, + }) + } + + evt = append(evt, &stgmod.BodyNewOrUpdateObject{ + Info: newObjs[i], + BlockDistribution: evtBlks, + }) + } return nil }) @@ -608,5 +742,9 @@ func (svc *Service) CloneObjects(msg *coormq.CloneObjects) (*coormq.CloneObjects return nil, mq.Failed(errorcode.OperationFailed, err.Error()) } + for _, e := range evt { + svc.evtPub.Publish(e) + } + return mq.ReplyOK(coormq.RespCloneObjects(ret)) } diff --git a/coordinator/internal/mq/package.go b/coordinator/internal/mq/package.go index 307a0cb..999234b 100644 --- a/coordinator/internal/mq/package.go +++ b/coordinator/internal/mq/package.go @@ -79,6 +79,10 @@ func (svc *Service) CreatePackage(msg *coormq.CreatePackage) (*coormq.CreatePack return nil, mq.Failed(errorcode.OperationFailed, err.Error()) } + svc.evtPub.Publish(&stgmod.BodyNewPackage{ + Info: pkg, + }) + return mq.ReplyOK(coormq.NewCreatePackageResp(pkg)) } @@ -90,21 +94,11 @@ func (svc *Service) UpdatePackage(msg *coormq.UpdatePackage) (*coormq.UpdatePack return fmt.Errorf("getting package by id: %w", err) } - // 先执行删除操作 - if len(msg.Deletes) > 0 { - if err := svc.db2.Object().BatchDelete(tx, msg.Deletes); err != nil { - return fmt.Errorf("deleting objects: %w", err) - } - } - - // 再执行添加操作 - if len(msg.Adds) > 0 { - ad, err := svc.db2.Object().BatchAdd(tx, msg.PackageID, msg.Adds) - if err != nil { - return fmt.Errorf("adding objects: %w", err) - } - added = ad + ad, err := svc.db2.Object().BatchAdd(tx, msg.PackageID, msg.Adds) + if err != nil { + return fmt.Errorf("adding objects: %w", err) } + added = ad return nil }) @@ -113,6 +107,26 @@ func (svc *Service) UpdatePackage(msg *coormq.UpdatePackage) (*coormq.UpdatePack return nil, mq.Failed(errorcode.OperationFailed, "update package failed") } + addedMp := make(map[string]cdssdk.Object) + for _, obj := range added { + addedMp[obj.Path] = obj + } + + for _, add := range msg.Adds { + var blks []stgmod.BlockDistributionObjectInfo + for _, stgID := range add.StorageIDs { + blks = append(blks, stgmod.BlockDistributionObjectInfo{ + BlockType: stgmod.BlockTypeRaw, + StorageID: stgID, + }) + } + + svc.evtPub.Publish(&stgmod.BodyNewOrUpdateObject{ + Info: addedMp[add.Path], + BlockDistribution: blks, + }) + } + return mq.ReplyOK(coormq.NewUpdatePackageResp(added)) } @@ -137,6 +151,10 @@ func (svc *Service) DeletePackage(msg *coormq.DeletePackage) (*coormq.DeletePack return nil, mq.Failed(errorcode.OperationFailed, "delete package failed") } + svc.evtPub.Publish(&stgmod.BodyPackageDeleted{ + PackageID: msg.PackageID, + }) + return mq.ReplyOK(coormq.NewDeletePackageResp()) } @@ -203,6 +221,11 @@ func (svc *Service) ClonePackage(msg *coormq.ClonePackage) (*coormq.ClonePackage return nil, mq.Failed(errorcode.OperationFailed, err.Error()) } + svc.evtPub.Publish(&stgmod.BodyPackageCloned{ + SourcePackageID: msg.PackageID, + NewPackage: pkg, + }) + return mq.ReplyOK(coormq.RespClonePackage(pkg)) } diff --git a/coordinator/internal/mq/service.go b/coordinator/internal/mq/service.go index c0fe4e5..d6f2481 100644 --- a/coordinator/internal/mq/service.go +++ b/coordinator/internal/mq/service.go @@ -2,14 +2,17 @@ package mq import ( "gitlink.org.cn/cloudream/storage/common/pkgs/db2" + "gitlink.org.cn/cloudream/storage/common/pkgs/sysevent" ) type Service struct { - db2 *db2.DB + db2 *db2.DB + evtPub *sysevent.Publisher } -func NewService(db2 *db2.DB) *Service { +func NewService(db2 *db2.DB, evtPub *sysevent.Publisher) *Service { return &Service{ - db2: db2, + db2: db2, + evtPub: evtPub, } } diff --git a/coordinator/internal/mq/utils.go b/coordinator/internal/mq/utils.go new file mode 100644 index 0000000..6b68c28 --- /dev/null +++ b/coordinator/internal/mq/utils.go @@ -0,0 +1,23 @@ +package mq + +import ( + cdssdk "gitlink.org.cn/cloudream/common/sdks/storage" + stgmod "gitlink.org.cn/cloudream/storage/common/models" +) + +func getBlockTypeFromRed(red cdssdk.Redundancy) string { + switch red.(type) { + case *cdssdk.NoneRedundancy: + return stgmod.BlockTypeRaw + + case *cdssdk.ECRedundancy: + return stgmod.BlockTypeEC + + case *cdssdk.LRCRedundancy: + return stgmod.BlockTypeEC + + case *cdssdk.SegmentRedundancy: + return stgmod.BlockTypeSegment + } + return "" +} diff --git a/datamap/internal/models/blockdistribution.go b/datamap/internal/models/blockdistribution.go index 12e1dc7..10e0d6e 100644 --- a/datamap/internal/models/blockdistribution.go +++ b/datamap/internal/models/blockdistribution.go @@ -170,7 +170,7 @@ func (w *BlockDistributionWatcher) OnEvent(event sysevent.SysEvent) { for _, dataTransfer := range body.DataTransfers { sourceStorageID, _ := strconv.ParseInt(string(dataTransfer.SourceStorageID), 10, 64) targetStorageID, _ := strconv.ParseInt(string(dataTransfer.TargetStorageID), 10, 64) - dataTransferCount, _ := strconv.ParseInt(dataTransfer.DataTransferCount, 10, 64) + dataTransferCount, _ := strconv.ParseInt(dataTransfer.TransferBytes, 10, 64) err := repoStorage.CreateStorageTransferCount(&StorageTransferCount{ ObjectID: int64(body.ObjectID), diff --git a/datamap/internal/models/object.go b/datamap/internal/models/object.go index 3cb2373..77d5ac2 100644 --- a/datamap/internal/models/object.go +++ b/datamap/internal/models/object.go @@ -75,7 +75,7 @@ type ObjectWatcher struct { func (w *ObjectWatcher) OnEvent(event sysevent.SysEvent) { if event.Category == "objectChange" { - if _, ok := event.Body.(*stgmod.BodyNewObject); ok { + if _, ok := event.Body.(*stgmod.BodyNewOrUpdateObject); ok { } else { fmt.Printf("Watcher %s: Unexpected Body type, expected *ObjectInfo, got %T\n", w.Name, event.Body) diff --git a/scanner/internal/event/check_package_redundancy.go b/scanner/internal/event/check_package_redundancy.go index 47e149a..9ceee86 100644 --- a/scanner/internal/event/check_package_redundancy.go +++ b/scanner/internal/event/check_package_redundancy.go @@ -483,6 +483,7 @@ func (t *CheckPackageRedundancy) noneToRep(ctx ExecuteContext, obj stgmod.Object } var blocks []stgmod.ObjectBlock + var blockChgs []stgmod.BlockChange for i, stg := range uploadStgs { blocks = append(blocks, stgmod.ObjectBlock{ ObjectID: obj.Object.ObjectID, @@ -490,8 +491,26 @@ func (t *CheckPackageRedundancy) noneToRep(ctx ExecuteContext, obj stgmod.Object StorageID: stg.Storage.Storage.StorageID, FileHash: ret[fmt.Sprintf("%d", i)].(*ops2.FileHashValue).Hash, }) + blockChgs = append(blockChgs, &stgmod.BlockChangeClone{ + BlockType: stgmod.BlockTypeRaw, + SourceStorageID: obj.Blocks[0].StorageID, + TargetStorageID: stg.Storage.Storage.StorageID, + TransferBytes: 1, + }) } + // 删除原本的文件块 + blockChgs = append(blockChgs, &stgmod.BlockChangeDeleted{ + Index: 0, + StorageID: obj.Blocks[0].StorageID, + }) + + ctx.Args.EvtPub.Publish(&stgmod.BodyBlockTransfer{ + ObjectID: obj.Object.ObjectID, + PackageID: obj.Object.PackageID, + BlockChanges: blockChgs, + }) + return &coormq.UpdatingObjectRedundancy{ ObjectID: obj.Object.ObjectID, Redundancy: red, @@ -532,6 +551,8 @@ func (t *CheckPackageRedundancy) noneToEC(ctx ExecuteContext, obj stgmod.ObjectD } var blocks []stgmod.ObjectBlock + var evtTargetBlocks []stgmod.Block + var evtBlockTrans []stgmod.DataTransfer for i := 0; i < red.N; i++ { blocks = append(blocks, stgmod.ObjectBlock{ ObjectID: obj.Object.ObjectID, @@ -539,8 +560,39 @@ func (t *CheckPackageRedundancy) noneToEC(ctx ExecuteContext, obj stgmod.ObjectD StorageID: uploadStgs[i].Storage.Storage.StorageID, FileHash: ioRet[fmt.Sprintf("%d", i)].(*ops2.FileHashValue).Hash, }) + evtTargetBlocks = append(evtTargetBlocks, stgmod.Block{ + BlockType: stgmod.BlockTypeEC, + Index: i, + StorageID: uploadStgs[i].Storage.Storage.StorageID, + }) + evtBlockTrans = append(evtBlockTrans, stgmod.DataTransfer{ + SourceStorageID: obj.Blocks[0].StorageID, + TargetStorageID: uploadStgs[i].Storage.Storage.StorageID, + TransferBytes: 1, + }) } + ctx.Args.EvtPub.Publish(&stgmod.BodyBlockTransfer{ + ObjectID: obj.Object.ObjectID, + PackageID: obj.Object.PackageID, + BlockChanges: []stgmod.BlockChange{ + &stgmod.BlockChangeEnDecode{ + SourceBlocks: []stgmod.Block{{ + BlockType: stgmod.BlockTypeRaw, + StorageID: obj.Blocks[0].StorageID, + }}, + TargetBlocks: evtTargetBlocks, + DataTransfers: evtBlockTrans, + }, + + // 删除原本的文件块 + &stgmod.BlockChangeDeleted{ + Index: 0, + StorageID: obj.Blocks[0].StorageID, + }, + }, + }) + return &coormq.UpdatingObjectRedundancy{ ObjectID: obj.Object.ObjectID, Redundancy: red, @@ -548,7 +600,7 @@ func (t *CheckPackageRedundancy) noneToEC(ctx ExecuteContext, obj stgmod.ObjectD }, nil } -func (t *CheckPackageRedundancy) noneToLRC(ctx ExecuteContext, obj stgmod.ObjectDetail, red *cdssdk.LRCRedundancy, uploadStorages []*StorageLoadInfo, allStgs map[cdssdk.StorageID]*StorageLoadInfo) (*coormq.UpdatingObjectRedundancy, error) { +func (t *CheckPackageRedundancy) noneToLRC(ctx ExecuteContext, obj stgmod.ObjectDetail, red *cdssdk.LRCRedundancy, uploadStgs []*StorageLoadInfo, allStgs map[cdssdk.StorageID]*StorageLoadInfo) (*coormq.UpdatingObjectRedundancy, error) { if len(obj.Blocks) == 0 { return nil, fmt.Errorf("object is not cached on any storages, cannot change its redundancy to ec") } @@ -563,7 +615,7 @@ func (t *CheckPackageRedundancy) noneToLRC(ctx ExecuteContext, obj stgmod.Object var toes []ioswitchlrc.To for i := 0; i < red.N; i++ { - toes = append(toes, ioswitchlrc.NewToStorage(*uploadStorages[i].Storage.MasterHub, uploadStorages[i].Storage.Storage, i, fmt.Sprintf("%d", i))) + toes = append(toes, ioswitchlrc.NewToStorage(*uploadStgs[i].Storage.MasterHub, uploadStgs[i].Storage.Storage, i, fmt.Sprintf("%d", i))) } plans := exec.NewPlanBuilder() @@ -580,15 +632,48 @@ func (t *CheckPackageRedundancy) noneToLRC(ctx ExecuteContext, obj stgmod.Object } var blocks []stgmod.ObjectBlock + var evtTargetBlocks []stgmod.Block + var evtBlockTrans []stgmod.DataTransfer for i := 0; i < red.N; i++ { blocks = append(blocks, stgmod.ObjectBlock{ ObjectID: obj.Object.ObjectID, Index: i, - StorageID: uploadStorages[i].Storage.Storage.StorageID, + StorageID: uploadStgs[i].Storage.Storage.StorageID, FileHash: ioRet[fmt.Sprintf("%d", i)].(*ops2.FileHashValue).Hash, }) + evtTargetBlocks = append(evtTargetBlocks, stgmod.Block{ + BlockType: stgmod.BlockTypeEC, + Index: i, + StorageID: uploadStgs[i].Storage.Storage.StorageID, + }) + evtBlockTrans = append(evtBlockTrans, stgmod.DataTransfer{ + SourceStorageID: obj.Blocks[0].StorageID, + TargetStorageID: uploadStgs[i].Storage.Storage.StorageID, + TransferBytes: 1, + }) } + ctx.Args.EvtPub.Publish(&stgmod.BodyBlockTransfer{ + ObjectID: obj.Object.ObjectID, + PackageID: obj.Object.PackageID, + BlockChanges: []stgmod.BlockChange{ + &stgmod.BlockChangeEnDecode{ + SourceBlocks: []stgmod.Block{{ + BlockType: stgmod.BlockTypeRaw, + StorageID: obj.Blocks[0].StorageID, + }}, + TargetBlocks: evtTargetBlocks, + DataTransfers: evtBlockTrans, + }, + + // 删除原本的文件块 + &stgmod.BlockChangeDeleted{ + Index: 0, + StorageID: obj.Blocks[0].StorageID, + }, + }, + }) + return &coormq.UpdatingObjectRedundancy{ ObjectID: obj.Object.ObjectID, Redundancy: red, @@ -634,6 +719,8 @@ func (t *CheckPackageRedundancy) noneToSeg(ctx ExecuteContext, obj stgmod.Object } var blocks []stgmod.ObjectBlock + var evtTargetBlocks []stgmod.Block + var evtBlockTrans []stgmod.DataTransfer for i, stg := range uploadStgs { blocks = append(blocks, stgmod.ObjectBlock{ ObjectID: obj.Object.ObjectID, @@ -641,8 +728,39 @@ func (t *CheckPackageRedundancy) noneToSeg(ctx ExecuteContext, obj stgmod.Object StorageID: stg.Storage.Storage.StorageID, FileHash: ret[fmt.Sprintf("%d", i)].(*ops2.FileHashValue).Hash, }) + evtTargetBlocks = append(evtTargetBlocks, stgmod.Block{ + BlockType: stgmod.BlockTypeSegment, + Index: i, + StorageID: uploadStgs[i].Storage.Storage.StorageID, + }) + evtBlockTrans = append(evtBlockTrans, stgmod.DataTransfer{ + SourceStorageID: obj.Blocks[0].StorageID, + TargetStorageID: uploadStgs[i].Storage.Storage.StorageID, + TransferBytes: 1, + }) } + ctx.Args.EvtPub.Publish(&stgmod.BodyBlockTransfer{ + ObjectID: obj.Object.ObjectID, + PackageID: obj.Object.PackageID, + BlockChanges: []stgmod.BlockChange{ + &stgmod.BlockChangeEnDecode{ + SourceBlocks: []stgmod.Block{{ + BlockType: stgmod.BlockTypeRaw, + StorageID: obj.Blocks[0].StorageID, + }}, + TargetBlocks: evtTargetBlocks, + DataTransfers: evtBlockTrans, + }, + + // 删除原本的文件块 + &stgmod.BlockChangeDeleted{ + Index: 0, + StorageID: obj.Blocks[0].StorageID, + }, + }, + }) + return &coormq.UpdatingObjectRedundancy{ ObjectID: obj.Object.ObjectID, Redundancy: red, @@ -687,6 +805,7 @@ func (t *CheckPackageRedundancy) repToRep(ctx ExecuteContext, obj stgmod.ObjectD } var blocks []stgmod.ObjectBlock + var blockChgs []stgmod.BlockChange for i, stg := range uploadStgs { blocks = append(blocks, stgmod.ObjectBlock{ ObjectID: obj.Object.ObjectID, @@ -694,8 +813,26 @@ func (t *CheckPackageRedundancy) repToRep(ctx ExecuteContext, obj stgmod.ObjectD StorageID: stg.Storage.Storage.StorageID, FileHash: ret[fmt.Sprintf("%d", i)].(*ops2.FileHashValue).Hash, }) + blockChgs = append(blockChgs, &stgmod.BlockChangeClone{ + BlockType: stgmod.BlockTypeRaw, + SourceStorageID: obj.Blocks[0].StorageID, + TargetStorageID: stg.Storage.Storage.StorageID, + TransferBytes: 1, + }) } + // 删除原本的文件块 + blockChgs = append(blockChgs, &stgmod.BlockChangeDeleted{ + Index: 0, + StorageID: obj.Blocks[0].StorageID, + }) + + ctx.Args.EvtPub.Publish(&stgmod.BodyBlockTransfer{ + ObjectID: obj.Object.ObjectID, + PackageID: obj.Object.PackageID, + BlockChanges: blockChgs, + }) + return &coormq.UpdatingObjectRedundancy{ ObjectID: obj.Object.ObjectID, Redundancy: red, @@ -739,26 +876,21 @@ func (t *CheckPackageRedundancy) ecToRep(ctx ExecuteContext, obj stgmod.ObjectDe // 如果选择的备份节点都是同一个，那么就只要上传一次 uploadStgs = lo.UniqBy(uploadStgs, func(item *StorageLoadInfo) cdssdk.StorageID { return item.Storage.Storage.StorageID }) - // 每个被选节点都在自己节点上重建原始数据 planBlder := exec.NewPlanBuilder() - for i := range uploadStgs { - ft := ioswitch2.NewFromTo() - ft.ECParam = srcRed + ft := ioswitch2.NewFromTo() + ft.ECParam = srcRed - for i2, block := range chosenBlocks { - ft.AddFrom(ioswitch2.NewFromShardstore(block.FileHash, *chosenBlockStg[i2].MasterHub, chosenBlockStg[i2], ioswitch2.ECStream(block.Index))) - } + for i, block := range chosenBlocks { + ft.AddFrom(ioswitch2.NewFromShardstore(block.FileHash, *chosenBlockStg[i].MasterHub, chosenBlockStg[i], ioswitch2.ECStream(block.Index))) + } - len := obj.Object.Size - ft.AddTo(ioswitch2.NewToShardStoreWithRange(*uploadStgs[i].Storage.MasterHub, uploadStgs[i].Storage, ioswitch2.RawStream(), fmt.Sprintf("%d", i), math2.Range{ - Offset: 0, - Length: &len, - })) + for i := range uploadStgs { + ft.AddTo(ioswitch2.NewToShardStoreWithRange(*uploadStgs[i].Storage.MasterHub, uploadStgs[i].Storage, ioswitch2.RawStream(), fmt.Sprintf("%d", i), math2.NewRange(0, obj.Object.Size))) + } - err := parser.Parse(ft, planBlder) - if err != nil { - return nil, fmt.Errorf("parsing plan: %w", err) - } + err := parser.Parse(ft, planBlder) + if err != nil { + return nil, fmt.Errorf("parsing plan: %w", err) } // TODO 添加依赖 @@ -770,6 +902,7 @@ func (t *CheckPackageRedundancy) ecToRep(ctx ExecuteContext, obj stgmod.ObjectDe } var blocks []stgmod.ObjectBlock + for i := range uploadStgs { blocks = append(blocks, stgmod.ObjectBlock{ ObjectID: obj.Object.ObjectID, @@ -779,6 +912,55 @@ func (t *CheckPackageRedundancy) ecToRep(ctx ExecuteContext, obj stgmod.ObjectDe }) } + var evtSrcBlocks []stgmod.Block + var evtTargetBlocks []stgmod.Block + for i2, block := range chosenBlocks { + evtSrcBlocks = append(evtSrcBlocks, stgmod.Block{ + BlockType: stgmod.BlockTypeEC, + Index: block.Index, + StorageID: chosenBlockStg[i2].Storage.StorageID, + }) + } + + for _, stg := range uploadStgs { + evtTargetBlocks = append(evtTargetBlocks, stgmod.Block{ + BlockType: stgmod.BlockTypeRaw, + Index: 0, + StorageID: stg.Storage.Storage.StorageID, + }) + } + + var evtBlockTrans []stgmod.DataTransfer + for _, stg := range uploadStgs { + for i2 := range chosenBlocks { + evtBlockTrans = append(evtBlockTrans, stgmod.DataTransfer{ + SourceStorageID: chosenBlockStg[i2].Storage.StorageID, + TargetStorageID: stg.Storage.Storage.StorageID, + TransferBytes: 1, + }) + } + } + + var blockChgs []stgmod.BlockChange + blockChgs = append(blockChgs, &stgmod.BlockChangeEnDecode{ + SourceBlocks: evtSrcBlocks, + TargetBlocks: evtTargetBlocks, + DataTransfers: evtBlockTrans, + }) + + for _, block := range obj.Blocks { + blockChgs = append(blockChgs, &stgmod.BlockChangeDeleted{ + Index: block.Index, + StorageID: block.StorageID, + }) + } + + ctx.Args.EvtPub.Publish(&stgmod.BodyBlockTransfer{ + ObjectID: obj.Object.ObjectID, + PackageID: obj.Object.PackageID, + BlockChanges: blockChgs, + }) + return &coormq.UpdatingObjectRedundancy{ ObjectID: obj.Object.ObjectID, Redundancy: tarRed, @@ -817,6 +999,22 @@ func (t *CheckPackageRedundancy) ecToEC(ctx ExecuteContext, obj stgmod.ObjectDet // 目前EC的参数都相同，所以可以不用重建出完整数据然后再分块，可以直接构建出目的节点需要的块 planBlder := exec.NewPlanBuilder() + var evtSrcBlocks []stgmod.Block + var evtTargetBlocks []stgmod.Block + + ft := ioswitch2.NewFromTo() + ft.ECParam = srcRed + + for i, block := range chosenBlocks { + ft.AddFrom(ioswitch2.NewFromShardstore(block.FileHash, *chosenBlockStg[i].MasterHub, chosenBlockStg[i], ioswitch2.ECStream(block.Index))) + + evtSrcBlocks = append(evtSrcBlocks, stgmod.Block{ + BlockType: stgmod.BlockTypeEC, + Index: block.Index, + StorageID: chosenBlockStg[i].Storage.StorageID, + }) + } + var newBlocks []stgmod.ObjectBlock shouldUpdateBlocks := false for i, stg := range uploadStorages { @@ -838,24 +1036,23 @@ func (t *CheckPackageRedundancy) ecToEC(ctx ExecuteContext, obj stgmod.ObjectDet shouldUpdateBlocks = true // 否则就要重建出这个节点需要的块 - - ft := ioswitch2.NewFromTo() - ft.ECParam = srcRed - for i2, block := range chosenBlocks { - ft.AddFrom(ioswitch2.NewFromShardstore(block.FileHash, *chosenBlockStg[i2].MasterHub, chosenBlockStg[i2], ioswitch2.ECStream(block.Index))) - } - // 输出只需要自己要保存的那一块 ft.AddTo(ioswitch2.NewToShardStore(*stg.Storage.MasterHub, stg.Storage, ioswitch2.ECStream(i), fmt.Sprintf("%d", i))) - err := parser.Parse(ft, planBlder) - if err != nil { - return nil, fmt.Errorf("parsing plan: %w", err) - } + evtTargetBlocks = append(evtTargetBlocks, stgmod.Block{ + BlockType: stgmod.BlockTypeEC, + Index: i, + StorageID: stg.Storage.Storage.StorageID, + }) newBlocks = append(newBlocks, newBlock) } + err := parser.Parse(ft, planBlder) + if err != nil { + return nil, fmt.Errorf("parsing plan: %w", err) + } + // 如果没有任何Plan，Wait会直接返回成功 execCtx := exec.NewExecContext() exec.SetValueByType(execCtx, ctx.Args.StgMgr) @@ -877,6 +1074,41 @@ func (t *CheckPackageRedundancy) ecToEC(ctx ExecuteContext, obj stgmod.ObjectDet newBlocks[idx].FileHash = v.(*ops2.FileHashValue).Hash } + var evtBlockTrans []stgmod.DataTransfer + for _, src := range evtSrcBlocks { + for _, tar := range evtTargetBlocks { + evtBlockTrans = append(evtBlockTrans, stgmod.DataTransfer{ + SourceStorageID: src.StorageID, + TargetStorageID: tar.StorageID, + TransferBytes: 1, + }) + } + } + + var blockChgs []stgmod.BlockChange + for _, block := range obj.Blocks { + keep := lo.ContainsBy(newBlocks, func(newBlock stgmod.ObjectBlock) bool { + return newBlock.Index == block.Index && newBlock.StorageID == block.StorageID + }) + if !keep { + blockChgs = append(blockChgs, &stgmod.BlockChangeDeleted{ + Index: block.Index, + StorageID: block.StorageID, + }) + } + } + blockChgs = append(blockChgs, &stgmod.BlockChangeEnDecode{ + SourceBlocks: evtSrcBlocks, + TargetBlocks: evtTargetBlocks, + DataTransfers: evtBlockTrans, + }) + + ctx.Args.EvtPub.Publish(&stgmod.BodyBlockTransfer{ + ObjectID: obj.Object.ObjectID, + PackageID: obj.Object.PackageID, + BlockChanges: blockChgs, + }) + return &coormq.UpdatingObjectRedundancy{ ObjectID: obj.Object.ObjectID, Redundancy: tarRed, @@ -915,6 +1147,8 @@ func (t *CheckPackageRedundancy) lrcToLRC(ctx ExecuteContext, obj stgmod.ObjectD } } + // TODO 产生BlockTransfer事件 + if canGroupReconstruct { // return t.groupReconstructLRC(obj, lostBlocks, lostBlockGrps, blocksGrpByIndex, srcRed, uploadStorages) } diff --git a/scanner/internal/event/clean_pinned.go b/scanner/internal/event/clean_pinned.go index 4af4c90..277222d 100644 --- a/scanner/internal/event/clean_pinned.go +++ b/scanner/internal/event/clean_pinned.go @@ -117,6 +117,8 @@ func (t *CleanPinned) Execute(execCtx ExecuteContext) { planBld := exec.NewPlanBuilder() planningStgIDs := make(map[cdssdk.StorageID]bool) + var sysEvents []stgmod.SysEventBody + // 对于rep对象，统计出所有对象块分布最多的两个节点，用这两个节点代表所有rep对象块的分布，去进行退火算法 var repObjectsUpdating []coormq.UpdatingObjectRedundancy repMostHubIDs := t.summaryRepObjectBlockNodes(repObjects) @@ -128,6 +130,7 @@ func (t *CleanPinned) Execute(execCtx ExecuteContext) { }) for _, obj := range repObjects { repObjectsUpdating = append(repObjectsUpdating, t.makePlansForRepObject(allStgInfos, solu, obj, planBld, planningStgIDs)) + sysEvents = append(sysEvents, t.generateSysEventForRepObject(solu, obj)...) } // 对于ec对象，则每个对象单独进行退火算法 @@ -141,6 +144,7 @@ func (t *CleanPinned) Execute(execCtx ExecuteContext) { blocks: obj.Blocks, }) ecObjectsUpdating = append(ecObjectsUpdating, t.makePlansForECObject(allStgInfos, solu, obj, planBld, planningStgIDs)) + sysEvents = append(sysEvents, t.generateSysEventForECObject(solu, obj)...) } ioSwRets, err := t.executePlans(execCtx, planBld, planningStgIDs) @@ -161,6 +165,10 @@ func (t *CleanPinned) Execute(execCtx ExecuteContext) { log.Warnf("changing object redundancy: %s", err.Error()) return } + + for _, e := range sysEvents { + execCtx.Args.EvtPub.Publish(e) + } } } @@ -227,9 +235,12 @@ type annealingState struct { maxScore float64 // 搜索过程中得到过的最大分数 maxScoreRmBlocks []bool // 最大分数对应的删除方案 - rmBlocks []bool // 当前删除方案 - inversedIndex int // 当前删除方案是从上一次的方案改动哪个flag而来的 - lastScore float64 // 上一次方案的分数 + rmBlocks []bool // 当前删除方案 + inversedIndex int // 当前删除方案是从上一次的方案改动哪个flag而来的 + lastDisasterTolerance float64 // 上一次方案的容灾度 + lastSpaceCost float64 // 上一次方案的冗余度 + lastMinAccessCost float64 // 上一次方案的最小访问费用 + lastScore float64 // 上一次方案的分数 } type objectBlock struct { @@ -464,8 +475,11 @@ type combinatorialTreeNode struct { } type annealingSolution struct { - blockList []objectBlock // 所有节点的块分布情况 - rmBlocks []bool // 要删除哪些块 + blockList []objectBlock // 所有节点的块分布情况 + rmBlocks []bool // 要删除哪些块 + disasterTolerance float64 // 本方案的容灾度 + spaceCost float64 // 本方案的冗余度 + minAccessCost float64 // 本方案的最小访问费用 } func (t *CleanPinned) startAnnealing(allStgInfos map[cdssdk.StorageID]*stgmod.StorageDetail, readerStgIDs []cdssdk.StorageID, object annealingObject) annealingSolution { @@ -529,8 +543,11 @@ func (t *CleanPinned) startAnnealing(allStgInfos map[cdssdk.StorageID]*stgmod.St } // fmt.Printf("final: %v\n", state.maxScoreRmBlocks) return annealingSolution{ - blockList: state.blockList, - rmBlocks: state.maxScoreRmBlocks, + blockList: state.blockList, + rmBlocks: state.maxScoreRmBlocks, + disasterTolerance: state.lastDisasterTolerance, + spaceCost: state.lastSpaceCost, + minAccessCost: state.lastMinAccessCost, } } @@ -640,6 +657,10 @@ func (t *CleanPinned) calcScore(state *annealingState) float64 { ac := t.calcMinAccessCost(state) sc := t.calcSpaceCost(state) + state.lastDisasterTolerance = dt + state.lastMinAccessCost = ac + state.lastSpaceCost = sc + dtSc := 1.0 if dt < 1 { dtSc = 0 @@ -730,6 +751,11 @@ func (t *CleanPinned) makePlansForRepObject(allStgInfos map[cdssdk.StorageID]*st Redundancy: obj.Object.Redundancy, } + ft := ioswitch2.NewFromTo() + + fromStg := allStgInfos[obj.Blocks[0].StorageID] + ft.AddFrom(ioswitch2.NewFromShardstore(obj.Object.FileHash, *fromStg.MasterHub, *fromStg, ioswitch2.RawStream())) + for i, f := range solu.rmBlocks { hasCache := lo.ContainsBy(obj.Blocks, func(b stgmod.ObjectBlock) bool { return b.StorageID == solu.blockList[i].StorageID }) || lo.ContainsBy(obj.PinnedAt, func(n cdssdk.StorageID) bool { return n == solu.blockList[i].StorageID }) @@ -738,18 +764,9 @@ func (t *CleanPinned) makePlansForRepObject(allStgInfos map[cdssdk.StorageID]*st if !willRm { // 如果对象在退火后要保留副本的节点没有副本，则需要在这个节点创建副本 if !hasCache { - ft := ioswitch2.NewFromTo() - - fromStg := allStgInfos[obj.Blocks[0].StorageID] - ft.AddFrom(ioswitch2.NewFromShardstore(obj.Object.FileHash, *fromStg.MasterHub, *fromStg, ioswitch2.RawStream())) toStg := allStgInfos[solu.blockList[i].StorageID] ft.AddTo(ioswitch2.NewToShardStore(*toStg.MasterHub, *toStg, ioswitch2.RawStream(), fmt.Sprintf("%d.0", obj.Object.ObjectID))) - err := parser.Parse(ft, planBld) - if err != nil { - // TODO 错误处理 - continue - } planningHubIDs[solu.blockList[i].StorageID] = true } entry.Blocks = append(entry.Blocks, stgmod.ObjectBlock{ @@ -761,9 +778,72 @@ func (t *CleanPinned) makePlansForRepObject(allStgInfos map[cdssdk.StorageID]*st } } + err := parser.Parse(ft, planBld) + if err != nil { + // TODO 错误处理 + } + return entry } +func (t *CleanPinned) generateSysEventForRepObject(solu annealingSolution, obj stgmod.ObjectDetail) []stgmod.SysEventBody { + var blockChgs []stgmod.BlockChange + + for i, f := range solu.rmBlocks { + hasCache := lo.ContainsBy(obj.Blocks, func(b stgmod.ObjectBlock) bool { return b.StorageID == solu.blockList[i].StorageID }) || + lo.ContainsBy(obj.PinnedAt, func(n cdssdk.StorageID) bool { return n == solu.blockList[i].StorageID }) + willRm := f + + if !willRm { + // 如果对象在退火后要保留副本的节点没有副本，则需要在这个节点创建副本 + if !hasCache { + blockChgs = append(blockChgs, &stgmod.BlockChangeClone{ + BlockType: stgmod.BlockTypeRaw, + SourceStorageID: obj.Blocks[0].StorageID, + TargetStorageID: solu.blockList[i].StorageID, + }) + } + } else { + blockChgs = append(blockChgs, &stgmod.BlockChangeDeleted{ + Index: 0, + StorageID: solu.blockList[i].StorageID, + }) + } + } + + transEvt := &stgmod.BodyBlockTransfer{ + ObjectID: obj.Object.ObjectID, + PackageID: obj.Object.PackageID, + BlockChanges: blockChgs, + } + + var blockDist []stgmod.BlockDistributionObjectInfo + for i, f := range solu.rmBlocks { + if !f { + blockDist = append(blockDist, stgmod.BlockDistributionObjectInfo{ + BlockType: stgmod.BlockTypeRaw, + Index: 0, + StorageID: solu.blockList[i].StorageID, + }) + } + } + + distEvt := &stgmod.BodyBlockDistribution{ + ObjectID: obj.Object.ObjectID, + PackageID: obj.Object.PackageID, + Path: obj.Object.Path, + Size: obj.Object.Size, + FileHash: obj.Object.FileHash, + FaultTolerance: solu.disasterTolerance, + Redundancy: solu.spaceCost, + AvgAccessCost: 0, // TODO 计算平均访问代价，从日常访问数据中统计 + BlockDistribution: blockDist, + // TODO 不好计算传输量 + } + + return []stgmod.SysEventBody{transEvt, distEvt} +} + func (t *CleanPinned) makePlansForECObject(allStgInfos map[cdssdk.StorageID]*stgmod.StorageDetail, solu annealingSolution, obj stgmod.ObjectDetail, planBld *exec.PlanBuilder, planningHubIDs map[cdssdk.StorageID]bool) coormq.UpdatingObjectRedundancy { entry := coormq.UpdatingObjectRedundancy{ ObjectID: obj.Object.ObjectID, @@ -797,6 +877,7 @@ func (t *CleanPinned) makePlansForECObject(allStgInfos map[cdssdk.StorageID]*stg ecRed := obj.Object.Redundancy.(*cdssdk.ECRedundancy) for id, idxs := range reconstrct { + // 依次生成每个节点上的执行计划，因为如果放到一个计划里一起生成，不能保证每个节点上的块用的都是本节点上的副本 ft := ioswitch2.NewFromTo() ft.ECParam = ecRed ft.AddFrom(ioswitch2.NewFromShardstore(obj.Object.FileHash, *allStgInfos[id].MasterHub, *allStgInfos[id], ioswitch2.RawStream())) @@ -816,6 +897,85 @@ func (t *CleanPinned) makePlansForECObject(allStgInfos map[cdssdk.StorageID]*stg return entry } +func (t *CleanPinned) generateSysEventForECObject(solu annealingSolution, obj stgmod.ObjectDetail) []stgmod.SysEventBody { + var blockChgs []stgmod.BlockChange + + reconstrct := make(map[cdssdk.StorageID]*[]int) + for i, f := range solu.rmBlocks { + block := solu.blockList[i] + if !f { + // 如果这个块是影子块，那么就要从完整对象里重建这个块 + if !block.HasEntity { + re, ok := reconstrct[block.StorageID] + if !ok { + re = &[]int{} + reconstrct[block.StorageID] = re + } + + *re = append(*re, block.Index) + } + } else { + blockChgs = append(blockChgs, &stgmod.BlockChangeDeleted{ + Index: block.Index, + StorageID: block.StorageID, + }) + } + } + + // 由于每一个需要被重建的块都是从同中心的副本里构建出来的，所以对于每一个中心都要产生一个BlockChangeEnDecode + for id, idxs := range reconstrct { + var tarBlocks []stgmod.Block + for _, idx := range *idxs { + tarBlocks = append(tarBlocks, stgmod.Block{ + BlockType: stgmod.BlockTypeEC, + Index: idx, + StorageID: id, + }) + } + blockChgs = append(blockChgs, &stgmod.BlockChangeEnDecode{ + SourceBlocks: []stgmod.Block{{ + BlockType: stgmod.BlockTypeRaw, + Index: 0, + StorageID: id, // 影子块的原始对象就在同一个节点上 + }}, + TargetBlocks: tarBlocks, + // 传输量为0 + }) + } + + transEvt := &stgmod.BodyBlockTransfer{ + ObjectID: obj.Object.ObjectID, + PackageID: obj.Object.PackageID, + BlockChanges: blockChgs, + } + + var blockDist []stgmod.BlockDistributionObjectInfo + for i, f := range solu.rmBlocks { + if !f { + blockDist = append(blockDist, stgmod.BlockDistributionObjectInfo{ + BlockType: stgmod.BlockTypeEC, + Index: solu.blockList[i].Index, + StorageID: solu.blockList[i].StorageID, + }) + } + } + + distEvt := &stgmod.BodyBlockDistribution{ + ObjectID: obj.Object.ObjectID, + PackageID: obj.Object.PackageID, + Path: obj.Object.Path, + Size: obj.Object.Size, + FileHash: obj.Object.FileHash, + FaultTolerance: solu.disasterTolerance, + Redundancy: solu.spaceCost, + AvgAccessCost: 0, // TODO 计算平均访问代价，从日常访问数据中统计 + BlockDistribution: blockDist, + // TODO 不好计算传输量 + } + + return []stgmod.SysEventBody{transEvt, distEvt} +} + func (t *CleanPinned) executePlans(ctx ExecuteContext, planBld *exec.PlanBuilder, planningStgIDs map[cdssdk.StorageID]bool) (map[string]exec.VarValue, error) { // 统一加锁，有重复也没关系 lockBld := reqbuilder.NewBuilder() diff --git a/scanner/internal/event/event.go b/scanner/internal/event/event.go index 900d404..c20efb5 100644 --- a/scanner/internal/event/event.go +++ b/scanner/internal/event/event.go @@ -10,12 +10,14 @@ import ( "gitlink.org.cn/cloudream/storage/common/pkgs/db2" scevt "gitlink.org.cn/cloudream/storage/common/pkgs/mq/scanner/event" "gitlink.org.cn/cloudream/storage/common/pkgs/storage/agtpool" + "gitlink.org.cn/cloudream/storage/common/pkgs/sysevent" ) type ExecuteArgs struct { DB *db2.DB DistLock *distlock.Service StgMgr *agtpool.AgentPool + EvtPub *sysevent.Publisher } type Executor = event.Executor[ExecuteArgs] @@ -26,11 +28,12 @@ type Event = event.Event[ExecuteArgs] type ExecuteOption = event.ExecuteOption -func NewExecutor(db *db2.DB, distLock *distlock.Service, stgAgts *agtpool.AgentPool) Executor { +func NewExecutor(db *db2.DB, distLock *distlock.Service, stgAgts *agtpool.AgentPool, evtPub *sysevent.Publisher) Executor { return event.NewExecutor(ExecuteArgs{ DB: db, DistLock: distLock, StgMgr: stgAgts, + EvtPub: evtPub, }) } diff --git a/scanner/main.go b/scanner/main.go index c9de497..80510ef 100644 --- a/scanner/main.go +++ b/scanner/main.go @@ -1,16 +1,19 @@ package main import ( + "context" "fmt" "os" "gitlink.org.cn/cloudream/common/pkgs/logger" stgglb "gitlink.org.cn/cloudream/storage/common/globals" + stgmod "gitlink.org.cn/cloudream/storage/common/models" "gitlink.org.cn/cloudream/storage/common/pkgs/db2" "gitlink.org.cn/cloudream/storage/common/pkgs/distlock" agtrpc "gitlink.org.cn/cloudream/storage/common/pkgs/grpc/agent" scmq "gitlink.org.cn/cloudream/storage/common/pkgs/mq/scanner" "gitlink.org.cn/cloudream/storage/common/pkgs/storage/agtpool" + "gitlink.org.cn/cloudream/storage/common/pkgs/sysevent" "gitlink.org.cn/cloudream/storage/scanner/internal/config" "gitlink.org.cn/cloudream/storage/scanner/internal/event" "gitlink.org.cn/cloudream/storage/scanner/internal/mq" @@ -50,8 +53,16 @@ func main() { // 启动存储服务管理器 stgAgts := agtpool.NewPool() + // 初始化系统事件发布器 + evtPub, err := sysevent.NewPublisher(sysevent.ConfigFromMQConfig(config.Cfg().RabbitMQ), &stgmod.SourceScanner{}) + if err != nil { + logger.Errorf("new sysevent publisher: %v", err) + os.Exit(1) + } + go servePublisher(evtPub) + // 启动事件执行器 - eventExecutor := event.NewExecutor(db, distlockSvc, stgAgts) + eventExecutor := event.NewExecutor(db, distlockSvc, stgAgts, evtPub) go serveEventExecutor(&eventExecutor) agtSvr, err := scmq.NewServer(mq.NewService(&eventExecutor), config.Cfg().RabbitMQ) @@ -88,6 +99,41 @@ func serveEventExecutor(executor *event.Executor) { os.Exit(1) } +func servePublisher(evtPub *sysevent.Publisher) { + logger.Info("start serving sysevent publisher") + + ch := evtPub.Start() + +loop: + for { + val, err := ch.Receive().Wait(context.Background()) + if err != nil { + logger.Errorf("sysevent publisher stopped with error: %s", err.Error()) + break + } + + switch val := val.(type) { + case sysevent.PublishError: + logger.Errorf("publishing event: %v", val) + + case sysevent.PublisherExited: + if val.Err != nil { + logger.Errorf("publisher exited with error: %v", val.Err) + } else { + logger.Info("publisher exited") + } + break loop + + case sysevent.OtherError: + logger.Errorf("sysevent: %v", val) + } + } + logger.Info("sysevent publisher stopped") + + // TODO 仅简单结束了程序 + os.Exit(1) +} + func serveScannerServer(server *scmq.Server) { logger.Info("start serving scanner server")