From 90afd5596f534412aec363d21a323f9321fda10c Mon Sep 17 00:00:00 2001 From: 17BaoH <963030965@qq.com> Date: Fri, 24 Mar 2023 20:45:48 +0800 Subject: [PATCH 01/68] =?UTF-8?q?=E5=A2=9E=E5=8A=A0scanner=E5=92=8C?= =?UTF-8?q?=E5=BF=83=E8=B7=B3=E5=8D=8F=E8=AE=AE=E6=A1=86=E6=9E=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main.go | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 main.go diff --git a/main.go b/main.go new file mode 100644 index 0000000..033aa6b --- /dev/null +++ b/main.go @@ -0,0 +1,29 @@ +package main + +import ( + + //"path/filepath" + //"sync" + + "time" + //agentcaller "proto" + //"github.com/pborman/uuid" + //"github.com/streadway/amqp" + //"google.golang.org/grpc" +) + +func main() { + for { + //jh:遍历对象副本表， + //-对于每一个rephash, + //--根据objectId查询对象表中的RepNum + //--查询缓存表中rephash对应的TempOrPin为false的nodeIp + //--如果查到的NodeIp数少于RepNum，需发起复制命令 + //jh:遍历对象编码块表， + //-对于每一个blockhash,获得其blockId、objectId、innerID + //--判断blockhash是否在ipfs网络中 + //--得到待修复object清单：记录下各个objectId对应的不在ipfs网络中的blockId、blockhash、innerID + //-查询待修复object清单中各个object的FileSizeInBytes、EcName等，并发出修复命令 + time.Sleep(time.Minute * 5) + } +} From d0def7da0324a53f57d22d030ace47f24afa7d7f Mon Sep 17 00:00:00 2001 From: 17BaoH <963030965@qq.com> Date: Wed, 29 Mar 2023 15:02:13 +0800 Subject: [PATCH 02/68] =?UTF-8?q?=E5=90=88=E5=B9=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/main.go b/main.go index 033aa6b..7b89034 100644 --- a/main.go +++ b/main.go @@ -1,3 +1,7 @@ +/* +1.判断需不需要进行修复，若需要则发起修复命令 +2.判断需不需要进行pin/unpin操作，若需要则发起相关命令 +*/ package main import ( From 36ba7bd88696453a3e6caff3368e5d712dae546d Mon Sep 17 00:00:00 2001 From: 17BaoH <963030965@qq.com> Date: Wed, 5 Apr 2023 21:49:45 +0800 Subject: [PATCH 03/68] =?UTF-8?q?=E8=A1=A5=E5=85=A8=E5=85=83=E6=95=B0?= =?UTF-8?q?=E6=8D=AE=E7=AE=A1=E7=90=86=E5=8F=8A=E7=BA=A0=E5=88=A0=E7=A0=81?= =?UTF-8?q?=E8=87=AA=E9=80=82=E5=BA=94=E8=AF=BB=E7=9B=B8=E5=85=B3=E4=BB=A3?= =?UTF-8?q?=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/main.go b/main.go index 7b89034..033aa6b 100644 --- a/main.go +++ b/main.go @@ -1,7 +1,3 @@ -/* -1.判断需不需要进行修复，若需要则发起修复命令 -2.判断需不需要进行pin/unpin操作，若需要则发起相关命令 -*/ package main import ( From a4fdb261cec2021958414c2151c1fcd073f8fb03 Mon Sep 17 00:00:00 2001 From: 17BaoH <963030965@qq.com> Date: Fri, 7 Apr 2023 14:33:32 +0800 Subject: [PATCH 04/68] =?UTF-8?q?=E5=A2=9E=E5=8A=A0repWrite=E9=83=A8?= =?UTF-8?q?=E5=88=86TODO?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- go.mod | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 go.mod diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..4ca70cb --- /dev/null +++ b/go.mod @@ -0,0 +1,3 @@ +module scanner + +go 1.18 From 1cdd6b67238306f321d8f14997a5479ea411ea1c Mon Sep 17 00:00:00 2001 From: Sydonian <794346190@qq.com> Date: Tue, 16 May 2023 16:33:26 +0800 Subject: [PATCH 05/68] =?UTF-8?q?=E6=8A=BD=E5=87=BAMessage=E5=85=AC?= =?UTF-8?q?=E7=94=A8=E7=9A=84=E7=BB=93=E6=9E=84=E4=BD=93=EF=BC=9B=E4=BC=98?= =?UTF-8?q?=E5=8C=96ObjectRep=E7=9B=B8=E5=85=B3=E7=9A=84=E8=A1=A8=E7=BB=93?= =?UTF-8?q?=E6=9E=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.go b/main.go index 033aa6b..b9b5171 100644 --- a/main.go +++ b/main.go @@ -23,7 +23,7 @@ func main() { //-对于每一个blockhash,获得其blockId、objectId、innerID //--判断blockhash是否在ipfs网络中 //--得到待修复object清单：记录下各个objectId对应的不在ipfs网络中的blockId、blockhash、innerID - //-查询待修复object清单中各个object的FileSizeInBytes、EcName等，并发出修复命令 + //-查询待修复object清单中各个object的FileSize、EcName等，并发出修复命令 time.Sleep(time.Minute * 5) } } From 230daf64d4431b105f04af0a974598a9655944ad Mon Sep 17 00:00:00 2001 From: Sydonian <794346190@qq.com> Date: Fri, 19 May 2023 10:01:33 +0800 Subject: [PATCH 06/68] =?UTF-8?q?=E5=A2=9E=E5=8A=A0Scanner=E6=A3=80?= =?UTF-8?q?=E6=9F=A5=E5=A4=87=E4=BB=BD=E6=95=B0=E5=92=8C=E8=A7=A6=E5=8F=91?= =?UTF-8?q?Agent=E6=A3=80=E6=9F=A5Cache=E7=9A=84=E4=BB=BB=E5=8A=A1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- go.mod | 39 +++++- go.sum | 53 +++++++ internal/config/config.go | 26 ++++ internal/task/agent_check_cache.go | 130 +++++++++++++++++ internal/task/check_rep_count.go | 192 ++++++++++++++++++++++++++ internal/task/check_rep_count_test.go | 155 +++++++++++++++++++++ internal/task/executor.go | 101 ++++++++++++++ internal/task/task.go | 6 + 8 files changed, 701 insertions(+), 1 deletion(-) create mode 100644 go.sum create mode 100644 internal/config/config.go create mode 100644 internal/task/agent_check_cache.go create mode 100644 internal/task/check_rep_count.go create mode 100644 internal/task/check_rep_count_test.go create mode 100644 internal/task/executor.go create mode 100644 internal/task/task.go diff --git a/go.mod b/go.mod index 4ca70cb..1bf8e96 100644 --- a/go.mod +++ b/go.mod @@ -1,3 +1,40 @@ -module scanner +module gitlink.org.cn/cloudream/scanner go 1.18 + +require ( + github.com/jmoiron/sqlx v1.3.5 + github.com/samber/lo v1.38.1 + github.com/smartystreets/goconvey v1.8.0 + gitlink.org.cn/cloudream/db v0.0.0 + gitlink.org.cn/cloudream/rabbitmq v0.0.0 + gitlink.org.cn/cloudream/utils v0.0.0 + golang.org/x/sync v0.1.0 +) + +require ( + github.com/gopherjs/gopherjs v1.17.2 // indirect + github.com/jtolds/gls v4.20.0+incompatible // indirect + github.com/smartystreets/assertions v1.13.1 // indirect +) + +require ( + github.com/antonfisher/nested-logrus-formatter v1.3.1 // indirect + github.com/go-sql-driver/mysql v1.6.0 // indirect + github.com/hashicorp/errwrap v1.0.0 // indirect + github.com/hashicorp/go-multierror v1.1.1 // indirect + github.com/imdario/mergo v0.3.15 // indirect + github.com/mitchellh/mapstructure v1.5.0 // indirect + github.com/sirupsen/logrus v1.9.0 // indirect + github.com/streadway/amqp v1.0.0 // indirect + github.com/zyedidia/generic v1.2.1 + golang.org/x/exp v0.0.0-20220303212507-bbda1eaf7a17 // indirect + golang.org/x/sys v0.7.0 // indirect +) + +// 运行go mod tidy时需要将下面几行取消注释 +// replace gitlink.org.cn/cloudream/rabbitmq => ../rabbitmq +// +// replace gitlink.org.cn/cloudream/utils => ../utils +// +// replace gitlink.org.cn/cloudream/db => ../db diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..8fc1746 --- /dev/null +++ b/go.sum @@ -0,0 +1,53 @@ +github.com/antonfisher/nested-logrus-formatter v1.3.1 h1:NFJIr+pzwv5QLHTPyKz9UMEoHck02Q9L0FP13b/xSbQ= +github.com/antonfisher/nested-logrus-formatter v1.3.1/go.mod h1:6WTfyWFkBc9+zyBaKIqRrg/KwMqBbodBjgbHjDz7zjA= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE= +github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= +github.com/gopherjs/gopherjs v1.17.2 h1:fQnZVsXk8uxXIStYb0N4bGk7jeyTalG/wsZjQ25dO0g= +github.com/gopherjs/gopherjs v1.17.2/go.mod h1:pRRIvn/QzFLrKfvEz3qUuEhtE/zLCWfreZ6J5gM2i+k= +github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= +github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= +github.com/imdario/mergo v0.3.15 h1:M8XP7IuFNsqUx6VPK2P9OSmsYsI/YFaGil0uD21V3dM= +github.com/imdario/mergo v0.3.15/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY= +github.com/jmoiron/sqlx v1.3.5 h1:vFFPA71p1o5gAeqtEAwLU4dnX2napprKtHr7PYIcN3g= +github.com/jmoiron/sqlx v1.3.5/go.mod h1:nRVWtLre0KfCLJvgxzCsLVMogSvQ1zNJtpYr2Ccp0mQ= +github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= +github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/lib/pq v1.2.0 h1:LXpIM/LZ5xGFhOpXAQUIMM1HdyqzVYM13zNdjCEEcA0= +github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/mattn/go-sqlite3 v1.14.6 h1:dNPt6NO46WmLVt2DLNpwczCmdV5boIZ6g/tlDrlRUbg= +github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= +github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= +github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/samber/lo v1.38.1 h1:j2XEAqXKb09Am4ebOg31SpvzUTTs6EN3VfgeLUhPdXM= +github.com/samber/lo v1.38.1/go.mod h1:+m/ZKRl6ClXCE2Lgf3MsQlWfh4bn1bz6CXEOxnEXnEA= +github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= +github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/smartystreets/assertions v1.13.1 h1:Ef7KhSmjZcK6AVf9YbJdvPYG9avaF0ZxudX+ThRdWfU= +github.com/smartystreets/assertions v1.13.1/go.mod h1:cXr/IwVfSo/RbCSPhoAPv73p3hlSdrBH/b3SdnW/LMY= +github.com/smartystreets/goconvey v1.8.0 h1:Oi49ha/2MURE0WexF052Z0m+BNSGirfjg5RL+JXWq3w= +github.com/smartystreets/goconvey v1.8.0/go.mod h1:EdX8jtrTIj26jmjCOVNMVSIYAtgexqXKHOXW2Dx9JLg= +github.com/streadway/amqp v1.0.0 h1:kuuDrUJFZL1QYL9hUNuCxNObNzB0bV/ZG5jV3RWAQgo= +github.com/streadway/amqp v1.0.0/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/zyedidia/generic v1.2.1 h1:Zv5KS/N2m0XZZiuLS82qheRG4X1o5gsWreGb0hR7XDc= +github.com/zyedidia/generic v1.2.1/go.mod h1:ly2RBz4mnz1yeuVbQA/VFwGjK3mnHGRj1JuoG336Bis= +golang.org/x/exp v0.0.0-20220303212507-bbda1eaf7a17 h1:3MTrJm4PyNL9NBqvYDSj3DHl46qQakyfqfWo4jgfaEM= +golang.org/x/exp v0.0.0-20220303212507-bbda1eaf7a17/go.mod h1:lgLbSvA5ygNOMpwM/9anMpWVlVJ7Z+cHWq/eFuinpGE= +golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU= +golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/config/config.go b/internal/config/config.go new file mode 100644 index 0000000..01c555b --- /dev/null +++ b/internal/config/config.go @@ -0,0 +1,26 @@ +package config + +import ( + db "gitlink.org.cn/cloudream/db/config" + racfg "gitlink.org.cn/cloudream/rabbitmq/config" + c "gitlink.org.cn/cloudream/utils/config" + log "gitlink.org.cn/cloudream/utils/logger" +) + +type Config struct { + MinAvailableRepProportion float32 `json:"minAvailableRepProportion"` // 可用的备份至少要占所有备份的比例，向上去整 + + Logger log.Config `json:"logger"` + DB db.Config `json:"db"` + RabbitMQ racfg.Config `json:"rabbitMQ"` +} + +var cfg Config + +func Init() error { + return c.DefaultLoad("scanner", &cfg) +} + +func Cfg() *Config { + return &cfg +} diff --git a/internal/task/agent_check_cache.go b/internal/task/agent_check_cache.go new file mode 100644 index 0000000..2768acb --- /dev/null +++ b/internal/task/agent_check_cache.go @@ -0,0 +1,130 @@ +package task + +import ( + "database/sql" + "fmt" + + "github.com/jmoiron/sqlx" + "github.com/samber/lo" + "gitlink.org.cn/cloudream/db/model" + mysql "gitlink.org.cn/cloudream/db/sql" + "gitlink.org.cn/cloudream/scanner/internal/config" + log "gitlink.org.cn/cloudream/utils/logger" + + agtcli "gitlink.org.cn/cloudream/rabbitmq/client/agent" + agtmsg "gitlink.org.cn/cloudream/rabbitmq/message/agent" + agttsk "gitlink.org.cn/cloudream/rabbitmq/message/agent/task" +) + +type AgentCheckCacheTaskEntry struct { + NodeID int + FileHashes []string // 需要检查的FileHash列表，如果为nil（不是为空），则代表进行全量检查 +} + +func NewAgentCheckCacheTaskEntry(nodeID int, fileHashes []string) AgentCheckCacheTaskEntry { + return AgentCheckCacheTaskEntry{ + NodeID: nodeID, + FileHashes: fileHashes, + } +} + +type AgentCheckCacheTask struct { + Entries []AgentCheckCacheTaskEntry +} + +func NewAgentCheckCacheTask(entries []AgentCheckCacheTaskEntry) *AgentCheckCacheTask { + return &AgentCheckCacheTask{ + Entries: entries, + } +} + +func (t *AgentCheckCacheTask) TryMerge(other Task) bool { + chkTask, ok := other.(*AgentCheckCacheTask) + if !ok { + return false + } + + for _, entry := range chkTask.Entries { + _, index, ok := lo.FindIndexOf(t.Entries, func(e AgentCheckCacheTaskEntry) bool { return e.NodeID == entry.NodeID }) + if ok { + myEntry := &t.Entries[index] + + // FileHashes为nil时代表全量检查 + if entry.FileHashes == nil { + myEntry.FileHashes = nil + } else if myEntry.FileHashes != nil { + myEntry.FileHashes = lo.Union(myEntry.FileHashes, entry.FileHashes) + } + + } else { + t.Entries = append(t.Entries, entry) + } + } + + return true +} + +func (t *AgentCheckCacheTask) Execute(execCtx *ExecuteContext, execOpts ExecuteOption) { + for _, entry := range t.Entries { + err := t.checkOneAgentCache(entry, execCtx, execOpts) + if err != nil { + log.Warnf("let agent check cache failed, err: %s", err.Error()) + continue + } + } +} + +func (t *AgentCheckCacheTask) checkOneAgentCache(entry AgentCheckCacheTaskEntry, execCtx *ExecuteContext, execOpts ExecuteOption) error { + var isComplete bool + var caches []model.Cache + + err := execCtx.MyDB.DoTx(sql.LevelSerializable, func(tx *sqlx.Tx) error { + // TODO unavailable的节点需不需要发送任务？ + + if entry.FileHashes == nil { + var err error + caches, err = mysql.Cache.GetNodeCaches(tx, entry.NodeID) + if err != nil { + return fmt.Errorf("get node caches failed, err: %w", err) + } + isComplete = true + + } else { + for _, hash := range entry.FileHashes { + ch, err := mysql.Cache.Get(tx, hash, entry.NodeID) + // 记录不存在则跳过 + if err == sql.ErrNoRows { + continue + } + + if err != nil { + return fmt.Errorf("get cache failed, err: %w", err) + } + + caches = append(caches, ch) + } + isComplete = false + } + return nil + }) + if err != nil { + return err + } + + // 然后向代理端发送移动文件的请求 + agentClient, err := agtcli.NewAgentClient(entry.NodeID, &config.Cfg().RabbitMQ) + if err != nil { + return fmt.Errorf("create agent client to %d failed, err: %w", entry.NodeID, err) + } + defer agentClient.Close() + + err = agentClient.PostTask(agtmsg.NewPostTaskBody( + agttsk.NewCheckCacheTask(isComplete, caches), + execOpts.IsEmergency, // 继承本任务的执行选项 + execOpts.DontMerge)) + if err != nil { + return fmt.Errorf("request to agent %d failed, err: %w", entry.NodeID, err) + } + + return nil +} diff --git a/internal/task/check_rep_count.go b/internal/task/check_rep_count.go new file mode 100644 index 0000000..4c18683 --- /dev/null +++ b/internal/task/check_rep_count.go @@ -0,0 +1,192 @@ +package task + +import ( + "database/sql" + "fmt" + "math" + + "github.com/jmoiron/sqlx" + "github.com/samber/lo" + "gitlink.org.cn/cloudream/scanner/internal/config" + "gitlink.org.cn/cloudream/utils/consts" + log "gitlink.org.cn/cloudream/utils/logger" + mymath "gitlink.org.cn/cloudream/utils/math" + mysort "gitlink.org.cn/cloudream/utils/sort" + + "gitlink.org.cn/cloudream/db/model" + mysql "gitlink.org.cn/cloudream/db/sql" +) + +type CheckRepCountTask struct { + FileHashes []string +} + +func (t *CheckRepCountTask) TryMerge(other Task) bool { + chkTask, ok := other.(*CheckRepCountTask) + if !ok { + return false + } + + t.FileHashes = lo.Union(t.FileHashes, chkTask.FileHashes) + return true +} + +func (t *CheckRepCountTask) Execute(execCtx *ExecuteContext, myOpts ExecuteOption) { + var updatedNodeAndHashes map[int][]string + + for _, fileHash := range t.FileHashes { + updatedNodeIDs, err := t.checkOneRepCount(fileHash, execCtx) + if err != nil { + log.WithField("FileHash", fileHash).Warnf("check file rep count failed, err: %s", err.Error()) + continue + } + + for _, id := range updatedNodeIDs { + hashes := updatedNodeAndHashes[id] + updatedNodeAndHashes[id] = append(hashes, fileHash) + } + } + + var agtChkEntries []AgentCheckCacheTaskEntry + for nodeID, hashes := range updatedNodeAndHashes { + agtChkEntries = append(agtChkEntries, NewAgentCheckCacheTaskEntry(nodeID, hashes)) + } + // 新任务继承本任务的执行设定（紧急任务依然保持紧急任务） + execCtx.Executor.Post(NewAgentCheckCacheTask(agtChkEntries), myOpts) +} + +func (t *CheckRepCountTask) checkOneRepCount(fileHash string, execCtx *ExecuteContext) ([]int, error) { + var updatedNodeIDs []int + err := execCtx.MyDB.DoTx(sql.LevelSerializable, func(tx *sqlx.Tx) error { + repMaxCnt, err := mysql.ObjectRep.GetFileMaxRepCount(tx, fileHash) + if err != nil { + return fmt.Errorf("get file max rep count failed, err: %w", err) + } + + blkCnt, err := mysql.ObjectBlock.CountBlockWithHash(tx, fileHash) + if err != nil { + return fmt.Errorf("count block with hash failed, err: %w", err) + } + + // 计算所需的最少备份数： + // ObjectRep中期望备份数的最大值 + // 如果ObjectBlock存在对此文件的引用，则至少为1 + needRepCount := mymath.Max(repMaxCnt, mymath.Max(1, blkCnt)) + + repNodes, err := mysql.Cache.GetCachingFileNodes(tx, fileHash) + if err != nil { + return fmt.Errorf("get caching file nodes failed, err: %w", err) + } + + allNodes, err := mysql.Node.GetAllNodes(tx) + if err != nil { + return fmt.Errorf("get all nodes failed, err: %w", err) + } + + var normalNodes, unavaiNodes []model.Node + for _, node := range repNodes { + if node.State == consts.NODE_STATE_NORMAL { + normalNodes = append(normalNodes, node) + } else if node.State == consts.NODE_STATE_UNAVAILABLE { + unavaiNodes = append(unavaiNodes, node) + } + } + + // 如果Available的备份数超过期望备份数，则让一些节点退出 + if len(normalNodes) > needRepCount { + delNodes := chooseDeleteAvaiRepNodes(allNodes, normalNodes, needRepCount-len(normalNodes)) + for _, node := range delNodes { + err := mysql.Cache.ChangeState(tx, fileHash, node.NodeID, consts.CACHE_STATE_TEMP) + if err != nil { + return fmt.Errorf("change cache state failed, err: %w", err) + } + updatedNodeIDs = append(updatedNodeIDs, node.NodeID) + } + return nil + } + + minAvaiNodeCnt := int(math.Ceil(float64(config.Cfg().MinAvailableRepProportion) * float64(needRepCount))) + + // 因为总备份数不够，而需要增加的备份数 + add1 := mymath.Max(0, needRepCount-len(repNodes)) + + // 因为Available的备份数占比过少，而需要增加的备份数 + add2 := mymath.Max(0, minAvaiNodeCnt-len(normalNodes)) + + // 最终需要增加的备份数，是以上两种情况的最大值 + finalAddCount := mymath.Max(add1, add2) + + if finalAddCount > 0 { + newNodes := chooseNewRepNodes(allNodes, repNodes, finalAddCount) + if len(newNodes) < finalAddCount { + // TODO 节点数不够，进行一个告警 + } + + for _, node := range newNodes { + err := mysql.Cache.Create(tx, fileHash, node.NodeID) + if err != nil { + return fmt.Errorf("create cache failed, err: %w", err) + } + updatedNodeIDs = append(updatedNodeIDs, node.NodeID) + } + } + + return nil + }) + + if err != nil { + return nil, err + } + + return updatedNodeIDs, nil +} + +func chooseNewRepNodes(allNodes []model.Node, curRepNodes []model.Node, newCount int) []model.Node { + noRepNodes := lo.Reject(allNodes, func(node model.Node, index int) bool { + return lo.ContainsBy(curRepNodes, func(n model.Node) bool { return node.NodeID == n.NodeID }) || + node.State != consts.NODE_STATE_NORMAL + }) + + repNodeLocationIDs := make(map[int]bool) + for _, node := range curRepNodes { + repNodeLocationIDs[node.LocationID] = true + } + + mysort.Sort(noRepNodes, func(l, r model.Node) int { + // LocationID不存在时为false，false - true < 0，所以LocationID不存在的会排在前面 + return mysort.CmpBool(repNodeLocationIDs[l.LocationID], repNodeLocationIDs[r.LocationID]) + }) + + return noRepNodes[:mymath.Min(newCount, len(noRepNodes))] +} + +func chooseDeleteAvaiRepNodes(allNodes []model.Node, curAvaiRepNodes []model.Node, delCount int) []model.Node { + // 按照地域ID分组 + locationGroupedNodes := make(map[int][]model.Node) + for _, node := range curAvaiRepNodes { + nodes := locationGroupedNodes[node.LocationID] + nodes = append(nodes, node) + locationGroupedNodes[node.LocationID] = nodes + } + + // 每次从每个分组中取出一个元素放入结果数组，并将这个元素从分组中删除 + // 最后结果数组中的元素会按照地域交错循环排列，比如：ABCABCBCC。同时还有一个特征：靠后的循环节中的元素都来自于元素数多的分组 + // 将结果数组反转（此处是用存放时就逆序的形式实现），就把元素数多的分组提前了，此时从头部取出要删除的节点即可 + alternatedNodes := make([]model.Node, len(curAvaiRepNodes)) + for i := len(curAvaiRepNodes) - 1; i >= 0; { + for id, nodes := range locationGroupedNodes { + alternatedNodes[i] = nodes[0] + + if len(nodes) == 1 { + delete(locationGroupedNodes, id) + } else { + locationGroupedNodes[id] = nodes[1:] + } + + // 放置一个元素就移动一下下一个存放点 + i-- + } + } + + return alternatedNodes[:mymath.Min(delCount, len(alternatedNodes))] +} diff --git a/internal/task/check_rep_count_test.go b/internal/task/check_rep_count_test.go new file mode 100644 index 0000000..3854171 --- /dev/null +++ b/internal/task/check_rep_count_test.go @@ -0,0 +1,155 @@ +package task + +import ( + "testing" + + "github.com/samber/lo" + . "github.com/smartystreets/goconvey/convey" + "gitlink.org.cn/cloudream/db/model" + "gitlink.org.cn/cloudream/utils/consts" + "gitlink.org.cn/cloudream/utils/sort" +) + +func Test_chooseNewRepNodes(t *testing.T) { + testcases := []struct { + title string + allNodes []model.Node + curRepNodes []model.Node + newCount int + wantNodeIDs []int + }{ + { + title: "优先选择不同地域的节点", + allNodes: []model.Node{ + { + NodeID: 1, + LocationID: 1, + State: consts.NODE_STATE_NORMAL, + }, + { + NodeID: 2, + LocationID: 1, + State: consts.NODE_STATE_NORMAL, + }, + { + NodeID: 3, + LocationID: 2, + State: consts.NODE_STATE_NORMAL, + }, + { + NodeID: 4, + LocationID: 3, + State: consts.NODE_STATE_NORMAL, + }, + }, + curRepNodes: []model.Node{ + { + NodeID: 1, + LocationID: 1, + }, + }, + newCount: 2, + wantNodeIDs: []int{3, 4}, + }, + { + title: "就算节点数不足，也不能选择重复节点", + allNodes: []model.Node{ + { + NodeID: 1, + LocationID: 1, + State: consts.NODE_STATE_NORMAL, + }, + { + NodeID: 2, + LocationID: 1, + State: consts.NODE_STATE_NORMAL, + }, + }, + curRepNodes: []model.Node{ + { + NodeID: 1, + LocationID: 1, + }, + }, + newCount: 2, + wantNodeIDs: []int{2}, + }, + { + title: "就算节点数不足，也不能选择状态unavailable的节点", + allNodes: []model.Node{ + { + NodeID: 1, + LocationID: 1, + State: consts.NODE_STATE_UNAVAILABLE, + }, + { + NodeID: 2, + LocationID: 1, + State: consts.NODE_STATE_NORMAL, + }, + }, + curRepNodes: []model.Node{ + { + NodeID: 3, + LocationID: 1, + }, + }, + newCount: 2, + wantNodeIDs: []int{2}, + }, + } + + for _, test := range testcases { + Convey(test.title, t, func() { + chooseNodes := chooseNewRepNodes(test.allNodes, test.curRepNodes, test.newCount) + chooseNodeIDs := lo.Map(chooseNodes, func(node model.Node, index int) int { return node.NodeID }) + + sort.Sort(chooseNodeIDs, sort.Cmp[int]) + + So(chooseNodeIDs, ShouldResemble, test.wantNodeIDs) + }) + } +} + +func Test_chooseDeleteAvaiRepNodes(t *testing.T) { + testcases := []struct { + title string + allNodes []model.Node + curRepNodes []model.Node + delCount int + wantNodeLocationIDs []int + }{ + { + title: "优先选择地域重复的节点", + allNodes: []model.Node{}, + curRepNodes: []model.Node{ + {NodeID: 1, LocationID: 1}, {NodeID: 2, LocationID: 1}, + {NodeID: 3, LocationID: 2}, {NodeID: 4, LocationID: 2}, + {NodeID: 5, LocationID: 3}, {NodeID: 6, LocationID: 3}, {NodeID: 7, LocationID: 3}, + {NodeID: 8, LocationID: 4}, + }, + delCount: 4, + wantNodeLocationIDs: []int{1, 2, 3, 3}, + }, + { + title: "节点不够删", + allNodes: []model.Node{}, + curRepNodes: []model.Node{ + {NodeID: 1, LocationID: 1}, + }, + delCount: 2, + wantNodeLocationIDs: []int{1}, + }, + } + + for _, test := range testcases { + Convey(test.title, t, func() { + chooseNodes := chooseDeleteAvaiRepNodes(test.allNodes, test.curRepNodes, test.delCount) + chooseNodeLocationIDs := lo.Map(chooseNodes, func(node model.Node, index int) int { return node.LocationID }) + + sort.Sort(chooseNodeLocationIDs, sort.Cmp[int]) + + So(chooseNodeLocationIDs, ShouldResemble, test.wantNodeLocationIDs) + }) + } +} diff --git a/internal/task/executor.go b/internal/task/executor.go new file mode 100644 index 0000000..aa93e23 --- /dev/null +++ b/internal/task/executor.go @@ -0,0 +1,101 @@ +package task + +import ( + "context" + "sync" + + "github.com/zyedidia/generic/list" + mydb "gitlink.org.cn/cloudream/db" + "golang.org/x/sync/semaphore" +) + +type ExecuteOption struct { + IsEmergency bool + DontMerge bool +} +type ExecuteContext struct { + Executor *Executor + MyDB *mydb.DB +} +type postedTask struct { + Task Task + Option ExecuteOption +} + +type Executor struct { + tasks list.List[postedTask] + locker sync.Mutex + taskSema semaphore.Weighted + execCtx *ExecuteContext +} + +func (e *Executor) Post(task Task, opts ...ExecuteOption) { + opt := ExecuteOption{ + IsEmergency: false, + DontMerge: false, + } + + if len(opts) > 0 { + opt = opts[0] + } + + e.locker.Lock() + defer e.locker.Unlock() + + // 紧急任务直接插入到队头，不进行合并 + if opt.IsEmergency { + e.tasks.PushFront(postedTask{ + Task: task, + Option: opt, + }) + e.taskSema.Release(1) + return + } + + // 合并任务 + if opt.DontMerge { + ptr := e.tasks.Front + for ptr != nil { + // 只与非紧急任务，且允许合并的任务进行合并 + if !ptr.Value.Option.IsEmergency && !ptr.Value.Option.DontMerge { + if ptr.Value.Task.TryMerge(task) { + return + } + } + + ptr = ptr.Next + } + } + + e.tasks.PushBack(postedTask{ + Task: task, + Option: opt, + }) + e.taskSema.Release(1) +} + +// Execute 开始执行任务 +func (e *Executor) Execute() error { + for { + // TODO 打印错误日志 + e.taskSema.Acquire(context.Background(), 1) + + task := e.popFrontTask() + if task == nil { + continue + } + + task.Task.Execute(e.execCtx, task.Option) + } +} + +func (e *Executor) popFrontTask() *postedTask { + e.locker.Lock() + defer e.locker.Unlock() + + if e.tasks.Front == nil { + return nil + } + + return &e.tasks.Front.Value +} diff --git a/internal/task/task.go b/internal/task/task.go new file mode 100644 index 0000000..40547df --- /dev/null +++ b/internal/task/task.go @@ -0,0 +1,6 @@ +package task + +type Task interface { + TryMerge(other Task) bool // 尝试将other任务与自身合并，如果成功返回true + Execute(ctx *ExecuteContext, myOpts ExecuteOption) +} From 1794bbafc65d7e74f0adf46b97cd15f928c216c4 Mon Sep 17 00:00:00 2001 From: Sydonian <794346190@qq.com> Date: Fri, 19 May 2023 10:53:56 +0800 Subject: [PATCH 07/68] =?UTF-8?q?=E5=B8=B8=E9=87=8F=E7=A7=BB=E5=8A=A8?= =?UTF-8?q?=E5=88=B0=E7=8B=AC=E7=AB=8B=E7=9A=84common=E6=A8=A1=E5=9D=97?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- go.mod | 3 +++ internal/task/check_rep_count_test.go | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 1bf8e96..de162ba 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,7 @@ require ( github.com/jmoiron/sqlx v1.3.5 github.com/samber/lo v1.38.1 github.com/smartystreets/goconvey v1.8.0 + gitlink.org.cn/cloudream/common v0.0.0 gitlink.org.cn/cloudream/db v0.0.0 gitlink.org.cn/cloudream/rabbitmq v0.0.0 gitlink.org.cn/cloudream/utils v0.0.0 @@ -38,3 +39,5 @@ require ( // replace gitlink.org.cn/cloudream/utils => ../utils // // replace gitlink.org.cn/cloudream/db => ../db +// +// replace gitlink.org.cn/cloudream/common => ../common diff --git a/internal/task/check_rep_count_test.go b/internal/task/check_rep_count_test.go index 3854171..cb81aa7 100644 --- a/internal/task/check_rep_count_test.go +++ b/internal/task/check_rep_count_test.go @@ -5,8 +5,8 @@ import ( "github.com/samber/lo" . "github.com/smartystreets/goconvey/convey" + "gitlink.org.cn/cloudream/common/consts" "gitlink.org.cn/cloudream/db/model" - "gitlink.org.cn/cloudream/utils/consts" "gitlink.org.cn/cloudream/utils/sort" ) From bf3cb7a6cf820763441ad05ff6656f46b910b7e2 Mon Sep 17 00:00:00 2001 From: Sydonian <794346190@qq.com> Date: Fri, 19 May 2023 11:22:20 +0800 Subject: [PATCH 08/68] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E6=9B=B4=E6=96=B0Cache?= =?UTF-8?q?=E7=9A=84=E4=BB=BB=E5=8A=A1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- internal/task/agent_check_cache.go | 2 +- internal/task/check_rep_count.go | 6 ++-- internal/task/executor.go | 2 +- internal/task/update_cache.go | 57 ++++++++++++++++++++++++++++++ 4 files changed, 62 insertions(+), 5 deletions(-) create mode 100644 internal/task/update_cache.go diff --git a/internal/task/agent_check_cache.go b/internal/task/agent_check_cache.go index 2768acb..a80e096 100644 --- a/internal/task/agent_check_cache.go +++ b/internal/task/agent_check_cache.go @@ -78,7 +78,7 @@ func (t *AgentCheckCacheTask) checkOneAgentCache(entry AgentCheckCacheTaskEntry, var isComplete bool var caches []model.Cache - err := execCtx.MyDB.DoTx(sql.LevelSerializable, func(tx *sqlx.Tx) error { + err := execCtx.DB.DoTx(sql.LevelSerializable, func(tx *sqlx.Tx) error { // TODO unavailable的节点需不需要发送任务？ if entry.FileHashes == nil { diff --git a/internal/task/check_rep_count.go b/internal/task/check_rep_count.go index 4c18683..eaa36ac 100644 --- a/internal/task/check_rep_count.go +++ b/internal/task/check_rep_count.go @@ -7,8 +7,8 @@ import ( "github.com/jmoiron/sqlx" "github.com/samber/lo" + "gitlink.org.cn/cloudream/common/consts" "gitlink.org.cn/cloudream/scanner/internal/config" - "gitlink.org.cn/cloudream/utils/consts" log "gitlink.org.cn/cloudream/utils/logger" mymath "gitlink.org.cn/cloudream/utils/math" mysort "gitlink.org.cn/cloudream/utils/sort" @@ -32,7 +32,7 @@ func (t *CheckRepCountTask) TryMerge(other Task) bool { } func (t *CheckRepCountTask) Execute(execCtx *ExecuteContext, myOpts ExecuteOption) { - var updatedNodeAndHashes map[int][]string + updatedNodeAndHashes := make(map[int][]string) for _, fileHash := range t.FileHashes { updatedNodeIDs, err := t.checkOneRepCount(fileHash, execCtx) @@ -57,7 +57,7 @@ func (t *CheckRepCountTask) Execute(execCtx *ExecuteContext, myOpts ExecuteOptio func (t *CheckRepCountTask) checkOneRepCount(fileHash string, execCtx *ExecuteContext) ([]int, error) { var updatedNodeIDs []int - err := execCtx.MyDB.DoTx(sql.LevelSerializable, func(tx *sqlx.Tx) error { + err := execCtx.DB.DoTx(sql.LevelSerializable, func(tx *sqlx.Tx) error { repMaxCnt, err := mysql.ObjectRep.GetFileMaxRepCount(tx, fileHash) if err != nil { return fmt.Errorf("get file max rep count failed, err: %w", err) diff --git a/internal/task/executor.go b/internal/task/executor.go index aa93e23..35f867f 100644 --- a/internal/task/executor.go +++ b/internal/task/executor.go @@ -15,7 +15,7 @@ type ExecuteOption struct { } type ExecuteContext struct { Executor *Executor - MyDB *mydb.DB + DB *mydb.DB } type postedTask struct { Task Task diff --git a/internal/task/update_cache.go b/internal/task/update_cache.go new file mode 100644 index 0000000..e54a0ea --- /dev/null +++ b/internal/task/update_cache.go @@ -0,0 +1,57 @@ +package task + +import ( + tskcst "gitlink.org.cn/cloudream/common/consts/task" + mysql "gitlink.org.cn/cloudream/db/sql" + "gitlink.org.cn/cloudream/utils/logger" +) + +type UpdateCacheTaskEntry struct { + FileHash string + NodeID int + Operation string +} + +func NewUpdateCacheTaskEntry(fileHash string, nodeID int, op string) UpdateCacheTaskEntry { + return UpdateCacheTaskEntry{ + FileHash: fileHash, + NodeID: nodeID, + Operation: op, + } +} + +type UpdateCacheTask struct { + Entries []UpdateCacheTaskEntry +} + +func NewUpdateCacheTask(entries []UpdateCacheTaskEntry) UpdateCacheTask { + return UpdateCacheTask{ + Entries: entries, + } +} + +func (t *UpdateCacheTask) TryMerge(other Task) bool { + chkTask, ok := other.(*UpdateCacheTask) + if !ok { + return false + } + + // TODO 可以考虑合并同FileHash和NodeID的记录 + t.Entries = append(t.Entries, chkTask.Entries...) + return true +} + +func (t *UpdateCacheTask) Execute(execCtx *ExecuteContext, execOpts ExecuteOption) { + for _, entry := range t.Entries { + switch entry.Operation { + case tskcst.UPDATE_CACHE_OP_UNTEMP: + err := mysql.Cache.DeleteTemp(execCtx.DB.SQLCtx(), entry.FileHash, entry.NodeID) + + if err != nil { + logger.WithField("FileHash", entry.FileHash). + WithField("NodeID", entry.NodeID). + Warnf("delete temp cache failed, err: %s", err.Error()) + } + } + } +} From 7250043896cf135da2229cddfb8aa3209f7365ce Mon Sep 17 00:00:00 2001 From: Sydonian <794346190@qq.com> Date: Fri, 19 May 2023 11:38:02 +0800 Subject: [PATCH 09/68] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E6=A3=80=E6=9F=A5Objec?= =?UTF-8?q?t=E7=9A=84=E4=BB=BB=E5=8A=A1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- internal/task/check_object.go | 36 ++++++++++++++++++++++++++++++++ internal/task/check_rep_count.go | 1 + 2 files changed, 37 insertions(+) create mode 100644 internal/task/check_object.go diff --git a/internal/task/check_object.go b/internal/task/check_object.go new file mode 100644 index 0000000..3f9cead --- /dev/null +++ b/internal/task/check_object.go @@ -0,0 +1,36 @@ +package task + +import ( + "github.com/samber/lo" + mysql "gitlink.org.cn/cloudream/db/sql" + "gitlink.org.cn/cloudream/utils/logger" +) + +type CheckObjectTask struct { + ObjectIDs []int +} + +func NewCheckObjectTask(objIDs []int) CheckObjectTask { + return CheckObjectTask{ + ObjectIDs: objIDs, + } +} + +func (t *CheckObjectTask) TryMerge(other Task) bool { + chkTask, ok := other.(*CheckObjectTask) + if !ok { + return false + } + + t.ObjectIDs = lo.Union(t.ObjectIDs, chkTask.ObjectIDs) + return true +} + +func (t *CheckObjectTask) Execute(execCtx *ExecuteContext, execOpts ExecuteOption) { + for _, objID := range t.ObjectIDs { + err := mysql.Object.DeleteUnused(execCtx.DB.SQLCtx(), objID) + if err != nil { + logger.WithField("ObjectID", objID).Warnf("delete unused object failed, err: %s", err.Error()) + } + } +} diff --git a/internal/task/check_rep_count.go b/internal/task/check_rep_count.go index eaa36ac..f843bb7 100644 --- a/internal/task/check_rep_count.go +++ b/internal/task/check_rep_count.go @@ -119,6 +119,7 @@ func (t *CheckRepCountTask) checkOneRepCount(fileHash string, execCtx *ExecuteCo if finalAddCount > 0 { newNodes := chooseNewRepNodes(allNodes, repNodes, finalAddCount) if len(newNodes) < finalAddCount { + log.WithField("FileHash", fileHash).Warnf("need %d more rep nodes, but get only %d nodes", finalAddCount, len(newNodes)) // TODO 节点数不够，进行一个告警 } From 2071a9fbae2dd51c6f97c585aadb9e3a8e282d14 Mon Sep 17 00:00:00 2001 From: Sydonian <794346190@qq.com> Date: Fri, 19 May 2023 15:11:16 +0800 Subject: [PATCH 10/68] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E6=A3=80=E6=9F=A5?= =?UTF-8?q?=E8=8A=82=E7=82=B9=E7=8A=B6=E6=80=81=E4=BB=BB=E5=8A=A1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- internal/config/config.go | 1 + internal/task/agent_check_state.go | 86 ++++++++++++++++++++++++++++++ internal/task/check_rep_count.go | 6 +++ 3 files changed, 93 insertions(+) create mode 100644 internal/task/agent_check_state.go diff --git a/internal/config/config.go b/internal/config/config.go index 01c555b..4301a0b 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -9,6 +9,7 @@ import ( type Config struct { MinAvailableRepProportion float32 `json:"minAvailableRepProportion"` // 可用的备份至少要占所有备份的比例，向上去整 + NodeUnavailableSeconds int `json:"nodeUnavailableSeconds"` // 如果节点上次上报时间超过这个值，则认为节点已经不可用 Logger log.Config `json:"logger"` DB db.Config `json:"db"` diff --git a/internal/task/agent_check_state.go b/internal/task/agent_check_state.go new file mode 100644 index 0000000..1addb95 --- /dev/null +++ b/internal/task/agent_check_state.go @@ -0,0 +1,86 @@ +package task + +import ( + "database/sql" + "time" + + "github.com/samber/lo" + "gitlink.org.cn/cloudream/common/consts" + "gitlink.org.cn/cloudream/db/model" + mysql "gitlink.org.cn/cloudream/db/sql" + agtcli "gitlink.org.cn/cloudream/rabbitmq/client/agent" + agtmsg "gitlink.org.cn/cloudream/rabbitmq/message/agent" + agttsk "gitlink.org.cn/cloudream/rabbitmq/message/agent/task" + "gitlink.org.cn/cloudream/scanner/internal/config" + "gitlink.org.cn/cloudream/utils/logger" +) + +type AgentCheckStateTask struct { + NodeIDs []int +} + +func NewAgentCheckStateTask(nodeIDs []int) AgentCheckStateTask { + return AgentCheckStateTask{ + NodeIDs: nodeIDs, + } +} + +func (t *AgentCheckStateTask) TryMerge(other Task) bool { + chkTask, ok := other.(*AgentCheckStateTask) + if !ok { + return false + } + + t.NodeIDs = lo.Union(t.NodeIDs, chkTask.NodeIDs) + return true +} + +func (t *AgentCheckStateTask) Execute(execCtx *ExecuteContext, execOpts ExecuteOption) { + for _, nodeID := range t.NodeIDs { + node, err := mysql.Node.GetByID(execCtx.DB.SQLCtx(), nodeID) + if err == sql.ErrNoRows { + continue + } + + if err != nil { + logger.WithField("NodeID", nodeID).Warnf("get node by id failed, err: %s", err.Error()) + continue + } + + if node.State != consts.NODE_STATE_NORMAL { + continue + } + + // 检查上次上报时间，超时的设置为不可用 + if time.Since(node.LastReportTime) > time.Duration(config.Cfg().NodeUnavailableSeconds)*time.Second { + err := mysql.Node.ChangeState(execCtx.DB.SQLCtx(), nodeID, consts.NODE_STATE_UNAVAILABLE) + if err != nil { + logger.WithField("NodeID", nodeID).Warnf("set node state failed, err: %s", err.Error()) + continue + } + + caches, err := mysql.Cache.GetNodeCaches(execCtx.DB.SQLCtx(), nodeID) + if err != nil { + logger.WithField("NodeID", nodeID).Warnf("get node caches failed, err: %s", err.Error()) + continue + } + + // 补充备份数 + execCtx.Executor.Post(NewCheckRepCountTask(lo.Map(caches, func(ch model.Cache, index int) string { return ch.HashValue }))) + continue + } + + agentClient, err := agtcli.NewAgentClient(nodeID, &config.Cfg().RabbitMQ) + if err != nil { + logger.WithField("NodeID", nodeID).Warnf("create agent client failed, err: %s", err.Error()) + continue + } + defer agentClient.Close() + + // 紧急任务 + err = agentClient.PostTask(agtmsg.NewPostTaskBody(agttsk.NewCheckStateTask(), true, true)) + if err != nil { + logger.WithField("NodeID", nodeID).Warnf("request to agent failed, err: %s", err.Error()) + } + } +} diff --git a/internal/task/check_rep_count.go b/internal/task/check_rep_count.go index f843bb7..99e1814 100644 --- a/internal/task/check_rep_count.go +++ b/internal/task/check_rep_count.go @@ -21,6 +21,12 @@ type CheckRepCountTask struct { FileHashes []string } +func NewCheckRepCountTask(fileHashes []string) *CheckRepCountTask { + return &CheckRepCountTask{ + FileHashes: fileHashes, + } +} + func (t *CheckRepCountTask) TryMerge(other Task) bool { chkTask, ok := other.(*CheckRepCountTask) if !ok { From 87144ed6f18c8da2153e1178325121205afc1aa5 Mon Sep 17 00:00:00 2001 From: Sydonian <794346190@qq.com> Date: Fri, 19 May 2023 15:19:50 +0800 Subject: [PATCH 11/68] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E6=9B=B4=E6=96=B0?= =?UTF-8?q?=E8=8A=82=E7=82=B9=E7=8A=B6=E6=80=81=E7=9A=84=E4=BB=BB=E5=8A=A1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- internal/task/update_agent_state.go | 34 +++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 internal/task/update_agent_state.go diff --git a/internal/task/update_agent_state.go b/internal/task/update_agent_state.go new file mode 100644 index 0000000..12846fe --- /dev/null +++ b/internal/task/update_agent_state.go @@ -0,0 +1,34 @@ +package task + +import ( + "gitlink.org.cn/cloudream/common/consts" + mysql "gitlink.org.cn/cloudream/db/sql" + "gitlink.org.cn/cloudream/utils/logger" +) + +type UpdateAgentStateTask struct { + NodeID int + IPFSStatus string +} + +func (t *UpdateAgentStateTask) TryMerge(other Task) bool { + return false +} + +func (t *UpdateAgentStateTask) Execute(execCtx *ExecuteContext, execOpts ExecuteOption) { + if t.IPFSStatus != consts.IPFS_STATUS_OK { + logger.WithField("NodeID", t.NodeID).Warnf("IPFS status is %s, set node state unavailable", t.IPFSStatus) + + err := mysql.Node.ChangeState(execCtx.DB.SQLCtx(), t.NodeID, consts.NODE_STATE_UNAVAILABLE) + if err != nil { + logger.WithField("NodeID", t.NodeID).Warnf("change node state failed, err: %s", err.Error()) + } + return + } + + // TODO 如果以后还有其他的状态，要判断哪些状态下能设置Normal + err := mysql.Node.ChangeState(execCtx.DB.SQLCtx(), t.NodeID, consts.NODE_STATE_NORMAL) + if err != nil { + logger.WithField("NodeID", t.NodeID).Warnf("change node state failed, err: %s", err.Error()) + } +} From acd01c5d0ce11c3190d6264de175afc62cc993d8 Mon Sep 17 00:00:00 2001 From: Sydonian <794346190@qq.com> Date: Fri, 19 May 2023 16:04:28 +0800 Subject: [PATCH 12/68] =?UTF-8?q?=E7=AE=80=E5=8C=96CheckCache=E4=BB=BB?= =?UTF-8?q?=E5=8A=A1=E7=9A=84=E9=80=BB=E8=BE=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- internal/task/agent_check_cache.go | 110 ++++++++++------------------- 1 file changed, 37 insertions(+), 73 deletions(-) diff --git a/internal/task/agent_check_cache.go b/internal/task/agent_check_cache.go index a80e096..7574511 100644 --- a/internal/task/agent_check_cache.go +++ b/internal/task/agent_check_cache.go @@ -2,119 +2,84 @@ package task import ( "database/sql" - "fmt" - "github.com/jmoiron/sqlx" "github.com/samber/lo" "gitlink.org.cn/cloudream/db/model" mysql "gitlink.org.cn/cloudream/db/sql" "gitlink.org.cn/cloudream/scanner/internal/config" - log "gitlink.org.cn/cloudream/utils/logger" + "gitlink.org.cn/cloudream/utils/logger" agtcli "gitlink.org.cn/cloudream/rabbitmq/client/agent" agtmsg "gitlink.org.cn/cloudream/rabbitmq/message/agent" agttsk "gitlink.org.cn/cloudream/rabbitmq/message/agent/task" ) -type AgentCheckCacheTaskEntry struct { +type AgentCheckCacheTask struct { NodeID int FileHashes []string // 需要检查的FileHash列表，如果为nil（不是为空），则代表进行全量检查 } -func NewAgentCheckCacheTaskEntry(nodeID int, fileHashes []string) AgentCheckCacheTaskEntry { - return AgentCheckCacheTaskEntry{ +func NewAgentCheckCacheTask(nodeID int, fileHashes []string) *AgentCheckCacheTask { + return &AgentCheckCacheTask{ NodeID: nodeID, FileHashes: fileHashes, } } -type AgentCheckCacheTask struct { - Entries []AgentCheckCacheTaskEntry -} - -func NewAgentCheckCacheTask(entries []AgentCheckCacheTaskEntry) *AgentCheckCacheTask { - return &AgentCheckCacheTask{ - Entries: entries, - } -} - func (t *AgentCheckCacheTask) TryMerge(other Task) bool { - chkTask, ok := other.(*AgentCheckCacheTask) + task, ok := other.(*AgentCheckCacheTask) if !ok { return false } - for _, entry := range chkTask.Entries { - _, index, ok := lo.FindIndexOf(t.Entries, func(e AgentCheckCacheTaskEntry) bool { return e.NodeID == entry.NodeID }) - if ok { - myEntry := &t.Entries[index] - - // FileHashes为nil时代表全量检查 - if entry.FileHashes == nil { - myEntry.FileHashes = nil - } else if myEntry.FileHashes != nil { - myEntry.FileHashes = lo.Union(myEntry.FileHashes, entry.FileHashes) - } - - } else { - t.Entries = append(t.Entries, entry) - } + // FileHashes为nil时代表全量检查 + if task.FileHashes == nil { + t.FileHashes = nil + } else if t.FileHashes != nil { + t.FileHashes = lo.Union(t.FileHashes, task.FileHashes) } return true } func (t *AgentCheckCacheTask) Execute(execCtx *ExecuteContext, execOpts ExecuteOption) { - for _, entry := range t.Entries { - err := t.checkOneAgentCache(entry, execCtx, execOpts) - if err != nil { - log.Warnf("let agent check cache failed, err: %s", err.Error()) - continue - } - } -} - -func (t *AgentCheckCacheTask) checkOneAgentCache(entry AgentCheckCacheTaskEntry, execCtx *ExecuteContext, execOpts ExecuteOption) error { var isComplete bool var caches []model.Cache - err := execCtx.DB.DoTx(sql.LevelSerializable, func(tx *sqlx.Tx) error { - // TODO unavailable的节点需不需要发送任务？ + // TODO unavailable的节点需不需要发送任务？ - if entry.FileHashes == nil { - var err error - caches, err = mysql.Cache.GetNodeCaches(tx, entry.NodeID) - if err != nil { - return fmt.Errorf("get node caches failed, err: %w", err) + if t.FileHashes == nil { + var err error + caches, err = mysql.Cache.GetNodeCaches(execCtx.DB.SQLCtx(), t.NodeID) + if err != nil { + logger.WithField("NodeID", t.NodeID).Warnf("get node caches failed, err: %s", err.Error()) + return + } + isComplete = true + + } else { + for _, hash := range t.FileHashes { + ch, err := mysql.Cache.Get(execCtx.DB.SQLCtx(), hash, t.NodeID) + // 记录不存在则跳过 + if err == sql.ErrNoRows { + continue } - isComplete = true - } else { - for _, hash := range entry.FileHashes { - ch, err := mysql.Cache.Get(tx, hash, entry.NodeID) - // 记录不存在则跳过 - if err == sql.ErrNoRows { - continue - } - - if err != nil { - return fmt.Errorf("get cache failed, err: %w", err) - } - - caches = append(caches, ch) + if err != nil { + logger.WithField("FileHash", hash).WithField("NodeID", t.NodeID).Warnf("get cache failed, err: %w", err) + return } - isComplete = false + + caches = append(caches, ch) } - return nil - }) - if err != nil { - return err + isComplete = false } // 然后向代理端发送移动文件的请求 - agentClient, err := agtcli.NewAgentClient(entry.NodeID, &config.Cfg().RabbitMQ) + agentClient, err := agtcli.NewAgentClient(t.NodeID, &config.Cfg().RabbitMQ) if err != nil { - return fmt.Errorf("create agent client to %d failed, err: %w", entry.NodeID, err) + logger.WithField("NodeID", t.NodeID).Warnf("create agent client failed, err: %s", err.Error()) + return } defer agentClient.Close() @@ -123,8 +88,7 @@ func (t *AgentCheckCacheTask) checkOneAgentCache(entry AgentCheckCacheTaskEntry, execOpts.IsEmergency, // 继承本任务的执行选项 execOpts.DontMerge)) if err != nil { - return fmt.Errorf("request to agent %d failed, err: %w", entry.NodeID, err) + logger.WithField("NodeID", t.NodeID).Warnf("request to agent failed, err: %s", err.Error()) + return } - - return nil } From c1bdf1f3eb0fb6e935a4f1413ddebf794fc33738 Mon Sep 17 00:00:00 2001 From: Sydonian <794346190@qq.com> Date: Fri, 19 May 2023 16:05:08 +0800 Subject: [PATCH 13/68] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E8=A7=A6=E5=8F=91?= =?UTF-8?q?=E8=8A=82=E7=82=B9=E6=A3=80=E6=9F=A5Storage=E7=9A=84=E4=BB=BB?= =?UTF-8?q?=E5=8A=A1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- internal/task/agent_check_storage.go | 108 +++++++++++++++++++++++++++ internal/task/check_rep_count.go | 6 +- 2 files changed, 110 insertions(+), 4 deletions(-) create mode 100644 internal/task/agent_check_storage.go diff --git a/internal/task/agent_check_storage.go b/internal/task/agent_check_storage.go new file mode 100644 index 0000000..d6d0e63 --- /dev/null +++ b/internal/task/agent_check_storage.go @@ -0,0 +1,108 @@ +package task + +import ( + "database/sql" + + "github.com/samber/lo" + "gitlink.org.cn/cloudream/common/consts" + "gitlink.org.cn/cloudream/db/model" + mysql "gitlink.org.cn/cloudream/db/sql" + agtcli "gitlink.org.cn/cloudream/rabbitmq/client/agent" + agtmsg "gitlink.org.cn/cloudream/rabbitmq/message/agent" + agttsk "gitlink.org.cn/cloudream/rabbitmq/message/agent/task" + "gitlink.org.cn/cloudream/scanner/internal/config" + "gitlink.org.cn/cloudream/utils/logger" +) + +type AgentCheckStorageTask struct { + StorageID int + ObjectIDs []int // 需要检查的Object文件列表，如果为nil（不是为空），则代表进行全量检查 +} + +func (t *AgentCheckStorageTask) TryMerge(other Task) bool { + task, ok := other.(*AgentCheckStorageTask) + if !ok { + return false + } + + if t.StorageID != task.StorageID { + return false + } + + // ObjectIDs为nil时代表全量检查 + if task.ObjectIDs == nil { + t.ObjectIDs = nil + } else if t.ObjectIDs != nil { + t.ObjectIDs = lo.Union(t.ObjectIDs, task.ObjectIDs) + } + + return true +} + +func (t *AgentCheckStorageTask) Execute(execCtx *ExecuteContext, execOpts ExecuteOption) { + stg, err := mysql.Storage.GetByID(execCtx.DB.SQLCtx(), t.StorageID) + if err != nil { + if err != sql.ErrNoRows { + logger.WithField("StorageID", t.StorageID).Warnf("get storage failed, err: %s", err.Error()) + } + return + } + + node, err := mysql.Node.GetByID(execCtx.DB.SQLCtx(), stg.NodeID) + if err != nil { + if err != sql.ErrNoRows { + logger.WithField("StorageID", t.StorageID).Warnf("get storage node failed, err: %s", err.Error()) + } + return + } + + // TODO unavailable的节点需不需要发送任务？ + if node.State != consts.NODE_STATE_NORMAL { + return + } + + // 获取对象信息 + var isComplete bool + var objects []model.StorageObject + if t.ObjectIDs == nil { + var err error + objects, err = mysql.StorageObject.GetAllByStorageID(execCtx.DB.SQLCtx(), t.StorageID) + if err != nil { + logger.WithField("StorageID", t.StorageID).Warnf("get storage objects failed, err: %s", err.Error()) + return + } + isComplete = true + } else { + for _, objID := range t.ObjectIDs { + obj, err := mysql.StorageObject.Get(execCtx.DB.SQLCtx(), t.StorageID, objID) + if err == sql.ErrNoRows { + continue + } + if err != nil { + logger.WithField("StorageID", t.StorageID). + WithField("ObjectID", objID). + Warnf("get storage object failed, err: %s", err.Error()) + return + } + + objects = append(objects, obj) + } + isComplete = false + } + + // 投递任务 + agentClient, err := agtcli.NewAgentClient(stg.NodeID, &config.Cfg().RabbitMQ) + if err != nil { + logger.WithField("NodeID", stg.NodeID).Warnf("create agent client failed, err: %s", err.Error()) + return + } + defer agentClient.Close() + + err = agentClient.PostTask(agtmsg.NewPostTaskBody( + agttsk.NewCheckStorageTask(isComplete, objects), + execOpts.IsEmergency, // 继承本任务的执行选项 + execOpts.DontMerge)) + if err != nil { + logger.WithField("NodeID", stg.NodeID).Warnf("request to agent failed, err: %s", stg.NodeID, err.Error()) + } +} diff --git a/internal/task/check_rep_count.go b/internal/task/check_rep_count.go index 99e1814..f4ce7a6 100644 --- a/internal/task/check_rep_count.go +++ b/internal/task/check_rep_count.go @@ -53,12 +53,10 @@ func (t *CheckRepCountTask) Execute(execCtx *ExecuteContext, myOpts ExecuteOptio } } - var agtChkEntries []AgentCheckCacheTaskEntry for nodeID, hashes := range updatedNodeAndHashes { - agtChkEntries = append(agtChkEntries, NewAgentCheckCacheTaskEntry(nodeID, hashes)) + // 新任务继承本任务的执行设定（紧急任务依然保持紧急任务） + execCtx.Executor.Post(NewAgentCheckCacheTask(nodeID, hashes), myOpts) } - // 新任务继承本任务的执行设定（紧急任务依然保持紧急任务） - execCtx.Executor.Post(NewAgentCheckCacheTask(agtChkEntries), myOpts) } func (t *CheckRepCountTask) checkOneRepCount(fileHash string, execCtx *ExecuteContext) ([]int, error) { From 1289ddbc8c6934a347ee1cf6463b1800848ac122 Mon Sep 17 00:00:00 2001 From: Sydonian <794346190@qq.com> Date: Fri, 19 May 2023 16:37:26 +0800 Subject: [PATCH 14/68] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E6=9B=B4=E6=96=B0Stora?= =?UTF-8?q?ge=E7=9A=84=E4=BB=BB=E5=8A=A1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- internal/task/update_storage.go | 76 +++++++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100644 internal/task/update_storage.go diff --git a/internal/task/update_storage.go b/internal/task/update_storage.go new file mode 100644 index 0000000..7f95de9 --- /dev/null +++ b/internal/task/update_storage.go @@ -0,0 +1,76 @@ +package task + +import ( + tskcst "gitlink.org.cn/cloudream/common/consts/task" + mysql "gitlink.org.cn/cloudream/db/sql" + "gitlink.org.cn/cloudream/utils/logger" +) + +type UpdateStorageTaskEntry struct { + ObjectID int + Operation string +} + +func NewUpdateStorageTaskEntry(objectID int, op string) UpdateStorageTaskEntry { + return UpdateStorageTaskEntry{ + ObjectID: objectID, + Operation: op, + } +} + +type UpdateStorageTask struct { + StorageID int + DirectoryState string + Entries []UpdateStorageTaskEntry +} + +func NewUpdateStorageTask(dirState string, entries []UpdateStorageTaskEntry) UpdateStorageTask { + return UpdateStorageTask{ + DirectoryState: dirState, + Entries: entries, + } +} + +func (t *UpdateStorageTask) TryMerge(other Task) bool { + task, ok := other.(*UpdateStorageTask) + if !ok { + return false + } + if task.StorageID != t.StorageID { + return false + } + + // 后投递的任务的状态更新一些 + t.DirectoryState = task.DirectoryState + // TODO 可以考虑合并同FileHash和NodeID的记录 + t.Entries = append(t.Entries, task.Entries...) + return true +} + +func (t *UpdateStorageTask) Execute(execCtx *ExecuteContext, execOpts ExecuteOption) { + + err := mysql.Storage.ChangeState(execCtx.DB.SQLCtx(), t.StorageID, t.DirectoryState) + if err != nil { + logger.WithField("StorageID", t.StorageID).Warnf("change storage state failed, err: %s", err.Error()) + } + + for _, entry := range t.Entries { + switch entry.Operation { + case tskcst.UPDATE_STORAGE_DELETE: + err := mysql.StorageObject.Delete(execCtx.DB.SQLCtx(), t.StorageID, entry.ObjectID) + if err != nil { + logger.WithField("StorageID", t.StorageID). + WithField("ObjectID", entry.ObjectID). + Warnf("delete storage object failed, err: %s", err.Error()) + } + + case tskcst.UPDATE_STORAGE_SET_NORMAL: + err := mysql.StorageObject.SetStateNormal(execCtx.DB.SQLCtx(), t.StorageID, entry.ObjectID) + if err != nil { + logger.WithField("StorageID", t.StorageID). + WithField("ObjectID", entry.ObjectID). + Warnf("change storage object state failed, err: %s", err.Error()) + } + } + } +} From 84ed914043faa37040eba6f4986d4a121fe048e5 Mon Sep 17 00:00:00 2001 From: Sydonian <794346190@qq.com> Date: Fri, 19 May 2023 16:48:01 +0800 Subject: [PATCH 15/68] =?UTF-8?q?=E7=AE=80=E5=8C=96UpdateCache=E4=BB=BB?= =?UTF-8?q?=E5=8A=A1=E9=80=BB=E8=BE=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- internal/task/update_cache.go | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/internal/task/update_cache.go b/internal/task/update_cache.go index e54a0ea..cdb17bc 100644 --- a/internal/task/update_cache.go +++ b/internal/task/update_cache.go @@ -8,36 +8,39 @@ import ( type UpdateCacheTaskEntry struct { FileHash string - NodeID int Operation string } -func NewUpdateCacheTaskEntry(fileHash string, nodeID int, op string) UpdateCacheTaskEntry { +func NewUpdateCacheTaskEntry(fileHash string, op string) UpdateCacheTaskEntry { return UpdateCacheTaskEntry{ FileHash: fileHash, - NodeID: nodeID, Operation: op, } } type UpdateCacheTask struct { + NodeID int Entries []UpdateCacheTaskEntry } -func NewUpdateCacheTask(entries []UpdateCacheTaskEntry) UpdateCacheTask { +func NewUpdateCacheTask(nodeID int, entries []UpdateCacheTaskEntry) UpdateCacheTask { return UpdateCacheTask{ + NodeID: nodeID, Entries: entries, } } func (t *UpdateCacheTask) TryMerge(other Task) bool { - chkTask, ok := other.(*UpdateCacheTask) + task, ok := other.(*UpdateCacheTask) if !ok { return false } + if task.NodeID != t.NodeID { + return false + } // TODO 可以考虑合并同FileHash和NodeID的记录 - t.Entries = append(t.Entries, chkTask.Entries...) + t.Entries = append(t.Entries, task.Entries...) return true } @@ -45,11 +48,11 @@ func (t *UpdateCacheTask) Execute(execCtx *ExecuteContext, execOpts ExecuteOptio for _, entry := range t.Entries { switch entry.Operation { case tskcst.UPDATE_CACHE_OP_UNTEMP: - err := mysql.Cache.DeleteTemp(execCtx.DB.SQLCtx(), entry.FileHash, entry.NodeID) + err := mysql.Cache.DeleteTemp(execCtx.DB.SQLCtx(), entry.FileHash, t.NodeID) if err != nil { logger.WithField("FileHash", entry.FileHash). - WithField("NodeID", entry.NodeID). + WithField("NodeID", t.NodeID). Warnf("delete temp cache failed, err: %s", err.Error()) } } From d6731a499cc051f92a248092fb5c4c6f4292a5e7 Mon Sep 17 00:00:00 2001 From: Sydonian <794346190@qq.com> Date: Fri, 19 May 2023 17:05:49 +0800 Subject: [PATCH 16/68] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E6=A3=80=E6=9F=A5?= =?UTF-8?q?=E4=B8=8D=E5=8F=AF=E7=94=A8Cache=E7=9A=84=E4=BB=BB=E5=8A=A1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- internal/task/agent_check_state.go | 4 +- internal/task/check_object.go | 4 +- internal/task/check_rep_count.go | 4 +- internal/task/check_unavailable_cache.go | 68 ++++++++++++++++++++++++ 4 files changed, 74 insertions(+), 6 deletions(-) create mode 100644 internal/task/check_unavailable_cache.go diff --git a/internal/task/agent_check_state.go b/internal/task/agent_check_state.go index 1addb95..e3f4b12 100644 --- a/internal/task/agent_check_state.go +++ b/internal/task/agent_check_state.go @@ -26,12 +26,12 @@ func NewAgentCheckStateTask(nodeIDs []int) AgentCheckStateTask { } func (t *AgentCheckStateTask) TryMerge(other Task) bool { - chkTask, ok := other.(*AgentCheckStateTask) + task, ok := other.(*AgentCheckStateTask) if !ok { return false } - t.NodeIDs = lo.Union(t.NodeIDs, chkTask.NodeIDs) + t.NodeIDs = lo.Union(t.NodeIDs, task.NodeIDs) return true } diff --git a/internal/task/check_object.go b/internal/task/check_object.go index 3f9cead..141c7dd 100644 --- a/internal/task/check_object.go +++ b/internal/task/check_object.go @@ -17,12 +17,12 @@ func NewCheckObjectTask(objIDs []int) CheckObjectTask { } func (t *CheckObjectTask) TryMerge(other Task) bool { - chkTask, ok := other.(*CheckObjectTask) + task, ok := other.(*CheckObjectTask) if !ok { return false } - t.ObjectIDs = lo.Union(t.ObjectIDs, chkTask.ObjectIDs) + t.ObjectIDs = lo.Union(t.ObjectIDs, task.ObjectIDs) return true } diff --git a/internal/task/check_rep_count.go b/internal/task/check_rep_count.go index f4ce7a6..41ce958 100644 --- a/internal/task/check_rep_count.go +++ b/internal/task/check_rep_count.go @@ -28,12 +28,12 @@ func NewCheckRepCountTask(fileHashes []string) *CheckRepCountTask { } func (t *CheckRepCountTask) TryMerge(other Task) bool { - chkTask, ok := other.(*CheckRepCountTask) + task, ok := other.(*CheckRepCountTask) if !ok { return false } - t.FileHashes = lo.Union(t.FileHashes, chkTask.FileHashes) + t.FileHashes = lo.Union(t.FileHashes, task.FileHashes) return true } diff --git a/internal/task/check_unavailable_cache.go b/internal/task/check_unavailable_cache.go new file mode 100644 index 0000000..d446faf --- /dev/null +++ b/internal/task/check_unavailable_cache.go @@ -0,0 +1,68 @@ +package task + +import ( + "database/sql" + "fmt" + + "github.com/jmoiron/sqlx" + "github.com/samber/lo" + "gitlink.org.cn/cloudream/common/consts" + "gitlink.org.cn/cloudream/db/model" + mysql "gitlink.org.cn/cloudream/db/sql" + "gitlink.org.cn/cloudream/utils/logger" +) + +type CheckUnavailableCacheTask struct { + NodeID int +} + +func NewCheckUnavailableCacheTask(nodeID int) CheckUnavailableCacheTask { + return CheckUnavailableCacheTask{ + NodeID: nodeID, + } +} + +func (t *CheckUnavailableCacheTask) TryMerge(other Task) bool { + task, ok := other.(*CheckUnavailableCacheTask) + if !ok { + return false + } + if task.NodeID != t.NodeID { + return false + } + + return true +} + +func (t *CheckUnavailableCacheTask) Execute(execCtx *ExecuteContext, myOpts ExecuteOption) { + err := execCtx.DB.DoTx(sql.LevelSerializable, func(tx *sqlx.Tx) error { + node, err := mysql.Node.GetByID(tx, t.NodeID) + if err == sql.ErrNoRows { + return nil + } + if err != nil { + return fmt.Errorf("get node failed, err: %w", err) + } + + if node.State != consts.NODE_STATE_UNAVAILABLE { + return nil + } + + caches, err := mysql.Cache.GetNodeCaches(tx, t.NodeID) + if err != nil { + return fmt.Errorf("get node caches failed, err: %w", err) + } + + err = mysql.Cache.DeleteNodeAll(tx, t.NodeID) + if err != nil { + return fmt.Errorf("delete node all caches failed, err: %w", err) + } + + execCtx.Executor.Post(NewCheckRepCountTask(lo.Map(caches, func(ch model.Cache, index int) string { return ch.HashValue }))) + return nil + }) + + if err != nil { + logger.WithField("NodeID", t.NodeID).Warn(err.Error()) + } +} From 4c458aaa403ffd25cbe4fad7d5ac0a69e9882481 Mon Sep 17 00:00:00 2001 From: Sydonian <794346190@qq.com> Date: Mon, 22 May 2023 14:41:12 +0800 Subject: [PATCH 17/68] =?UTF-8?q?=E8=B0=83=E6=95=B4CheckStorage=E4=BB=BB?= =?UTF-8?q?=E5=8A=A1=E7=9A=84=E5=8F=82=E6=95=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- internal/task/agent_check_storage.go | 2 +- internal/task/update_storage.go | 16 ++++++++-------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/internal/task/agent_check_storage.go b/internal/task/agent_check_storage.go index d6d0e63..8675860 100644 --- a/internal/task/agent_check_storage.go +++ b/internal/task/agent_check_storage.go @@ -99,7 +99,7 @@ func (t *AgentCheckStorageTask) Execute(execCtx *ExecuteContext, execOpts Execut defer agentClient.Close() err = agentClient.PostTask(agtmsg.NewPostTaskBody( - agttsk.NewCheckStorageTask(isComplete, objects), + agttsk.NewCheckStorageTask(stg.Directory, isComplete, objects), execOpts.IsEmergency, // 继承本任务的执行选项 execOpts.DontMerge)) if err != nil { diff --git a/internal/task/update_storage.go b/internal/task/update_storage.go index 7f95de9..36a5639 100644 --- a/internal/task/update_storage.go +++ b/internal/task/update_storage.go @@ -19,15 +19,15 @@ func NewUpdateStorageTaskEntry(objectID int, op string) UpdateStorageTaskEntry { } type UpdateStorageTask struct { - StorageID int - DirectoryState string - Entries []UpdateStorageTaskEntry + StorageID int + DirectoryStatus string + Entries []UpdateStorageTaskEntry } -func NewUpdateStorageTask(dirState string, entries []UpdateStorageTaskEntry) UpdateStorageTask { +func NewUpdateStorageTask(dirStatus string, entries []UpdateStorageTaskEntry) UpdateStorageTask { return UpdateStorageTask{ - DirectoryState: dirState, - Entries: entries, + DirectoryStatus: dirStatus, + Entries: entries, } } @@ -41,7 +41,7 @@ func (t *UpdateStorageTask) TryMerge(other Task) bool { } // 后投递的任务的状态更新一些 - t.DirectoryState = task.DirectoryState + t.DirectoryStatus = task.DirectoryStatus // TODO 可以考虑合并同FileHash和NodeID的记录 t.Entries = append(t.Entries, task.Entries...) return true @@ -49,7 +49,7 @@ func (t *UpdateStorageTask) TryMerge(other Task) bool { func (t *UpdateStorageTask) Execute(execCtx *ExecuteContext, execOpts ExecuteOption) { - err := mysql.Storage.ChangeState(execCtx.DB.SQLCtx(), t.StorageID, t.DirectoryState) + err := mysql.Storage.ChangeState(execCtx.DB.SQLCtx(), t.StorageID, t.DirectoryStatus) if err != nil { logger.WithField("StorageID", t.StorageID).Warnf("change storage state failed, err: %s", err.Error()) } From de57db533ad27acb482b4b31b682390be7490570 Mon Sep 17 00:00:00 2001 From: Sydonian <794346190@qq.com> Date: Mon, 22 May 2023 14:54:03 +0800 Subject: [PATCH 18/68] =?UTF-8?q?utils=E7=A7=BB=E5=8A=A8=E5=88=B0common?= =?UTF-8?q?=E5=BA=93=E4=B8=8B=E7=9A=84=E4=B8=80=E4=B8=AA=E5=8C=85=E4=B8=AD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- go.mod | 3 --- internal/config/config.go | 4 ++-- internal/task/agent_check_cache.go | 2 +- internal/task/agent_check_state.go | 2 +- internal/task/agent_check_storage.go | 2 +- internal/task/check_object.go | 2 +- internal/task/check_rep_count.go | 6 +++--- internal/task/check_rep_count_test.go | 2 +- internal/task/check_unavailable_cache.go | 2 +- internal/task/update_agent_state.go | 2 +- internal/task/update_cache.go | 2 +- internal/task/update_storage.go | 2 +- 12 files changed, 14 insertions(+), 17 deletions(-) diff --git a/go.mod b/go.mod index de162ba..cd5fcaa 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,6 @@ require ( gitlink.org.cn/cloudream/common v0.0.0 gitlink.org.cn/cloudream/db v0.0.0 gitlink.org.cn/cloudream/rabbitmq v0.0.0 - gitlink.org.cn/cloudream/utils v0.0.0 golang.org/x/sync v0.1.0 ) @@ -36,8 +35,6 @@ require ( // 运行go mod tidy时需要将下面几行取消注释 // replace gitlink.org.cn/cloudream/rabbitmq => ../rabbitmq // -// replace gitlink.org.cn/cloudream/utils => ../utils -// // replace gitlink.org.cn/cloudream/db => ../db // // replace gitlink.org.cn/cloudream/common => ../common diff --git a/internal/config/config.go b/internal/config/config.go index 4301a0b..f7b5b22 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -1,10 +1,10 @@ package config import ( + c "gitlink.org.cn/cloudream/common/utils/config" + log "gitlink.org.cn/cloudream/common/utils/logger" db "gitlink.org.cn/cloudream/db/config" racfg "gitlink.org.cn/cloudream/rabbitmq/config" - c "gitlink.org.cn/cloudream/utils/config" - log "gitlink.org.cn/cloudream/utils/logger" ) type Config struct { diff --git a/internal/task/agent_check_cache.go b/internal/task/agent_check_cache.go index 7574511..1460841 100644 --- a/internal/task/agent_check_cache.go +++ b/internal/task/agent_check_cache.go @@ -4,10 +4,10 @@ import ( "database/sql" "github.com/samber/lo" + "gitlink.org.cn/cloudream/common/utils/logger" "gitlink.org.cn/cloudream/db/model" mysql "gitlink.org.cn/cloudream/db/sql" "gitlink.org.cn/cloudream/scanner/internal/config" - "gitlink.org.cn/cloudream/utils/logger" agtcli "gitlink.org.cn/cloudream/rabbitmq/client/agent" agtmsg "gitlink.org.cn/cloudream/rabbitmq/message/agent" diff --git a/internal/task/agent_check_state.go b/internal/task/agent_check_state.go index e3f4b12..a9a7dd7 100644 --- a/internal/task/agent_check_state.go +++ b/internal/task/agent_check_state.go @@ -6,13 +6,13 @@ import ( "github.com/samber/lo" "gitlink.org.cn/cloudream/common/consts" + "gitlink.org.cn/cloudream/common/utils/logger" "gitlink.org.cn/cloudream/db/model" mysql "gitlink.org.cn/cloudream/db/sql" agtcli "gitlink.org.cn/cloudream/rabbitmq/client/agent" agtmsg "gitlink.org.cn/cloudream/rabbitmq/message/agent" agttsk "gitlink.org.cn/cloudream/rabbitmq/message/agent/task" "gitlink.org.cn/cloudream/scanner/internal/config" - "gitlink.org.cn/cloudream/utils/logger" ) type AgentCheckStateTask struct { diff --git a/internal/task/agent_check_storage.go b/internal/task/agent_check_storage.go index 8675860..cdd6bb9 100644 --- a/internal/task/agent_check_storage.go +++ b/internal/task/agent_check_storage.go @@ -5,13 +5,13 @@ import ( "github.com/samber/lo" "gitlink.org.cn/cloudream/common/consts" + "gitlink.org.cn/cloudream/common/utils/logger" "gitlink.org.cn/cloudream/db/model" mysql "gitlink.org.cn/cloudream/db/sql" agtcli "gitlink.org.cn/cloudream/rabbitmq/client/agent" agtmsg "gitlink.org.cn/cloudream/rabbitmq/message/agent" agttsk "gitlink.org.cn/cloudream/rabbitmq/message/agent/task" "gitlink.org.cn/cloudream/scanner/internal/config" - "gitlink.org.cn/cloudream/utils/logger" ) type AgentCheckStorageTask struct { diff --git a/internal/task/check_object.go b/internal/task/check_object.go index 141c7dd..80463bf 100644 --- a/internal/task/check_object.go +++ b/internal/task/check_object.go @@ -2,8 +2,8 @@ package task import ( "github.com/samber/lo" + "gitlink.org.cn/cloudream/common/utils/logger" mysql "gitlink.org.cn/cloudream/db/sql" - "gitlink.org.cn/cloudream/utils/logger" ) type CheckObjectTask struct { diff --git a/internal/task/check_rep_count.go b/internal/task/check_rep_count.go index 41ce958..14f35e4 100644 --- a/internal/task/check_rep_count.go +++ b/internal/task/check_rep_count.go @@ -8,10 +8,10 @@ import ( "github.com/jmoiron/sqlx" "github.com/samber/lo" "gitlink.org.cn/cloudream/common/consts" + log "gitlink.org.cn/cloudream/common/utils/logger" + mymath "gitlink.org.cn/cloudream/common/utils/math" + mysort "gitlink.org.cn/cloudream/common/utils/sort" "gitlink.org.cn/cloudream/scanner/internal/config" - log "gitlink.org.cn/cloudream/utils/logger" - mymath "gitlink.org.cn/cloudream/utils/math" - mysort "gitlink.org.cn/cloudream/utils/sort" "gitlink.org.cn/cloudream/db/model" mysql "gitlink.org.cn/cloudream/db/sql" diff --git a/internal/task/check_rep_count_test.go b/internal/task/check_rep_count_test.go index cb81aa7..830b75d 100644 --- a/internal/task/check_rep_count_test.go +++ b/internal/task/check_rep_count_test.go @@ -6,8 +6,8 @@ import ( "github.com/samber/lo" . "github.com/smartystreets/goconvey/convey" "gitlink.org.cn/cloudream/common/consts" + "gitlink.org.cn/cloudream/common/utils/sort" "gitlink.org.cn/cloudream/db/model" - "gitlink.org.cn/cloudream/utils/sort" ) func Test_chooseNewRepNodes(t *testing.T) { diff --git a/internal/task/check_unavailable_cache.go b/internal/task/check_unavailable_cache.go index d446faf..13bc18f 100644 --- a/internal/task/check_unavailable_cache.go +++ b/internal/task/check_unavailable_cache.go @@ -7,9 +7,9 @@ import ( "github.com/jmoiron/sqlx" "github.com/samber/lo" "gitlink.org.cn/cloudream/common/consts" + "gitlink.org.cn/cloudream/common/utils/logger" "gitlink.org.cn/cloudream/db/model" mysql "gitlink.org.cn/cloudream/db/sql" - "gitlink.org.cn/cloudream/utils/logger" ) type CheckUnavailableCacheTask struct { diff --git a/internal/task/update_agent_state.go b/internal/task/update_agent_state.go index 12846fe..4dcb17c 100644 --- a/internal/task/update_agent_state.go +++ b/internal/task/update_agent_state.go @@ -2,8 +2,8 @@ package task import ( "gitlink.org.cn/cloudream/common/consts" + "gitlink.org.cn/cloudream/common/utils/logger" mysql "gitlink.org.cn/cloudream/db/sql" - "gitlink.org.cn/cloudream/utils/logger" ) type UpdateAgentStateTask struct { diff --git a/internal/task/update_cache.go b/internal/task/update_cache.go index cdb17bc..243f1cf 100644 --- a/internal/task/update_cache.go +++ b/internal/task/update_cache.go @@ -2,8 +2,8 @@ package task import ( tskcst "gitlink.org.cn/cloudream/common/consts/task" + "gitlink.org.cn/cloudream/common/utils/logger" mysql "gitlink.org.cn/cloudream/db/sql" - "gitlink.org.cn/cloudream/utils/logger" ) type UpdateCacheTaskEntry struct { diff --git a/internal/task/update_storage.go b/internal/task/update_storage.go index 36a5639..9f5ff5a 100644 --- a/internal/task/update_storage.go +++ b/internal/task/update_storage.go @@ -2,8 +2,8 @@ package task import ( tskcst "gitlink.org.cn/cloudream/common/consts/task" + "gitlink.org.cn/cloudream/common/utils/logger" mysql "gitlink.org.cn/cloudream/db/sql" - "gitlink.org.cn/cloudream/utils/logger" ) type UpdateStorageTaskEntry struct { From c52a1114417bf45f264e67eb50dd672968b8d61f Mon Sep 17 00:00:00 2001 From: Sydonian <794346190@qq.com> Date: Mon, 22 May 2023 16:51:37 +0800 Subject: [PATCH 19/68] =?UTF-8?q?Task=E6=94=B9=E5=90=8D=E6=88=90Event?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- internal/{task => event}/agent_check_cache.go | 32 +++--- internal/{task => event}/agent_check_state.go | 28 ++--- .../{task => event}/agent_check_storage.go | 34 +++--- internal/event/check_object.go | 36 +++++++ internal/{task => event}/check_rep_count.go | 22 ++-- .../{task => event}/check_rep_count_test.go | 2 +- .../check_unavailable_cache.go | 20 ++-- internal/event/task.go | 16 +++ .../{task => event}/update_agent_state.go | 12 +-- internal/event/update_cache.go | 60 +++++++++++ internal/{task => event}/update_storage.go | 36 +++---- internal/task/check_object.go | 36 ------- internal/task/executor.go | 101 ------------------ internal/task/task.go | 6 -- internal/task/update_cache.go | 60 ----------- 15 files changed, 205 insertions(+), 296 deletions(-) rename internal/{task => event}/agent_check_cache.go (67%) rename internal/{task => event}/agent_check_state.go (64%) rename internal/{task => event}/agent_check_storage.go (69%) create mode 100644 internal/event/check_object.go rename internal/{task => event}/check_rep_count.go (90%) rename internal/{task => event}/check_rep_count_test.go (99%) rename internal/{task => event}/check_unavailable_cache.go (62%) create mode 100644 internal/event/task.go rename internal/{task => event}/update_agent_state.go (64%) create mode 100644 internal/event/update_cache.go rename internal/{task => event}/update_storage.go (52%) delete mode 100644 internal/task/check_object.go delete mode 100644 internal/task/executor.go delete mode 100644 internal/task/task.go delete mode 100644 internal/task/update_cache.go diff --git a/internal/task/agent_check_cache.go b/internal/event/agent_check_cache.go similarity index 67% rename from internal/task/agent_check_cache.go rename to internal/event/agent_check_cache.go index 1460841..c895bcc 100644 --- a/internal/task/agent_check_cache.go +++ b/internal/event/agent_check_cache.go @@ -1,4 +1,4 @@ -package task +package event import ( "database/sql" @@ -11,38 +11,38 @@ import ( agtcli "gitlink.org.cn/cloudream/rabbitmq/client/agent" agtmsg "gitlink.org.cn/cloudream/rabbitmq/message/agent" - agttsk "gitlink.org.cn/cloudream/rabbitmq/message/agent/task" + agttsk "gitlink.org.cn/cloudream/rabbitmq/message/agent/event" ) -type AgentCheckCacheTask struct { +type AgentCheckCache struct { NodeID int FileHashes []string // 需要检查的FileHash列表，如果为nil（不是为空），则代表进行全量检查 } -func NewAgentCheckCacheTask(nodeID int, fileHashes []string) *AgentCheckCacheTask { - return &AgentCheckCacheTask{ +func NewAgentCheckCache(nodeID int, fileHashes []string) *AgentCheckCache { + return &AgentCheckCache{ NodeID: nodeID, FileHashes: fileHashes, } } -func (t *AgentCheckCacheTask) TryMerge(other Task) bool { - task, ok := other.(*AgentCheckCacheTask) +func (t *AgentCheckCache) TryMerge(other Event) bool { + event, ok := other.(*AgentCheckCache) if !ok { return false } // FileHashes为nil时代表全量检查 - if task.FileHashes == nil { + if event.FileHashes == nil { t.FileHashes = nil } else if t.FileHashes != nil { - t.FileHashes = lo.Union(t.FileHashes, task.FileHashes) + t.FileHashes = lo.Union(t.FileHashes, event.FileHashes) } return true } -func (t *AgentCheckCacheTask) Execute(execCtx *ExecuteContext, execOpts ExecuteOption) { +func (t *AgentCheckCache) Execute(execCtx ExecuteContext) { var isComplete bool var caches []model.Cache @@ -50,7 +50,7 @@ func (t *AgentCheckCacheTask) Execute(execCtx *ExecuteContext, execOpts ExecuteO if t.FileHashes == nil { var err error - caches, err = mysql.Cache.GetNodeCaches(execCtx.DB.SQLCtx(), t.NodeID) + caches, err = mysql.Cache.GetNodeCaches(execCtx.Args.DB.SQLCtx(), t.NodeID) if err != nil { logger.WithField("NodeID", t.NodeID).Warnf("get node caches failed, err: %s", err.Error()) return @@ -59,7 +59,7 @@ func (t *AgentCheckCacheTask) Execute(execCtx *ExecuteContext, execOpts ExecuteO } else { for _, hash := range t.FileHashes { - ch, err := mysql.Cache.Get(execCtx.DB.SQLCtx(), hash, t.NodeID) + ch, err := mysql.Cache.Get(execCtx.Args.DB.SQLCtx(), hash, t.NodeID) // 记录不存在则跳过 if err == sql.ErrNoRows { continue @@ -83,10 +83,10 @@ func (t *AgentCheckCacheTask) Execute(execCtx *ExecuteContext, execOpts ExecuteO } defer agentClient.Close() - err = agentClient.PostTask(agtmsg.NewPostTaskBody( - agttsk.NewCheckCacheTask(isComplete, caches), - execOpts.IsEmergency, // 继承本任务的执行选项 - execOpts.DontMerge)) + err = agentClient.PostEvent(agtmsg.NewPostEventBody( + agttsk.NewCheckCache(isComplete, caches), + execCtx.Option.IsEmergency, // 继承本任务的执行选项 + execCtx.Option.DontMerge)) if err != nil { logger.WithField("NodeID", t.NodeID).Warnf("request to agent failed, err: %s", err.Error()) return diff --git a/internal/task/agent_check_state.go b/internal/event/agent_check_state.go similarity index 64% rename from internal/task/agent_check_state.go rename to internal/event/agent_check_state.go index a9a7dd7..ec1e947 100644 --- a/internal/task/agent_check_state.go +++ b/internal/event/agent_check_state.go @@ -1,4 +1,4 @@ -package task +package event import ( "database/sql" @@ -11,33 +11,33 @@ import ( mysql "gitlink.org.cn/cloudream/db/sql" agtcli "gitlink.org.cn/cloudream/rabbitmq/client/agent" agtmsg "gitlink.org.cn/cloudream/rabbitmq/message/agent" - agttsk "gitlink.org.cn/cloudream/rabbitmq/message/agent/task" + agttsk "gitlink.org.cn/cloudream/rabbitmq/message/agent/event" "gitlink.org.cn/cloudream/scanner/internal/config" ) -type AgentCheckStateTask struct { +type AgentCheckState struct { NodeIDs []int } -func NewAgentCheckStateTask(nodeIDs []int) AgentCheckStateTask { - return AgentCheckStateTask{ +func NewAgentCheckState(nodeIDs []int) AgentCheckState { + return AgentCheckState{ NodeIDs: nodeIDs, } } -func (t *AgentCheckStateTask) TryMerge(other Task) bool { - task, ok := other.(*AgentCheckStateTask) +func (t *AgentCheckState) TryMerge(other Event) bool { + event, ok := other.(*AgentCheckState) if !ok { return false } - t.NodeIDs = lo.Union(t.NodeIDs, task.NodeIDs) + t.NodeIDs = lo.Union(t.NodeIDs, event.NodeIDs) return true } -func (t *AgentCheckStateTask) Execute(execCtx *ExecuteContext, execOpts ExecuteOption) { +func (t *AgentCheckState) Execute(execCtx ExecuteContext) { for _, nodeID := range t.NodeIDs { - node, err := mysql.Node.GetByID(execCtx.DB.SQLCtx(), nodeID) + node, err := mysql.Node.GetByID(execCtx.Args.DB.SQLCtx(), nodeID) if err == sql.ErrNoRows { continue } @@ -53,20 +53,20 @@ func (t *AgentCheckStateTask) Execute(execCtx *ExecuteContext, execOpts ExecuteO // 检查上次上报时间，超时的设置为不可用 if time.Since(node.LastReportTime) > time.Duration(config.Cfg().NodeUnavailableSeconds)*time.Second { - err := mysql.Node.ChangeState(execCtx.DB.SQLCtx(), nodeID, consts.NODE_STATE_UNAVAILABLE) + err := mysql.Node.ChangeState(execCtx.Args.DB.SQLCtx(), nodeID, consts.NODE_STATE_UNAVAILABLE) if err != nil { logger.WithField("NodeID", nodeID).Warnf("set node state failed, err: %s", err.Error()) continue } - caches, err := mysql.Cache.GetNodeCaches(execCtx.DB.SQLCtx(), nodeID) + caches, err := mysql.Cache.GetNodeCaches(execCtx.Args.DB.SQLCtx(), nodeID) if err != nil { logger.WithField("NodeID", nodeID).Warnf("get node caches failed, err: %s", err.Error()) continue } // 补充备份数 - execCtx.Executor.Post(NewCheckRepCountTask(lo.Map(caches, func(ch model.Cache, index int) string { return ch.HashValue }))) + execCtx.Executor.Post(NewCheckRepCount(lo.Map(caches, func(ch model.Cache, index int) string { return ch.HashValue }))) continue } @@ -78,7 +78,7 @@ func (t *AgentCheckStateTask) Execute(execCtx *ExecuteContext, execOpts ExecuteO defer agentClient.Close() // 紧急任务 - err = agentClient.PostTask(agtmsg.NewPostTaskBody(agttsk.NewCheckStateTask(), true, true)) + err = agentClient.PostEvent(agtmsg.NewPostEventBody(agttsk.NewCheckState(), true, true)) if err != nil { logger.WithField("NodeID", nodeID).Warnf("request to agent failed, err: %s", err.Error()) } diff --git a/internal/task/agent_check_storage.go b/internal/event/agent_check_storage.go similarity index 69% rename from internal/task/agent_check_storage.go rename to internal/event/agent_check_storage.go index cdd6bb9..05a0705 100644 --- a/internal/task/agent_check_storage.go +++ b/internal/event/agent_check_storage.go @@ -1,4 +1,4 @@ -package task +package event import ( "database/sql" @@ -10,37 +10,37 @@ import ( mysql "gitlink.org.cn/cloudream/db/sql" agtcli "gitlink.org.cn/cloudream/rabbitmq/client/agent" agtmsg "gitlink.org.cn/cloudream/rabbitmq/message/agent" - agttsk "gitlink.org.cn/cloudream/rabbitmq/message/agent/task" + agttsk "gitlink.org.cn/cloudream/rabbitmq/message/agent/event" "gitlink.org.cn/cloudream/scanner/internal/config" ) -type AgentCheckStorageTask struct { +type AgentCheckStorage struct { StorageID int ObjectIDs []int // 需要检查的Object文件列表，如果为nil（不是为空），则代表进行全量检查 } -func (t *AgentCheckStorageTask) TryMerge(other Task) bool { - task, ok := other.(*AgentCheckStorageTask) +func (t *AgentCheckStorage) TryMerge(other Event) bool { + event, ok := other.(*AgentCheckStorage) if !ok { return false } - if t.StorageID != task.StorageID { + if t.StorageID != event.StorageID { return false } // ObjectIDs为nil时代表全量检查 - if task.ObjectIDs == nil { + if event.ObjectIDs == nil { t.ObjectIDs = nil } else if t.ObjectIDs != nil { - t.ObjectIDs = lo.Union(t.ObjectIDs, task.ObjectIDs) + t.ObjectIDs = lo.Union(t.ObjectIDs, event.ObjectIDs) } return true } -func (t *AgentCheckStorageTask) Execute(execCtx *ExecuteContext, execOpts ExecuteOption) { - stg, err := mysql.Storage.GetByID(execCtx.DB.SQLCtx(), t.StorageID) +func (t *AgentCheckStorage) Execute(execCtx ExecuteContext) { + stg, err := mysql.Storage.GetByID(execCtx.Args.DB.SQLCtx(), t.StorageID) if err != nil { if err != sql.ErrNoRows { logger.WithField("StorageID", t.StorageID).Warnf("get storage failed, err: %s", err.Error()) @@ -48,7 +48,7 @@ func (t *AgentCheckStorageTask) Execute(execCtx *ExecuteContext, execOpts Execut return } - node, err := mysql.Node.GetByID(execCtx.DB.SQLCtx(), stg.NodeID) + node, err := mysql.Node.GetByID(execCtx.Args.DB.SQLCtx(), stg.NodeID) if err != nil { if err != sql.ErrNoRows { logger.WithField("StorageID", t.StorageID).Warnf("get storage node failed, err: %s", err.Error()) @@ -66,7 +66,7 @@ func (t *AgentCheckStorageTask) Execute(execCtx *ExecuteContext, execOpts Execut var objects []model.StorageObject if t.ObjectIDs == nil { var err error - objects, err = mysql.StorageObject.GetAllByStorageID(execCtx.DB.SQLCtx(), t.StorageID) + objects, err = mysql.StorageObject.GetAllByStorageID(execCtx.Args.DB.SQLCtx(), t.StorageID) if err != nil { logger.WithField("StorageID", t.StorageID).Warnf("get storage objects failed, err: %s", err.Error()) return @@ -74,7 +74,7 @@ func (t *AgentCheckStorageTask) Execute(execCtx *ExecuteContext, execOpts Execut isComplete = true } else { for _, objID := range t.ObjectIDs { - obj, err := mysql.StorageObject.Get(execCtx.DB.SQLCtx(), t.StorageID, objID) + obj, err := mysql.StorageObject.Get(execCtx.Args.DB.SQLCtx(), t.StorageID, objID) if err == sql.ErrNoRows { continue } @@ -98,10 +98,10 @@ func (t *AgentCheckStorageTask) Execute(execCtx *ExecuteContext, execOpts Execut } defer agentClient.Close() - err = agentClient.PostTask(agtmsg.NewPostTaskBody( - agttsk.NewCheckStorageTask(stg.Directory, isComplete, objects), - execOpts.IsEmergency, // 继承本任务的执行选项 - execOpts.DontMerge)) + err = agentClient.PostEvent(agtmsg.NewPostEventBody( + agttsk.NewCheckStorage(stg.Directory, isComplete, objects), + execCtx.Option.IsEmergency, // 继承本任务的执行选项 + execCtx.Option.DontMerge)) if err != nil { logger.WithField("NodeID", stg.NodeID).Warnf("request to agent failed, err: %s", stg.NodeID, err.Error()) } diff --git a/internal/event/check_object.go b/internal/event/check_object.go new file mode 100644 index 0000000..bb986c9 --- /dev/null +++ b/internal/event/check_object.go @@ -0,0 +1,36 @@ +package event + +import ( + "github.com/samber/lo" + "gitlink.org.cn/cloudream/common/utils/logger" + mysql "gitlink.org.cn/cloudream/db/sql" +) + +type CheckObject struct { + ObjectIDs []int +} + +func NewCheckObject(objIDs []int) CheckObject { + return CheckObject{ + ObjectIDs: objIDs, + } +} + +func (t *CheckObject) TryMerge(other Event) bool { + event, ok := other.(*CheckObject) + if !ok { + return false + } + + t.ObjectIDs = lo.Union(t.ObjectIDs, event.ObjectIDs) + return true +} + +func (t *CheckObject) Execute(execCtx ExecuteContext) { + for _, objID := range t.ObjectIDs { + err := mysql.Object.DeleteUnused(execCtx.Args.DB.SQLCtx(), objID) + if err != nil { + logger.WithField("ObjectID", objID).Warnf("delete unused object failed, err: %s", err.Error()) + } + } +} diff --git a/internal/task/check_rep_count.go b/internal/event/check_rep_count.go similarity index 90% rename from internal/task/check_rep_count.go rename to internal/event/check_rep_count.go index 14f35e4..69bcdf4 100644 --- a/internal/task/check_rep_count.go +++ b/internal/event/check_rep_count.go @@ -1,4 +1,4 @@ -package task +package event import ( "database/sql" @@ -17,27 +17,27 @@ import ( mysql "gitlink.org.cn/cloudream/db/sql" ) -type CheckRepCountTask struct { +type CheckRepCount struct { FileHashes []string } -func NewCheckRepCountTask(fileHashes []string) *CheckRepCountTask { - return &CheckRepCountTask{ +func NewCheckRepCount(fileHashes []string) *CheckRepCount { + return &CheckRepCount{ FileHashes: fileHashes, } } -func (t *CheckRepCountTask) TryMerge(other Task) bool { - task, ok := other.(*CheckRepCountTask) +func (t *CheckRepCount) TryMerge(other Event) bool { + event, ok := other.(*CheckRepCount) if !ok { return false } - t.FileHashes = lo.Union(t.FileHashes, task.FileHashes) + t.FileHashes = lo.Union(t.FileHashes, event.FileHashes) return true } -func (t *CheckRepCountTask) Execute(execCtx *ExecuteContext, myOpts ExecuteOption) { +func (t *CheckRepCount) Execute(execCtx ExecuteContext) { updatedNodeAndHashes := make(map[int][]string) for _, fileHash := range t.FileHashes { @@ -55,13 +55,13 @@ func (t *CheckRepCountTask) Execute(execCtx *ExecuteContext, myOpts ExecuteOptio for nodeID, hashes := range updatedNodeAndHashes { // 新任务继承本任务的执行设定（紧急任务依然保持紧急任务） - execCtx.Executor.Post(NewAgentCheckCacheTask(nodeID, hashes), myOpts) + execCtx.Executor.Post(NewAgentCheckCache(nodeID, hashes), execCtx.Option) } } -func (t *CheckRepCountTask) checkOneRepCount(fileHash string, execCtx *ExecuteContext) ([]int, error) { +func (t *CheckRepCount) checkOneRepCount(fileHash string, execCtx ExecuteContext) ([]int, error) { var updatedNodeIDs []int - err := execCtx.DB.DoTx(sql.LevelSerializable, func(tx *sqlx.Tx) error { + err := execCtx.Args.DB.DoTx(sql.LevelSerializable, func(tx *sqlx.Tx) error { repMaxCnt, err := mysql.ObjectRep.GetFileMaxRepCount(tx, fileHash) if err != nil { return fmt.Errorf("get file max rep count failed, err: %w", err) diff --git a/internal/task/check_rep_count_test.go b/internal/event/check_rep_count_test.go similarity index 99% rename from internal/task/check_rep_count_test.go rename to internal/event/check_rep_count_test.go index 830b75d..3c0beab 100644 --- a/internal/task/check_rep_count_test.go +++ b/internal/event/check_rep_count_test.go @@ -1,4 +1,4 @@ -package task +package event import ( "testing" diff --git a/internal/task/check_unavailable_cache.go b/internal/event/check_unavailable_cache.go similarity index 62% rename from internal/task/check_unavailable_cache.go rename to internal/event/check_unavailable_cache.go index 13bc18f..189b4f4 100644 --- a/internal/task/check_unavailable_cache.go +++ b/internal/event/check_unavailable_cache.go @@ -1,4 +1,4 @@ -package task +package event import ( "database/sql" @@ -12,30 +12,30 @@ import ( mysql "gitlink.org.cn/cloudream/db/sql" ) -type CheckUnavailableCacheTask struct { +type CheckUnavailableCache struct { NodeID int } -func NewCheckUnavailableCacheTask(nodeID int) CheckUnavailableCacheTask { - return CheckUnavailableCacheTask{ +func NewCheckUnavailableCache(nodeID int) CheckUnavailableCache { + return CheckUnavailableCache{ NodeID: nodeID, } } -func (t *CheckUnavailableCacheTask) TryMerge(other Task) bool { - task, ok := other.(*CheckUnavailableCacheTask) +func (t *CheckUnavailableCache) TryMerge(other Event) bool { + event, ok := other.(*CheckUnavailableCache) if !ok { return false } - if task.NodeID != t.NodeID { + if event.NodeID != t.NodeID { return false } return true } -func (t *CheckUnavailableCacheTask) Execute(execCtx *ExecuteContext, myOpts ExecuteOption) { - err := execCtx.DB.DoTx(sql.LevelSerializable, func(tx *sqlx.Tx) error { +func (t *CheckUnavailableCache) Execute(execCtx ExecuteContext) { + err := execCtx.Args.DB.DoTx(sql.LevelSerializable, func(tx *sqlx.Tx) error { node, err := mysql.Node.GetByID(tx, t.NodeID) if err == sql.ErrNoRows { return nil @@ -58,7 +58,7 @@ func (t *CheckUnavailableCacheTask) Execute(execCtx *ExecuteContext, myOpts Exec return fmt.Errorf("delete node all caches failed, err: %w", err) } - execCtx.Executor.Post(NewCheckRepCountTask(lo.Map(caches, func(ch model.Cache, index int) string { return ch.HashValue }))) + execCtx.Executor.Post(NewCheckRepCount(lo.Map(caches, func(ch model.Cache, index int) string { return ch.HashValue }))) return nil }) diff --git a/internal/event/task.go b/internal/event/task.go new file mode 100644 index 0000000..a1bd1f5 --- /dev/null +++ b/internal/event/task.go @@ -0,0 +1,16 @@ +package event + +import ( + event "gitlink.org.cn/cloudream/common/pkg/event" + mydb "gitlink.org.cn/cloudream/db" +) + +type ExecuteArgs struct { + DB *mydb.DB +} + +type Executor = event.Executor[ExecuteArgs] + +type ExecuteContext = event.ExecuteContext[ExecuteArgs] + +type Event = event.Event[ExecuteArgs] diff --git a/internal/task/update_agent_state.go b/internal/event/update_agent_state.go similarity index 64% rename from internal/task/update_agent_state.go rename to internal/event/update_agent_state.go index 4dcb17c..24d6f46 100644 --- a/internal/task/update_agent_state.go +++ b/internal/event/update_agent_state.go @@ -1,4 +1,4 @@ -package task +package event import ( "gitlink.org.cn/cloudream/common/consts" @@ -6,20 +6,20 @@ import ( mysql "gitlink.org.cn/cloudream/db/sql" ) -type UpdateAgentStateTask struct { +type UpdateAgentState struct { NodeID int IPFSStatus string } -func (t *UpdateAgentStateTask) TryMerge(other Task) bool { +func (t *UpdateAgentState) TryMerge(other Event) bool { return false } -func (t *UpdateAgentStateTask) Execute(execCtx *ExecuteContext, execOpts ExecuteOption) { +func (t *UpdateAgentState) Execute(execCtx ExecuteContext) { if t.IPFSStatus != consts.IPFS_STATUS_OK { logger.WithField("NodeID", t.NodeID).Warnf("IPFS status is %s, set node state unavailable", t.IPFSStatus) - err := mysql.Node.ChangeState(execCtx.DB.SQLCtx(), t.NodeID, consts.NODE_STATE_UNAVAILABLE) + err := mysql.Node.ChangeState(execCtx.Args.DB.SQLCtx(), t.NodeID, consts.NODE_STATE_UNAVAILABLE) if err != nil { logger.WithField("NodeID", t.NodeID).Warnf("change node state failed, err: %s", err.Error()) } @@ -27,7 +27,7 @@ func (t *UpdateAgentStateTask) Execute(execCtx *ExecuteContext, execOpts Execute } // TODO 如果以后还有其他的状态，要判断哪些状态下能设置Normal - err := mysql.Node.ChangeState(execCtx.DB.SQLCtx(), t.NodeID, consts.NODE_STATE_NORMAL) + err := mysql.Node.ChangeState(execCtx.Args.DB.SQLCtx(), t.NodeID, consts.NODE_STATE_NORMAL) if err != nil { logger.WithField("NodeID", t.NodeID).Warnf("change node state failed, err: %s", err.Error()) } diff --git a/internal/event/update_cache.go b/internal/event/update_cache.go new file mode 100644 index 0000000..b3ac62f --- /dev/null +++ b/internal/event/update_cache.go @@ -0,0 +1,60 @@ +package event + +import ( + tskcst "gitlink.org.cn/cloudream/common/consts/event" + "gitlink.org.cn/cloudream/common/utils/logger" + mysql "gitlink.org.cn/cloudream/db/sql" +) + +type UpdateCacheEntry struct { + FileHash string + Operation string +} + +func NewUpdateCacheEntry(fileHash string, op string) UpdateCacheEntry { + return UpdateCacheEntry{ + FileHash: fileHash, + Operation: op, + } +} + +type UpdateCache struct { + NodeID int + Entries []UpdateCacheEntry +} + +func NewUpdateCache(nodeID int, entries []UpdateCacheEntry) UpdateCache { + return UpdateCache{ + NodeID: nodeID, + Entries: entries, + } +} + +func (t *UpdateCache) TryMerge(other Event) bool { + event, ok := other.(*UpdateCache) + if !ok { + return false + } + if event.NodeID != t.NodeID { + return false + } + + // TODO 可以考虑合并同FileHash和NodeID的记录 + t.Entries = append(t.Entries, event.Entries...) + return true +} + +func (t *UpdateCache) Execute(execCtx ExecuteContext) { + for _, entry := range t.Entries { + switch entry.Operation { + case tskcst.UPDATE_CACHE_OP_UNTEMP: + err := mysql.Cache.DeleteTemp(execCtx.Args.DB.SQLCtx(), entry.FileHash, t.NodeID) + + if err != nil { + logger.WithField("FileHash", entry.FileHash). + WithField("NodeID", t.NodeID). + Warnf("delete temp cache failed, err: %s", err.Error()) + } + } + } +} diff --git a/internal/task/update_storage.go b/internal/event/update_storage.go similarity index 52% rename from internal/task/update_storage.go rename to internal/event/update_storage.go index 9f5ff5a..848285e 100644 --- a/internal/task/update_storage.go +++ b/internal/event/update_storage.go @@ -1,55 +1,55 @@ -package task +package event import ( - tskcst "gitlink.org.cn/cloudream/common/consts/task" + tskcst "gitlink.org.cn/cloudream/common/consts/event" "gitlink.org.cn/cloudream/common/utils/logger" mysql "gitlink.org.cn/cloudream/db/sql" ) -type UpdateStorageTaskEntry struct { +type UpdateStorageEntry struct { ObjectID int Operation string } -func NewUpdateStorageTaskEntry(objectID int, op string) UpdateStorageTaskEntry { - return UpdateStorageTaskEntry{ +func NewUpdateStorageEntry(objectID int, op string) UpdateStorageEntry { + return UpdateStorageEntry{ ObjectID: objectID, Operation: op, } } -type UpdateStorageTask struct { +type UpdateStorage struct { StorageID int DirectoryStatus string - Entries []UpdateStorageTaskEntry + Entries []UpdateStorageEntry } -func NewUpdateStorageTask(dirStatus string, entries []UpdateStorageTaskEntry) UpdateStorageTask { - return UpdateStorageTask{ +func NewUpdateStorage(dirStatus string, entries []UpdateStorageEntry) UpdateStorage { + return UpdateStorage{ DirectoryStatus: dirStatus, Entries: entries, } } -func (t *UpdateStorageTask) TryMerge(other Task) bool { - task, ok := other.(*UpdateStorageTask) +func (t *UpdateStorage) TryMerge(other Event) bool { + event, ok := other.(*UpdateStorage) if !ok { return false } - if task.StorageID != t.StorageID { + if event.StorageID != t.StorageID { return false } // 后投递的任务的状态更新一些 - t.DirectoryStatus = task.DirectoryStatus + t.DirectoryStatus = event.DirectoryStatus // TODO 可以考虑合并同FileHash和NodeID的记录 - t.Entries = append(t.Entries, task.Entries...) + t.Entries = append(t.Entries, event.Entries...) return true } -func (t *UpdateStorageTask) Execute(execCtx *ExecuteContext, execOpts ExecuteOption) { +func (t *UpdateStorage) Execute(execCtx ExecuteContext) { - err := mysql.Storage.ChangeState(execCtx.DB.SQLCtx(), t.StorageID, t.DirectoryStatus) + err := mysql.Storage.ChangeState(execCtx.Args.DB.SQLCtx(), t.StorageID, t.DirectoryStatus) if err != nil { logger.WithField("StorageID", t.StorageID).Warnf("change storage state failed, err: %s", err.Error()) } @@ -57,7 +57,7 @@ func (t *UpdateStorageTask) Execute(execCtx *ExecuteContext, execOpts ExecuteOpt for _, entry := range t.Entries { switch entry.Operation { case tskcst.UPDATE_STORAGE_DELETE: - err := mysql.StorageObject.Delete(execCtx.DB.SQLCtx(), t.StorageID, entry.ObjectID) + err := mysql.StorageObject.Delete(execCtx.Args.DB.SQLCtx(), t.StorageID, entry.ObjectID) if err != nil { logger.WithField("StorageID", t.StorageID). WithField("ObjectID", entry.ObjectID). @@ -65,7 +65,7 @@ func (t *UpdateStorageTask) Execute(execCtx *ExecuteContext, execOpts ExecuteOpt } case tskcst.UPDATE_STORAGE_SET_NORMAL: - err := mysql.StorageObject.SetStateNormal(execCtx.DB.SQLCtx(), t.StorageID, entry.ObjectID) + err := mysql.StorageObject.SetStateNormal(execCtx.Args.DB.SQLCtx(), t.StorageID, entry.ObjectID) if err != nil { logger.WithField("StorageID", t.StorageID). WithField("ObjectID", entry.ObjectID). diff --git a/internal/task/check_object.go b/internal/task/check_object.go deleted file mode 100644 index 80463bf..0000000 --- a/internal/task/check_object.go +++ /dev/null @@ -1,36 +0,0 @@ -package task - -import ( - "github.com/samber/lo" - "gitlink.org.cn/cloudream/common/utils/logger" - mysql "gitlink.org.cn/cloudream/db/sql" -) - -type CheckObjectTask struct { - ObjectIDs []int -} - -func NewCheckObjectTask(objIDs []int) CheckObjectTask { - return CheckObjectTask{ - ObjectIDs: objIDs, - } -} - -func (t *CheckObjectTask) TryMerge(other Task) bool { - task, ok := other.(*CheckObjectTask) - if !ok { - return false - } - - t.ObjectIDs = lo.Union(t.ObjectIDs, task.ObjectIDs) - return true -} - -func (t *CheckObjectTask) Execute(execCtx *ExecuteContext, execOpts ExecuteOption) { - for _, objID := range t.ObjectIDs { - err := mysql.Object.DeleteUnused(execCtx.DB.SQLCtx(), objID) - if err != nil { - logger.WithField("ObjectID", objID).Warnf("delete unused object failed, err: %s", err.Error()) - } - } -} diff --git a/internal/task/executor.go b/internal/task/executor.go deleted file mode 100644 index 35f867f..0000000 --- a/internal/task/executor.go +++ /dev/null @@ -1,101 +0,0 @@ -package task - -import ( - "context" - "sync" - - "github.com/zyedidia/generic/list" - mydb "gitlink.org.cn/cloudream/db" - "golang.org/x/sync/semaphore" -) - -type ExecuteOption struct { - IsEmergency bool - DontMerge bool -} -type ExecuteContext struct { - Executor *Executor - DB *mydb.DB -} -type postedTask struct { - Task Task - Option ExecuteOption -} - -type Executor struct { - tasks list.List[postedTask] - locker sync.Mutex - taskSema semaphore.Weighted - execCtx *ExecuteContext -} - -func (e *Executor) Post(task Task, opts ...ExecuteOption) { - opt := ExecuteOption{ - IsEmergency: false, - DontMerge: false, - } - - if len(opts) > 0 { - opt = opts[0] - } - - e.locker.Lock() - defer e.locker.Unlock() - - // 紧急任务直接插入到队头，不进行合并 - if opt.IsEmergency { - e.tasks.PushFront(postedTask{ - Task: task, - Option: opt, - }) - e.taskSema.Release(1) - return - } - - // 合并任务 - if opt.DontMerge { - ptr := e.tasks.Front - for ptr != nil { - // 只与非紧急任务，且允许合并的任务进行合并 - if !ptr.Value.Option.IsEmergency && !ptr.Value.Option.DontMerge { - if ptr.Value.Task.TryMerge(task) { - return - } - } - - ptr = ptr.Next - } - } - - e.tasks.PushBack(postedTask{ - Task: task, - Option: opt, - }) - e.taskSema.Release(1) -} - -// Execute 开始执行任务 -func (e *Executor) Execute() error { - for { - // TODO 打印错误日志 - e.taskSema.Acquire(context.Background(), 1) - - task := e.popFrontTask() - if task == nil { - continue - } - - task.Task.Execute(e.execCtx, task.Option) - } -} - -func (e *Executor) popFrontTask() *postedTask { - e.locker.Lock() - defer e.locker.Unlock() - - if e.tasks.Front == nil { - return nil - } - - return &e.tasks.Front.Value -} diff --git a/internal/task/task.go b/internal/task/task.go deleted file mode 100644 index 40547df..0000000 --- a/internal/task/task.go +++ /dev/null @@ -1,6 +0,0 @@ -package task - -type Task interface { - TryMerge(other Task) bool // 尝试将other任务与自身合并，如果成功返回true - Execute(ctx *ExecuteContext, myOpts ExecuteOption) -} diff --git a/internal/task/update_cache.go b/internal/task/update_cache.go deleted file mode 100644 index 243f1cf..0000000 --- a/internal/task/update_cache.go +++ /dev/null @@ -1,60 +0,0 @@ -package task - -import ( - tskcst "gitlink.org.cn/cloudream/common/consts/task" - "gitlink.org.cn/cloudream/common/utils/logger" - mysql "gitlink.org.cn/cloudream/db/sql" -) - -type UpdateCacheTaskEntry struct { - FileHash string - Operation string -} - -func NewUpdateCacheTaskEntry(fileHash string, op string) UpdateCacheTaskEntry { - return UpdateCacheTaskEntry{ - FileHash: fileHash, - Operation: op, - } -} - -type UpdateCacheTask struct { - NodeID int - Entries []UpdateCacheTaskEntry -} - -func NewUpdateCacheTask(nodeID int, entries []UpdateCacheTaskEntry) UpdateCacheTask { - return UpdateCacheTask{ - NodeID: nodeID, - Entries: entries, - } -} - -func (t *UpdateCacheTask) TryMerge(other Task) bool { - task, ok := other.(*UpdateCacheTask) - if !ok { - return false - } - if task.NodeID != t.NodeID { - return false - } - - // TODO 可以考虑合并同FileHash和NodeID的记录 - t.Entries = append(t.Entries, task.Entries...) - return true -} - -func (t *UpdateCacheTask) Execute(execCtx *ExecuteContext, execOpts ExecuteOption) { - for _, entry := range t.Entries { - switch entry.Operation { - case tskcst.UPDATE_CACHE_OP_UNTEMP: - err := mysql.Cache.DeleteTemp(execCtx.DB.SQLCtx(), entry.FileHash, t.NodeID) - - if err != nil { - logger.WithField("FileHash", entry.FileHash). - WithField("NodeID", t.NodeID). - Warnf("delete temp cache failed, err: %s", err.Error()) - } - } - } -} From 89d47f88ca892dede4b8c3b3ef1a3277e5f1026f Mon Sep 17 00:00:00 2001 From: Sydonian <794346190@qq.com> Date: Tue, 23 May 2023 10:28:22 +0800 Subject: [PATCH 20/68] =?UTF-8?q?scanner=E5=92=8Cagent=E5=90=AF=E5=8A=A8Ev?= =?UTF-8?q?entExecutor?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Makefile | 15 +++++ internal/event/agent_check_cache.go | 4 +- internal/event/agent_check_state.go | 4 +- internal/event/agent_check_storage.go | 4 +- internal/event/event.go | 43 +++++++++++++ internal/event/task.go | 16 ----- internal/services/event.go | 28 +++++++++ internal/services/service.go | 15 +++++ main.go | 89 +++++++++++++++++++++------ 9 files changed, 176 insertions(+), 42 deletions(-) create mode 100644 Makefile create mode 100644 internal/event/event.go delete mode 100644 internal/event/task.go create mode 100644 internal/services/event.go create mode 100644 internal/services/service.go diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..a8bb2e8 --- /dev/null +++ b/Makefile @@ -0,0 +1,15 @@ +OUTPUT_BINARY_NAME = "scanner" +OUTPUT_DIR_NAME = "scanner" + + +ASSETS_DIR_NAME = "assets" +BUILD_DIR = "../../build" + +build: + go build -o ${BUILD_DIR}/${OUTPUT_DIR_NAME}/${OUTPUT_BINARY_NAME} + @if [ -d ${ASSETS_DIR_NAME} ] && [ -n "`ls -A ${ASSETS_DIR_NAME}`" ] ;then \ + cp -r ${ASSETS_DIR_NAME}/* ${BUILD_DIR}/${OUTPUT_DIR_NAME}/; \ + fi + +clean: + rm -f ${BUILD_DIR}/${OUTPUT_DIR_NAME}/${OUTPUT_BINARY_NAME} \ No newline at end of file diff --git a/internal/event/agent_check_cache.go b/internal/event/agent_check_cache.go index c895bcc..942ec5a 100644 --- a/internal/event/agent_check_cache.go +++ b/internal/event/agent_check_cache.go @@ -11,7 +11,7 @@ import ( agtcli "gitlink.org.cn/cloudream/rabbitmq/client/agent" agtmsg "gitlink.org.cn/cloudream/rabbitmq/message/agent" - agttsk "gitlink.org.cn/cloudream/rabbitmq/message/agent/event" + agtevt "gitlink.org.cn/cloudream/rabbitmq/message/agent/event" ) type AgentCheckCache struct { @@ -84,7 +84,7 @@ func (t *AgentCheckCache) Execute(execCtx ExecuteContext) { defer agentClient.Close() err = agentClient.PostEvent(agtmsg.NewPostEventBody( - agttsk.NewCheckCache(isComplete, caches), + agtevt.NewCheckCache(isComplete, caches), execCtx.Option.IsEmergency, // 继承本任务的执行选项 execCtx.Option.DontMerge)) if err != nil { diff --git a/internal/event/agent_check_state.go b/internal/event/agent_check_state.go index ec1e947..6df6bcc 100644 --- a/internal/event/agent_check_state.go +++ b/internal/event/agent_check_state.go @@ -11,7 +11,7 @@ import ( mysql "gitlink.org.cn/cloudream/db/sql" agtcli "gitlink.org.cn/cloudream/rabbitmq/client/agent" agtmsg "gitlink.org.cn/cloudream/rabbitmq/message/agent" - agttsk "gitlink.org.cn/cloudream/rabbitmq/message/agent/event" + agtevt "gitlink.org.cn/cloudream/rabbitmq/message/agent/event" "gitlink.org.cn/cloudream/scanner/internal/config" ) @@ -78,7 +78,7 @@ func (t *AgentCheckState) Execute(execCtx ExecuteContext) { defer agentClient.Close() // 紧急任务 - err = agentClient.PostEvent(agtmsg.NewPostEventBody(agttsk.NewCheckState(), true, true)) + err = agentClient.PostEvent(agtmsg.NewPostEventBody(agtevt.NewCheckState(), true, true)) if err != nil { logger.WithField("NodeID", nodeID).Warnf("request to agent failed, err: %s", err.Error()) } diff --git a/internal/event/agent_check_storage.go b/internal/event/agent_check_storage.go index 05a0705..05dbba1 100644 --- a/internal/event/agent_check_storage.go +++ b/internal/event/agent_check_storage.go @@ -10,7 +10,7 @@ import ( mysql "gitlink.org.cn/cloudream/db/sql" agtcli "gitlink.org.cn/cloudream/rabbitmq/client/agent" agtmsg "gitlink.org.cn/cloudream/rabbitmq/message/agent" - agttsk "gitlink.org.cn/cloudream/rabbitmq/message/agent/event" + agtevt "gitlink.org.cn/cloudream/rabbitmq/message/agent/event" "gitlink.org.cn/cloudream/scanner/internal/config" ) @@ -99,7 +99,7 @@ func (t *AgentCheckStorage) Execute(execCtx ExecuteContext) { defer agentClient.Close() err = agentClient.PostEvent(agtmsg.NewPostEventBody( - agttsk.NewCheckStorage(stg.Directory, isComplete, objects), + agtevt.NewCheckStorage(stg.Directory, isComplete, objects), execCtx.Option.IsEmergency, // 继承本任务的执行选项 execCtx.Option.DontMerge)) if err != nil { diff --git a/internal/event/event.go b/internal/event/event.go new file mode 100644 index 0000000..f64ed98 --- /dev/null +++ b/internal/event/event.go @@ -0,0 +1,43 @@ +package event + +import ( + "fmt" + "reflect" + + event "gitlink.org.cn/cloudream/common/pkg/event" + "gitlink.org.cn/cloudream/common/pkg/typedispatcher" + mydb "gitlink.org.cn/cloudream/db" +) + +type ExecuteArgs struct { + DB *mydb.DB +} + +type Executor = event.Executor[ExecuteArgs] + +type ExecuteContext = event.ExecuteContext[ExecuteArgs] + +type Event = event.Event[ExecuteArgs] + +type ExecuteOption = event.ExecuteOption + +func NewExecutor(db *mydb.DB) Executor { + return event.NewExecutor(ExecuteArgs{ + DB: db, + }) +} + +var msgDispatcher typedispatcher.TypeDispatcher[Event] + +func FromMessage(msg any) (Event, error) { + event, ok := msgDispatcher.Dispatch(msg) + if !ok { + return nil, fmt.Errorf("unknow event message type: %s", reflect.TypeOf(msg).Name()) + } + + return event, nil +} + +func Register[T any](converter func(msg T) Event) { + typedispatcher.Add(msgDispatcher, converter) +} diff --git a/internal/event/task.go b/internal/event/task.go deleted file mode 100644 index a1bd1f5..0000000 --- a/internal/event/task.go +++ /dev/null @@ -1,16 +0,0 @@ -package event - -import ( - event "gitlink.org.cn/cloudream/common/pkg/event" - mydb "gitlink.org.cn/cloudream/db" -) - -type ExecuteArgs struct { - DB *mydb.DB -} - -type Executor = event.Executor[ExecuteArgs] - -type ExecuteContext = event.ExecuteContext[ExecuteArgs] - -type Event = event.Event[ExecuteArgs] diff --git a/internal/services/event.go b/internal/services/event.go new file mode 100644 index 0000000..3d57a42 --- /dev/null +++ b/internal/services/event.go @@ -0,0 +1,28 @@ +package services + +import ( + "gitlink.org.cn/cloudream/common/utils/logger" + scmsg "gitlink.org.cn/cloudream/rabbitmq/message/scanner" + scevt "gitlink.org.cn/cloudream/rabbitmq/message/scanner/event" + "gitlink.org.cn/cloudream/scanner/internal/event" +) + +func (svc *Service) PostEvent(msg *scmsg.PostEvent) { + + evtMsg, err := scevt.MapToMessage(msg.Body.Event.(map[string]any)) + if err != nil { + logger.Warnf("convert map to event message failed, err: %s", err.Error()) + return + } + + evt, err := event.FromMessage(evtMsg) + if err != nil { + logger.Warnf("create event from event message failed, err: %s", err.Error()) + return + } + + svc.eventExecutor.Post(evt, event.ExecuteOption{ + IsEmergency: msg.Body.IsEmergency, + DontMerge: msg.Body.DontMerge, + }) +} diff --git a/internal/services/service.go b/internal/services/service.go new file mode 100644 index 0000000..9c5af8a --- /dev/null +++ b/internal/services/service.go @@ -0,0 +1,15 @@ +package services + +import ( + "gitlink.org.cn/cloudream/scanner/internal/event" +) + +type Service struct { + eventExecutor *event.Executor +} + +func NewService(eventExecutor *event.Executor) *Service { + return &Service{ + eventExecutor: eventExecutor, + } +} diff --git a/main.go b/main.go index b9b5171..5bd1d0f 100644 --- a/main.go +++ b/main.go @@ -1,29 +1,78 @@ package main import ( + "fmt" + "os" + "sync" - //"path/filepath" - //"sync" - - "time" - //agentcaller "proto" - //"github.com/pborman/uuid" - //"github.com/streadway/amqp" - //"google.golang.org/grpc" + log "gitlink.org.cn/cloudream/common/utils/logger" + "gitlink.org.cn/cloudream/db" + scsvr "gitlink.org.cn/cloudream/rabbitmq/server/scanner" + "gitlink.org.cn/cloudream/scanner/internal/config" + "gitlink.org.cn/cloudream/scanner/internal/event" + "gitlink.org.cn/cloudream/scanner/internal/services" ) func main() { - for { - //jh:遍历对象副本表， - //-对于每一个rephash, - //--根据objectId查询对象表中的RepNum - //--查询缓存表中rephash对应的TempOrPin为false的nodeIp - //--如果查到的NodeIp数少于RepNum，需发起复制命令 - //jh:遍历对象编码块表， - //-对于每一个blockhash,获得其blockId、objectId、innerID - //--判断blockhash是否在ipfs网络中 - //--得到待修复object清单：记录下各个objectId对应的不在ipfs网络中的blockId、blockhash、innerID - //-查询待修复object清单中各个object的FileSize、EcName等，并发出修复命令 - time.Sleep(time.Minute * 5) + err := config.Init() + if err != nil { + fmt.Printf("init config failed, err: %s", err.Error()) + os.Exit(1) + } + + err = log.Init(&config.Cfg().Logger) + if err != nil { + fmt.Printf("init logger failed, err: %s", err.Error()) + os.Exit(1) } + + db, err := db.NewDB(&config.Cfg().DB) + if err != nil { + log.Fatalf("new db failed, err: %s", err.Error()) + } + + wg := sync.WaitGroup{} + wg.Add(2) + + eventExecutor := event.NewExecutor(db) + go serveEventExecutor(&eventExecutor, &wg) + + agtSvr, err := scsvr.NewScannerServer(services.NewService(&eventExecutor), &config.Cfg().RabbitMQ) + if err != nil { + log.Fatalf("new agent server failed, err: %s", err.Error()) + } + agtSvr.OnError = func(err error) { + log.Warnf("agent server err: %s", err.Error()) + } + go serveScannerServer(agtSvr, &wg) + + wg.Wait() +} + +func serveEventExecutor(executor *event.Executor, wg *sync.WaitGroup) { + log.Info("start serving event executor") + + err := executor.Execute() + + if err != nil { + log.Errorf("event executor stopped with error: %s", err.Error()) + } + + log.Info("event executor stopped") + + wg.Done() +} + +func serveScannerServer(server *scsvr.ScannerServer, wg *sync.WaitGroup) { + log.Info("start serving scanner server") + + err := server.Serve() + + if err != nil { + log.Errorf("scanner server stopped with error: %s", err.Error()) + } + + log.Info("scanner server stopped") + + wg.Done() } From a99c5c78f2fae7db943a4e346d830de20072b73a Mon Sep 17 00:00:00 2001 From: Sydonian <794346190@qq.com> Date: Tue, 23 May 2023 15:47:30 +0800 Subject: [PATCH 21/68] =?UTF-8?q?=E4=BC=98=E5=8C=96=E6=A3=80=E6=9F=A5?= =?UTF-8?q?=E8=8A=82=E7=82=B9=E7=8A=B6=E6=80=81=E7=9A=84=E4=BA=8B=E4=BB=B6?= =?UTF-8?q?=E9=80=BB=E8=BE=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- internal/event/agent_check_state.go | 85 +++++++++++------------ internal/event/check_unavailable_cache.go | 4 +- 2 files changed, 43 insertions(+), 46 deletions(-) diff --git a/internal/event/agent_check_state.go b/internal/event/agent_check_state.go index 6df6bcc..b37c92b 100644 --- a/internal/event/agent_check_state.go +++ b/internal/event/agent_check_state.go @@ -16,12 +16,12 @@ import ( ) type AgentCheckState struct { - NodeIDs []int + NodeID int } -func NewAgentCheckState(nodeIDs []int) AgentCheckState { - return AgentCheckState{ - NodeIDs: nodeIDs, +func NewAgentCheckState(nodeID int) *AgentCheckState { + return &AgentCheckState{ + NodeID: nodeID, } } @@ -31,56 +31,53 @@ func (t *AgentCheckState) TryMerge(other Event) bool { return false } - t.NodeIDs = lo.Union(t.NodeIDs, event.NodeIDs) - return true + return t.NodeID == event.NodeID } func (t *AgentCheckState) Execute(execCtx ExecuteContext) { - for _, nodeID := range t.NodeIDs { - node, err := mysql.Node.GetByID(execCtx.Args.DB.SQLCtx(), nodeID) - if err == sql.ErrNoRows { - continue - } - - if err != nil { - logger.WithField("NodeID", nodeID).Warnf("get node by id failed, err: %s", err.Error()) - continue - } - - if node.State != consts.NODE_STATE_NORMAL { - continue - } - - // 检查上次上报时间，超时的设置为不可用 - if time.Since(node.LastReportTime) > time.Duration(config.Cfg().NodeUnavailableSeconds)*time.Second { - err := mysql.Node.ChangeState(execCtx.Args.DB.SQLCtx(), nodeID, consts.NODE_STATE_UNAVAILABLE) - if err != nil { - logger.WithField("NodeID", nodeID).Warnf("set node state failed, err: %s", err.Error()) - continue - } + node, err := mysql.Node.GetByID(execCtx.Args.DB.SQLCtx(), t.NodeID) + if err == sql.ErrNoRows { + return + } - caches, err := mysql.Cache.GetNodeCaches(execCtx.Args.DB.SQLCtx(), nodeID) - if err != nil { - logger.WithField("NodeID", nodeID).Warnf("get node caches failed, err: %s", err.Error()) - continue - } + if err != nil { + logger.WithField("NodeID", t.NodeID).Warnf("get node by id failed, err: %s", err.Error()) + return + } - // 补充备份数 - execCtx.Executor.Post(NewCheckRepCount(lo.Map(caches, func(ch model.Cache, index int) string { return ch.HashValue }))) - continue - } + if node.State != consts.NODE_STATE_NORMAL { + return + } - agentClient, err := agtcli.NewAgentClient(nodeID, &config.Cfg().RabbitMQ) + // 检查上次上报时间，超时的设置为不可用 + if time.Since(node.LastReportTime) > time.Duration(config.Cfg().NodeUnavailableSeconds)*time.Second { + err := mysql.Node.ChangeState(execCtx.Args.DB.SQLCtx(), t.NodeID, consts.NODE_STATE_UNAVAILABLE) if err != nil { - logger.WithField("NodeID", nodeID).Warnf("create agent client failed, err: %s", err.Error()) - continue + logger.WithField("NodeID", t.NodeID).Warnf("set node state failed, err: %s", err.Error()) + return } - defer agentClient.Close() - // 紧急任务 - err = agentClient.PostEvent(agtmsg.NewPostEventBody(agtevt.NewCheckState(), true, true)) + caches, err := mysql.Cache.GetNodeCaches(execCtx.Args.DB.SQLCtx(), t.NodeID) if err != nil { - logger.WithField("NodeID", nodeID).Warnf("request to agent failed, err: %s", err.Error()) + logger.WithField("NodeID", t.NodeID).Warnf("get node caches failed, err: %s", err.Error()) + return } + + // 补充备份数 + execCtx.Executor.Post(NewCheckRepCount(lo.Map(caches, func(ch model.Cache, index int) string { return ch.HashValue }))) + return + } + + agentClient, err := agtcli.NewAgentClient(t.NodeID, &config.Cfg().RabbitMQ) + if err != nil { + logger.WithField("NodeID", t.NodeID).Warnf("create agent client failed, err: %s", err.Error()) + return + } + defer agentClient.Close() + + // 紧急任务 + err = agentClient.PostEvent(agtmsg.NewPostEventBody(agtevt.NewCheckState(), true, true)) + if err != nil { + logger.WithField("NodeID", t.NodeID).Warnf("request to agent failed, err: %s", err.Error()) } } diff --git a/internal/event/check_unavailable_cache.go b/internal/event/check_unavailable_cache.go index 189b4f4..b503412 100644 --- a/internal/event/check_unavailable_cache.go +++ b/internal/event/check_unavailable_cache.go @@ -16,8 +16,8 @@ type CheckUnavailableCache struct { NodeID int } -func NewCheckUnavailableCache(nodeID int) CheckUnavailableCache { - return CheckUnavailableCache{ +func NewCheckUnavailableCache(nodeID int) *CheckUnavailableCache { + return &CheckUnavailableCache{ NodeID: nodeID, } } From b28033953a27973337cdce0492843cd50f9e96dc Mon Sep 17 00:00:00 2001 From: Sydonian <794346190@qq.com> Date: Tue, 23 May 2023 15:47:53 +0800 Subject: [PATCH 22/68] =?UTF-8?q?=E5=A2=9E=E5=8A=A0scanner=E7=9A=84?= =?UTF-8?q?=E5=AE=9A=E6=97=B6=E4=BA=8B=E4=BB=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../tickevent/batch_check_all_rep_count.go | 32 +++++++++++++++++++ internal/tickevent/check_agent_state.go | 25 +++++++++++++++ internal/tickevent/check_unavailable_cache.go | 22 +++++++++++++ internal/tickevent/tick_event.go | 18 +++++++++++ 4 files changed, 97 insertions(+) create mode 100644 internal/tickevent/batch_check_all_rep_count.go create mode 100644 internal/tickevent/check_agent_state.go create mode 100644 internal/tickevent/check_unavailable_cache.go create mode 100644 internal/tickevent/tick_event.go diff --git a/internal/tickevent/batch_check_all_rep_count.go b/internal/tickevent/batch_check_all_rep_count.go new file mode 100644 index 0000000..8370f0c --- /dev/null +++ b/internal/tickevent/batch_check_all_rep_count.go @@ -0,0 +1,32 @@ +package tickevent + +import ( + "gitlink.org.cn/cloudream/common/utils/logger" + mysql "gitlink.org.cn/cloudream/db/sql" + "gitlink.org.cn/cloudream/scanner/internal/event" +) + +const CHECK_CACHE_BATCH_SIZE = 100 + +type BatchCheckAllRepCount struct { + lastCheckStart int +} + +func (e *BatchCheckAllRepCount) Execute(ctx ExecuteContext) { + fileHashes, err := mysql.Cache.BatchGetAllFileHashes(ctx.Args.DB.SQLCtx(), e.lastCheckStart, CHECK_CACHE_BATCH_SIZE) + if err != nil { + logger.Warnf("batch get file hashes failed, err: %s", err.Error()) + return + } + + ctx.Args.EventExecutor.Post(event.NewCheckRepCount(fileHashes)) + + // 如果结果的长度小于预期的长度，则认为已经查询了所有，下次从头再来 + if len(fileHashes) < CHECK_CACHE_BATCH_SIZE { + e.lastCheckStart = 0 + logger.Debugf("all rep count checked, next time will start check at 0") + + } else { + e.lastCheckStart += CHECK_CACHE_BATCH_SIZE + } +} diff --git a/internal/tickevent/check_agent_state.go b/internal/tickevent/check_agent_state.go new file mode 100644 index 0000000..e93a3a6 --- /dev/null +++ b/internal/tickevent/check_agent_state.go @@ -0,0 +1,25 @@ +package tickevent + +import ( + "gitlink.org.cn/cloudream/common/utils/logger" + mysql "gitlink.org.cn/cloudream/db/sql" + "gitlink.org.cn/cloudream/scanner/internal/event" +) + +type CheckAgentState struct { +} + +func (e *CheckAgentState) Execute(ctx ExecuteContext) { + nodes, err := mysql.Node.GetAllNodes(ctx.Args.DB.SQLCtx()) + if err != nil { + logger.Warnf("get all nodes failed, err: %s", err.Error()) + return + } + + for _, node := range nodes { + ctx.Args.EventExecutor.Post(event.NewAgentCheckState(node.NodeID), event.ExecuteOption{ + IsEmergency: true, + DontMerge: true, + }) + } +} diff --git a/internal/tickevent/check_unavailable_cache.go b/internal/tickevent/check_unavailable_cache.go new file mode 100644 index 0000000..2bf75d4 --- /dev/null +++ b/internal/tickevent/check_unavailable_cache.go @@ -0,0 +1,22 @@ +package tickevent + +import ( + "gitlink.org.cn/cloudream/common/utils/logger" + mysql "gitlink.org.cn/cloudream/db/sql" + "gitlink.org.cn/cloudream/scanner/internal/event" +) + +type CheckUnavailableCache struct { +} + +func (e *CheckUnavailableCache) Execute(ctx ExecuteContext) { + nodes, err := mysql.Node.GetAllNodes(ctx.Args.DB.SQLCtx()) + if err != nil { + logger.Warnf("get all nodes failed, err: %s", err.Error()) + return + } + + for _, node := range nodes { + ctx.Args.EventExecutor.Post(event.NewCheckUnavailableCache(node.NodeID)) + } +} diff --git a/internal/tickevent/tick_event.go b/internal/tickevent/tick_event.go new file mode 100644 index 0000000..d981402 --- /dev/null +++ b/internal/tickevent/tick_event.go @@ -0,0 +1,18 @@ +package tickevent + +import ( + tickevent "gitlink.org.cn/cloudream/common/pkg/tickevent" + mydb "gitlink.org.cn/cloudream/db" + "gitlink.org.cn/cloudream/scanner/internal/event" +) + +type ExecuteArgs struct { + EventExecutor *event.Executor + DB *mydb.DB +} + +type Executor = tickevent.Executor[ExecuteArgs] + +type ExecuteContext = tickevent.ExecuteContext[ExecuteArgs] + +type Event = tickevent.TickEvent[ExecuteArgs] From caf8c152121b14140f9e750f3d94c2e38e7e4c08 Mon Sep 17 00:00:00 2001 From: Sydonian <794346190@qq.com> Date: Tue, 23 May 2023 16:53:59 +0800 Subject: [PATCH 23/68] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E5=88=86=E6=89=B9?= =?UTF-8?q?=E6=A3=80=E6=9F=A5Object=E7=8A=B6=E6=80=81=E7=9A=84=E4=BB=BB?= =?UTF-8?q?=E5=8A=A1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- internal/event/check_object.go | 4 +-- internal/tickevent/batch_check_all_object.go | 32 ++++++++++++++++++++ 2 files changed, 34 insertions(+), 2 deletions(-) create mode 100644 internal/tickevent/batch_check_all_object.go diff --git a/internal/event/check_object.go b/internal/event/check_object.go index bb986c9..1d72c29 100644 --- a/internal/event/check_object.go +++ b/internal/event/check_object.go @@ -10,8 +10,8 @@ type CheckObject struct { ObjectIDs []int } -func NewCheckObject(objIDs []int) CheckObject { - return CheckObject{ +func NewCheckObject(objIDs []int) *CheckObject { + return &CheckObject{ ObjectIDs: objIDs, } } diff --git a/internal/tickevent/batch_check_all_object.go b/internal/tickevent/batch_check_all_object.go new file mode 100644 index 0000000..9578196 --- /dev/null +++ b/internal/tickevent/batch_check_all_object.go @@ -0,0 +1,32 @@ +package tickevent + +import ( + "gitlink.org.cn/cloudream/common/utils/logger" + mysql "gitlink.org.cn/cloudream/db/sql" + "gitlink.org.cn/cloudream/scanner/internal/event" +) + +const CHECK_OBJECT_BATCH_SIZE = 100 + +type BatchCheckObject struct { + lastCheckStart int +} + +func (e *BatchCheckObject) Execute(ctx ExecuteContext) { + objectIDs, err := mysql.Object.BatchGetAllObjectIDs(ctx.Args.DB.SQLCtx(), e.lastCheckStart, CHECK_OBJECT_BATCH_SIZE) + if err != nil { + logger.Warnf("batch get object ids failed, err: %s", err.Error()) + return + } + + ctx.Args.EventExecutor.Post(event.NewCheckObject(objectIDs)) + + // 如果结果的长度小于预期的长度，则认为已经查询了所有，下次从头再来 + if len(objectIDs) < CHECK_OBJECT_BATCH_SIZE { + e.lastCheckStart = 0 + logger.Debugf("all object checked, next time will start check at 0") + + } else { + e.lastCheckStart += CHECK_OBJECT_BATCH_SIZE + } +} From 6d07b5a6efcd25932af18a135d929baff18c3eef Mon Sep 17 00:00:00 2001 From: Sydonian <794346190@qq.com> Date: Tue, 23 May 2023 17:03:13 +0800 Subject: [PATCH 24/68] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E5=88=86=E6=89=B9?= =?UTF-8?q?=E6=A3=80=E6=9F=A5Storage=E7=9A=84=E5=AE=9A=E6=97=B6=E4=BB=BB?= =?UTF-8?q?=E5=8A=A1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- internal/event/agent_check_storage.go | 7 ++++ internal/tickevent/batch_check_all_storage.go | 35 +++++++++++++++++++ 2 files changed, 42 insertions(+) create mode 100644 internal/tickevent/batch_check_all_storage.go diff --git a/internal/event/agent_check_storage.go b/internal/event/agent_check_storage.go index 05dbba1..345ef8b 100644 --- a/internal/event/agent_check_storage.go +++ b/internal/event/agent_check_storage.go @@ -19,6 +19,13 @@ type AgentCheckStorage struct { ObjectIDs []int // 需要检查的Object文件列表，如果为nil（不是为空），则代表进行全量检查 } +func NewAgentCheckStorage(storageID int, objectIDs []int) *AgentCheckStorage { + return &AgentCheckStorage{ + StorageID: storageID, + ObjectIDs: objectIDs, + } +} + func (t *AgentCheckStorage) TryMerge(other Event) bool { event, ok := other.(*AgentCheckStorage) if !ok { diff --git a/internal/tickevent/batch_check_all_storage.go b/internal/tickevent/batch_check_all_storage.go new file mode 100644 index 0000000..075d057 --- /dev/null +++ b/internal/tickevent/batch_check_all_storage.go @@ -0,0 +1,35 @@ +package tickevent + +import ( + "gitlink.org.cn/cloudream/common/utils/logger" + mysql "gitlink.org.cn/cloudream/db/sql" + "gitlink.org.cn/cloudream/scanner/internal/event" +) + +const CHECK_STORAGE_BATCH_SIZE = 5 + +type BatchCheckStorage struct { + lastCheckStart int +} + +func (e *BatchCheckStorage) Execute(ctx ExecuteContext) { + storageIDs, err := mysql.Storage.BatchGetAllStorageIDs(ctx.Args.DB.SQLCtx(), e.lastCheckStart, CHECK_STORAGE_BATCH_SIZE) + if err != nil { + logger.Warnf("batch get storage ids failed, err: %s", err.Error()) + return + } + + for _, stgID := range storageIDs { + // 设置nil代表进行全量检查 + ctx.Args.EventExecutor.Post(event.NewAgentCheckStorage(stgID, nil)) + } + + // 如果结果的长度小于预期的长度，则认为已经查询了所有，下次从头再来 + if len(storageIDs) < CHECK_STORAGE_BATCH_SIZE { + e.lastCheckStart = 0 + logger.Debugf("all storage checked, next time will start check at 0") + + } else { + e.lastCheckStart += CHECK_STORAGE_BATCH_SIZE + } +} From 8aa8f68a6e3506eadd0fcf35457abc9ee5359273 Mon Sep 17 00:00:00 2001 From: Sydonian <794346190@qq.com> Date: Wed, 24 May 2023 09:21:32 +0800 Subject: [PATCH 25/68] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E5=88=86=E6=89=B9?= =?UTF-8?q?=E6=A3=80=E6=9F=A5Agent=E7=BC=93=E5=AD=98=E7=9A=84=E5=AE=9A?= =?UTF-8?q?=E6=97=B6=E4=BB=BB=E5=8A=A1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../tickevent/batch_all_agent_check_cache.go | 35 +++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 internal/tickevent/batch_all_agent_check_cache.go diff --git a/internal/tickevent/batch_all_agent_check_cache.go b/internal/tickevent/batch_all_agent_check_cache.go new file mode 100644 index 0000000..88bad18 --- /dev/null +++ b/internal/tickevent/batch_all_agent_check_cache.go @@ -0,0 +1,35 @@ +package tickevent + +import ( + "github.com/samber/lo" + "gitlink.org.cn/cloudream/common/utils/logger" + "gitlink.org.cn/cloudream/db/model" + mysql "gitlink.org.cn/cloudream/db/sql" + "gitlink.org.cn/cloudream/scanner/internal/event" +) + +const AGENT_CHECK_CACHE_BATCH_SIZE = 2 + +type BatchAgentCheckCache struct { + nodeIDs []int +} + +func (e *BatchAgentCheckCache) Execute(ctx ExecuteContext) { + if e.nodeIDs == nil || len(e.nodeIDs) == 0 { + nodes, err := mysql.Node.GetAllNodes(ctx.Args.DB.SQLCtx()) + if err != nil { + logger.Warnf("get all nodes failed, err: %s", err.Error()) + return + } + + e.nodeIDs = lo.Map(nodes, func(node model.Node, index int) int { return node.NodeID }) + + logger.Debugf("new check start, get all nodes") + } + + for i := 0; i < len(e.nodeIDs) && i < AGENT_CHECK_CACHE_BATCH_SIZE; i++ { + // nil代表进行全量检查 + ctx.Args.EventExecutor.Post(event.NewAgentCheckCache(e.nodeIDs[i], nil)) + } + e.nodeIDs = e.nodeIDs[AGENT_CHECK_CACHE_BATCH_SIZE:] +} From b84ed51ce54fd09bfb15e11d484f1ea212080a01 Mon Sep 17 00:00:00 2001 From: Sydonian <794346190@qq.com> Date: Wed, 24 May 2023 10:28:56 +0800 Subject: [PATCH 26/68] =?UTF-8?q?=E5=AE=8C=E6=88=90=E8=8A=82=E7=82=B9?= =?UTF-8?q?=E6=A3=80=E6=9F=A5Cache=E7=9A=84=E4=BB=BB=E5=8A=A1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- internal/event/update_cache.go | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/internal/event/update_cache.go b/internal/event/update_cache.go index b3ac62f..8d9598d 100644 --- a/internal/event/update_cache.go +++ b/internal/event/update_cache.go @@ -1,7 +1,7 @@ package event import ( - tskcst "gitlink.org.cn/cloudream/common/consts/event" + evtcst "gitlink.org.cn/cloudream/common/consts/event" "gitlink.org.cn/cloudream/common/utils/logger" mysql "gitlink.org.cn/cloudream/db/sql" ) @@ -47,7 +47,7 @@ func (t *UpdateCache) TryMerge(other Event) bool { func (t *UpdateCache) Execute(execCtx ExecuteContext) { for _, entry := range t.Entries { switch entry.Operation { - case tskcst.UPDATE_CACHE_OP_UNTEMP: + case evtcst.UPDATE_CACHE_UNTEMP: err := mysql.Cache.DeleteTemp(execCtx.Args.DB.SQLCtx(), entry.FileHash, t.NodeID) if err != nil { @@ -55,6 +55,14 @@ func (t *UpdateCache) Execute(execCtx ExecuteContext) { WithField("NodeID", t.NodeID). Warnf("delete temp cache failed, err: %s", err.Error()) } + + case evtcst.UPDATE_CACHE_CREATE_TEMP: + err := mysql.Cache.CreateTemp(execCtx.Args.DB.SQLCtx(), entry.FileHash, t.NodeID) + if err != nil { + logger.WithField("FileHash", entry.FileHash). + WithField("NodeID", t.NodeID). + Warnf("create temp cache failed, err: %s", err.Error()) + } } } } From 49fdd20df51855c95abe84abf5e26af424e8be19 Mon Sep 17 00:00:00 2001 From: Sydonian <794346190@qq.com> Date: Wed, 24 May 2023 10:56:50 +0800 Subject: [PATCH 27/68] =?UTF-8?q?StorageObject=E8=A1=A8=E5=A2=9E=E5=8A=A0U?= =?UTF-8?q?serID=EF=BC=9B=E5=AE=8C=E6=88=90=E8=8A=82=E7=82=B9=E6=A3=80?= =?UTF-8?q?=E6=9F=A5Storage=E7=9A=84=E4=BB=BB=E5=8A=A1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- internal/event/agent_check_storage.go | 7 ++----- internal/event/update_storage.go | 8 +++++--- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/internal/event/agent_check_storage.go b/internal/event/agent_check_storage.go index 345ef8b..6b2808a 100644 --- a/internal/event/agent_check_storage.go +++ b/internal/event/agent_check_storage.go @@ -81,10 +81,7 @@ func (t *AgentCheckStorage) Execute(execCtx ExecuteContext) { isComplete = true } else { for _, objID := range t.ObjectIDs { - obj, err := mysql.StorageObject.Get(execCtx.Args.DB.SQLCtx(), t.StorageID, objID) - if err == sql.ErrNoRows { - continue - } + objs, err := mysql.StorageObject.GetAllByStorageAndObjectID(execCtx.Args.DB.SQLCtx(), t.StorageID, objID) if err != nil { logger.WithField("StorageID", t.StorageID). WithField("ObjectID", objID). @@ -92,7 +89,7 @@ func (t *AgentCheckStorage) Execute(execCtx ExecuteContext) { return } - objects = append(objects, obj) + objects = append(objects, objs...) } isComplete = false } diff --git a/internal/event/update_storage.go b/internal/event/update_storage.go index 848285e..a133364 100644 --- a/internal/event/update_storage.go +++ b/internal/event/update_storage.go @@ -8,12 +8,14 @@ import ( type UpdateStorageEntry struct { ObjectID int + UserID int Operation string } -func NewUpdateStorageEntry(objectID int, op string) UpdateStorageEntry { +func NewUpdateStorageEntry(objectID int, userID int, op string) UpdateStorageEntry { return UpdateStorageEntry{ ObjectID: objectID, + UserID: userID, Operation: op, } } @@ -57,7 +59,7 @@ func (t *UpdateStorage) Execute(execCtx ExecuteContext) { for _, entry := range t.Entries { switch entry.Operation { case tskcst.UPDATE_STORAGE_DELETE: - err := mysql.StorageObject.Delete(execCtx.Args.DB.SQLCtx(), t.StorageID, entry.ObjectID) + err := mysql.StorageObject.Delete(execCtx.Args.DB.SQLCtx(), t.StorageID, entry.ObjectID, entry.UserID) if err != nil { logger.WithField("StorageID", t.StorageID). WithField("ObjectID", entry.ObjectID). @@ -65,7 +67,7 @@ func (t *UpdateStorage) Execute(execCtx ExecuteContext) { } case tskcst.UPDATE_STORAGE_SET_NORMAL: - err := mysql.StorageObject.SetStateNormal(execCtx.Args.DB.SQLCtx(), t.StorageID, entry.ObjectID) + err := mysql.StorageObject.SetStateNormal(execCtx.Args.DB.SQLCtx(), t.StorageID, entry.ObjectID, entry.UserID) if err != nil { logger.WithField("StorageID", t.StorageID). WithField("ObjectID", entry.ObjectID). From a1ef2fcf974ba9c6ea3af5ee8a5b6c97a42c2bbf Mon Sep 17 00:00:00 2001 From: Sydonian <794346190@qq.com> Date: Wed, 24 May 2023 15:14:18 +0800 Subject: [PATCH 28/68] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E5=90=AF=E5=8A=A8?= =?UTF-8?q?=E5=AE=9A=E6=97=B6=E4=BB=BB=E5=8A=A1=E7=9A=84=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../tickevent/batch_all_agent_check_cache.go | 8 +++++-- internal/tickevent/batch_check_all_object.go | 8 +++++-- .../tickevent/batch_check_all_rep_count.go | 4 ++++ internal/tickevent/batch_check_all_storage.go | 8 +++++-- internal/tickevent/check_agent_state.go | 4 ++++ internal/tickevent/check_unavailable_cache.go | 4 ++++ internal/tickevent/tick_event.go | 4 ++++ main.go | 22 +++++++++++++++++++ 8 files changed, 56 insertions(+), 6 deletions(-) diff --git a/internal/tickevent/batch_all_agent_check_cache.go b/internal/tickevent/batch_all_agent_check_cache.go index 88bad18..61090b0 100644 --- a/internal/tickevent/batch_all_agent_check_cache.go +++ b/internal/tickevent/batch_all_agent_check_cache.go @@ -10,11 +10,15 @@ import ( const AGENT_CHECK_CACHE_BATCH_SIZE = 2 -type BatchAgentCheckCache struct { +type BatchAllAgentCheckCache struct { nodeIDs []int } -func (e *BatchAgentCheckCache) Execute(ctx ExecuteContext) { +func NewBatchAllAgentCheckCache() *BatchAllAgentCheckCache { + return &BatchAllAgentCheckCache{} +} + +func (e *BatchAllAgentCheckCache) Execute(ctx ExecuteContext) { if e.nodeIDs == nil || len(e.nodeIDs) == 0 { nodes, err := mysql.Node.GetAllNodes(ctx.Args.DB.SQLCtx()) if err != nil { diff --git a/internal/tickevent/batch_check_all_object.go b/internal/tickevent/batch_check_all_object.go index 9578196..b970fa0 100644 --- a/internal/tickevent/batch_check_all_object.go +++ b/internal/tickevent/batch_check_all_object.go @@ -8,11 +8,15 @@ import ( const CHECK_OBJECT_BATCH_SIZE = 100 -type BatchCheckObject struct { +type BatchCheckAllObject struct { lastCheckStart int } -func (e *BatchCheckObject) Execute(ctx ExecuteContext) { +func NewBatchCheckAllObject() *BatchCheckAllObject { + return &BatchCheckAllObject{} +} + +func (e *BatchCheckAllObject) Execute(ctx ExecuteContext) { objectIDs, err := mysql.Object.BatchGetAllObjectIDs(ctx.Args.DB.SQLCtx(), e.lastCheckStart, CHECK_OBJECT_BATCH_SIZE) if err != nil { logger.Warnf("batch get object ids failed, err: %s", err.Error()) diff --git a/internal/tickevent/batch_check_all_rep_count.go b/internal/tickevent/batch_check_all_rep_count.go index 8370f0c..a0d5626 100644 --- a/internal/tickevent/batch_check_all_rep_count.go +++ b/internal/tickevent/batch_check_all_rep_count.go @@ -12,6 +12,10 @@ type BatchCheckAllRepCount struct { lastCheckStart int } +func NewBatchCheckAllRepCount() *BatchCheckAllRepCount { + return &BatchCheckAllRepCount{} +} + func (e *BatchCheckAllRepCount) Execute(ctx ExecuteContext) { fileHashes, err := mysql.Cache.BatchGetAllFileHashes(ctx.Args.DB.SQLCtx(), e.lastCheckStart, CHECK_CACHE_BATCH_SIZE) if err != nil { diff --git a/internal/tickevent/batch_check_all_storage.go b/internal/tickevent/batch_check_all_storage.go index 075d057..5550671 100644 --- a/internal/tickevent/batch_check_all_storage.go +++ b/internal/tickevent/batch_check_all_storage.go @@ -8,11 +8,15 @@ import ( const CHECK_STORAGE_BATCH_SIZE = 5 -type BatchCheckStorage struct { +type BatchCheckAllStorage struct { lastCheckStart int } -func (e *BatchCheckStorage) Execute(ctx ExecuteContext) { +func NewBatchCheckAllStorage() *BatchCheckAllStorage { + return &BatchCheckAllStorage{} +} + +func (e *BatchCheckAllStorage) Execute(ctx ExecuteContext) { storageIDs, err := mysql.Storage.BatchGetAllStorageIDs(ctx.Args.DB.SQLCtx(), e.lastCheckStart, CHECK_STORAGE_BATCH_SIZE) if err != nil { logger.Warnf("batch get storage ids failed, err: %s", err.Error()) diff --git a/internal/tickevent/check_agent_state.go b/internal/tickevent/check_agent_state.go index e93a3a6..07ed28d 100644 --- a/internal/tickevent/check_agent_state.go +++ b/internal/tickevent/check_agent_state.go @@ -9,6 +9,10 @@ import ( type CheckAgentState struct { } +func NewCheckAgentState() *CheckAgentState { + return &CheckAgentState{} +} + func (e *CheckAgentState) Execute(ctx ExecuteContext) { nodes, err := mysql.Node.GetAllNodes(ctx.Args.DB.SQLCtx()) if err != nil { diff --git a/internal/tickevent/check_unavailable_cache.go b/internal/tickevent/check_unavailable_cache.go index 2bf75d4..57158eb 100644 --- a/internal/tickevent/check_unavailable_cache.go +++ b/internal/tickevent/check_unavailable_cache.go @@ -9,6 +9,10 @@ import ( type CheckUnavailableCache struct { } +func NewCheckUnavailableCache() *CheckUnavailableCache { + return &CheckUnavailableCache{} +} + func (e *CheckUnavailableCache) Execute(ctx ExecuteContext) { nodes, err := mysql.Node.GetAllNodes(ctx.Args.DB.SQLCtx()) if err != nil { diff --git a/internal/tickevent/tick_event.go b/internal/tickevent/tick_event.go index d981402..da51b6e 100644 --- a/internal/tickevent/tick_event.go +++ b/internal/tickevent/tick_event.go @@ -16,3 +16,7 @@ type Executor = tickevent.Executor[ExecuteArgs] type ExecuteContext = tickevent.ExecuteContext[ExecuteArgs] type Event = tickevent.TickEvent[ExecuteArgs] + +func NewExecutor(args ExecuteArgs) Executor { + return tickevent.NewExecutor(args) +} diff --git a/main.go b/main.go index 5bd1d0f..fbefede 100644 --- a/main.go +++ b/main.go @@ -11,6 +11,7 @@ import ( "gitlink.org.cn/cloudream/scanner/internal/config" "gitlink.org.cn/cloudream/scanner/internal/event" "gitlink.org.cn/cloudream/scanner/internal/services" + "gitlink.org.cn/cloudream/scanner/internal/tickevent" ) func main() { @@ -46,6 +47,11 @@ func main() { } go serveScannerServer(agtSvr, &wg) + tickExecutor := tickevent.NewExecutor(tickevent.ExecuteArgs{ + DB: db, + }) + startTickEvent(&tickExecutor) + wg.Wait() } @@ -76,3 +82,19 @@ func serveScannerServer(server *scsvr.ScannerServer, wg *sync.WaitGroup) { wg.Done() } + +func startTickEvent(tickExecutor *tickevent.Executor) { + // TODO 可以考虑增加配置文件，配置这些任务间隔时间 + + tickExecutor.Start(tickevent.NewBatchAllAgentCheckCache(), 5*60*100) + + tickExecutor.Start(tickevent.NewBatchCheckAllObject(), 5*60*100) + + tickExecutor.Start(tickevent.NewBatchCheckAllRepCount(), 5*60*100) + + tickExecutor.Start(tickevent.NewBatchCheckAllStorage(), 5*60*100) + + tickExecutor.Start(tickevent.NewCheckAgentState(), 5*60*100) + + tickExecutor.Start(tickevent.NewCheckUnavailableCache(), 5*60*100) +} From 0268c5bc8e82c97f9d2ee92ed177c31019e5bcaa Mon Sep 17 00:00:00 2001 From: Sydonian <794346190@qq.com> Date: Wed, 24 May 2023 15:31:23 +0800 Subject: [PATCH 29/68] =?UTF-8?q?=E4=BA=8B=E4=BB=B6=E5=AF=B9=E8=B1=A1?= =?UTF-8?q?=E5=A4=8D=E7=94=A8=E7=9B=B8=E5=90=8C=E7=9A=84=E7=BB=93=E6=9E=84?= =?UTF-8?q?=E4=BD=93?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- internal/event/agent_check_storage.go | 7 +++---- internal/event/check_object.go | 5 +++-- internal/event/check_rep_count.go | 5 +++-- internal/event/update_agent_state.go | 14 +++++++++---- internal/event/update_cache.go | 21 +++++-------------- internal/event/update_storage.go | 29 +++++++-------------------- 6 files changed, 31 insertions(+), 50 deletions(-) diff --git a/internal/event/agent_check_storage.go b/internal/event/agent_check_storage.go index 6b2808a..b4d29e1 100644 --- a/internal/event/agent_check_storage.go +++ b/internal/event/agent_check_storage.go @@ -11,18 +11,17 @@ import ( agtcli "gitlink.org.cn/cloudream/rabbitmq/client/agent" agtmsg "gitlink.org.cn/cloudream/rabbitmq/message/agent" agtevt "gitlink.org.cn/cloudream/rabbitmq/message/agent/event" + scevt "gitlink.org.cn/cloudream/rabbitmq/message/scanner/event" "gitlink.org.cn/cloudream/scanner/internal/config" ) type AgentCheckStorage struct { - StorageID int - ObjectIDs []int // 需要检查的Object文件列表，如果为nil（不是为空），则代表进行全量检查 + scevt.AgentCheckStorage } func NewAgentCheckStorage(storageID int, objectIDs []int) *AgentCheckStorage { return &AgentCheckStorage{ - StorageID: storageID, - ObjectIDs: objectIDs, + AgentCheckStorage: scevt.NewAgentCheckStorage(storageID, objectIDs), } } diff --git a/internal/event/check_object.go b/internal/event/check_object.go index 1d72c29..448e109 100644 --- a/internal/event/check_object.go +++ b/internal/event/check_object.go @@ -4,15 +4,16 @@ import ( "github.com/samber/lo" "gitlink.org.cn/cloudream/common/utils/logger" mysql "gitlink.org.cn/cloudream/db/sql" + scevt "gitlink.org.cn/cloudream/rabbitmq/message/scanner/event" ) type CheckObject struct { - ObjectIDs []int + scevt.CheckObject } func NewCheckObject(objIDs []int) *CheckObject { return &CheckObject{ - ObjectIDs: objIDs, + CheckObject: scevt.NewCheckObject(objIDs), } } diff --git a/internal/event/check_rep_count.go b/internal/event/check_rep_count.go index 69bcdf4..9fe012a 100644 --- a/internal/event/check_rep_count.go +++ b/internal/event/check_rep_count.go @@ -15,15 +15,16 @@ import ( "gitlink.org.cn/cloudream/db/model" mysql "gitlink.org.cn/cloudream/db/sql" + scevt "gitlink.org.cn/cloudream/rabbitmq/message/scanner/event" ) type CheckRepCount struct { - FileHashes []string + scevt.CheckRepCount } func NewCheckRepCount(fileHashes []string) *CheckRepCount { return &CheckRepCount{ - FileHashes: fileHashes, + CheckRepCount: scevt.NewCheckRepCount(fileHashes), } } diff --git a/internal/event/update_agent_state.go b/internal/event/update_agent_state.go index 24d6f46..613fee9 100644 --- a/internal/event/update_agent_state.go +++ b/internal/event/update_agent_state.go @@ -4,11 +4,17 @@ import ( "gitlink.org.cn/cloudream/common/consts" "gitlink.org.cn/cloudream/common/utils/logger" mysql "gitlink.org.cn/cloudream/db/sql" + scevt "gitlink.org.cn/cloudream/rabbitmq/message/scanner/event" ) type UpdateAgentState struct { - NodeID int - IPFSStatus string + scevt.UpdateAgentState +} + +func NewUpdateAgentState(nodeID int, ipfsState string) *UpdateAgentState { + return &UpdateAgentState{ + UpdateAgentState: scevt.NewUpdateAgentState(nodeID, ipfsState), + } } func (t *UpdateAgentState) TryMerge(other Event) bool { @@ -16,8 +22,8 @@ func (t *UpdateAgentState) TryMerge(other Event) bool { } func (t *UpdateAgentState) Execute(execCtx ExecuteContext) { - if t.IPFSStatus != consts.IPFS_STATUS_OK { - logger.WithField("NodeID", t.NodeID).Warnf("IPFS status is %s, set node state unavailable", t.IPFSStatus) + if t.IPFSState != consts.IPFS_STATUS_OK { + logger.WithField("NodeID", t.NodeID).Warnf("IPFS status is %s, set node state unavailable", t.IPFSState) err := mysql.Node.ChangeState(execCtx.Args.DB.SQLCtx(), t.NodeID, consts.NODE_STATE_UNAVAILABLE) if err != nil { diff --git a/internal/event/update_cache.go b/internal/event/update_cache.go index 8d9598d..3dab9de 100644 --- a/internal/event/update_cache.go +++ b/internal/event/update_cache.go @@ -4,29 +4,18 @@ import ( evtcst "gitlink.org.cn/cloudream/common/consts/event" "gitlink.org.cn/cloudream/common/utils/logger" mysql "gitlink.org.cn/cloudream/db/sql" + scevt "gitlink.org.cn/cloudream/rabbitmq/message/scanner/event" ) -type UpdateCacheEntry struct { - FileHash string - Operation string -} - -func NewUpdateCacheEntry(fileHash string, op string) UpdateCacheEntry { - return UpdateCacheEntry{ - FileHash: fileHash, - Operation: op, - } -} +type UpdateCacheEntry = scevt.UpdateCacheEntry type UpdateCache struct { - NodeID int - Entries []UpdateCacheEntry + scevt.UpdateCache } -func NewUpdateCache(nodeID int, entries []UpdateCacheEntry) UpdateCache { +func NewUpdateCache(nodeID int, entries []scevt.UpdateCacheEntry) UpdateCache { return UpdateCache{ - NodeID: nodeID, - Entries: entries, + UpdateCache: scevt.NewUpdateCache(nodeID, entries), } } diff --git a/internal/event/update_storage.go b/internal/event/update_storage.go index a133364..4e82871 100644 --- a/internal/event/update_storage.go +++ b/internal/event/update_storage.go @@ -4,32 +4,17 @@ import ( tskcst "gitlink.org.cn/cloudream/common/consts/event" "gitlink.org.cn/cloudream/common/utils/logger" mysql "gitlink.org.cn/cloudream/db/sql" + scevt "gitlink.org.cn/cloudream/rabbitmq/message/scanner/event" ) -type UpdateStorageEntry struct { - ObjectID int - UserID int - Operation string -} - -func NewUpdateStorageEntry(objectID int, userID int, op string) UpdateStorageEntry { - return UpdateStorageEntry{ - ObjectID: objectID, - UserID: userID, - Operation: op, - } -} - +type UpdateStorageEntry = scevt.UpdateStorageEntry type UpdateStorage struct { - StorageID int - DirectoryStatus string - Entries []UpdateStorageEntry + scevt.UpdateStorage } -func NewUpdateStorage(dirStatus string, entries []UpdateStorageEntry) UpdateStorage { +func NewUpdateStorage(dirState string, entries []UpdateStorageEntry) UpdateStorage { return UpdateStorage{ - DirectoryStatus: dirStatus, - Entries: entries, + UpdateStorage: scevt.NewUpdateStorage(dirState, entries), } } @@ -43,7 +28,7 @@ func (t *UpdateStorage) TryMerge(other Event) bool { } // 后投递的任务的状态更新一些 - t.DirectoryStatus = event.DirectoryStatus + t.DirectoryState = event.DirectoryState // TODO 可以考虑合并同FileHash和NodeID的记录 t.Entries = append(t.Entries, event.Entries...) return true @@ -51,7 +36,7 @@ func (t *UpdateStorage) TryMerge(other Event) bool { func (t *UpdateStorage) Execute(execCtx ExecuteContext) { - err := mysql.Storage.ChangeState(execCtx.Args.DB.SQLCtx(), t.StorageID, t.DirectoryStatus) + err := mysql.Storage.ChangeState(execCtx.Args.DB.SQLCtx(), t.StorageID, t.DirectoryState) if err != nil { logger.WithField("StorageID", t.StorageID).Warnf("change storage state failed, err: %s", err.Error()) } From c792a97c529f478ec8e08815fd784dcc51c61aea Mon Sep 17 00:00:00 2001 From: Sydonian <794346190@qq.com> Date: Wed, 24 May 2023 15:39:49 +0800 Subject: [PATCH 30/68] =?UTF-8?q?=E6=B3=A8=E5=86=8Cscanner=E7=9A=84?= =?UTF-8?q?=E4=BA=8B=E4=BB=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- internal/event/agent_check_cache.go | 4 ++++ internal/event/agent_check_state.go | 4 ++++ internal/event/agent_check_storage.go | 4 ++++ ...check_unavailable_cache.go => check_cache.go} | 16 ++++++++++------ internal/event/check_object.go | 4 ++++ internal/event/check_rep_count.go | 4 ++++ internal/event/update_agent_state.go | 4 ++++ internal/event/update_cache.go | 8 ++++++-- internal/event/update_storage.go | 8 ++++++-- ...check_unavailable_cache.go => check_cache.go} | 10 +++++----- 10 files changed, 51 insertions(+), 15 deletions(-) rename internal/event/{check_unavailable_cache.go => check_cache.go} (79%) rename internal/tickevent/{check_unavailable_cache.go => check_cache.go} (57%) diff --git a/internal/event/agent_check_cache.go b/internal/event/agent_check_cache.go index 942ec5a..890255b 100644 --- a/internal/event/agent_check_cache.go +++ b/internal/event/agent_check_cache.go @@ -92,3 +92,7 @@ func (t *AgentCheckCache) Execute(execCtx ExecuteContext) { return } } + +func init() { + Register(func(msg AgentCheckCache) Event { return NewAgentCheckCache(msg.NodeID, msg.FileHashes) }) +} diff --git a/internal/event/agent_check_state.go b/internal/event/agent_check_state.go index b37c92b..41445a9 100644 --- a/internal/event/agent_check_state.go +++ b/internal/event/agent_check_state.go @@ -81,3 +81,7 @@ func (t *AgentCheckState) Execute(execCtx ExecuteContext) { logger.WithField("NodeID", t.NodeID).Warnf("request to agent failed, err: %s", err.Error()) } } + +func init() { + Register(func(msg AgentCheckState) Event { return NewAgentCheckState(msg.NodeID) }) +} diff --git a/internal/event/agent_check_storage.go b/internal/event/agent_check_storage.go index b4d29e1..c7861de 100644 --- a/internal/event/agent_check_storage.go +++ b/internal/event/agent_check_storage.go @@ -109,3 +109,7 @@ func (t *AgentCheckStorage) Execute(execCtx ExecuteContext) { logger.WithField("NodeID", stg.NodeID).Warnf("request to agent failed, err: %s", stg.NodeID, err.Error()) } } + +func init() { + Register(func(msg AgentCheckStorage) Event { return NewAgentCheckStorage(msg.StorageID, msg.ObjectIDs) }) +} diff --git a/internal/event/check_unavailable_cache.go b/internal/event/check_cache.go similarity index 79% rename from internal/event/check_unavailable_cache.go rename to internal/event/check_cache.go index b503412..d4fa7f8 100644 --- a/internal/event/check_unavailable_cache.go +++ b/internal/event/check_cache.go @@ -12,18 +12,18 @@ import ( mysql "gitlink.org.cn/cloudream/db/sql" ) -type CheckUnavailableCache struct { +type CheckCache struct { NodeID int } -func NewCheckUnavailableCache(nodeID int) *CheckUnavailableCache { - return &CheckUnavailableCache{ +func NewCheckCache(nodeID int) *CheckCache { + return &CheckCache{ NodeID: nodeID, } } -func (t *CheckUnavailableCache) TryMerge(other Event) bool { - event, ok := other.(*CheckUnavailableCache) +func (t *CheckCache) TryMerge(other Event) bool { + event, ok := other.(*CheckCache) if !ok { return false } @@ -34,7 +34,7 @@ func (t *CheckUnavailableCache) TryMerge(other Event) bool { return true } -func (t *CheckUnavailableCache) Execute(execCtx ExecuteContext) { +func (t *CheckCache) Execute(execCtx ExecuteContext) { err := execCtx.Args.DB.DoTx(sql.LevelSerializable, func(tx *sqlx.Tx) error { node, err := mysql.Node.GetByID(tx, t.NodeID) if err == sql.ErrNoRows { @@ -66,3 +66,7 @@ func (t *CheckUnavailableCache) Execute(execCtx ExecuteContext) { logger.WithField("NodeID", t.NodeID).Warn(err.Error()) } } + +func init() { + Register(func(msg CheckCache) Event { return NewCheckCache(msg.NodeID) }) +} diff --git a/internal/event/check_object.go b/internal/event/check_object.go index 448e109..b969a1d 100644 --- a/internal/event/check_object.go +++ b/internal/event/check_object.go @@ -35,3 +35,7 @@ func (t *CheckObject) Execute(execCtx ExecuteContext) { } } } + +func init() { + Register(func(msg CheckObject) Event { return NewCheckObject(msg.ObjectIDs) }) +} diff --git a/internal/event/check_rep_count.go b/internal/event/check_rep_count.go index 9fe012a..fa4fc26 100644 --- a/internal/event/check_rep_count.go +++ b/internal/event/check_rep_count.go @@ -196,3 +196,7 @@ func chooseDeleteAvaiRepNodes(allNodes []model.Node, curAvaiRepNodes []model.Nod return alternatedNodes[:mymath.Min(delCount, len(alternatedNodes))] } + +func init() { + Register(func(msg CheckRepCount) Event { return NewCheckRepCount(msg.FileHashes) }) +} diff --git a/internal/event/update_agent_state.go b/internal/event/update_agent_state.go index 613fee9..154cab8 100644 --- a/internal/event/update_agent_state.go +++ b/internal/event/update_agent_state.go @@ -38,3 +38,7 @@ func (t *UpdateAgentState) Execute(execCtx ExecuteContext) { logger.WithField("NodeID", t.NodeID).Warnf("change node state failed, err: %s", err.Error()) } } + +func init() { + Register(func(msg UpdateAgentState) Event { return NewUpdateAgentState(msg.NodeID, msg.IPFSState) }) +} diff --git a/internal/event/update_cache.go b/internal/event/update_cache.go index 3dab9de..3a4f997 100644 --- a/internal/event/update_cache.go +++ b/internal/event/update_cache.go @@ -13,8 +13,8 @@ type UpdateCache struct { scevt.UpdateCache } -func NewUpdateCache(nodeID int, entries []scevt.UpdateCacheEntry) UpdateCache { - return UpdateCache{ +func NewUpdateCache(nodeID int, entries []scevt.UpdateCacheEntry) *UpdateCache { + return &UpdateCache{ UpdateCache: scevt.NewUpdateCache(nodeID, entries), } } @@ -55,3 +55,7 @@ func (t *UpdateCache) Execute(execCtx ExecuteContext) { } } } + +func init() { + Register(func(msg UpdateCache) Event { return NewUpdateCache(msg.NodeID, msg.Entries) }) +} diff --git a/internal/event/update_storage.go b/internal/event/update_storage.go index 4e82871..e54c65d 100644 --- a/internal/event/update_storage.go +++ b/internal/event/update_storage.go @@ -12,8 +12,8 @@ type UpdateStorage struct { scevt.UpdateStorage } -func NewUpdateStorage(dirState string, entries []UpdateStorageEntry) UpdateStorage { - return UpdateStorage{ +func NewUpdateStorage(dirState string, entries []UpdateStorageEntry) *UpdateStorage { + return &UpdateStorage{ UpdateStorage: scevt.NewUpdateStorage(dirState, entries), } } @@ -61,3 +61,7 @@ func (t *UpdateStorage) Execute(execCtx ExecuteContext) { } } } + +func init() { + Register(func(msg UpdateStorage) Event { return NewUpdateStorage(msg.DirectoryState, msg.Entries) }) +} diff --git a/internal/tickevent/check_unavailable_cache.go b/internal/tickevent/check_cache.go similarity index 57% rename from internal/tickevent/check_unavailable_cache.go rename to internal/tickevent/check_cache.go index 57158eb..c03651b 100644 --- a/internal/tickevent/check_unavailable_cache.go +++ b/internal/tickevent/check_cache.go @@ -6,14 +6,14 @@ import ( "gitlink.org.cn/cloudream/scanner/internal/event" ) -type CheckUnavailableCache struct { +type CheckCache struct { } -func NewCheckUnavailableCache() *CheckUnavailableCache { - return &CheckUnavailableCache{} +func NewCheckUnavailableCache() *CheckCache { + return &CheckCache{} } -func (e *CheckUnavailableCache) Execute(ctx ExecuteContext) { +func (e *CheckCache) Execute(ctx ExecuteContext) { nodes, err := mysql.Node.GetAllNodes(ctx.Args.DB.SQLCtx()) if err != nil { logger.Warnf("get all nodes failed, err: %s", err.Error()) @@ -21,6 +21,6 @@ func (e *CheckUnavailableCache) Execute(ctx ExecuteContext) { } for _, node := range nodes { - ctx.Args.EventExecutor.Post(event.NewCheckUnavailableCache(node.NodeID)) + ctx.Args.EventExecutor.Post(event.NewCheckCache(node.NodeID)) } } From 3715d53db7ea0b295331bcc58c5e71899ed81a41 Mon Sep 17 00:00:00 2001 From: Sydonian <794346190@qq.com> Date: Wed, 24 May 2023 16:05:35 +0800 Subject: [PATCH 31/68] =?UTF-8?q?=E6=9B=B4=E6=96=B0StorageObject=E4=B9=8B?= =?UTF-8?q?=E5=90=8E=E6=8A=95=E9=80=92CheckObject=E4=BB=BB=E5=8A=A1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- internal/event/check_object.go | 3 +-- internal/event/update_storage.go | 6 ++++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/internal/event/check_object.go b/internal/event/check_object.go index b969a1d..e8a5163 100644 --- a/internal/event/check_object.go +++ b/internal/event/check_object.go @@ -3,7 +3,6 @@ package event import ( "github.com/samber/lo" "gitlink.org.cn/cloudream/common/utils/logger" - mysql "gitlink.org.cn/cloudream/db/sql" scevt "gitlink.org.cn/cloudream/rabbitmq/message/scanner/event" ) @@ -29,7 +28,7 @@ func (t *CheckObject) TryMerge(other Event) bool { func (t *CheckObject) Execute(execCtx ExecuteContext) { for _, objID := range t.ObjectIDs { - err := mysql.Object.DeleteUnused(execCtx.Args.DB.SQLCtx(), objID) + err := execCtx.Args.DB.Object().DeleteUnused(execCtx.Args.DB.SQLCtx(), objID) if err != nil { logger.WithField("ObjectID", objID).Warnf("delete unused object failed, err: %s", err.Error()) } diff --git a/internal/event/update_storage.go b/internal/event/update_storage.go index e54c65d..ad8f393 100644 --- a/internal/event/update_storage.go +++ b/internal/event/update_storage.go @@ -41,6 +41,7 @@ func (t *UpdateStorage) Execute(execCtx ExecuteContext) { logger.WithField("StorageID", t.StorageID).Warnf("change storage state failed, err: %s", err.Error()) } + var chkObjIDs []int for _, entry := range t.Entries { switch entry.Operation { case tskcst.UPDATE_STORAGE_DELETE: @@ -50,6 +51,7 @@ func (t *UpdateStorage) Execute(execCtx ExecuteContext) { WithField("ObjectID", entry.ObjectID). Warnf("delete storage object failed, err: %s", err.Error()) } + chkObjIDs = append(chkObjIDs, entry.ObjectID) case tskcst.UPDATE_STORAGE_SET_NORMAL: err := mysql.StorageObject.SetStateNormal(execCtx.Args.DB.SQLCtx(), t.StorageID, entry.ObjectID, entry.UserID) @@ -60,6 +62,10 @@ func (t *UpdateStorage) Execute(execCtx ExecuteContext) { } } } + + if len(chkObjIDs) > 0 { + execCtx.Executor.Post(NewCheckObject(chkObjIDs)) + } } func init() { From 249188084557dab7a8c41332acabbf3a59088d86 Mon Sep 17 00:00:00 2001 From: Sydonian <794346190@qq.com> Date: Wed, 24 May 2023 17:20:15 +0800 Subject: [PATCH 32/68] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E8=B0=83=E8=AF=95?= =?UTF-8?q?=E5=8F=91=E7=8E=B0=E7=9A=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- internal/event/agent_check_cache.go | 10 ++++++++-- internal/event/agent_check_state.go | 7 ++++++- internal/event/agent_check_storage.go | 10 ++++++++-- internal/event/event.go | 2 +- internal/services/event.go | 2 +- internal/tickevent/batch_check_all_object.go | 4 +++- internal/tickevent/batch_check_all_storage.go | 3 ++- main.go | 3 ++- 8 files changed, 31 insertions(+), 10 deletions(-) diff --git a/internal/event/agent_check_cache.go b/internal/event/agent_check_cache.go index 890255b..1277d0c 100644 --- a/internal/event/agent_check_cache.go +++ b/internal/event/agent_check_cache.go @@ -83,10 +83,16 @@ func (t *AgentCheckCache) Execute(execCtx ExecuteContext) { } defer agentClient.Close() - err = agentClient.PostEvent(agtmsg.NewPostEventBody( + evtmsg, err := agtmsg.NewPostEventBody( agtevt.NewCheckCache(isComplete, caches), execCtx.Option.IsEmergency, // 继承本任务的执行选项 - execCtx.Option.DontMerge)) + execCtx.Option.DontMerge) + if err != nil { + logger.Warnf("new post event body failed, err: %s", err.Error()) + return + } + + err = agentClient.PostEvent(evtmsg) if err != nil { logger.WithField("NodeID", t.NodeID).Warnf("request to agent failed, err: %s", err.Error()) return diff --git a/internal/event/agent_check_state.go b/internal/event/agent_check_state.go index 41445a9..c56d0c3 100644 --- a/internal/event/agent_check_state.go +++ b/internal/event/agent_check_state.go @@ -76,7 +76,12 @@ func (t *AgentCheckState) Execute(execCtx ExecuteContext) { defer agentClient.Close() // 紧急任务 - err = agentClient.PostEvent(agtmsg.NewPostEventBody(agtevt.NewCheckState(), true, true)) + evtmsg, err := agtmsg.NewPostEventBody(agtevt.NewCheckState(), true, true) + if err != nil { + logger.Warnf("new post event body failed, err: %s", err.Error()) + return + } + err = agentClient.PostEvent(evtmsg) if err != nil { logger.WithField("NodeID", t.NodeID).Warnf("request to agent failed, err: %s", err.Error()) } diff --git a/internal/event/agent_check_storage.go b/internal/event/agent_check_storage.go index c7861de..9898b9b 100644 --- a/internal/event/agent_check_storage.go +++ b/internal/event/agent_check_storage.go @@ -101,10 +101,16 @@ func (t *AgentCheckStorage) Execute(execCtx ExecuteContext) { } defer agentClient.Close() - err = agentClient.PostEvent(agtmsg.NewPostEventBody( + evtmsg, err := agtmsg.NewPostEventBody( agtevt.NewCheckStorage(stg.Directory, isComplete, objects), execCtx.Option.IsEmergency, // 继承本任务的执行选项 - execCtx.Option.DontMerge)) + execCtx.Option.DontMerge) + if err != nil { + logger.Warnf("new post event body failed, err: %s", err.Error()) + return + } + + err = agentClient.PostEvent(evtmsg) if err != nil { logger.WithField("NodeID", stg.NodeID).Warnf("request to agent failed, err: %s", stg.NodeID, err.Error()) } diff --git a/internal/event/event.go b/internal/event/event.go index f64ed98..e6aab1d 100644 --- a/internal/event/event.go +++ b/internal/event/event.go @@ -27,7 +27,7 @@ func NewExecutor(db *mydb.DB) Executor { }) } -var msgDispatcher typedispatcher.TypeDispatcher[Event] +var msgDispatcher = typedispatcher.NewTypeDispatcher[Event]() func FromMessage(msg any) (Event, error) { event, ok := msgDispatcher.Dispatch(msg) diff --git a/internal/services/event.go b/internal/services/event.go index 3d57a42..5264883 100644 --- a/internal/services/event.go +++ b/internal/services/event.go @@ -9,7 +9,7 @@ import ( func (svc *Service) PostEvent(msg *scmsg.PostEvent) { - evtMsg, err := scevt.MapToMessage(msg.Body.Event.(map[string]any)) + evtMsg, err := scevt.MapToMessage(msg.Body.Event) if err != nil { logger.Warnf("convert map to event message failed, err: %s", err.Error()) return diff --git a/internal/tickevent/batch_check_all_object.go b/internal/tickevent/batch_check_all_object.go index b970fa0..b57e3ed 100644 --- a/internal/tickevent/batch_check_all_object.go +++ b/internal/tickevent/batch_check_all_object.go @@ -1,6 +1,7 @@ package tickevent import ( + "github.com/samber/lo" "gitlink.org.cn/cloudream/common/utils/logger" mysql "gitlink.org.cn/cloudream/db/sql" "gitlink.org.cn/cloudream/scanner/internal/event" @@ -23,7 +24,8 @@ func (e *BatchCheckAllObject) Execute(ctx ExecuteContext) { return } - ctx.Args.EventExecutor.Post(event.NewCheckObject(objectIDs)) + // TODO 将ID字段换成int64类型 + ctx.Args.EventExecutor.Post(event.NewCheckObject(lo.Map(objectIDs, func(id int64, index int) int { return int(id) }))) // 如果结果的长度小于预期的长度，则认为已经查询了所有，下次从头再来 if len(objectIDs) < CHECK_OBJECT_BATCH_SIZE { diff --git a/internal/tickevent/batch_check_all_storage.go b/internal/tickevent/batch_check_all_storage.go index 5550671..51f46a8 100644 --- a/internal/tickevent/batch_check_all_storage.go +++ b/internal/tickevent/batch_check_all_storage.go @@ -25,7 +25,8 @@ func (e *BatchCheckAllStorage) Execute(ctx ExecuteContext) { for _, stgID := range storageIDs { // 设置nil代表进行全量检查 - ctx.Args.EventExecutor.Post(event.NewAgentCheckStorage(stgID, nil)) + // TODO 将ID字段换成int64类型 + ctx.Args.EventExecutor.Post(event.NewAgentCheckStorage(int(stgID), nil)) } // 如果结果的长度小于预期的长度，则认为已经查询了所有，下次从头再来 diff --git a/main.go b/main.go index fbefede..026cb57 100644 --- a/main.go +++ b/main.go @@ -48,7 +48,8 @@ func main() { go serveScannerServer(agtSvr, &wg) tickExecutor := tickevent.NewExecutor(tickevent.ExecuteArgs{ - DB: db, + EventExecutor: &eventExecutor, + DB: db, }) startTickEvent(&tickExecutor) From 4b8b1851434706b89e3a9e4c7d067113f0442aec Mon Sep 17 00:00:00 2001 From: Sydonian <794346190@qq.com> Date: Thu, 25 May 2023 11:35:33 +0800 Subject: [PATCH 33/68] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E8=B0=83=E8=AF=95?= =?UTF-8?q?=E6=97=A5=E5=BF=97=EF=BC=9B=E8=A7=A3=E5=86=B3=E8=B0=83=E8=AF=95?= =?UTF-8?q?=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- internal/event/agent_check_cache.go | 10 ++++++---- internal/event/agent_check_state.go | 13 ++++++++----- internal/event/agent_check_storage.go | 6 ++++-- internal/event/check_cache.go | 10 ++++++---- internal/event/check_object.go | 4 +++- internal/event/check_rep_count.go | 7 +++++-- internal/event/event.go | 2 +- internal/event/update_agent_state.go | 4 +++- internal/event/update_cache.go | 4 +++- internal/event/update_storage.go | 3 ++- internal/tickevent/batch_all_agent_check_cache.go | 7 ++++--- 11 files changed, 45 insertions(+), 25 deletions(-) diff --git a/internal/event/agent_check_cache.go b/internal/event/agent_check_cache.go index 1277d0c..7ff7dd3 100644 --- a/internal/event/agent_check_cache.go +++ b/internal/event/agent_check_cache.go @@ -43,6 +43,8 @@ func (t *AgentCheckCache) TryMerge(other Event) bool { } func (t *AgentCheckCache) Execute(execCtx ExecuteContext) { + logger.Debugf("begin agent check cache") + var isComplete bool var caches []model.Cache @@ -66,7 +68,7 @@ func (t *AgentCheckCache) Execute(execCtx ExecuteContext) { } if err != nil { - logger.WithField("FileHash", hash).WithField("NodeID", t.NodeID).Warnf("get cache failed, err: %w", err) + logger.WithField("FileHash", hash).WithField("NodeID", t.NodeID).Warnf("get cache failed, err: %s", err.Error()) return } @@ -99,6 +101,6 @@ func (t *AgentCheckCache) Execute(execCtx ExecuteContext) { } } -func init() { - Register(func(msg AgentCheckCache) Event { return NewAgentCheckCache(msg.NodeID, msg.FileHashes) }) -} +//func init() { +// Register(func(msg scevt.AgentCheckCache) Event { return NewAgentCheckCache(msg.NodeID, msg.FileHashes) }) +//} diff --git a/internal/event/agent_check_state.go b/internal/event/agent_check_state.go index c56d0c3..98280ae 100644 --- a/internal/event/agent_check_state.go +++ b/internal/event/agent_check_state.go @@ -35,6 +35,8 @@ func (t *AgentCheckState) TryMerge(other Event) bool { } func (t *AgentCheckState) Execute(execCtx ExecuteContext) { + logger.Debugf("begin agent check state") + node, err := mysql.Node.GetByID(execCtx.Args.DB.SQLCtx(), t.NodeID) if err == sql.ErrNoRows { return @@ -50,7 +52,8 @@ func (t *AgentCheckState) Execute(execCtx ExecuteContext) { } // 检查上次上报时间，超时的设置为不可用 - if time.Since(node.LastReportTime) > time.Duration(config.Cfg().NodeUnavailableSeconds)*time.Second { + // TODO 没有上报过是否要特殊处理？ + if node.LastReportTime == nil && time.Since(*node.LastReportTime) > time.Duration(config.Cfg().NodeUnavailableSeconds)*time.Second { err := mysql.Node.ChangeState(execCtx.Args.DB.SQLCtx(), t.NodeID, consts.NODE_STATE_UNAVAILABLE) if err != nil { logger.WithField("NodeID", t.NodeID).Warnf("set node state failed, err: %s", err.Error()) @@ -64,7 +67,7 @@ func (t *AgentCheckState) Execute(execCtx ExecuteContext) { } // 补充备份数 - execCtx.Executor.Post(NewCheckRepCount(lo.Map(caches, func(ch model.Cache, index int) string { return ch.HashValue }))) + execCtx.Executor.Post(NewCheckRepCount(lo.Map(caches, func(ch model.Cache, index int) string { return ch.FileHash }))) return } @@ -87,6 +90,6 @@ func (t *AgentCheckState) Execute(execCtx ExecuteContext) { } } -func init() { - Register(func(msg AgentCheckState) Event { return NewAgentCheckState(msg.NodeID) }) -} +// func init() { +// Register(func(msg scevt.AgentCheckState) Event { return NewAgentCheckState(msg.NodeID) }) +// } diff --git a/internal/event/agent_check_storage.go b/internal/event/agent_check_storage.go index 9898b9b..837656a 100644 --- a/internal/event/agent_check_storage.go +++ b/internal/event/agent_check_storage.go @@ -46,6 +46,8 @@ func (t *AgentCheckStorage) TryMerge(other Event) bool { } func (t *AgentCheckStorage) Execute(execCtx ExecuteContext) { + logger.Debugf("begin agent check storage") + stg, err := mysql.Storage.GetByID(execCtx.Args.DB.SQLCtx(), t.StorageID) if err != nil { if err != sql.ErrNoRows { @@ -112,10 +114,10 @@ func (t *AgentCheckStorage) Execute(execCtx ExecuteContext) { err = agentClient.PostEvent(evtmsg) if err != nil { - logger.WithField("NodeID", stg.NodeID).Warnf("request to agent failed, err: %s", stg.NodeID, err.Error()) + logger.WithField("NodeID", stg.NodeID).Warnf("request to agent failed, err: %s", err.Error()) } } func init() { - Register(func(msg AgentCheckStorage) Event { return NewAgentCheckStorage(msg.StorageID, msg.ObjectIDs) }) + RegisterMessageConvertor(func(msg scevt.AgentCheckStorage) Event { return NewAgentCheckStorage(msg.StorageID, msg.ObjectIDs) }) } diff --git a/internal/event/check_cache.go b/internal/event/check_cache.go index d4fa7f8..cc832bc 100644 --- a/internal/event/check_cache.go +++ b/internal/event/check_cache.go @@ -35,6 +35,8 @@ func (t *CheckCache) TryMerge(other Event) bool { } func (t *CheckCache) Execute(execCtx ExecuteContext) { + logger.Debugf("begin check cache") + err := execCtx.Args.DB.DoTx(sql.LevelSerializable, func(tx *sqlx.Tx) error { node, err := mysql.Node.GetByID(tx, t.NodeID) if err == sql.ErrNoRows { @@ -58,7 +60,7 @@ func (t *CheckCache) Execute(execCtx ExecuteContext) { return fmt.Errorf("delete node all caches failed, err: %w", err) } - execCtx.Executor.Post(NewCheckRepCount(lo.Map(caches, func(ch model.Cache, index int) string { return ch.HashValue }))) + execCtx.Executor.Post(NewCheckRepCount(lo.Map(caches, func(ch model.Cache, index int) string { return ch.FileHash }))) return nil }) @@ -67,6 +69,6 @@ func (t *CheckCache) Execute(execCtx ExecuteContext) { } } -func init() { - Register(func(msg CheckCache) Event { return NewCheckCache(msg.NodeID) }) -} +// func init() { +// Register(func(msg scevt.CheckCache) Event { return NewCheckCache(msg.NodeID) }) +// } diff --git a/internal/event/check_object.go b/internal/event/check_object.go index e8a5163..f94c795 100644 --- a/internal/event/check_object.go +++ b/internal/event/check_object.go @@ -27,6 +27,8 @@ func (t *CheckObject) TryMerge(other Event) bool { } func (t *CheckObject) Execute(execCtx ExecuteContext) { + logger.Debugf("begin check object") + for _, objID := range t.ObjectIDs { err := execCtx.Args.DB.Object().DeleteUnused(execCtx.Args.DB.SQLCtx(), objID) if err != nil { @@ -36,5 +38,5 @@ func (t *CheckObject) Execute(execCtx ExecuteContext) { } func init() { - Register(func(msg CheckObject) Event { return NewCheckObject(msg.ObjectIDs) }) + RegisterMessageConvertor(func(msg scevt.CheckObject) Event { return NewCheckObject(msg.ObjectIDs) }) } diff --git a/internal/event/check_rep_count.go b/internal/event/check_rep_count.go index fa4fc26..c09ff0a 100644 --- a/internal/event/check_rep_count.go +++ b/internal/event/check_rep_count.go @@ -8,6 +8,7 @@ import ( "github.com/jmoiron/sqlx" "github.com/samber/lo" "gitlink.org.cn/cloudream/common/consts" + "gitlink.org.cn/cloudream/common/utils/logger" log "gitlink.org.cn/cloudream/common/utils/logger" mymath "gitlink.org.cn/cloudream/common/utils/math" mysort "gitlink.org.cn/cloudream/common/utils/sort" @@ -39,6 +40,8 @@ func (t *CheckRepCount) TryMerge(other Event) bool { } func (t *CheckRepCount) Execute(execCtx ExecuteContext) { + logger.Debugf("begin check rep count") + updatedNodeAndHashes := make(map[int][]string) for _, fileHash := range t.FileHashes { @@ -99,7 +102,7 @@ func (t *CheckRepCount) checkOneRepCount(fileHash string, execCtx ExecuteContext // 如果Available的备份数超过期望备份数，则让一些节点退出 if len(normalNodes) > needRepCount { - delNodes := chooseDeleteAvaiRepNodes(allNodes, normalNodes, needRepCount-len(normalNodes)) + delNodes := chooseDeleteAvaiRepNodes(allNodes, normalNodes, len(normalNodes)-needRepCount) for _, node := range delNodes { err := mysql.Cache.ChangeState(tx, fileHash, node.NodeID, consts.CACHE_STATE_TEMP) if err != nil { @@ -198,5 +201,5 @@ func chooseDeleteAvaiRepNodes(allNodes []model.Node, curAvaiRepNodes []model.Nod } func init() { - Register(func(msg CheckRepCount) Event { return NewCheckRepCount(msg.FileHashes) }) + RegisterMessageConvertor(func(msg scevt.CheckRepCount) Event { return NewCheckRepCount(msg.FileHashes) }) } diff --git a/internal/event/event.go b/internal/event/event.go index e6aab1d..6296252 100644 --- a/internal/event/event.go +++ b/internal/event/event.go @@ -38,6 +38,6 @@ func FromMessage(msg any) (Event, error) { return event, nil } -func Register[T any](converter func(msg T) Event) { +func RegisterMessageConvertor[T any](converter func(msg T) Event) { typedispatcher.Add(msgDispatcher, converter) } diff --git a/internal/event/update_agent_state.go b/internal/event/update_agent_state.go index 154cab8..bef11a3 100644 --- a/internal/event/update_agent_state.go +++ b/internal/event/update_agent_state.go @@ -22,6 +22,8 @@ func (t *UpdateAgentState) TryMerge(other Event) bool { } func (t *UpdateAgentState) Execute(execCtx ExecuteContext) { + logger.Debugf("begin update agent state") + if t.IPFSState != consts.IPFS_STATUS_OK { logger.WithField("NodeID", t.NodeID).Warnf("IPFS status is %s, set node state unavailable", t.IPFSState) @@ -40,5 +42,5 @@ func (t *UpdateAgentState) Execute(execCtx ExecuteContext) { } func init() { - Register(func(msg UpdateAgentState) Event { return NewUpdateAgentState(msg.NodeID, msg.IPFSState) }) + RegisterMessageConvertor(func(msg scevt.UpdateAgentState) Event { return NewUpdateAgentState(msg.NodeID, msg.IPFSState) }) } diff --git a/internal/event/update_cache.go b/internal/event/update_cache.go index 3a4f997..22a2283 100644 --- a/internal/event/update_cache.go +++ b/internal/event/update_cache.go @@ -34,6 +34,8 @@ func (t *UpdateCache) TryMerge(other Event) bool { } func (t *UpdateCache) Execute(execCtx ExecuteContext) { + logger.Debugf("begin update cache") + for _, entry := range t.Entries { switch entry.Operation { case evtcst.UPDATE_CACHE_UNTEMP: @@ -57,5 +59,5 @@ func (t *UpdateCache) Execute(execCtx ExecuteContext) { } func init() { - Register(func(msg UpdateCache) Event { return NewUpdateCache(msg.NodeID, msg.Entries) }) + RegisterMessageConvertor(func(msg scevt.UpdateCache) Event { return NewUpdateCache(msg.NodeID, msg.Entries) }) } diff --git a/internal/event/update_storage.go b/internal/event/update_storage.go index ad8f393..e55305c 100644 --- a/internal/event/update_storage.go +++ b/internal/event/update_storage.go @@ -35,6 +35,7 @@ func (t *UpdateStorage) TryMerge(other Event) bool { } func (t *UpdateStorage) Execute(execCtx ExecuteContext) { + logger.Debugf("begin update storage") err := mysql.Storage.ChangeState(execCtx.Args.DB.SQLCtx(), t.StorageID, t.DirectoryState) if err != nil { @@ -69,5 +70,5 @@ func (t *UpdateStorage) Execute(execCtx ExecuteContext) { } func init() { - Register(func(msg UpdateStorage) Event { return NewUpdateStorage(msg.DirectoryState, msg.Entries) }) + RegisterMessageConvertor(func(msg scevt.UpdateStorage) Event { return NewUpdateStorage(msg.DirectoryState, msg.Entries) }) } diff --git a/internal/tickevent/batch_all_agent_check_cache.go b/internal/tickevent/batch_all_agent_check_cache.go index 61090b0..b20552d 100644 --- a/internal/tickevent/batch_all_agent_check_cache.go +++ b/internal/tickevent/batch_all_agent_check_cache.go @@ -31,9 +31,10 @@ func (e *BatchAllAgentCheckCache) Execute(ctx ExecuteContext) { logger.Debugf("new check start, get all nodes") } - for i := 0; i < len(e.nodeIDs) && i < AGENT_CHECK_CACHE_BATCH_SIZE; i++ { + checkedCnt := 0 + for ; checkedCnt < len(e.nodeIDs) && checkedCnt < AGENT_CHECK_CACHE_BATCH_SIZE; checkedCnt++ { // nil代表进行全量检查 - ctx.Args.EventExecutor.Post(event.NewAgentCheckCache(e.nodeIDs[i], nil)) + ctx.Args.EventExecutor.Post(event.NewAgentCheckCache(e.nodeIDs[checkedCnt], nil)) } - e.nodeIDs = e.nodeIDs[AGENT_CHECK_CACHE_BATCH_SIZE:] + e.nodeIDs = e.nodeIDs[checkedCnt:] } From 0e0f0b8b331e7637a2958a739e77a6c68a8ea007 Mon Sep 17 00:00:00 2001 From: Sydonian <794346190@qq.com> Date: Thu, 25 May 2023 15:14:48 +0800 Subject: [PATCH 34/68] =?UTF-8?q?CheckStorage=E4=BB=BB=E5=8A=A1=E5=A2=9E?= =?UTF-8?q?=E5=8A=A0StorageID?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- internal/event/agent_check_storage.go | 2 +- internal/event/update_cache.go | 11 +++++++++-- internal/event/update_storage.go | 18 +++++++++++++++--- internal/tickevent/check_cache.go | 2 +- 4 files changed, 26 insertions(+), 7 deletions(-) diff --git a/internal/event/agent_check_storage.go b/internal/event/agent_check_storage.go index 837656a..358e30e 100644 --- a/internal/event/agent_check_storage.go +++ b/internal/event/agent_check_storage.go @@ -104,7 +104,7 @@ func (t *AgentCheckStorage) Execute(execCtx ExecuteContext) { defer agentClient.Close() evtmsg, err := agtmsg.NewPostEventBody( - agtevt.NewCheckStorage(stg.Directory, isComplete, objects), + agtevt.NewCheckStorage(stg.StorageID, stg.Directory, isComplete, objects), execCtx.Option.IsEmergency, // 继承本任务的执行选项 execCtx.Option.DontMerge) if err != nil { diff --git a/internal/event/update_cache.go b/internal/event/update_cache.go index 22a2283..0a03e69 100644 --- a/internal/event/update_cache.go +++ b/internal/event/update_cache.go @@ -38,15 +38,18 @@ func (t *UpdateCache) Execute(execCtx ExecuteContext) { for _, entry := range t.Entries { switch entry.Operation { - case evtcst.UPDATE_CACHE_UNTEMP: + case evtcst.UPDATE_CACHE_DELETE_TEMP: err := mysql.Cache.DeleteTemp(execCtx.Args.DB.SQLCtx(), entry.FileHash, t.NodeID) - if err != nil { logger.WithField("FileHash", entry.FileHash). WithField("NodeID", t.NodeID). Warnf("delete temp cache failed, err: %s", err.Error()) } + logger.WithField("FileHash", entry.FileHash). + WithField("NodeID", t.NodeID). + Debugf("delete temp cache") + case evtcst.UPDATE_CACHE_CREATE_TEMP: err := mysql.Cache.CreateTemp(execCtx.Args.DB.SQLCtx(), entry.FileHash, t.NodeID) if err != nil { @@ -54,6 +57,10 @@ func (t *UpdateCache) Execute(execCtx ExecuteContext) { WithField("NodeID", t.NodeID). Warnf("create temp cache failed, err: %s", err.Error()) } + + logger.WithField("FileHash", entry.FileHash). + WithField("NodeID", t.NodeID). + Debugf("create temp cache") } } } diff --git a/internal/event/update_storage.go b/internal/event/update_storage.go index e55305c..aeedb36 100644 --- a/internal/event/update_storage.go +++ b/internal/event/update_storage.go @@ -12,9 +12,9 @@ type UpdateStorage struct { scevt.UpdateStorage } -func NewUpdateStorage(dirState string, entries []UpdateStorageEntry) *UpdateStorage { +func NewUpdateStorage(storageID int, dirState string, entries []UpdateStorageEntry) *UpdateStorage { return &UpdateStorage{ - UpdateStorage: scevt.NewUpdateStorage(dirState, entries), + UpdateStorage: scevt.NewUpdateStorage(storageID, dirState, entries), } } @@ -54,6 +54,11 @@ func (t *UpdateStorage) Execute(execCtx ExecuteContext) { } chkObjIDs = append(chkObjIDs, entry.ObjectID) + logger.WithField("StorageID", t.StorageID). + WithField("ObjectID", entry.ObjectID). + WithField("UserID", entry.UserID). + Debugf("delete storage object") + case tskcst.UPDATE_STORAGE_SET_NORMAL: err := mysql.StorageObject.SetStateNormal(execCtx.Args.DB.SQLCtx(), t.StorageID, entry.ObjectID, entry.UserID) if err != nil { @@ -61,6 +66,11 @@ func (t *UpdateStorage) Execute(execCtx ExecuteContext) { WithField("ObjectID", entry.ObjectID). Warnf("change storage object state failed, err: %s", err.Error()) } + + logger.WithField("StorageID", t.StorageID). + WithField("ObjectID", entry.ObjectID). + WithField("UserID", entry.UserID). + Debugf("set storage object normal") } } @@ -70,5 +80,7 @@ func (t *UpdateStorage) Execute(execCtx ExecuteContext) { } func init() { - RegisterMessageConvertor(func(msg scevt.UpdateStorage) Event { return NewUpdateStorage(msg.DirectoryState, msg.Entries) }) + RegisterMessageConvertor(func(msg scevt.UpdateStorage) Event { + return NewUpdateStorage(msg.StorageID, msg.DirectoryState, msg.Entries) + }) } diff --git a/internal/tickevent/check_cache.go b/internal/tickevent/check_cache.go index c03651b..c5037eb 100644 --- a/internal/tickevent/check_cache.go +++ b/internal/tickevent/check_cache.go @@ -9,7 +9,7 @@ import ( type CheckCache struct { } -func NewCheckUnavailableCache() *CheckCache { +func NewCheckCache() *CheckCache { return &CheckCache{} } From 867fbe3ecf7937fd1137b7bbbbf1b64b03aecea1 Mon Sep 17 00:00:00 2001 From: Sydonian <794346190@qq.com> Date: Thu, 25 May 2023 15:21:40 +0800 Subject: [PATCH 35/68] =?UTF-8?q?=E7=A7=BB=E5=8A=A8=E6=97=A5=E5=BF=97?= =?UTF-8?q?=E5=BA=93=E7=9A=84=E4=BD=8D=E7=BD=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- internal/config/config.go | 2 +- internal/event/agent_check_cache.go | 2 +- internal/event/agent_check_state.go | 2 +- internal/event/agent_check_storage.go | 2 +- internal/event/check_cache.go | 2 +- internal/event/check_object.go | 2 +- internal/event/check_rep_count.go | 4 ++-- internal/event/update_agent_state.go | 2 +- internal/event/update_cache.go | 2 +- internal/event/update_storage.go | 2 +- internal/services/event.go | 2 +- internal/tickevent/batch_all_agent_check_cache.go | 2 +- internal/tickevent/batch_check_all_object.go | 2 +- internal/tickevent/batch_check_all_rep_count.go | 2 +- internal/tickevent/batch_check_all_storage.go | 2 +- internal/tickevent/check_agent_state.go | 2 +- internal/tickevent/check_cache.go | 2 +- main.go | 14 +++++++------- 18 files changed, 25 insertions(+), 25 deletions(-) diff --git a/internal/config/config.go b/internal/config/config.go index f7b5b22..0d654e1 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -1,8 +1,8 @@ package config import ( + log "gitlink.org.cn/cloudream/common/pkg/logger" c "gitlink.org.cn/cloudream/common/utils/config" - log "gitlink.org.cn/cloudream/common/utils/logger" db "gitlink.org.cn/cloudream/db/config" racfg "gitlink.org.cn/cloudream/rabbitmq/config" ) diff --git a/internal/event/agent_check_cache.go b/internal/event/agent_check_cache.go index 7ff7dd3..3afa2c5 100644 --- a/internal/event/agent_check_cache.go +++ b/internal/event/agent_check_cache.go @@ -4,7 +4,7 @@ import ( "database/sql" "github.com/samber/lo" - "gitlink.org.cn/cloudream/common/utils/logger" + "gitlink.org.cn/cloudream/common/pkg/logger" "gitlink.org.cn/cloudream/db/model" mysql "gitlink.org.cn/cloudream/db/sql" "gitlink.org.cn/cloudream/scanner/internal/config" diff --git a/internal/event/agent_check_state.go b/internal/event/agent_check_state.go index 98280ae..aa3874a 100644 --- a/internal/event/agent_check_state.go +++ b/internal/event/agent_check_state.go @@ -6,7 +6,7 @@ import ( "github.com/samber/lo" "gitlink.org.cn/cloudream/common/consts" - "gitlink.org.cn/cloudream/common/utils/logger" + "gitlink.org.cn/cloudream/common/pkg/logger" "gitlink.org.cn/cloudream/db/model" mysql "gitlink.org.cn/cloudream/db/sql" agtcli "gitlink.org.cn/cloudream/rabbitmq/client/agent" diff --git a/internal/event/agent_check_storage.go b/internal/event/agent_check_storage.go index 358e30e..27d258f 100644 --- a/internal/event/agent_check_storage.go +++ b/internal/event/agent_check_storage.go @@ -5,7 +5,7 @@ import ( "github.com/samber/lo" "gitlink.org.cn/cloudream/common/consts" - "gitlink.org.cn/cloudream/common/utils/logger" + "gitlink.org.cn/cloudream/common/pkg/logger" "gitlink.org.cn/cloudream/db/model" mysql "gitlink.org.cn/cloudream/db/sql" agtcli "gitlink.org.cn/cloudream/rabbitmq/client/agent" diff --git a/internal/event/check_cache.go b/internal/event/check_cache.go index cc832bc..ea60866 100644 --- a/internal/event/check_cache.go +++ b/internal/event/check_cache.go @@ -7,7 +7,7 @@ import ( "github.com/jmoiron/sqlx" "github.com/samber/lo" "gitlink.org.cn/cloudream/common/consts" - "gitlink.org.cn/cloudream/common/utils/logger" + "gitlink.org.cn/cloudream/common/pkg/logger" "gitlink.org.cn/cloudream/db/model" mysql "gitlink.org.cn/cloudream/db/sql" ) diff --git a/internal/event/check_object.go b/internal/event/check_object.go index f94c795..e2c45c0 100644 --- a/internal/event/check_object.go +++ b/internal/event/check_object.go @@ -2,7 +2,7 @@ package event import ( "github.com/samber/lo" - "gitlink.org.cn/cloudream/common/utils/logger" + "gitlink.org.cn/cloudream/common/pkg/logger" scevt "gitlink.org.cn/cloudream/rabbitmq/message/scanner/event" ) diff --git a/internal/event/check_rep_count.go b/internal/event/check_rep_count.go index c09ff0a..8f7c828 100644 --- a/internal/event/check_rep_count.go +++ b/internal/event/check_rep_count.go @@ -8,8 +8,8 @@ import ( "github.com/jmoiron/sqlx" "github.com/samber/lo" "gitlink.org.cn/cloudream/common/consts" - "gitlink.org.cn/cloudream/common/utils/logger" - log "gitlink.org.cn/cloudream/common/utils/logger" + "gitlink.org.cn/cloudream/common/pkg/logger" + log "gitlink.org.cn/cloudream/common/pkg/logger" mymath "gitlink.org.cn/cloudream/common/utils/math" mysort "gitlink.org.cn/cloudream/common/utils/sort" "gitlink.org.cn/cloudream/scanner/internal/config" diff --git a/internal/event/update_agent_state.go b/internal/event/update_agent_state.go index bef11a3..3b78c6b 100644 --- a/internal/event/update_agent_state.go +++ b/internal/event/update_agent_state.go @@ -2,7 +2,7 @@ package event import ( "gitlink.org.cn/cloudream/common/consts" - "gitlink.org.cn/cloudream/common/utils/logger" + "gitlink.org.cn/cloudream/common/pkg/logger" mysql "gitlink.org.cn/cloudream/db/sql" scevt "gitlink.org.cn/cloudream/rabbitmq/message/scanner/event" ) diff --git a/internal/event/update_cache.go b/internal/event/update_cache.go index 0a03e69..d945977 100644 --- a/internal/event/update_cache.go +++ b/internal/event/update_cache.go @@ -2,7 +2,7 @@ package event import ( evtcst "gitlink.org.cn/cloudream/common/consts/event" - "gitlink.org.cn/cloudream/common/utils/logger" + "gitlink.org.cn/cloudream/common/pkg/logger" mysql "gitlink.org.cn/cloudream/db/sql" scevt "gitlink.org.cn/cloudream/rabbitmq/message/scanner/event" ) diff --git a/internal/event/update_storage.go b/internal/event/update_storage.go index aeedb36..210a475 100644 --- a/internal/event/update_storage.go +++ b/internal/event/update_storage.go @@ -2,7 +2,7 @@ package event import ( tskcst "gitlink.org.cn/cloudream/common/consts/event" - "gitlink.org.cn/cloudream/common/utils/logger" + "gitlink.org.cn/cloudream/common/pkg/logger" mysql "gitlink.org.cn/cloudream/db/sql" scevt "gitlink.org.cn/cloudream/rabbitmq/message/scanner/event" ) diff --git a/internal/services/event.go b/internal/services/event.go index 5264883..5790c20 100644 --- a/internal/services/event.go +++ b/internal/services/event.go @@ -1,7 +1,7 @@ package services import ( - "gitlink.org.cn/cloudream/common/utils/logger" + "gitlink.org.cn/cloudream/common/pkg/logger" scmsg "gitlink.org.cn/cloudream/rabbitmq/message/scanner" scevt "gitlink.org.cn/cloudream/rabbitmq/message/scanner/event" "gitlink.org.cn/cloudream/scanner/internal/event" diff --git a/internal/tickevent/batch_all_agent_check_cache.go b/internal/tickevent/batch_all_agent_check_cache.go index b20552d..da169dd 100644 --- a/internal/tickevent/batch_all_agent_check_cache.go +++ b/internal/tickevent/batch_all_agent_check_cache.go @@ -2,7 +2,7 @@ package tickevent import ( "github.com/samber/lo" - "gitlink.org.cn/cloudream/common/utils/logger" + "gitlink.org.cn/cloudream/common/pkg/logger" "gitlink.org.cn/cloudream/db/model" mysql "gitlink.org.cn/cloudream/db/sql" "gitlink.org.cn/cloudream/scanner/internal/event" diff --git a/internal/tickevent/batch_check_all_object.go b/internal/tickevent/batch_check_all_object.go index b57e3ed..277aa27 100644 --- a/internal/tickevent/batch_check_all_object.go +++ b/internal/tickevent/batch_check_all_object.go @@ -2,7 +2,7 @@ package tickevent import ( "github.com/samber/lo" - "gitlink.org.cn/cloudream/common/utils/logger" + "gitlink.org.cn/cloudream/common/pkg/logger" mysql "gitlink.org.cn/cloudream/db/sql" "gitlink.org.cn/cloudream/scanner/internal/event" ) diff --git a/internal/tickevent/batch_check_all_rep_count.go b/internal/tickevent/batch_check_all_rep_count.go index a0d5626..68f6f29 100644 --- a/internal/tickevent/batch_check_all_rep_count.go +++ b/internal/tickevent/batch_check_all_rep_count.go @@ -1,7 +1,7 @@ package tickevent import ( - "gitlink.org.cn/cloudream/common/utils/logger" + "gitlink.org.cn/cloudream/common/pkg/logger" mysql "gitlink.org.cn/cloudream/db/sql" "gitlink.org.cn/cloudream/scanner/internal/event" ) diff --git a/internal/tickevent/batch_check_all_storage.go b/internal/tickevent/batch_check_all_storage.go index 51f46a8..9635644 100644 --- a/internal/tickevent/batch_check_all_storage.go +++ b/internal/tickevent/batch_check_all_storage.go @@ -1,7 +1,7 @@ package tickevent import ( - "gitlink.org.cn/cloudream/common/utils/logger" + "gitlink.org.cn/cloudream/common/pkg/logger" mysql "gitlink.org.cn/cloudream/db/sql" "gitlink.org.cn/cloudream/scanner/internal/event" ) diff --git a/internal/tickevent/check_agent_state.go b/internal/tickevent/check_agent_state.go index 07ed28d..cc89a48 100644 --- a/internal/tickevent/check_agent_state.go +++ b/internal/tickevent/check_agent_state.go @@ -1,7 +1,7 @@ package tickevent import ( - "gitlink.org.cn/cloudream/common/utils/logger" + "gitlink.org.cn/cloudream/common/pkg/logger" mysql "gitlink.org.cn/cloudream/db/sql" "gitlink.org.cn/cloudream/scanner/internal/event" ) diff --git a/internal/tickevent/check_cache.go b/internal/tickevent/check_cache.go index c5037eb..04160ab 100644 --- a/internal/tickevent/check_cache.go +++ b/internal/tickevent/check_cache.go @@ -1,7 +1,7 @@ package tickevent import ( - "gitlink.org.cn/cloudream/common/utils/logger" + "gitlink.org.cn/cloudream/common/pkg/logger" mysql "gitlink.org.cn/cloudream/db/sql" "gitlink.org.cn/cloudream/scanner/internal/event" ) diff --git a/main.go b/main.go index 026cb57..c1a3253 100644 --- a/main.go +++ b/main.go @@ -5,7 +5,7 @@ import ( "os" "sync" - log "gitlink.org.cn/cloudream/common/utils/logger" + log "gitlink.org.cn/cloudream/common/pkg/logger" "gitlink.org.cn/cloudream/db" scsvr "gitlink.org.cn/cloudream/rabbitmq/server/scanner" "gitlink.org.cn/cloudream/scanner/internal/config" @@ -87,15 +87,15 @@ func serveScannerServer(server *scsvr.ScannerServer, wg *sync.WaitGroup) { func startTickEvent(tickExecutor *tickevent.Executor) { // TODO 可以考虑增加配置文件，配置这些任务间隔时间 - tickExecutor.Start(tickevent.NewBatchAllAgentCheckCache(), 5*60*100) + tickExecutor.Start(tickevent.NewBatchAllAgentCheckCache(), 5*60*1000) - tickExecutor.Start(tickevent.NewBatchCheckAllObject(), 5*60*100) + tickExecutor.Start(tickevent.NewBatchCheckAllObject(), 5*60*1000) - tickExecutor.Start(tickevent.NewBatchCheckAllRepCount(), 5*60*100) + tickExecutor.Start(tickevent.NewBatchCheckAllRepCount(), 5*60*1000) - tickExecutor.Start(tickevent.NewBatchCheckAllStorage(), 5*60*100) + tickExecutor.Start(tickevent.NewBatchCheckAllStorage(), 5*60*1000) - tickExecutor.Start(tickevent.NewCheckAgentState(), 5*60*100) + tickExecutor.Start(tickevent.NewCheckAgentState(), 5*60*1000) - tickExecutor.Start(tickevent.NewCheckUnavailableCache(), 5*60*100) + tickExecutor.Start(tickevent.NewCheckCache(), 5*60*1000) } From 9ecfa96e2d207c102bbfc84c558f64192e57ae8e Mon Sep 17 00:00:00 2001 From: Sydonian <794346190@qq.com> Date: Fri, 26 May 2023 09:48:34 +0800 Subject: [PATCH 36/68] =?UTF-8?q?=E4=BA=8B=E4=BB=B6=E5=A2=9E=E5=8A=A0?= =?UTF-8?q?=E8=B0=83=E8=AF=95=E6=97=A5=E5=BF=97?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- internal/event/agent_check_cache.go | 21 ++++++++++--------- internal/event/agent_check_state.go | 19 +++++++---------- internal/event/agent_check_storage.go | 17 ++++++++------- internal/event/check_cache.go | 9 +++----- internal/event/check_object.go | 5 +++-- internal/event/check_rep_count.go | 6 ++++-- internal/event/update_agent_state.go | 9 ++++---- internal/event/update_cache.go | 11 +++++----- internal/event/update_storage.go | 13 ++++++------ .../tickevent/batch_all_agent_check_cache.go | 7 +++++-- internal/tickevent/batch_check_all_object.go | 7 +++++-- .../tickevent/batch_check_all_rep_count.go | 7 +++++-- internal/tickevent/batch_check_all_storage.go | 7 +++++-- internal/tickevent/check_agent_state.go | 5 ++++- internal/tickevent/check_cache.go | 5 ++++- main.go | 2 +- 16 files changed, 85 insertions(+), 65 deletions(-) diff --git a/internal/event/agent_check_cache.go b/internal/event/agent_check_cache.go index 3afa2c5..380d039 100644 --- a/internal/event/agent_check_cache.go +++ b/internal/event/agent_check_cache.go @@ -32,6 +32,10 @@ func (t *AgentCheckCache) TryMerge(other Event) bool { return false } + if event.NodeID != t.NodeID { + return false + } + // FileHashes为nil时代表全量检查 if event.FileHashes == nil { t.FileHashes = nil @@ -43,7 +47,8 @@ func (t *AgentCheckCache) TryMerge(other Event) bool { } func (t *AgentCheckCache) Execute(execCtx ExecuteContext) { - logger.Debugf("begin agent check cache") + log := logger.WithType[AgentCheckCache]("Event") + log.Debugf("begin with %v", logger.FormatStruct(t)) var isComplete bool var caches []model.Cache @@ -54,7 +59,7 @@ func (t *AgentCheckCache) Execute(execCtx ExecuteContext) { var err error caches, err = mysql.Cache.GetNodeCaches(execCtx.Args.DB.SQLCtx(), t.NodeID) if err != nil { - logger.WithField("NodeID", t.NodeID).Warnf("get node caches failed, err: %s", err.Error()) + log.WithField("NodeID", t.NodeID).Warnf("get node caches failed, err: %s", err.Error()) return } isComplete = true @@ -68,7 +73,7 @@ func (t *AgentCheckCache) Execute(execCtx ExecuteContext) { } if err != nil { - logger.WithField("FileHash", hash).WithField("NodeID", t.NodeID).Warnf("get cache failed, err: %s", err.Error()) + log.WithField("FileHash", hash).WithField("NodeID", t.NodeID).Warnf("get cache failed, err: %s", err.Error()) return } @@ -80,7 +85,7 @@ func (t *AgentCheckCache) Execute(execCtx ExecuteContext) { // 然后向代理端发送移动文件的请求 agentClient, err := agtcli.NewAgentClient(t.NodeID, &config.Cfg().RabbitMQ) if err != nil { - logger.WithField("NodeID", t.NodeID).Warnf("create agent client failed, err: %s", err.Error()) + log.WithField("NodeID", t.NodeID).Warnf("create agent client failed, err: %s", err.Error()) return } defer agentClient.Close() @@ -90,17 +95,13 @@ func (t *AgentCheckCache) Execute(execCtx ExecuteContext) { execCtx.Option.IsEmergency, // 继承本任务的执行选项 execCtx.Option.DontMerge) if err != nil { - logger.Warnf("new post event body failed, err: %s", err.Error()) + log.Warnf("new post event body failed, err: %s", err.Error()) return } err = agentClient.PostEvent(evtmsg) if err != nil { - logger.WithField("NodeID", t.NodeID).Warnf("request to agent failed, err: %s", err.Error()) + log.WithField("NodeID", t.NodeID).Warnf("request to agent failed, err: %s", err.Error()) return } } - -//func init() { -// Register(func(msg scevt.AgentCheckCache) Event { return NewAgentCheckCache(msg.NodeID, msg.FileHashes) }) -//} diff --git a/internal/event/agent_check_state.go b/internal/event/agent_check_state.go index aa3874a..811cdd2 100644 --- a/internal/event/agent_check_state.go +++ b/internal/event/agent_check_state.go @@ -35,7 +35,8 @@ func (t *AgentCheckState) TryMerge(other Event) bool { } func (t *AgentCheckState) Execute(execCtx ExecuteContext) { - logger.Debugf("begin agent check state") + log := logger.WithType[AgentCheckState]("Event") + log.Debugf("begin with %v", logger.FormatStruct(t)) node, err := mysql.Node.GetByID(execCtx.Args.DB.SQLCtx(), t.NodeID) if err == sql.ErrNoRows { @@ -43,7 +44,7 @@ func (t *AgentCheckState) Execute(execCtx ExecuteContext) { } if err != nil { - logger.WithField("NodeID", t.NodeID).Warnf("get node by id failed, err: %s", err.Error()) + log.WithField("NodeID", t.NodeID).Warnf("get node by id failed, err: %s", err.Error()) return } @@ -56,13 +57,13 @@ func (t *AgentCheckState) Execute(execCtx ExecuteContext) { if node.LastReportTime == nil && time.Since(*node.LastReportTime) > time.Duration(config.Cfg().NodeUnavailableSeconds)*time.Second { err := mysql.Node.ChangeState(execCtx.Args.DB.SQLCtx(), t.NodeID, consts.NODE_STATE_UNAVAILABLE) if err != nil { - logger.WithField("NodeID", t.NodeID).Warnf("set node state failed, err: %s", err.Error()) + log.WithField("NodeID", t.NodeID).Warnf("set node state failed, err: %s", err.Error()) return } caches, err := mysql.Cache.GetNodeCaches(execCtx.Args.DB.SQLCtx(), t.NodeID) if err != nil { - logger.WithField("NodeID", t.NodeID).Warnf("get node caches failed, err: %s", err.Error()) + log.WithField("NodeID", t.NodeID).Warnf("get node caches failed, err: %s", err.Error()) return } @@ -73,7 +74,7 @@ func (t *AgentCheckState) Execute(execCtx ExecuteContext) { agentClient, err := agtcli.NewAgentClient(t.NodeID, &config.Cfg().RabbitMQ) if err != nil { - logger.WithField("NodeID", t.NodeID).Warnf("create agent client failed, err: %s", err.Error()) + log.WithField("NodeID", t.NodeID).Warnf("create agent client failed, err: %s", err.Error()) return } defer agentClient.Close() @@ -81,15 +82,11 @@ func (t *AgentCheckState) Execute(execCtx ExecuteContext) { // 紧急任务 evtmsg, err := agtmsg.NewPostEventBody(agtevt.NewCheckState(), true, true) if err != nil { - logger.Warnf("new post event body failed, err: %s", err.Error()) + log.Warnf("new post event body failed, err: %s", err.Error()) return } err = agentClient.PostEvent(evtmsg) if err != nil { - logger.WithField("NodeID", t.NodeID).Warnf("request to agent failed, err: %s", err.Error()) + log.WithField("NodeID", t.NodeID).Warnf("request to agent failed, err: %s", err.Error()) } } - -// func init() { -// Register(func(msg scevt.AgentCheckState) Event { return NewAgentCheckState(msg.NodeID) }) -// } diff --git a/internal/event/agent_check_storage.go b/internal/event/agent_check_storage.go index 27d258f..58968e0 100644 --- a/internal/event/agent_check_storage.go +++ b/internal/event/agent_check_storage.go @@ -46,12 +46,13 @@ func (t *AgentCheckStorage) TryMerge(other Event) bool { } func (t *AgentCheckStorage) Execute(execCtx ExecuteContext) { - logger.Debugf("begin agent check storage") + log := logger.WithType[AgentCheckStorage]("Event") + log.Debugf("begin with %v", logger.FormatStruct(t)) stg, err := mysql.Storage.GetByID(execCtx.Args.DB.SQLCtx(), t.StorageID) if err != nil { if err != sql.ErrNoRows { - logger.WithField("StorageID", t.StorageID).Warnf("get storage failed, err: %s", err.Error()) + log.WithField("StorageID", t.StorageID).Warnf("get storage failed, err: %s", err.Error()) } return } @@ -59,7 +60,7 @@ func (t *AgentCheckStorage) Execute(execCtx ExecuteContext) { node, err := mysql.Node.GetByID(execCtx.Args.DB.SQLCtx(), stg.NodeID) if err != nil { if err != sql.ErrNoRows { - logger.WithField("StorageID", t.StorageID).Warnf("get storage node failed, err: %s", err.Error()) + log.WithField("StorageID", t.StorageID).Warnf("get storage node failed, err: %s", err.Error()) } return } @@ -76,7 +77,7 @@ func (t *AgentCheckStorage) Execute(execCtx ExecuteContext) { var err error objects, err = mysql.StorageObject.GetAllByStorageID(execCtx.Args.DB.SQLCtx(), t.StorageID) if err != nil { - logger.WithField("StorageID", t.StorageID).Warnf("get storage objects failed, err: %s", err.Error()) + log.WithField("StorageID", t.StorageID).Warnf("get storage objects failed, err: %s", err.Error()) return } isComplete = true @@ -84,7 +85,7 @@ func (t *AgentCheckStorage) Execute(execCtx ExecuteContext) { for _, objID := range t.ObjectIDs { objs, err := mysql.StorageObject.GetAllByStorageAndObjectID(execCtx.Args.DB.SQLCtx(), t.StorageID, objID) if err != nil { - logger.WithField("StorageID", t.StorageID). + log.WithField("StorageID", t.StorageID). WithField("ObjectID", objID). Warnf("get storage object failed, err: %s", err.Error()) return @@ -98,7 +99,7 @@ func (t *AgentCheckStorage) Execute(execCtx ExecuteContext) { // 投递任务 agentClient, err := agtcli.NewAgentClient(stg.NodeID, &config.Cfg().RabbitMQ) if err != nil { - logger.WithField("NodeID", stg.NodeID).Warnf("create agent client failed, err: %s", err.Error()) + log.WithField("NodeID", stg.NodeID).Warnf("create agent client failed, err: %s", err.Error()) return } defer agentClient.Close() @@ -108,13 +109,13 @@ func (t *AgentCheckStorage) Execute(execCtx ExecuteContext) { execCtx.Option.IsEmergency, // 继承本任务的执行选项 execCtx.Option.DontMerge) if err != nil { - logger.Warnf("new post event body failed, err: %s", err.Error()) + log.Warnf("new post event body failed, err: %s", err.Error()) return } err = agentClient.PostEvent(evtmsg) if err != nil { - logger.WithField("NodeID", stg.NodeID).Warnf("request to agent failed, err: %s", err.Error()) + log.WithField("NodeID", stg.NodeID).Warnf("request to agent failed, err: %s", err.Error()) } } diff --git a/internal/event/check_cache.go b/internal/event/check_cache.go index ea60866..c61000e 100644 --- a/internal/event/check_cache.go +++ b/internal/event/check_cache.go @@ -35,7 +35,8 @@ func (t *CheckCache) TryMerge(other Event) bool { } func (t *CheckCache) Execute(execCtx ExecuteContext) { - logger.Debugf("begin check cache") + log := logger.WithType[AgentCheckStorage]("Event") + log.Debugf("begin with %v", logger.FormatStruct(t)) err := execCtx.Args.DB.DoTx(sql.LevelSerializable, func(tx *sqlx.Tx) error { node, err := mysql.Node.GetByID(tx, t.NodeID) @@ -65,10 +66,6 @@ func (t *CheckCache) Execute(execCtx ExecuteContext) { }) if err != nil { - logger.WithField("NodeID", t.NodeID).Warn(err.Error()) + log.WithField("NodeID", t.NodeID).Warn(err.Error()) } } - -// func init() { -// Register(func(msg scevt.CheckCache) Event { return NewCheckCache(msg.NodeID) }) -// } diff --git a/internal/event/check_object.go b/internal/event/check_object.go index e2c45c0..85e9d78 100644 --- a/internal/event/check_object.go +++ b/internal/event/check_object.go @@ -27,12 +27,13 @@ func (t *CheckObject) TryMerge(other Event) bool { } func (t *CheckObject) Execute(execCtx ExecuteContext) { - logger.Debugf("begin check object") + log := logger.WithType[CheckObject]("Event") + log.Debugf("begin with %v", logger.FormatStruct(t)) for _, objID := range t.ObjectIDs { err := execCtx.Args.DB.Object().DeleteUnused(execCtx.Args.DB.SQLCtx(), objID) if err != nil { - logger.WithField("ObjectID", objID).Warnf("delete unused object failed, err: %s", err.Error()) + log.WithField("ObjectID", objID).Warnf("delete unused object failed, err: %s", err.Error()) } } } diff --git a/internal/event/check_rep_count.go b/internal/event/check_rep_count.go index 8f7c828..ac3d237 100644 --- a/internal/event/check_rep_count.go +++ b/internal/event/check_rep_count.go @@ -9,7 +9,6 @@ import ( "github.com/samber/lo" "gitlink.org.cn/cloudream/common/consts" "gitlink.org.cn/cloudream/common/pkg/logger" - log "gitlink.org.cn/cloudream/common/pkg/logger" mymath "gitlink.org.cn/cloudream/common/utils/math" mysort "gitlink.org.cn/cloudream/common/utils/sort" "gitlink.org.cn/cloudream/scanner/internal/config" @@ -40,7 +39,8 @@ func (t *CheckRepCount) TryMerge(other Event) bool { } func (t *CheckRepCount) Execute(execCtx ExecuteContext) { - logger.Debugf("begin check rep count") + log := logger.WithType[CheckRepCount]("Event") + log.Debugf("begin with %v", logger.FormatStruct(t)) updatedNodeAndHashes := make(map[int][]string) @@ -64,6 +64,8 @@ func (t *CheckRepCount) Execute(execCtx ExecuteContext) { } func (t *CheckRepCount) checkOneRepCount(fileHash string, execCtx ExecuteContext) ([]int, error) { + log := logger.WithType[CheckRepCount]("Event") + var updatedNodeIDs []int err := execCtx.Args.DB.DoTx(sql.LevelSerializable, func(tx *sqlx.Tx) error { repMaxCnt, err := mysql.ObjectRep.GetFileMaxRepCount(tx, fileHash) diff --git a/internal/event/update_agent_state.go b/internal/event/update_agent_state.go index 3b78c6b..1669738 100644 --- a/internal/event/update_agent_state.go +++ b/internal/event/update_agent_state.go @@ -22,14 +22,15 @@ func (t *UpdateAgentState) TryMerge(other Event) bool { } func (t *UpdateAgentState) Execute(execCtx ExecuteContext) { - logger.Debugf("begin update agent state") + log := logger.WithType[UpdateAgentState]("Event") + log.Debugf("begin with %v", logger.FormatStruct(t)) if t.IPFSState != consts.IPFS_STATUS_OK { - logger.WithField("NodeID", t.NodeID).Warnf("IPFS status is %s, set node state unavailable", t.IPFSState) + log.WithField("NodeID", t.NodeID).Warnf("IPFS status is %s, set node state unavailable", t.IPFSState) err := mysql.Node.ChangeState(execCtx.Args.DB.SQLCtx(), t.NodeID, consts.NODE_STATE_UNAVAILABLE) if err != nil { - logger.WithField("NodeID", t.NodeID).Warnf("change node state failed, err: %s", err.Error()) + log.WithField("NodeID", t.NodeID).Warnf("change node state failed, err: %s", err.Error()) } return } @@ -37,7 +38,7 @@ func (t *UpdateAgentState) Execute(execCtx ExecuteContext) { // TODO 如果以后还有其他的状态，要判断哪些状态下能设置Normal err := mysql.Node.ChangeState(execCtx.Args.DB.SQLCtx(), t.NodeID, consts.NODE_STATE_NORMAL) if err != nil { - logger.WithField("NodeID", t.NodeID).Warnf("change node state failed, err: %s", err.Error()) + log.WithField("NodeID", t.NodeID).Warnf("change node state failed, err: %s", err.Error()) } } diff --git a/internal/event/update_cache.go b/internal/event/update_cache.go index d945977..a72fa3f 100644 --- a/internal/event/update_cache.go +++ b/internal/event/update_cache.go @@ -34,31 +34,32 @@ func (t *UpdateCache) TryMerge(other Event) bool { } func (t *UpdateCache) Execute(execCtx ExecuteContext) { - logger.Debugf("begin update cache") + log := logger.WithType[UpdateCache]("Event") + log.Debugf("begin with %v", logger.FormatStruct(t)) for _, entry := range t.Entries { switch entry.Operation { case evtcst.UPDATE_CACHE_DELETE_TEMP: err := mysql.Cache.DeleteTemp(execCtx.Args.DB.SQLCtx(), entry.FileHash, t.NodeID) if err != nil { - logger.WithField("FileHash", entry.FileHash). + log.WithField("FileHash", entry.FileHash). WithField("NodeID", t.NodeID). Warnf("delete temp cache failed, err: %s", err.Error()) } - logger.WithField("FileHash", entry.FileHash). + log.WithField("FileHash", entry.FileHash). WithField("NodeID", t.NodeID). Debugf("delete temp cache") case evtcst.UPDATE_CACHE_CREATE_TEMP: err := mysql.Cache.CreateTemp(execCtx.Args.DB.SQLCtx(), entry.FileHash, t.NodeID) if err != nil { - logger.WithField("FileHash", entry.FileHash). + log.WithField("FileHash", entry.FileHash). WithField("NodeID", t.NodeID). Warnf("create temp cache failed, err: %s", err.Error()) } - logger.WithField("FileHash", entry.FileHash). + log.WithField("FileHash", entry.FileHash). WithField("NodeID", t.NodeID). Debugf("create temp cache") } diff --git a/internal/event/update_storage.go b/internal/event/update_storage.go index 210a475..61e9851 100644 --- a/internal/event/update_storage.go +++ b/internal/event/update_storage.go @@ -35,11 +35,12 @@ func (t *UpdateStorage) TryMerge(other Event) bool { } func (t *UpdateStorage) Execute(execCtx ExecuteContext) { - logger.Debugf("begin update storage") + log := logger.WithType[UpdateStorage]("Event") + log.Debugf("begin with %v", logger.FormatStruct(t)) err := mysql.Storage.ChangeState(execCtx.Args.DB.SQLCtx(), t.StorageID, t.DirectoryState) if err != nil { - logger.WithField("StorageID", t.StorageID).Warnf("change storage state failed, err: %s", err.Error()) + log.WithField("StorageID", t.StorageID).Warnf("change storage state failed, err: %s", err.Error()) } var chkObjIDs []int @@ -48,13 +49,13 @@ func (t *UpdateStorage) Execute(execCtx ExecuteContext) { case tskcst.UPDATE_STORAGE_DELETE: err := mysql.StorageObject.Delete(execCtx.Args.DB.SQLCtx(), t.StorageID, entry.ObjectID, entry.UserID) if err != nil { - logger.WithField("StorageID", t.StorageID). + log.WithField("StorageID", t.StorageID). WithField("ObjectID", entry.ObjectID). Warnf("delete storage object failed, err: %s", err.Error()) } chkObjIDs = append(chkObjIDs, entry.ObjectID) - logger.WithField("StorageID", t.StorageID). + log.WithField("StorageID", t.StorageID). WithField("ObjectID", entry.ObjectID). WithField("UserID", entry.UserID). Debugf("delete storage object") @@ -62,12 +63,12 @@ func (t *UpdateStorage) Execute(execCtx ExecuteContext) { case tskcst.UPDATE_STORAGE_SET_NORMAL: err := mysql.StorageObject.SetStateNormal(execCtx.Args.DB.SQLCtx(), t.StorageID, entry.ObjectID, entry.UserID) if err != nil { - logger.WithField("StorageID", t.StorageID). + log.WithField("StorageID", t.StorageID). WithField("ObjectID", entry.ObjectID). Warnf("change storage object state failed, err: %s", err.Error()) } - logger.WithField("StorageID", t.StorageID). + log.WithField("StorageID", t.StorageID). WithField("ObjectID", entry.ObjectID). WithField("UserID", entry.UserID). Debugf("set storage object normal") diff --git a/internal/tickevent/batch_all_agent_check_cache.go b/internal/tickevent/batch_all_agent_check_cache.go index da169dd..748dc50 100644 --- a/internal/tickevent/batch_all_agent_check_cache.go +++ b/internal/tickevent/batch_all_agent_check_cache.go @@ -19,16 +19,19 @@ func NewBatchAllAgentCheckCache() *BatchAllAgentCheckCache { } func (e *BatchAllAgentCheckCache) Execute(ctx ExecuteContext) { + log := logger.WithType[BatchAllAgentCheckCache]("TickEvent") + log.Debugf("begin") + if e.nodeIDs == nil || len(e.nodeIDs) == 0 { nodes, err := mysql.Node.GetAllNodes(ctx.Args.DB.SQLCtx()) if err != nil { - logger.Warnf("get all nodes failed, err: %s", err.Error()) + log.Warnf("get all nodes failed, err: %s", err.Error()) return } e.nodeIDs = lo.Map(nodes, func(node model.Node, index int) int { return node.NodeID }) - logger.Debugf("new check start, get all nodes") + log.Debugf("new check start, get all nodes") } checkedCnt := 0 diff --git a/internal/tickevent/batch_check_all_object.go b/internal/tickevent/batch_check_all_object.go index 277aa27..9d759c7 100644 --- a/internal/tickevent/batch_check_all_object.go +++ b/internal/tickevent/batch_check_all_object.go @@ -18,9 +18,12 @@ func NewBatchCheckAllObject() *BatchCheckAllObject { } func (e *BatchCheckAllObject) Execute(ctx ExecuteContext) { + log := logger.WithType[BatchCheckAllObject]("TickEvent") + log.Debugf("begin") + objectIDs, err := mysql.Object.BatchGetAllObjectIDs(ctx.Args.DB.SQLCtx(), e.lastCheckStart, CHECK_OBJECT_BATCH_SIZE) if err != nil { - logger.Warnf("batch get object ids failed, err: %s", err.Error()) + log.Warnf("batch get object ids failed, err: %s", err.Error()) return } @@ -30,7 +33,7 @@ func (e *BatchCheckAllObject) Execute(ctx ExecuteContext) { // 如果结果的长度小于预期的长度，则认为已经查询了所有，下次从头再来 if len(objectIDs) < CHECK_OBJECT_BATCH_SIZE { e.lastCheckStart = 0 - logger.Debugf("all object checked, next time will start check at 0") + log.Debugf("all object checked, next time will start check at 0") } else { e.lastCheckStart += CHECK_OBJECT_BATCH_SIZE diff --git a/internal/tickevent/batch_check_all_rep_count.go b/internal/tickevent/batch_check_all_rep_count.go index 68f6f29..07f476d 100644 --- a/internal/tickevent/batch_check_all_rep_count.go +++ b/internal/tickevent/batch_check_all_rep_count.go @@ -17,9 +17,12 @@ func NewBatchCheckAllRepCount() *BatchCheckAllRepCount { } func (e *BatchCheckAllRepCount) Execute(ctx ExecuteContext) { + log := logger.WithType[BatchCheckAllRepCount]("TickEvent") + log.Debugf("begin") + fileHashes, err := mysql.Cache.BatchGetAllFileHashes(ctx.Args.DB.SQLCtx(), e.lastCheckStart, CHECK_CACHE_BATCH_SIZE) if err != nil { - logger.Warnf("batch get file hashes failed, err: %s", err.Error()) + log.Warnf("batch get file hashes failed, err: %s", err.Error()) return } @@ -28,7 +31,7 @@ func (e *BatchCheckAllRepCount) Execute(ctx ExecuteContext) { // 如果结果的长度小于预期的长度，则认为已经查询了所有，下次从头再来 if len(fileHashes) < CHECK_CACHE_BATCH_SIZE { e.lastCheckStart = 0 - logger.Debugf("all rep count checked, next time will start check at 0") + log.Debugf("all rep count checked, next time will start check at 0") } else { e.lastCheckStart += CHECK_CACHE_BATCH_SIZE diff --git a/internal/tickevent/batch_check_all_storage.go b/internal/tickevent/batch_check_all_storage.go index 9635644..1142f04 100644 --- a/internal/tickevent/batch_check_all_storage.go +++ b/internal/tickevent/batch_check_all_storage.go @@ -17,9 +17,12 @@ func NewBatchCheckAllStorage() *BatchCheckAllStorage { } func (e *BatchCheckAllStorage) Execute(ctx ExecuteContext) { + log := logger.WithType[BatchCheckAllStorage]("TickEvent") + log.Debugf("begin") + storageIDs, err := mysql.Storage.BatchGetAllStorageIDs(ctx.Args.DB.SQLCtx(), e.lastCheckStart, CHECK_STORAGE_BATCH_SIZE) if err != nil { - logger.Warnf("batch get storage ids failed, err: %s", err.Error()) + log.Warnf("batch get storage ids failed, err: %s", err.Error()) return } @@ -32,7 +35,7 @@ func (e *BatchCheckAllStorage) Execute(ctx ExecuteContext) { // 如果结果的长度小于预期的长度，则认为已经查询了所有，下次从头再来 if len(storageIDs) < CHECK_STORAGE_BATCH_SIZE { e.lastCheckStart = 0 - logger.Debugf("all storage checked, next time will start check at 0") + log.Debugf("all storage checked, next time will start check at 0") } else { e.lastCheckStart += CHECK_STORAGE_BATCH_SIZE diff --git a/internal/tickevent/check_agent_state.go b/internal/tickevent/check_agent_state.go index cc89a48..8300a4d 100644 --- a/internal/tickevent/check_agent_state.go +++ b/internal/tickevent/check_agent_state.go @@ -14,9 +14,12 @@ func NewCheckAgentState() *CheckAgentState { } func (e *CheckAgentState) Execute(ctx ExecuteContext) { + log := logger.WithType[CheckAgentState]("TickEvent") + log.Debugf("begin") + nodes, err := mysql.Node.GetAllNodes(ctx.Args.DB.SQLCtx()) if err != nil { - logger.Warnf("get all nodes failed, err: %s", err.Error()) + log.Warnf("get all nodes failed, err: %s", err.Error()) return } diff --git a/internal/tickevent/check_cache.go b/internal/tickevent/check_cache.go index 04160ab..5628027 100644 --- a/internal/tickevent/check_cache.go +++ b/internal/tickevent/check_cache.go @@ -14,9 +14,12 @@ func NewCheckCache() *CheckCache { } func (e *CheckCache) Execute(ctx ExecuteContext) { + log := logger.WithType[CheckCache]("TickEvent") + log.Debugf("begin") + nodes, err := mysql.Node.GetAllNodes(ctx.Args.DB.SQLCtx()) if err != nil { - logger.Warnf("get all nodes failed, err: %s", err.Error()) + log.Warnf("get all nodes failed, err: %s", err.Error()) return } diff --git a/main.go b/main.go index c1a3253..d47fa63 100644 --- a/main.go +++ b/main.go @@ -95,7 +95,7 @@ func startTickEvent(tickExecutor *tickevent.Executor) { tickExecutor.Start(tickevent.NewBatchCheckAllStorage(), 5*60*1000) - tickExecutor.Start(tickevent.NewCheckAgentState(), 5*60*1000) + //tickExecutor.Start(tickevent.NewCheckAgentState(), 5*60*1000) tickExecutor.Start(tickevent.NewCheckCache(), 5*60*1000) } From 5010436332b51ffdea669967ea0e252ae14006e6 Mon Sep 17 00:00:00 2001 From: Sydonian <794346190@qq.com> Date: Fri, 26 May 2023 14:08:22 +0800 Subject: [PATCH 37/68] =?UTF-8?q?=E4=BC=98=E5=8C=96PostEvent=E7=9A=84?= =?UTF-8?q?=E6=8E=A5=E5=8F=A3=EF=BC=9Bclient=E5=A2=9E=E5=8A=A0=E7=9B=B4?= =?UTF-8?q?=E6=8E=A5=E8=A7=A6=E5=8F=91=E4=BA=8B=E4=BB=B6=E7=9A=84=E6=8E=A5?= =?UTF-8?q?=E5=8F=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- internal/event/agent_check_cache.go | 13 +++++-------- internal/event/agent_check_state.go | 13 ++++++------- internal/event/agent_check_storage.go | 9 +-------- internal/event/check_cache.go | 5 +++++ 4 files changed, 17 insertions(+), 23 deletions(-) diff --git a/internal/event/agent_check_cache.go b/internal/event/agent_check_cache.go index 380d039..458f4c1 100644 --- a/internal/event/agent_check_cache.go +++ b/internal/event/agent_check_cache.go @@ -10,8 +10,8 @@ import ( "gitlink.org.cn/cloudream/scanner/internal/config" agtcli "gitlink.org.cn/cloudream/rabbitmq/client/agent" - agtmsg "gitlink.org.cn/cloudream/rabbitmq/message/agent" agtevt "gitlink.org.cn/cloudream/rabbitmq/message/agent/event" + scevt "gitlink.org.cn/cloudream/rabbitmq/message/scanner/event" ) type AgentCheckCache struct { @@ -90,18 +90,15 @@ func (t *AgentCheckCache) Execute(execCtx ExecuteContext) { } defer agentClient.Close() - evtmsg, err := agtmsg.NewPostEventBody( + err = agentClient.PostEvent( agtevt.NewCheckCache(isComplete, caches), execCtx.Option.IsEmergency, // 继承本任务的执行选项 execCtx.Option.DontMerge) - if err != nil { - log.Warnf("new post event body failed, err: %s", err.Error()) - return - } - - err = agentClient.PostEvent(evtmsg) if err != nil { log.WithField("NodeID", t.NodeID).Warnf("request to agent failed, err: %s", err.Error()) return } } +func init() { + RegisterMessageConvertor(func(msg scevt.AgentCheckCache) Event { return NewAgentCheckCache(msg.NodeID, msg.FileHashes) }) +} diff --git a/internal/event/agent_check_state.go b/internal/event/agent_check_state.go index 811cdd2..fcefd84 100644 --- a/internal/event/agent_check_state.go +++ b/internal/event/agent_check_state.go @@ -10,8 +10,8 @@ import ( "gitlink.org.cn/cloudream/db/model" mysql "gitlink.org.cn/cloudream/db/sql" agtcli "gitlink.org.cn/cloudream/rabbitmq/client/agent" - agtmsg "gitlink.org.cn/cloudream/rabbitmq/message/agent" agtevt "gitlink.org.cn/cloudream/rabbitmq/message/agent/event" + scevt "gitlink.org.cn/cloudream/rabbitmq/message/scanner/event" "gitlink.org.cn/cloudream/scanner/internal/config" ) @@ -80,13 +80,12 @@ func (t *AgentCheckState) Execute(execCtx ExecuteContext) { defer agentClient.Close() // 紧急任务 - evtmsg, err := agtmsg.NewPostEventBody(agtevt.NewCheckState(), true, true) - if err != nil { - log.Warnf("new post event body failed, err: %s", err.Error()) - return - } - err = agentClient.PostEvent(evtmsg) + err = agentClient.PostEvent(agtevt.NewCheckState(), true, true) if err != nil { log.WithField("NodeID", t.NodeID).Warnf("request to agent failed, err: %s", err.Error()) } } + +func init() { + RegisterMessageConvertor(func(msg scevt.AgentCheckState) Event { return NewAgentCheckState(msg.NodeID) }) +} diff --git a/internal/event/agent_check_storage.go b/internal/event/agent_check_storage.go index 58968e0..ea01a5e 100644 --- a/internal/event/agent_check_storage.go +++ b/internal/event/agent_check_storage.go @@ -9,7 +9,6 @@ import ( "gitlink.org.cn/cloudream/db/model" mysql "gitlink.org.cn/cloudream/db/sql" agtcli "gitlink.org.cn/cloudream/rabbitmq/client/agent" - agtmsg "gitlink.org.cn/cloudream/rabbitmq/message/agent" agtevt "gitlink.org.cn/cloudream/rabbitmq/message/agent/event" scevt "gitlink.org.cn/cloudream/rabbitmq/message/scanner/event" "gitlink.org.cn/cloudream/scanner/internal/config" @@ -104,16 +103,10 @@ func (t *AgentCheckStorage) Execute(execCtx ExecuteContext) { } defer agentClient.Close() - evtmsg, err := agtmsg.NewPostEventBody( + err = agentClient.PostEvent( agtevt.NewCheckStorage(stg.StorageID, stg.Directory, isComplete, objects), execCtx.Option.IsEmergency, // 继承本任务的执行选项 execCtx.Option.DontMerge) - if err != nil { - log.Warnf("new post event body failed, err: %s", err.Error()) - return - } - - err = agentClient.PostEvent(evtmsg) if err != nil { log.WithField("NodeID", stg.NodeID).Warnf("request to agent failed, err: %s", err.Error()) } diff --git a/internal/event/check_cache.go b/internal/event/check_cache.go index c61000e..2760d02 100644 --- a/internal/event/check_cache.go +++ b/internal/event/check_cache.go @@ -10,6 +10,7 @@ import ( "gitlink.org.cn/cloudream/common/pkg/logger" "gitlink.org.cn/cloudream/db/model" mysql "gitlink.org.cn/cloudream/db/sql" + scevt "gitlink.org.cn/cloudream/rabbitmq/message/scanner/event" ) type CheckCache struct { @@ -69,3 +70,7 @@ func (t *CheckCache) Execute(execCtx ExecuteContext) { log.WithField("NodeID", t.NodeID).Warn(err.Error()) } } + +func init() { + RegisterMessageConvertor(func(msg scevt.CheckCache) Event { return NewCheckCache(msg.NodeID) }) +} From 77372d91a6e285cfcfbaea9d7081b36cb1d945af Mon Sep 17 00:00:00 2001 From: Sydonian <794346190@qq.com> Date: Tue, 30 May 2023 15:49:37 +0800 Subject: [PATCH 38/68] =?UTF-8?q?=E5=B0=86=E6=89=80=E6=9C=89=E5=91=BD?= =?UTF-8?q?=E4=BB=A4=E9=83=BD=E4=BD=BF=E7=94=A8cmdtrie=E6=9D=A5=E7=AE=A1?= =?UTF-8?q?=E7=90=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- internal/event/check_rep_count.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/event/check_rep_count.go b/internal/event/check_rep_count.go index ac3d237..ffdc37e 100644 --- a/internal/event/check_rep_count.go +++ b/internal/event/check_rep_count.go @@ -134,7 +134,7 @@ func (t *CheckRepCount) checkOneRepCount(fileHash string, execCtx ExecuteContext } for _, node := range newNodes { - err := mysql.Cache.Create(tx, fileHash, node.NodeID) + err := mysql.Cache.CreatePinned(tx, fileHash, node.NodeID) if err != nil { return fmt.Errorf("create cache failed, err: %w", err) } From 015bfd8c1c7e7e6651574a2cff06852cb59cf4c2 Mon Sep 17 00:00:00 2001 From: Sydonian <794346190@qq.com> Date: Tue, 30 May 2023 16:28:16 +0800 Subject: [PATCH 39/68] =?UTF-8?q?=E5=AE=9A=E6=97=B6=E4=BB=BB=E5=8A=A1?= =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E9=9A=8F=E6=9C=BA=E5=90=AF=E5=8A=A8=E5=BB=B6?= =?UTF-8?q?=E6=97=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- internal/tickevent/tick_event.go | 2 ++ main.go | 12 ++++++------ 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/internal/tickevent/tick_event.go b/internal/tickevent/tick_event.go index da51b6e..4620624 100644 --- a/internal/tickevent/tick_event.go +++ b/internal/tickevent/tick_event.go @@ -11,6 +11,8 @@ type ExecuteArgs struct { DB *mydb.DB } +type StartOption = tickevent.StartOption + type Executor = tickevent.Executor[ExecuteArgs] type ExecuteContext = tickevent.ExecuteContext[ExecuteArgs] diff --git a/main.go b/main.go index d47fa63..c40ad05 100644 --- a/main.go +++ b/main.go @@ -87,15 +87,15 @@ func serveScannerServer(server *scsvr.ScannerServer, wg *sync.WaitGroup) { func startTickEvent(tickExecutor *tickevent.Executor) { // TODO 可以考虑增加配置文件，配置这些任务间隔时间 - tickExecutor.Start(tickevent.NewBatchAllAgentCheckCache(), 5*60*1000) + tickExecutor.Start(tickevent.NewBatchAllAgentCheckCache(), 5*60*1000, tickevent.StartOption{RandomStartDelayMs: 60 * 1000}) - tickExecutor.Start(tickevent.NewBatchCheckAllObject(), 5*60*1000) + tickExecutor.Start(tickevent.NewBatchCheckAllObject(), 5*60*1000, tickevent.StartOption{RandomStartDelayMs: 60 * 1000}) - tickExecutor.Start(tickevent.NewBatchCheckAllRepCount(), 5*60*1000) + tickExecutor.Start(tickevent.NewBatchCheckAllRepCount(), 5*60*1000, tickevent.StartOption{RandomStartDelayMs: 60 * 1000}) - tickExecutor.Start(tickevent.NewBatchCheckAllStorage(), 5*60*1000) + tickExecutor.Start(tickevent.NewBatchCheckAllStorage(), 5*60*1000, tickevent.StartOption{RandomStartDelayMs: 60 * 1000}) - //tickExecutor.Start(tickevent.NewCheckAgentState(), 5*60*1000) + //tickExecutor.Start(tickevent.NewCheckAgentState(), 5*60*1000, tickevent.StartOption{RandomFirstStartDelayMs: 60 * 1000}) - tickExecutor.Start(tickevent.NewCheckCache(), 5*60*1000) + tickExecutor.Start(tickevent.NewCheckCache(), 5*60*1000, tickevent.StartOption{RandomStartDelayMs: 60 * 1000}) } From 429e7385dd1c5be76b3616ac339c0be3e4d47834 Mon Sep 17 00:00:00 2001 From: Sydonian <794346190@qq.com> Date: Tue, 30 May 2023 16:44:31 +0800 Subject: [PATCH 40/68] =?UTF-8?q?=E5=B0=86sql=E5=8C=85=E4=B8=ADcache?= =?UTF-8?q?=E7=9A=84=E4=BB=A3=E7=A0=81=E7=A7=BB=E5=8A=A8=E5=88=B0=E5=A4=96?= =?UTF-8?q?=E5=B1=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- internal/event/agent_check_cache.go | 5 ++--- internal/event/agent_check_state.go | 2 +- internal/event/check_cache.go | 4 ++-- internal/event/check_rep_count.go | 8 ++++---- internal/event/update_cache.go | 5 ++--- internal/tickevent/batch_check_all_rep_count.go | 3 +-- 6 files changed, 12 insertions(+), 15 deletions(-) diff --git a/internal/event/agent_check_cache.go b/internal/event/agent_check_cache.go index 458f4c1..9c30cd1 100644 --- a/internal/event/agent_check_cache.go +++ b/internal/event/agent_check_cache.go @@ -6,7 +6,6 @@ import ( "github.com/samber/lo" "gitlink.org.cn/cloudream/common/pkg/logger" "gitlink.org.cn/cloudream/db/model" - mysql "gitlink.org.cn/cloudream/db/sql" "gitlink.org.cn/cloudream/scanner/internal/config" agtcli "gitlink.org.cn/cloudream/rabbitmq/client/agent" @@ -57,7 +56,7 @@ func (t *AgentCheckCache) Execute(execCtx ExecuteContext) { if t.FileHashes == nil { var err error - caches, err = mysql.Cache.GetNodeCaches(execCtx.Args.DB.SQLCtx(), t.NodeID) + caches, err = execCtx.Args.DB.Cache().GetNodeCaches(execCtx.Args.DB.SQLCtx(), t.NodeID) if err != nil { log.WithField("NodeID", t.NodeID).Warnf("get node caches failed, err: %s", err.Error()) return @@ -66,7 +65,7 @@ func (t *AgentCheckCache) Execute(execCtx ExecuteContext) { } else { for _, hash := range t.FileHashes { - ch, err := mysql.Cache.Get(execCtx.Args.DB.SQLCtx(), hash, t.NodeID) + ch, err := execCtx.Args.DB.Cache().Get(execCtx.Args.DB.SQLCtx(), hash, t.NodeID) // 记录不存在则跳过 if err == sql.ErrNoRows { continue diff --git a/internal/event/agent_check_state.go b/internal/event/agent_check_state.go index fcefd84..c5f38db 100644 --- a/internal/event/agent_check_state.go +++ b/internal/event/agent_check_state.go @@ -61,7 +61,7 @@ func (t *AgentCheckState) Execute(execCtx ExecuteContext) { return } - caches, err := mysql.Cache.GetNodeCaches(execCtx.Args.DB.SQLCtx(), t.NodeID) + caches, err := execCtx.Args.DB.Cache().GetNodeCaches(execCtx.Args.DB.SQLCtx(), t.NodeID) if err != nil { log.WithField("NodeID", t.NodeID).Warnf("get node caches failed, err: %s", err.Error()) return diff --git a/internal/event/check_cache.go b/internal/event/check_cache.go index 2760d02..0c5a839 100644 --- a/internal/event/check_cache.go +++ b/internal/event/check_cache.go @@ -52,12 +52,12 @@ func (t *CheckCache) Execute(execCtx ExecuteContext) { return nil } - caches, err := mysql.Cache.GetNodeCaches(tx, t.NodeID) + caches, err := execCtx.Args.DB.Cache().GetNodeCaches(tx, t.NodeID) if err != nil { return fmt.Errorf("get node caches failed, err: %w", err) } - err = mysql.Cache.DeleteNodeAll(tx, t.NodeID) + err = execCtx.Args.DB.Cache().DeleteNodeAll(tx, t.NodeID) if err != nil { return fmt.Errorf("delete node all caches failed, err: %w", err) } diff --git a/internal/event/check_rep_count.go b/internal/event/check_rep_count.go index ffdc37e..cf6cb62 100644 --- a/internal/event/check_rep_count.go +++ b/internal/event/check_rep_count.go @@ -81,9 +81,9 @@ func (t *CheckRepCount) checkOneRepCount(fileHash string, execCtx ExecuteContext // 计算所需的最少备份数： // ObjectRep中期望备份数的最大值 // 如果ObjectBlock存在对此文件的引用，则至少为1 - needRepCount := mymath.Max(repMaxCnt, mymath.Max(1, blkCnt)) + needRepCount := mymath.Max(repMaxCnt, mymath.Min(1, blkCnt)) - repNodes, err := mysql.Cache.GetCachingFileNodes(tx, fileHash) + repNodes, err := execCtx.Args.DB.Cache().GetCachingFileNodes(tx, fileHash) if err != nil { return fmt.Errorf("get caching file nodes failed, err: %w", err) } @@ -106,7 +106,7 @@ func (t *CheckRepCount) checkOneRepCount(fileHash string, execCtx ExecuteContext if len(normalNodes) > needRepCount { delNodes := chooseDeleteAvaiRepNodes(allNodes, normalNodes, len(normalNodes)-needRepCount) for _, node := range delNodes { - err := mysql.Cache.ChangeState(tx, fileHash, node.NodeID, consts.CACHE_STATE_TEMP) + err := execCtx.Args.DB.Cache().SetTemp(tx, fileHash, node.NodeID) if err != nil { return fmt.Errorf("change cache state failed, err: %w", err) } @@ -134,7 +134,7 @@ func (t *CheckRepCount) checkOneRepCount(fileHash string, execCtx ExecuteContext } for _, node := range newNodes { - err := mysql.Cache.CreatePinned(tx, fileHash, node.NodeID) + err := execCtx.Args.DB.Cache().CreatePinned(tx, fileHash, node.NodeID) if err != nil { return fmt.Errorf("create cache failed, err: %w", err) } diff --git a/internal/event/update_cache.go b/internal/event/update_cache.go index a72fa3f..173838c 100644 --- a/internal/event/update_cache.go +++ b/internal/event/update_cache.go @@ -3,7 +3,6 @@ package event import ( evtcst "gitlink.org.cn/cloudream/common/consts/event" "gitlink.org.cn/cloudream/common/pkg/logger" - mysql "gitlink.org.cn/cloudream/db/sql" scevt "gitlink.org.cn/cloudream/rabbitmq/message/scanner/event" ) @@ -40,7 +39,7 @@ func (t *UpdateCache) Execute(execCtx ExecuteContext) { for _, entry := range t.Entries { switch entry.Operation { case evtcst.UPDATE_CACHE_DELETE_TEMP: - err := mysql.Cache.DeleteTemp(execCtx.Args.DB.SQLCtx(), entry.FileHash, t.NodeID) + err := execCtx.Args.DB.Cache().DeleteTemp(execCtx.Args.DB.SQLCtx(), entry.FileHash, t.NodeID) if err != nil { log.WithField("FileHash", entry.FileHash). WithField("NodeID", t.NodeID). @@ -52,7 +51,7 @@ func (t *UpdateCache) Execute(execCtx ExecuteContext) { Debugf("delete temp cache") case evtcst.UPDATE_CACHE_CREATE_TEMP: - err := mysql.Cache.CreateTemp(execCtx.Args.DB.SQLCtx(), entry.FileHash, t.NodeID) + err := execCtx.Args.DB.Cache().CreateTemp(execCtx.Args.DB.SQLCtx(), entry.FileHash, t.NodeID) if err != nil { log.WithField("FileHash", entry.FileHash). WithField("NodeID", t.NodeID). diff --git a/internal/tickevent/batch_check_all_rep_count.go b/internal/tickevent/batch_check_all_rep_count.go index 07f476d..cdb6902 100644 --- a/internal/tickevent/batch_check_all_rep_count.go +++ b/internal/tickevent/batch_check_all_rep_count.go @@ -2,7 +2,6 @@ package tickevent import ( "gitlink.org.cn/cloudream/common/pkg/logger" - mysql "gitlink.org.cn/cloudream/db/sql" "gitlink.org.cn/cloudream/scanner/internal/event" ) @@ -20,7 +19,7 @@ func (e *BatchCheckAllRepCount) Execute(ctx ExecuteContext) { log := logger.WithType[BatchCheckAllRepCount]("TickEvent") log.Debugf("begin") - fileHashes, err := mysql.Cache.BatchGetAllFileHashes(ctx.Args.DB.SQLCtx(), e.lastCheckStart, CHECK_CACHE_BATCH_SIZE) + fileHashes, err := ctx.Args.DB.Cache().BatchGetAllFileHashes(ctx.Args.DB.SQLCtx(), e.lastCheckStart, CHECK_CACHE_BATCH_SIZE) if err != nil { log.Warnf("batch get file hashes failed, err: %s", err.Error()) return From e362a16e9237b1e8d64b2162d4bc859c87780042 Mon Sep 17 00:00:00 2001 From: Sydonian <794346190@qq.com> Date: Wed, 7 Jun 2023 10:27:06 +0800 Subject: [PATCH 41/68] =?UTF-8?q?=E9=83=A8=E5=88=86=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E4=BC=98=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- internal/event/agent_check_cache.go | 6 ++---- internal/event/agent_check_state.go | 4 ++-- internal/event/check_cache.go | 4 ++-- 3 files changed, 6 insertions(+), 8 deletions(-) diff --git a/internal/event/agent_check_cache.go b/internal/event/agent_check_cache.go index 9c30cd1..11865d4 100644 --- a/internal/event/agent_check_cache.go +++ b/internal/event/agent_check_cache.go @@ -14,14 +14,12 @@ import ( ) type AgentCheckCache struct { - NodeID int - FileHashes []string // 需要检查的FileHash列表，如果为nil（不是为空），则代表进行全量检查 + scevt.AgentCheckCache } func NewAgentCheckCache(nodeID int, fileHashes []string) *AgentCheckCache { return &AgentCheckCache{ - NodeID: nodeID, - FileHashes: fileHashes, + AgentCheckCache: scevt.NewAgentCheckCache(nodeID, fileHashes), } } diff --git a/internal/event/agent_check_state.go b/internal/event/agent_check_state.go index c5f38db..6fa20fe 100644 --- a/internal/event/agent_check_state.go +++ b/internal/event/agent_check_state.go @@ -16,12 +16,12 @@ import ( ) type AgentCheckState struct { - NodeID int + scevt.AgentCheckState } func NewAgentCheckState(nodeID int) *AgentCheckState { return &AgentCheckState{ - NodeID: nodeID, + AgentCheckState: scevt.NewAgentCheckState(nodeID), } } diff --git a/internal/event/check_cache.go b/internal/event/check_cache.go index 0c5a839..c201aed 100644 --- a/internal/event/check_cache.go +++ b/internal/event/check_cache.go @@ -14,12 +14,12 @@ import ( ) type CheckCache struct { - NodeID int + scevt.CheckCache } func NewCheckCache(nodeID int) *CheckCache { return &CheckCache{ - NodeID: nodeID, + CheckCache: scevt.NewCheckCache(nodeID), } } From 2cfe8fa35cfe52c2aa1feb54100c747f934d16c3 Mon Sep 17 00:00:00 2001 From: Sydonian <794346190@qq.com> Date: Tue, 13 Jun 2023 16:05:01 +0800 Subject: [PATCH 42/68] =?UTF-8?q?=E5=8F=96=E6=B6=88agent=E7=9A=84event?= =?UTF-8?q?=E6=9C=BA=E5=88=B6=EF=BC=8C=E6=94=B9=E4=B8=BA=E5=AF=B9=E5=A4=96?= =?UTF-8?q?=E6=8F=90=E4=BE=9BAPI?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- internal/event/agent_check_cache.go | 42 +++++++++++-- internal/event/agent_check_state.go | 26 +++++++- internal/event/agent_check_storage.go | 49 +++++++++++++-- internal/event/update_agent_state.go | 47 --------------- internal/event/update_cache.go | 70 --------------------- internal/event/update_storage.go | 87 --------------------------- main.go | 4 +- 7 files changed, 104 insertions(+), 221 deletions(-) delete mode 100644 internal/event/update_agent_state.go delete mode 100644 internal/event/update_cache.go delete mode 100644 internal/event/update_storage.go diff --git a/internal/event/agent_check_cache.go b/internal/event/agent_check_cache.go index 11865d4..27891fc 100644 --- a/internal/event/agent_check_cache.go +++ b/internal/event/agent_check_cache.go @@ -9,7 +9,7 @@ import ( "gitlink.org.cn/cloudream/scanner/internal/config" agtcli "gitlink.org.cn/cloudream/rabbitmq/client/agent" - agtevt "gitlink.org.cn/cloudream/rabbitmq/message/agent/event" + agtmsg "gitlink.org.cn/cloudream/rabbitmq/message/agent" scevt "gitlink.org.cn/cloudream/rabbitmq/message/scanner/event" ) @@ -80,21 +80,51 @@ func (t *AgentCheckCache) Execute(execCtx ExecuteContext) { } // 然后向代理端发送移动文件的请求 - agentClient, err := agtcli.NewAgentClient(t.NodeID, &config.Cfg().RabbitMQ) + agentClient, err := agtcli.NewClient(t.NodeID, &config.Cfg().RabbitMQ) if err != nil { log.WithField("NodeID", t.NodeID).Warnf("create agent client failed, err: %s", err.Error()) return } defer agentClient.Close() - err = agentClient.PostEvent( - agtevt.NewCheckCache(isComplete, caches), - execCtx.Option.IsEmergency, // 继承本任务的执行选项 - execCtx.Option.DontMerge) + checkResp, err := agentClient.CheckIPFS(agtmsg.NewCheckIPFSBody(isComplete, caches)) if err != nil { log.WithField("NodeID", t.NodeID).Warnf("request to agent failed, err: %s", err.Error()) return } + if checkResp.IsFailed() { + log.WithField("NodeID", t.NodeID).Warnf("agent operation failed, err: %s", err.Error()) + return + } + + // 根据返回结果修改数据库 + for _, entry := range checkResp.Body.Entries { + switch entry.Operation { + case agtmsg.CHECK_IPFS_RESP_OP_DELETE_TEMP: + err := execCtx.Args.DB.Cache().DeleteTemp(execCtx.Args.DB.SQLCtx(), entry.FileHash, t.NodeID) + if err != nil { + log.WithField("FileHash", entry.FileHash). + WithField("NodeID", t.NodeID). + Warnf("delete temp cache failed, err: %s", err.Error()) + } + + log.WithField("FileHash", entry.FileHash). + WithField("NodeID", t.NodeID). + Debugf("delete temp cache") + + case agtmsg.CHECK_IPFS_RESP_OP_CREATE_TEMP: + err := execCtx.Args.DB.Cache().CreateTemp(execCtx.Args.DB.SQLCtx(), entry.FileHash, t.NodeID) + if err != nil { + log.WithField("FileHash", entry.FileHash). + WithField("NodeID", t.NodeID). + Warnf("create temp cache failed, err: %s", err.Error()) + } + + log.WithField("FileHash", entry.FileHash). + WithField("NodeID", t.NodeID). + Debugf("create temp cache") + } + } } func init() { RegisterMessageConvertor(func(msg scevt.AgentCheckCache) Event { return NewAgentCheckCache(msg.NodeID, msg.FileHashes) }) diff --git a/internal/event/agent_check_state.go b/internal/event/agent_check_state.go index 6fa20fe..1ea3d04 100644 --- a/internal/event/agent_check_state.go +++ b/internal/event/agent_check_state.go @@ -10,7 +10,7 @@ import ( "gitlink.org.cn/cloudream/db/model" mysql "gitlink.org.cn/cloudream/db/sql" agtcli "gitlink.org.cn/cloudream/rabbitmq/client/agent" - agtevt "gitlink.org.cn/cloudream/rabbitmq/message/agent/event" + agtmsg "gitlink.org.cn/cloudream/rabbitmq/message/agent" scevt "gitlink.org.cn/cloudream/rabbitmq/message/scanner/event" "gitlink.org.cn/cloudream/scanner/internal/config" ) @@ -72,7 +72,7 @@ func (t *AgentCheckState) Execute(execCtx ExecuteContext) { return } - agentClient, err := agtcli.NewAgentClient(t.NodeID, &config.Cfg().RabbitMQ) + agentClient, err := agtcli.NewClient(t.NodeID, &config.Cfg().RabbitMQ) if err != nil { log.WithField("NodeID", t.NodeID).Warnf("create agent client failed, err: %s", err.Error()) return @@ -80,10 +80,30 @@ func (t *AgentCheckState) Execute(execCtx ExecuteContext) { defer agentClient.Close() // 紧急任务 - err = agentClient.PostEvent(agtevt.NewCheckState(), true, true) + getResp, err := agentClient.GetState(agtmsg.NewGetStateBody()) if err != nil { log.WithField("NodeID", t.NodeID).Warnf("request to agent failed, err: %s", err.Error()) } + if getResp.IsFailed() { + log.WithField("NodeID", t.NodeID).Warnf("agent operation failed, err: %s", err.Error()) + return + } + + if getResp.Body.IPFSState != consts.IPFS_STATE_OK { + log.WithField("NodeID", t.NodeID).Warnf("IPFS status is %s, set node state unavailable", getResp.Body.IPFSState) + + err := mysql.Node.ChangeState(execCtx.Args.DB.SQLCtx(), t.NodeID, consts.NODE_STATE_UNAVAILABLE) + if err != nil { + log.WithField("NodeID", t.NodeID).Warnf("change node state failed, err: %s", err.Error()) + } + return + } + + // TODO 如果以后还有其他的状态，要判断哪些状态下能设置Normal + err = mysql.Node.ChangeState(execCtx.Args.DB.SQLCtx(), t.NodeID, consts.NODE_STATE_NORMAL) + if err != nil { + log.WithField("NodeID", t.NodeID).Warnf("change node state failed, err: %s", err.Error()) + } } func init() { diff --git a/internal/event/agent_check_storage.go b/internal/event/agent_check_storage.go index ea01a5e..4b19d6c 100644 --- a/internal/event/agent_check_storage.go +++ b/internal/event/agent_check_storage.go @@ -9,7 +9,7 @@ import ( "gitlink.org.cn/cloudream/db/model" mysql "gitlink.org.cn/cloudream/db/sql" agtcli "gitlink.org.cn/cloudream/rabbitmq/client/agent" - agtevt "gitlink.org.cn/cloudream/rabbitmq/message/agent/event" + agtmsg "gitlink.org.cn/cloudream/rabbitmq/message/agent" scevt "gitlink.org.cn/cloudream/rabbitmq/message/scanner/event" "gitlink.org.cn/cloudream/scanner/internal/config" ) @@ -96,20 +96,57 @@ func (t *AgentCheckStorage) Execute(execCtx ExecuteContext) { } // 投递任务 - agentClient, err := agtcli.NewAgentClient(stg.NodeID, &config.Cfg().RabbitMQ) + agentClient, err := agtcli.NewClient(stg.NodeID, &config.Cfg().RabbitMQ) if err != nil { log.WithField("NodeID", stg.NodeID).Warnf("create agent client failed, err: %s", err.Error()) return } defer agentClient.Close() - err = agentClient.PostEvent( - agtevt.NewCheckStorage(stg.StorageID, stg.Directory, isComplete, objects), - execCtx.Option.IsEmergency, // 继承本任务的执行选项 - execCtx.Option.DontMerge) + checkResp, err := agentClient.CheckStorage(agtmsg.NewCheckStorageBody(stg.StorageID, stg.Directory, isComplete, objects)) if err != nil { log.WithField("NodeID", stg.NodeID).Warnf("request to agent failed, err: %s", err.Error()) } + if checkResp.IsFailed() { + log.WithField("NodeID", stg.NodeID).Warnf("agent operation failed, err: %s", err.Error()) + return + } + + var chkObjIDs []int + for _, entry := range checkResp.Body.Entries { + switch entry.Operation { + case agtmsg.CHECK_STORAGE_RESP_OP_DELETE: + err := mysql.StorageObject.Delete(execCtx.Args.DB.SQLCtx(), t.StorageID, entry.ObjectID, entry.UserID) + if err != nil { + log.WithField("StorageID", t.StorageID). + WithField("ObjectID", entry.ObjectID). + Warnf("delete storage object failed, err: %s", err.Error()) + } + chkObjIDs = append(chkObjIDs, entry.ObjectID) + + log.WithField("StorageID", t.StorageID). + WithField("ObjectID", entry.ObjectID). + WithField("UserID", entry.UserID). + Debugf("delete storage object") + + case agtmsg.CHECK_STORAGE_RESP_OP_SET_NORMAL: + err := mysql.StorageObject.SetStateNormal(execCtx.Args.DB.SQLCtx(), t.StorageID, entry.ObjectID, entry.UserID) + if err != nil { + log.WithField("StorageID", t.StorageID). + WithField("ObjectID", entry.ObjectID). + Warnf("change storage object state failed, err: %s", err.Error()) + } + + log.WithField("StorageID", t.StorageID). + WithField("ObjectID", entry.ObjectID). + WithField("UserID", entry.UserID). + Debugf("set storage object normal") + } + } + + if len(chkObjIDs) > 0 { + execCtx.Executor.Post(NewCheckObject(chkObjIDs)) + } } func init() { diff --git a/internal/event/update_agent_state.go b/internal/event/update_agent_state.go deleted file mode 100644 index 1669738..0000000 --- a/internal/event/update_agent_state.go +++ /dev/null @@ -1,47 +0,0 @@ -package event - -import ( - "gitlink.org.cn/cloudream/common/consts" - "gitlink.org.cn/cloudream/common/pkg/logger" - mysql "gitlink.org.cn/cloudream/db/sql" - scevt "gitlink.org.cn/cloudream/rabbitmq/message/scanner/event" -) - -type UpdateAgentState struct { - scevt.UpdateAgentState -} - -func NewUpdateAgentState(nodeID int, ipfsState string) *UpdateAgentState { - return &UpdateAgentState{ - UpdateAgentState: scevt.NewUpdateAgentState(nodeID, ipfsState), - } -} - -func (t *UpdateAgentState) TryMerge(other Event) bool { - return false -} - -func (t *UpdateAgentState) Execute(execCtx ExecuteContext) { - log := logger.WithType[UpdateAgentState]("Event") - log.Debugf("begin with %v", logger.FormatStruct(t)) - - if t.IPFSState != consts.IPFS_STATUS_OK { - log.WithField("NodeID", t.NodeID).Warnf("IPFS status is %s, set node state unavailable", t.IPFSState) - - err := mysql.Node.ChangeState(execCtx.Args.DB.SQLCtx(), t.NodeID, consts.NODE_STATE_UNAVAILABLE) - if err != nil { - log.WithField("NodeID", t.NodeID).Warnf("change node state failed, err: %s", err.Error()) - } - return - } - - // TODO 如果以后还有其他的状态，要判断哪些状态下能设置Normal - err := mysql.Node.ChangeState(execCtx.Args.DB.SQLCtx(), t.NodeID, consts.NODE_STATE_NORMAL) - if err != nil { - log.WithField("NodeID", t.NodeID).Warnf("change node state failed, err: %s", err.Error()) - } -} - -func init() { - RegisterMessageConvertor(func(msg scevt.UpdateAgentState) Event { return NewUpdateAgentState(msg.NodeID, msg.IPFSState) }) -} diff --git a/internal/event/update_cache.go b/internal/event/update_cache.go deleted file mode 100644 index 173838c..0000000 --- a/internal/event/update_cache.go +++ /dev/null @@ -1,70 +0,0 @@ -package event - -import ( - evtcst "gitlink.org.cn/cloudream/common/consts/event" - "gitlink.org.cn/cloudream/common/pkg/logger" - scevt "gitlink.org.cn/cloudream/rabbitmq/message/scanner/event" -) - -type UpdateCacheEntry = scevt.UpdateCacheEntry - -type UpdateCache struct { - scevt.UpdateCache -} - -func NewUpdateCache(nodeID int, entries []scevt.UpdateCacheEntry) *UpdateCache { - return &UpdateCache{ - UpdateCache: scevt.NewUpdateCache(nodeID, entries), - } -} - -func (t *UpdateCache) TryMerge(other Event) bool { - event, ok := other.(*UpdateCache) - if !ok { - return false - } - if event.NodeID != t.NodeID { - return false - } - - // TODO 可以考虑合并同FileHash和NodeID的记录 - t.Entries = append(t.Entries, event.Entries...) - return true -} - -func (t *UpdateCache) Execute(execCtx ExecuteContext) { - log := logger.WithType[UpdateCache]("Event") - log.Debugf("begin with %v", logger.FormatStruct(t)) - - for _, entry := range t.Entries { - switch entry.Operation { - case evtcst.UPDATE_CACHE_DELETE_TEMP: - err := execCtx.Args.DB.Cache().DeleteTemp(execCtx.Args.DB.SQLCtx(), entry.FileHash, t.NodeID) - if err != nil { - log.WithField("FileHash", entry.FileHash). - WithField("NodeID", t.NodeID). - Warnf("delete temp cache failed, err: %s", err.Error()) - } - - log.WithField("FileHash", entry.FileHash). - WithField("NodeID", t.NodeID). - Debugf("delete temp cache") - - case evtcst.UPDATE_CACHE_CREATE_TEMP: - err := execCtx.Args.DB.Cache().CreateTemp(execCtx.Args.DB.SQLCtx(), entry.FileHash, t.NodeID) - if err != nil { - log.WithField("FileHash", entry.FileHash). - WithField("NodeID", t.NodeID). - Warnf("create temp cache failed, err: %s", err.Error()) - } - - log.WithField("FileHash", entry.FileHash). - WithField("NodeID", t.NodeID). - Debugf("create temp cache") - } - } -} - -func init() { - RegisterMessageConvertor(func(msg scevt.UpdateCache) Event { return NewUpdateCache(msg.NodeID, msg.Entries) }) -} diff --git a/internal/event/update_storage.go b/internal/event/update_storage.go deleted file mode 100644 index 61e9851..0000000 --- a/internal/event/update_storage.go +++ /dev/null @@ -1,87 +0,0 @@ -package event - -import ( - tskcst "gitlink.org.cn/cloudream/common/consts/event" - "gitlink.org.cn/cloudream/common/pkg/logger" - mysql "gitlink.org.cn/cloudream/db/sql" - scevt "gitlink.org.cn/cloudream/rabbitmq/message/scanner/event" -) - -type UpdateStorageEntry = scevt.UpdateStorageEntry -type UpdateStorage struct { - scevt.UpdateStorage -} - -func NewUpdateStorage(storageID int, dirState string, entries []UpdateStorageEntry) *UpdateStorage { - return &UpdateStorage{ - UpdateStorage: scevt.NewUpdateStorage(storageID, dirState, entries), - } -} - -func (t *UpdateStorage) TryMerge(other Event) bool { - event, ok := other.(*UpdateStorage) - if !ok { - return false - } - if event.StorageID != t.StorageID { - return false - } - - // 后投递的任务的状态更新一些 - t.DirectoryState = event.DirectoryState - // TODO 可以考虑合并同FileHash和NodeID的记录 - t.Entries = append(t.Entries, event.Entries...) - return true -} - -func (t *UpdateStorage) Execute(execCtx ExecuteContext) { - log := logger.WithType[UpdateStorage]("Event") - log.Debugf("begin with %v", logger.FormatStruct(t)) - - err := mysql.Storage.ChangeState(execCtx.Args.DB.SQLCtx(), t.StorageID, t.DirectoryState) - if err != nil { - log.WithField("StorageID", t.StorageID).Warnf("change storage state failed, err: %s", err.Error()) - } - - var chkObjIDs []int - for _, entry := range t.Entries { - switch entry.Operation { - case tskcst.UPDATE_STORAGE_DELETE: - err := mysql.StorageObject.Delete(execCtx.Args.DB.SQLCtx(), t.StorageID, entry.ObjectID, entry.UserID) - if err != nil { - log.WithField("StorageID", t.StorageID). - WithField("ObjectID", entry.ObjectID). - Warnf("delete storage object failed, err: %s", err.Error()) - } - chkObjIDs = append(chkObjIDs, entry.ObjectID) - - log.WithField("StorageID", t.StorageID). - WithField("ObjectID", entry.ObjectID). - WithField("UserID", entry.UserID). - Debugf("delete storage object") - - case tskcst.UPDATE_STORAGE_SET_NORMAL: - err := mysql.StorageObject.SetStateNormal(execCtx.Args.DB.SQLCtx(), t.StorageID, entry.ObjectID, entry.UserID) - if err != nil { - log.WithField("StorageID", t.StorageID). - WithField("ObjectID", entry.ObjectID). - Warnf("change storage object state failed, err: %s", err.Error()) - } - - log.WithField("StorageID", t.StorageID). - WithField("ObjectID", entry.ObjectID). - WithField("UserID", entry.UserID). - Debugf("set storage object normal") - } - } - - if len(chkObjIDs) > 0 { - execCtx.Executor.Post(NewCheckObject(chkObjIDs)) - } -} - -func init() { - RegisterMessageConvertor(func(msg scevt.UpdateStorage) Event { - return NewUpdateStorage(msg.StorageID, msg.DirectoryState, msg.Entries) - }) -} diff --git a/main.go b/main.go index c40ad05..624e537 100644 --- a/main.go +++ b/main.go @@ -38,7 +38,7 @@ func main() { eventExecutor := event.NewExecutor(db) go serveEventExecutor(&eventExecutor, &wg) - agtSvr, err := scsvr.NewScannerServer(services.NewService(&eventExecutor), &config.Cfg().RabbitMQ) + agtSvr, err := scsvr.NewServer(services.NewService(&eventExecutor), &config.Cfg().RabbitMQ) if err != nil { log.Fatalf("new agent server failed, err: %s", err.Error()) } @@ -70,7 +70,7 @@ func serveEventExecutor(executor *event.Executor, wg *sync.WaitGroup) { wg.Done() } -func serveScannerServer(server *scsvr.ScannerServer, wg *sync.WaitGroup) { +func serveScannerServer(server *scsvr.Server, wg *sync.WaitGroup) { log.Info("start serving scanner server") err := server.Serve() From 18ed91c074118b2d55c689937fadee66c90703fd Mon Sep 17 00:00:00 2001 From: Sydonian <794346190@qq.com> Date: Wed, 14 Jun 2023 10:44:37 +0800 Subject: [PATCH 43/68] =?UTF-8?q?=E5=8F=96=E6=B6=88db=E6=A8=A1=E5=9D=97?= =?UTF-8?q?=E7=9A=84sql=E5=8C=85=EF=BC=9B=E7=BB=99CreateBucket=E6=8E=A5?= =?UTF-8?q?=E5=8F=A3=E5=A2=9E=E5=8A=A0=E4=B8=8A=E9=94=81=E8=BF=87=E7=A8=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- internal/event/agent_check_state.go | 11 +++++------ internal/event/agent_check_storage.go | 14 +++++++------- 2 files changed, 12 insertions(+), 13 deletions(-) diff --git a/internal/event/agent_check_state.go b/internal/event/agent_check_state.go index 1ea3d04..0df1f1a 100644 --- a/internal/event/agent_check_state.go +++ b/internal/event/agent_check_state.go @@ -8,7 +8,6 @@ import ( "gitlink.org.cn/cloudream/common/consts" "gitlink.org.cn/cloudream/common/pkg/logger" "gitlink.org.cn/cloudream/db/model" - mysql "gitlink.org.cn/cloudream/db/sql" agtcli "gitlink.org.cn/cloudream/rabbitmq/client/agent" agtmsg "gitlink.org.cn/cloudream/rabbitmq/message/agent" scevt "gitlink.org.cn/cloudream/rabbitmq/message/scanner/event" @@ -38,7 +37,7 @@ func (t *AgentCheckState) Execute(execCtx ExecuteContext) { log := logger.WithType[AgentCheckState]("Event") log.Debugf("begin with %v", logger.FormatStruct(t)) - node, err := mysql.Node.GetByID(execCtx.Args.DB.SQLCtx(), t.NodeID) + node, err := execCtx.Args.DB.Node().GetByID(execCtx.Args.DB.SQLCtx(), t.NodeID) if err == sql.ErrNoRows { return } @@ -55,7 +54,7 @@ func (t *AgentCheckState) Execute(execCtx ExecuteContext) { // 检查上次上报时间，超时的设置为不可用 // TODO 没有上报过是否要特殊处理？ if node.LastReportTime == nil && time.Since(*node.LastReportTime) > time.Duration(config.Cfg().NodeUnavailableSeconds)*time.Second { - err := mysql.Node.ChangeState(execCtx.Args.DB.SQLCtx(), t.NodeID, consts.NODE_STATE_UNAVAILABLE) + err := execCtx.Args.DB.Node().ChangeState(execCtx.Args.DB.SQLCtx(), t.NodeID, consts.NODE_STATE_UNAVAILABLE) if err != nil { log.WithField("NodeID", t.NodeID).Warnf("set node state failed, err: %s", err.Error()) return @@ -79,7 +78,6 @@ func (t *AgentCheckState) Execute(execCtx ExecuteContext) { } defer agentClient.Close() - // 紧急任务 getResp, err := agentClient.GetState(agtmsg.NewGetStateBody()) if err != nil { log.WithField("NodeID", t.NodeID).Warnf("request to agent failed, err: %s", err.Error()) @@ -89,10 +87,11 @@ func (t *AgentCheckState) Execute(execCtx ExecuteContext) { return } + // 根据返回结果修改节点状态 if getResp.Body.IPFSState != consts.IPFS_STATE_OK { log.WithField("NodeID", t.NodeID).Warnf("IPFS status is %s, set node state unavailable", getResp.Body.IPFSState) - err := mysql.Node.ChangeState(execCtx.Args.DB.SQLCtx(), t.NodeID, consts.NODE_STATE_UNAVAILABLE) + err := execCtx.Args.DB.Node().ChangeState(execCtx.Args.DB.SQLCtx(), t.NodeID, consts.NODE_STATE_UNAVAILABLE) if err != nil { log.WithField("NodeID", t.NodeID).Warnf("change node state failed, err: %s", err.Error()) } @@ -100,7 +99,7 @@ func (t *AgentCheckState) Execute(execCtx ExecuteContext) { } // TODO 如果以后还有其他的状态，要判断哪些状态下能设置Normal - err = mysql.Node.ChangeState(execCtx.Args.DB.SQLCtx(), t.NodeID, consts.NODE_STATE_NORMAL) + err = execCtx.Args.DB.Node().ChangeState(execCtx.Args.DB.SQLCtx(), t.NodeID, consts.NODE_STATE_NORMAL) if err != nil { log.WithField("NodeID", t.NodeID).Warnf("change node state failed, err: %s", err.Error()) } diff --git a/internal/event/agent_check_storage.go b/internal/event/agent_check_storage.go index 4b19d6c..9f54c1a 100644 --- a/internal/event/agent_check_storage.go +++ b/internal/event/agent_check_storage.go @@ -7,7 +7,6 @@ import ( "gitlink.org.cn/cloudream/common/consts" "gitlink.org.cn/cloudream/common/pkg/logger" "gitlink.org.cn/cloudream/db/model" - mysql "gitlink.org.cn/cloudream/db/sql" agtcli "gitlink.org.cn/cloudream/rabbitmq/client/agent" agtmsg "gitlink.org.cn/cloudream/rabbitmq/message/agent" scevt "gitlink.org.cn/cloudream/rabbitmq/message/scanner/event" @@ -48,7 +47,7 @@ func (t *AgentCheckStorage) Execute(execCtx ExecuteContext) { log := logger.WithType[AgentCheckStorage]("Event") log.Debugf("begin with %v", logger.FormatStruct(t)) - stg, err := mysql.Storage.GetByID(execCtx.Args.DB.SQLCtx(), t.StorageID) + stg, err := execCtx.Args.DB.Storage().GetByID(execCtx.Args.DB.SQLCtx(), t.StorageID) if err != nil { if err != sql.ErrNoRows { log.WithField("StorageID", t.StorageID).Warnf("get storage failed, err: %s", err.Error()) @@ -56,7 +55,7 @@ func (t *AgentCheckStorage) Execute(execCtx ExecuteContext) { return } - node, err := mysql.Node.GetByID(execCtx.Args.DB.SQLCtx(), stg.NodeID) + node, err := execCtx.Args.DB.Node().GetByID(execCtx.Args.DB.SQLCtx(), stg.NodeID) if err != nil { if err != sql.ErrNoRows { log.WithField("StorageID", t.StorageID).Warnf("get storage node failed, err: %s", err.Error()) @@ -74,7 +73,7 @@ func (t *AgentCheckStorage) Execute(execCtx ExecuteContext) { var objects []model.StorageObject if t.ObjectIDs == nil { var err error - objects, err = mysql.StorageObject.GetAllByStorageID(execCtx.Args.DB.SQLCtx(), t.StorageID) + objects, err = execCtx.Args.DB.StorageObject().GetAllByStorageID(execCtx.Args.DB.SQLCtx(), t.StorageID) if err != nil { log.WithField("StorageID", t.StorageID).Warnf("get storage objects failed, err: %s", err.Error()) return @@ -82,7 +81,7 @@ func (t *AgentCheckStorage) Execute(execCtx ExecuteContext) { isComplete = true } else { for _, objID := range t.ObjectIDs { - objs, err := mysql.StorageObject.GetAllByStorageAndObjectID(execCtx.Args.DB.SQLCtx(), t.StorageID, objID) + objs, err := execCtx.Args.DB.StorageObject().GetAllByStorageAndObjectID(execCtx.Args.DB.SQLCtx(), t.StorageID, objID) if err != nil { log.WithField("StorageID", t.StorageID). WithField("ObjectID", objID). @@ -112,11 +111,12 @@ func (t *AgentCheckStorage) Execute(execCtx ExecuteContext) { return } + // 根据返回结果修改数据库 var chkObjIDs []int for _, entry := range checkResp.Body.Entries { switch entry.Operation { case agtmsg.CHECK_STORAGE_RESP_OP_DELETE: - err := mysql.StorageObject.Delete(execCtx.Args.DB.SQLCtx(), t.StorageID, entry.ObjectID, entry.UserID) + err := execCtx.Args.DB.StorageObject().Delete(execCtx.Args.DB.SQLCtx(), t.StorageID, entry.ObjectID, entry.UserID) if err != nil { log.WithField("StorageID", t.StorageID). WithField("ObjectID", entry.ObjectID). @@ -130,7 +130,7 @@ func (t *AgentCheckStorage) Execute(execCtx ExecuteContext) { Debugf("delete storage object") case agtmsg.CHECK_STORAGE_RESP_OP_SET_NORMAL: - err := mysql.StorageObject.SetStateNormal(execCtx.Args.DB.SQLCtx(), t.StorageID, entry.ObjectID, entry.UserID) + err := execCtx.Args.DB.StorageObject().SetStateNormal(execCtx.Args.DB.SQLCtx(), t.StorageID, entry.ObjectID, entry.UserID) if err != nil { log.WithField("StorageID", t.StorageID). WithField("ObjectID", entry.ObjectID). From 8b90a57373402d1e504e33e24bef972bda8e7e44 Mon Sep 17 00:00:00 2001 From: Sydonian <794346190@qq.com> Date: Wed, 14 Jun 2023 10:47:33 +0800 Subject: [PATCH 44/68] =?UTF-8?q?=E8=A7=A3=E5=86=B3=E7=BC=96=E8=AF=91?= =?UTF-8?q?=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- internal/event/check_cache.go | 3 +-- internal/event/check_rep_count.go | 7 +++---- internal/tickevent/batch_all_agent_check_cache.go | 3 +-- internal/tickevent/batch_check_all_object.go | 3 +-- internal/tickevent/batch_check_all_storage.go | 3 +-- internal/tickevent/check_agent_state.go | 3 +-- internal/tickevent/check_cache.go | 3 +-- 7 files changed, 9 insertions(+), 16 deletions(-) diff --git a/internal/event/check_cache.go b/internal/event/check_cache.go index c201aed..05d22fe 100644 --- a/internal/event/check_cache.go +++ b/internal/event/check_cache.go @@ -9,7 +9,6 @@ import ( "gitlink.org.cn/cloudream/common/consts" "gitlink.org.cn/cloudream/common/pkg/logger" "gitlink.org.cn/cloudream/db/model" - mysql "gitlink.org.cn/cloudream/db/sql" scevt "gitlink.org.cn/cloudream/rabbitmq/message/scanner/event" ) @@ -40,7 +39,7 @@ func (t *CheckCache) Execute(execCtx ExecuteContext) { log.Debugf("begin with %v", logger.FormatStruct(t)) err := execCtx.Args.DB.DoTx(sql.LevelSerializable, func(tx *sqlx.Tx) error { - node, err := mysql.Node.GetByID(tx, t.NodeID) + node, err := execCtx.Args.DB.Node().GetByID(tx, t.NodeID) if err == sql.ErrNoRows { return nil } diff --git a/internal/event/check_rep_count.go b/internal/event/check_rep_count.go index cf6cb62..b219f1b 100644 --- a/internal/event/check_rep_count.go +++ b/internal/event/check_rep_count.go @@ -14,7 +14,6 @@ import ( "gitlink.org.cn/cloudream/scanner/internal/config" "gitlink.org.cn/cloudream/db/model" - mysql "gitlink.org.cn/cloudream/db/sql" scevt "gitlink.org.cn/cloudream/rabbitmq/message/scanner/event" ) @@ -68,12 +67,12 @@ func (t *CheckRepCount) checkOneRepCount(fileHash string, execCtx ExecuteContext var updatedNodeIDs []int err := execCtx.Args.DB.DoTx(sql.LevelSerializable, func(tx *sqlx.Tx) error { - repMaxCnt, err := mysql.ObjectRep.GetFileMaxRepCount(tx, fileHash) + repMaxCnt, err := execCtx.Args.DB.ObjectRep().GetFileMaxRepCount(tx, fileHash) if err != nil { return fmt.Errorf("get file max rep count failed, err: %w", err) } - blkCnt, err := mysql.ObjectBlock.CountBlockWithHash(tx, fileHash) + blkCnt, err := execCtx.Args.DB.ObjectBlock().CountBlockWithHash(tx, fileHash) if err != nil { return fmt.Errorf("count block with hash failed, err: %w", err) } @@ -88,7 +87,7 @@ func (t *CheckRepCount) checkOneRepCount(fileHash string, execCtx ExecuteContext return fmt.Errorf("get caching file nodes failed, err: %w", err) } - allNodes, err := mysql.Node.GetAllNodes(tx) + allNodes, err := execCtx.Args.DB.Node().GetAllNodes(tx) if err != nil { return fmt.Errorf("get all nodes failed, err: %w", err) } diff --git a/internal/tickevent/batch_all_agent_check_cache.go b/internal/tickevent/batch_all_agent_check_cache.go index 748dc50..931b999 100644 --- a/internal/tickevent/batch_all_agent_check_cache.go +++ b/internal/tickevent/batch_all_agent_check_cache.go @@ -4,7 +4,6 @@ import ( "github.com/samber/lo" "gitlink.org.cn/cloudream/common/pkg/logger" "gitlink.org.cn/cloudream/db/model" - mysql "gitlink.org.cn/cloudream/db/sql" "gitlink.org.cn/cloudream/scanner/internal/event" ) @@ -23,7 +22,7 @@ func (e *BatchAllAgentCheckCache) Execute(ctx ExecuteContext) { log.Debugf("begin") if e.nodeIDs == nil || len(e.nodeIDs) == 0 { - nodes, err := mysql.Node.GetAllNodes(ctx.Args.DB.SQLCtx()) + nodes, err := ctx.Args.DB.Node().GetAllNodes(ctx.Args.DB.SQLCtx()) if err != nil { log.Warnf("get all nodes failed, err: %s", err.Error()) return diff --git a/internal/tickevent/batch_check_all_object.go b/internal/tickevent/batch_check_all_object.go index 9d759c7..43d611b 100644 --- a/internal/tickevent/batch_check_all_object.go +++ b/internal/tickevent/batch_check_all_object.go @@ -3,7 +3,6 @@ package tickevent import ( "github.com/samber/lo" "gitlink.org.cn/cloudream/common/pkg/logger" - mysql "gitlink.org.cn/cloudream/db/sql" "gitlink.org.cn/cloudream/scanner/internal/event" ) @@ -21,7 +20,7 @@ func (e *BatchCheckAllObject) Execute(ctx ExecuteContext) { log := logger.WithType[BatchCheckAllObject]("TickEvent") log.Debugf("begin") - objectIDs, err := mysql.Object.BatchGetAllObjectIDs(ctx.Args.DB.SQLCtx(), e.lastCheckStart, CHECK_OBJECT_BATCH_SIZE) + objectIDs, err := ctx.Args.DB.Object().BatchGetAllObjectIDs(ctx.Args.DB.SQLCtx(), e.lastCheckStart, CHECK_OBJECT_BATCH_SIZE) if err != nil { log.Warnf("batch get object ids failed, err: %s", err.Error()) return diff --git a/internal/tickevent/batch_check_all_storage.go b/internal/tickevent/batch_check_all_storage.go index 1142f04..81a94da 100644 --- a/internal/tickevent/batch_check_all_storage.go +++ b/internal/tickevent/batch_check_all_storage.go @@ -2,7 +2,6 @@ package tickevent import ( "gitlink.org.cn/cloudream/common/pkg/logger" - mysql "gitlink.org.cn/cloudream/db/sql" "gitlink.org.cn/cloudream/scanner/internal/event" ) @@ -20,7 +19,7 @@ func (e *BatchCheckAllStorage) Execute(ctx ExecuteContext) { log := logger.WithType[BatchCheckAllStorage]("TickEvent") log.Debugf("begin") - storageIDs, err := mysql.Storage.BatchGetAllStorageIDs(ctx.Args.DB.SQLCtx(), e.lastCheckStart, CHECK_STORAGE_BATCH_SIZE) + storageIDs, err := ctx.Args.DB.Storage().BatchGetAllStorageIDs(ctx.Args.DB.SQLCtx(), e.lastCheckStart, CHECK_STORAGE_BATCH_SIZE) if err != nil { log.Warnf("batch get storage ids failed, err: %s", err.Error()) return diff --git a/internal/tickevent/check_agent_state.go b/internal/tickevent/check_agent_state.go index 8300a4d..9adddea 100644 --- a/internal/tickevent/check_agent_state.go +++ b/internal/tickevent/check_agent_state.go @@ -2,7 +2,6 @@ package tickevent import ( "gitlink.org.cn/cloudream/common/pkg/logger" - mysql "gitlink.org.cn/cloudream/db/sql" "gitlink.org.cn/cloudream/scanner/internal/event" ) @@ -17,7 +16,7 @@ func (e *CheckAgentState) Execute(ctx ExecuteContext) { log := logger.WithType[CheckAgentState]("TickEvent") log.Debugf("begin") - nodes, err := mysql.Node.GetAllNodes(ctx.Args.DB.SQLCtx()) + nodes, err := ctx.Args.DB.Node().GetAllNodes(ctx.Args.DB.SQLCtx()) if err != nil { log.Warnf("get all nodes failed, err: %s", err.Error()) return diff --git a/internal/tickevent/check_cache.go b/internal/tickevent/check_cache.go index 5628027..ed459da 100644 --- a/internal/tickevent/check_cache.go +++ b/internal/tickevent/check_cache.go @@ -2,7 +2,6 @@ package tickevent import ( "gitlink.org.cn/cloudream/common/pkg/logger" - mysql "gitlink.org.cn/cloudream/db/sql" "gitlink.org.cn/cloudream/scanner/internal/event" ) @@ -17,7 +16,7 @@ func (e *CheckCache) Execute(ctx ExecuteContext) { log := logger.WithType[CheckCache]("TickEvent") log.Debugf("begin") - nodes, err := mysql.Node.GetAllNodes(ctx.Args.DB.SQLCtx()) + nodes, err := ctx.Args.DB.Node().GetAllNodes(ctx.Args.DB.SQLCtx()) if err != nil { log.Warnf("get all nodes failed, err: %s", err.Error()) return From a70beea0968f623fd9bac16dbb9899db1c53e7e9 Mon Sep 17 00:00:00 2001 From: Sydonian <794346190@qq.com> Date: Wed, 14 Jun 2023 16:10:04 +0800 Subject: [PATCH 45/68] =?UTF-8?q?=E7=BB=99scanner=E7=9A=84Event=E5=8A=A0?= =?UTF-8?q?=E9=94=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- internal/config/config.go | 8 ++- internal/event/agent_check_cache.go | 89 ++++++++++++++++++++------- internal/event/agent_check_state.go | 12 ++++ internal/event/agent_check_storage.go | 84 +++++++++++++++++++------ internal/event/check_cache.go | 60 ++++++++++-------- internal/event/check_object.go | 13 ++++ internal/event/check_rep_count.go | 35 +++++++---- internal/event/event.go | 9 ++- main.go | 9 ++- 9 files changed, 234 insertions(+), 85 deletions(-) diff --git a/internal/config/config.go b/internal/config/config.go index 0d654e1..d933cd4 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -1,6 +1,7 @@ package config import ( + "gitlink.org.cn/cloudream/common/pkg/distlock" log "gitlink.org.cn/cloudream/common/pkg/logger" c "gitlink.org.cn/cloudream/common/utils/config" db "gitlink.org.cn/cloudream/db/config" @@ -11,9 +12,10 @@ type Config struct { MinAvailableRepProportion float32 `json:"minAvailableRepProportion"` // 可用的备份至少要占所有备份的比例，向上去整 NodeUnavailableSeconds int `json:"nodeUnavailableSeconds"` // 如果节点上次上报时间超过这个值，则认为节点已经不可用 - Logger log.Config `json:"logger"` - DB db.Config `json:"db"` - RabbitMQ racfg.Config `json:"rabbitMQ"` + Logger log.Config `json:"logger"` + DB db.Config `json:"db"` + RabbitMQ racfg.Config `json:"rabbitMQ"` + DistLock distlock.Config `json:"distlock"` } var cfg Config diff --git a/internal/event/agent_check_cache.go b/internal/event/agent_check_cache.go index 27891fc..3176913 100644 --- a/internal/event/agent_check_cache.go +++ b/internal/event/agent_check_cache.go @@ -4,6 +4,7 @@ import ( "database/sql" "github.com/samber/lo" + "gitlink.org.cn/cloudream/common/pkg/distlock/reqbuilder" "gitlink.org.cn/cloudream/common/pkg/logger" "gitlink.org.cn/cloudream/db/model" "gitlink.org.cn/cloudream/scanner/internal/config" @@ -47,37 +48,80 @@ func (t *AgentCheckCache) Execute(execCtx ExecuteContext) { log := logger.WithType[AgentCheckCache]("Event") log.Debugf("begin with %v", logger.FormatStruct(t)) - var isComplete bool - var caches []model.Cache - // TODO unavailable的节点需不需要发送任务？ if t.FileHashes == nil { - var err error - caches, err = execCtx.Args.DB.Cache().GetNodeCaches(execCtx.Args.DB.SQLCtx(), t.NodeID) + t.checkComplete(execCtx) + } else { + t.checkIncrement(execCtx) + } +} + +func (t *AgentCheckCache) checkComplete(execCtx ExecuteContext) { + log := logger.WithType[AgentCheckCache]("Event") + + mutex, err := reqbuilder.NewBuilder(). + Metadata(). + // 全量模式下修改某个节点所有的Cache记录 + Cache().WriteAny(). + IPFS(). + // 全量模式下修改某个节点所有的副本数据 + WriteAnyRep(t.NodeID). + MutexLock(execCtx.Args.DistLock) + if err != nil { + log.Warnf("acquire locks failed, err: %s", err.Error()) + return + } + defer mutex.Unlock() + + caches, err := execCtx.Args.DB.Cache().GetNodeCaches(execCtx.Args.DB.SQLCtx(), t.NodeID) + if err != nil { + log.WithField("NodeID", t.NodeID).Warnf("get node caches failed, err: %s", err.Error()) + return + } + + t.startCheck(execCtx, true, caches) +} + +func (t *AgentCheckCache) checkIncrement(execCtx ExecuteContext) { + log := logger.WithType[AgentCheckCache]("Event") + + builder := reqbuilder.NewBuilder() + for _, hash := range t.FileHashes { + builder. + // 增量模式下，不会有改动到Cache记录的操作 + Metadata().Cache().ReadOne(t.NodeID, hash). + // 由于副本Write锁的特点，Pin文件（创建文件）不需要Create锁 + IPFS().WriteOneRep(t.NodeID, hash) + } + mutex, err := builder.MutexLock(execCtx.Args.DistLock) + if err != nil { + log.Warnf("acquire locks failed, err: %s", err.Error()) + return + } + defer mutex.Unlock() + + var caches []model.Cache + for _, hash := range t.FileHashes { + ch, err := execCtx.Args.DB.Cache().Get(execCtx.Args.DB.SQLCtx(), hash, t.NodeID) + // 记录不存在则跳过 + if err == sql.ErrNoRows { + continue + } + if err != nil { - log.WithField("NodeID", t.NodeID).Warnf("get node caches failed, err: %s", err.Error()) + log.WithField("FileHash", hash).WithField("NodeID", t.NodeID).Warnf("get cache failed, err: %s", err.Error()) return } - isComplete = true - } else { - for _, hash := range t.FileHashes { - ch, err := execCtx.Args.DB.Cache().Get(execCtx.Args.DB.SQLCtx(), hash, t.NodeID) - // 记录不存在则跳过 - if err == sql.ErrNoRows { - continue - } + caches = append(caches, ch) + } - if err != nil { - log.WithField("FileHash", hash).WithField("NodeID", t.NodeID).Warnf("get cache failed, err: %s", err.Error()) - return - } + t.startCheck(execCtx, true, caches) +} - caches = append(caches, ch) - } - isComplete = false - } +func (t *AgentCheckCache) startCheck(execCtx ExecuteContext, isComplete bool, caches []model.Cache) { + log := logger.WithType[AgentCheckCache]("Event") // 然后向代理端发送移动文件的请求 agentClient, err := agtcli.NewClient(t.NodeID, &config.Cfg().RabbitMQ) @@ -126,6 +170,7 @@ func (t *AgentCheckCache) Execute(execCtx ExecuteContext) { } } } + func init() { RegisterMessageConvertor(func(msg scevt.AgentCheckCache) Event { return NewAgentCheckCache(msg.NodeID, msg.FileHashes) }) } diff --git a/internal/event/agent_check_state.go b/internal/event/agent_check_state.go index 0df1f1a..5e344cd 100644 --- a/internal/event/agent_check_state.go +++ b/internal/event/agent_check_state.go @@ -6,6 +6,7 @@ import ( "github.com/samber/lo" "gitlink.org.cn/cloudream/common/consts" + "gitlink.org.cn/cloudream/common/pkg/distlock/reqbuilder" "gitlink.org.cn/cloudream/common/pkg/logger" "gitlink.org.cn/cloudream/db/model" agtcli "gitlink.org.cn/cloudream/rabbitmq/client/agent" @@ -37,6 +38,17 @@ func (t *AgentCheckState) Execute(execCtx ExecuteContext) { log := logger.WithType[AgentCheckState]("Event") log.Debugf("begin with %v", logger.FormatStruct(t)) + mutex, err := reqbuilder.NewBuilder(). + Metadata(). + // 查询、修改节点状态 + Node().WriteOne(t.NodeID). + MutexLock(execCtx.Args.DistLock) + if err != nil { + log.Warnf("acquire locks failed, err: %s", err.Error()) + return + } + defer mutex.Unlock() + node, err := execCtx.Args.DB.Node().GetByID(execCtx.Args.DB.SQLCtx(), t.NodeID) if err == sql.ErrNoRows { return diff --git a/internal/event/agent_check_storage.go b/internal/event/agent_check_storage.go index 9f54c1a..652c6e5 100644 --- a/internal/event/agent_check_storage.go +++ b/internal/event/agent_check_storage.go @@ -5,6 +5,7 @@ import ( "github.com/samber/lo" "gitlink.org.cn/cloudream/common/consts" + "gitlink.org.cn/cloudream/common/pkg/distlock/reqbuilder" "gitlink.org.cn/cloudream/common/pkg/logger" "gitlink.org.cn/cloudream/db/model" agtcli "gitlink.org.cn/cloudream/rabbitmq/client/agent" @@ -47,6 +48,8 @@ func (t *AgentCheckStorage) Execute(execCtx ExecuteContext) { log := logger.WithType[AgentCheckStorage]("Event") log.Debugf("begin with %v", logger.FormatStruct(t)) + // 读取数据的地方就不加锁了，因为check任务会反复执行，单次失败问题不大 + stg, err := execCtx.Args.DB.Storage().GetByID(execCtx.Args.DB.SQLCtx(), t.StorageID) if err != nil { if err != sql.ErrNoRows { @@ -68,32 +71,75 @@ func (t *AgentCheckStorage) Execute(execCtx ExecuteContext) { return } - // 获取对象信息 - var isComplete bool - var objects []model.StorageObject if t.ObjectIDs == nil { - var err error - objects, err = execCtx.Args.DB.StorageObject().GetAllByStorageID(execCtx.Args.DB.SQLCtx(), t.StorageID) + t.checkComplete(execCtx, stg) + } else { + t.checkIncrement(execCtx, stg) + } +} + +func (t *AgentCheckStorage) checkComplete(execCtx ExecuteContext, stg model.Storage) { + log := logger.WithType[AgentCheckStorage]("Event") + + mutex, err := reqbuilder.NewBuilder(). + Metadata(). + // 全量模式下查询、修改Move记录 + StorageObject().WriteAny(). + Storage(). + // 全量模式下删除对象文件 + WriteAnyObject(t.StorageID). + MutexLock(execCtx.Args.DistLock) + if err != nil { + log.Warnf("acquire locks failed, err: %s", err.Error()) + return + } + defer mutex.Unlock() + + objects, err := execCtx.Args.DB.StorageObject().GetAllByStorageID(execCtx.Args.DB.SQLCtx(), t.StorageID) + if err != nil { + log.WithField("StorageID", t.StorageID).Warnf("get storage objects failed, err: %s", err.Error()) + return + } + + t.startCheck(execCtx, stg, true, objects) +} + +func (t *AgentCheckStorage) checkIncrement(execCtx ExecuteContext, stg model.Storage) { + log := logger.WithType[AgentCheckStorage]("Event") + + mutex, err := reqbuilder.NewBuilder(). + Metadata(). + // 全量模式下查询、修改Move记录。因为可能有多个User Move相同的文件，所以只能用集合Write锁 + StorageObject().WriteAny(). + Storage(). + // 全量模式下删除对象文件。因为可能有多个User Move相同的文件，所以只能用集合Write锁 + WriteAnyObject(t.StorageID). + MutexLock(execCtx.Args.DistLock) + if err != nil { + log.Warnf("acquire locks failed, err: %s", err.Error()) + return + } + defer mutex.Unlock() + + var objects []model.StorageObject + for _, objID := range t.ObjectIDs { + objs, err := execCtx.Args.DB.StorageObject().GetAllByStorageAndObjectID(execCtx.Args.DB.SQLCtx(), t.StorageID, objID) if err != nil { - log.WithField("StorageID", t.StorageID).Warnf("get storage objects failed, err: %s", err.Error()) + log.WithField("StorageID", t.StorageID). + WithField("ObjectID", objID). + Warnf("get storage object failed, err: %s", err.Error()) return } - isComplete = true - } else { - for _, objID := range t.ObjectIDs { - objs, err := execCtx.Args.DB.StorageObject().GetAllByStorageAndObjectID(execCtx.Args.DB.SQLCtx(), t.StorageID, objID) - if err != nil { - log.WithField("StorageID", t.StorageID). - WithField("ObjectID", objID). - Warnf("get storage object failed, err: %s", err.Error()) - return - } - objects = append(objects, objs...) - } - isComplete = false + objects = append(objects, objs...) } + t.startCheck(execCtx, stg, false, objects) +} + +func (t *AgentCheckStorage) startCheck(execCtx ExecuteContext, stg model.Storage, isComplete bool, objects []model.StorageObject) { + log := logger.WithType[AgentCheckStorage]("Event") + // 投递任务 agentClient, err := agtcli.NewClient(stg.NodeID, &config.Cfg().RabbitMQ) if err != nil { diff --git a/internal/event/check_cache.go b/internal/event/check_cache.go index 05d22fe..c7acf6f 100644 --- a/internal/event/check_cache.go +++ b/internal/event/check_cache.go @@ -2,11 +2,10 @@ package event import ( "database/sql" - "fmt" - "github.com/jmoiron/sqlx" "github.com/samber/lo" "gitlink.org.cn/cloudream/common/consts" + "gitlink.org.cn/cloudream/common/pkg/distlock/reqbuilder" "gitlink.org.cn/cloudream/common/pkg/logger" "gitlink.org.cn/cloudream/db/model" scevt "gitlink.org.cn/cloudream/rabbitmq/message/scanner/event" @@ -38,36 +37,45 @@ func (t *CheckCache) Execute(execCtx ExecuteContext) { log := logger.WithType[AgentCheckStorage]("Event") log.Debugf("begin with %v", logger.FormatStruct(t)) - err := execCtx.Args.DB.DoTx(sql.LevelSerializable, func(tx *sqlx.Tx) error { - node, err := execCtx.Args.DB.Node().GetByID(tx, t.NodeID) - if err == sql.ErrNoRows { - return nil - } - if err != nil { - return fmt.Errorf("get node failed, err: %w", err) - } - - if node.State != consts.NODE_STATE_UNAVAILABLE { - return nil - } + mutex, err := reqbuilder.NewBuilder(). + Metadata(). + // 查询节点状态 + Node().ReadOne(t.NodeID). + // 删除节点所有的Cache记录 + Cache().WriteAny(). + MutexLock(execCtx.Args.DistLock) + if err != nil { + log.Warnf("acquire locks failed, err: %s", err.Error()) + return + } + defer mutex.Unlock() - caches, err := execCtx.Args.DB.Cache().GetNodeCaches(tx, t.NodeID) - if err != nil { - return fmt.Errorf("get node caches failed, err: %w", err) - } + node, err := execCtx.Args.DB.Node().GetByID(execCtx.Args.DB.SQLCtx(), t.NodeID) + if err == sql.ErrNoRows { + return + } + if err != nil { + log.WithField("NodeID", t.NodeID).Warnf("get node failed, err: %s", err.Error()) + return + } - err = execCtx.Args.DB.Cache().DeleteNodeAll(tx, t.NodeID) - if err != nil { - return fmt.Errorf("delete node all caches failed, err: %w", err) - } + if node.State != consts.NODE_STATE_UNAVAILABLE { + return + } - execCtx.Executor.Post(NewCheckRepCount(lo.Map(caches, func(ch model.Cache, index int) string { return ch.FileHash }))) - return nil - }) + caches, err := execCtx.Args.DB.Cache().GetNodeCaches(execCtx.Args.DB.SQLCtx(), t.NodeID) + if err != nil { + log.WithField("NodeID", t.NodeID).Warnf("get node caches failed, err: %s", err.Error()) + return + } + err = execCtx.Args.DB.Cache().DeleteNodeAll(execCtx.Args.DB.SQLCtx(), t.NodeID) if err != nil { - log.WithField("NodeID", t.NodeID).Warn(err.Error()) + log.WithField("NodeID", t.NodeID).Warnf("delete node all caches failed, err: %s", err.Error()) + return } + + execCtx.Executor.Post(NewCheckRepCount(lo.Map(caches, func(ch model.Cache, index int) string { return ch.FileHash }))) } func init() { diff --git a/internal/event/check_object.go b/internal/event/check_object.go index 85e9d78..c6ddb3c 100644 --- a/internal/event/check_object.go +++ b/internal/event/check_object.go @@ -2,6 +2,7 @@ package event import ( "github.com/samber/lo" + "gitlink.org.cn/cloudream/common/pkg/distlock/reqbuilder" "gitlink.org.cn/cloudream/common/pkg/logger" scevt "gitlink.org.cn/cloudream/rabbitmq/message/scanner/event" ) @@ -30,6 +31,18 @@ func (t *CheckObject) Execute(execCtx ExecuteContext) { log := logger.WithType[CheckObject]("Event") log.Debugf("begin with %v", logger.FormatStruct(t)) + // 检查对象是否没有被引用的时候，需要读取StorageObject表 + builder := reqbuilder.NewBuilder().Metadata().StorageObject().ReadAny() + for _, objID := range t.ObjectIDs { + builder.Metadata().Object().WriteOne(objID) + } + mutex, err := builder.MutexLock(execCtx.Args.DistLock) + if err != nil { + log.Warnf("acquire locks failed, err: %s", err.Error()) + return + } + defer mutex.Unlock() + for _, objID := range t.ObjectIDs { err := execCtx.Args.DB.Object().DeleteUnused(execCtx.Args.DB.SQLCtx(), objID) if err != nil { diff --git a/internal/event/check_rep_count.go b/internal/event/check_rep_count.go index b219f1b..cdbecae 100644 --- a/internal/event/check_rep_count.go +++ b/internal/event/check_rep_count.go @@ -8,6 +8,7 @@ import ( "github.com/jmoiron/sqlx" "github.com/samber/lo" "gitlink.org.cn/cloudream/common/consts" + "gitlink.org.cn/cloudream/common/pkg/distlock/reqbuilder" "gitlink.org.cn/cloudream/common/pkg/logger" mymath "gitlink.org.cn/cloudream/common/utils/math" mysort "gitlink.org.cn/cloudream/common/utils/sort" @@ -41,6 +42,23 @@ func (t *CheckRepCount) Execute(execCtx ExecuteContext) { log := logger.WithType[CheckRepCount]("Event") log.Debugf("begin with %v", logger.FormatStruct(t)) + mutex, err := reqbuilder.NewBuilder(). + Metadata(). + // 读取某个FileHash的备份数设定 + ObjectRep().ReadAny(). + // 读取某个FileHash是否被Block引用 + ObjectBlock().ReadAny(). + // 获取所有可用的节点 + Node().ReadAny(). + // 增加或修改FileHash关联的Cache记录 + Cache().WriteAny(). + MutexLock(execCtx.Args.DistLock) + if err != nil { + log.Warnf("acquire locks failed, err: %s", err.Error()) + return + } + defer mutex.Unlock() + updatedNodeAndHashes := make(map[int][]string) for _, fileHash := range t.FileHashes { @@ -66,7 +84,10 @@ func (t *CheckRepCount) checkOneRepCount(fileHash string, execCtx ExecuteContext log := logger.WithType[CheckRepCount]("Event") var updatedNodeIDs []int - err := execCtx.Args.DB.DoTx(sql.LevelSerializable, func(tx *sqlx.Tx) error { + err := execCtx.Args.DB.DoTx(sql.LevelDefault, func(tx *sqlx.Tx) error { + // 计算所需的最少备份数： + // 1. ObjectRep中期望备份数的最大值 + // 2. 如果ObjectBlock存在对此文件的引用，则至少为1 repMaxCnt, err := execCtx.Args.DB.ObjectRep().GetFileMaxRepCount(tx, fileHash) if err != nil { return fmt.Errorf("get file max rep count failed, err: %w", err) @@ -77,9 +98,6 @@ func (t *CheckRepCount) checkOneRepCount(fileHash string, execCtx ExecuteContext return fmt.Errorf("count block with hash failed, err: %w", err) } - // 计算所需的最少备份数： - // ObjectRep中期望备份数的最大值 - // 如果ObjectBlock存在对此文件的引用，则至少为1 needRepCount := mymath.Max(repMaxCnt, mymath.Min(1, blkCnt)) repNodes, err := execCtx.Args.DB.Cache().GetCachingFileNodes(tx, fileHash) @@ -114,12 +132,11 @@ func (t *CheckRepCount) checkOneRepCount(fileHash string, execCtx ExecuteContext return nil } - minAvaiNodeCnt := int(math.Ceil(float64(config.Cfg().MinAvailableRepProportion) * float64(needRepCount))) - // 因为总备份数不够，而需要增加的备份数 add1 := mymath.Max(0, needRepCount-len(repNodes)) // 因为Available的备份数占比过少，而需要增加的备份数 + minAvaiNodeCnt := int(math.Ceil(float64(config.Cfg().MinAvailableRepProportion) * float64(needRepCount))) add2 := mymath.Max(0, minAvaiNodeCnt-len(normalNodes)) // 最终需要增加的备份数，是以上两种情况的最大值 @@ -144,11 +161,7 @@ func (t *CheckRepCount) checkOneRepCount(fileHash string, execCtx ExecuteContext return nil }) - if err != nil { - return nil, err - } - - return updatedNodeIDs, nil + return updatedNodeIDs, err } func chooseNewRepNodes(allNodes []model.Node, curRepNodes []model.Node, newCount int) []model.Node { diff --git a/internal/event/event.go b/internal/event/event.go index 6296252..7bf9076 100644 --- a/internal/event/event.go +++ b/internal/event/event.go @@ -4,13 +4,15 @@ import ( "fmt" "reflect" + distlocksvc "gitlink.org.cn/cloudream/common/pkg/distlock/service" event "gitlink.org.cn/cloudream/common/pkg/event" "gitlink.org.cn/cloudream/common/pkg/typedispatcher" mydb "gitlink.org.cn/cloudream/db" ) type ExecuteArgs struct { - DB *mydb.DB + DB *mydb.DB + DistLock *distlocksvc.Service } type Executor = event.Executor[ExecuteArgs] @@ -21,9 +23,10 @@ type Event = event.Event[ExecuteArgs] type ExecuteOption = event.ExecuteOption -func NewExecutor(db *mydb.DB) Executor { +func NewExecutor(db *mydb.DB, distLock *distlocksvc.Service) Executor { return event.NewExecutor(ExecuteArgs{ - DB: db, + DB: db, + DistLock: distLock, }) } diff --git a/main.go b/main.go index 624e537..f9a5e40 100644 --- a/main.go +++ b/main.go @@ -5,6 +5,7 @@ import ( "os" "sync" + distlocksvc "gitlink.org.cn/cloudream/common/pkg/distlock/service" log "gitlink.org.cn/cloudream/common/pkg/logger" "gitlink.org.cn/cloudream/db" scsvr "gitlink.org.cn/cloudream/rabbitmq/server/scanner" @@ -32,10 +33,16 @@ func main() { log.Fatalf("new db failed, err: %s", err.Error()) } + distlockSvc, err := distlocksvc.NewService(&config.Cfg().DistLock) + if err != nil { + log.Warnf("new distlock service failed, err: %s", err.Error()) + os.Exit(1) + } + wg := sync.WaitGroup{} wg.Add(2) - eventExecutor := event.NewExecutor(db) + eventExecutor := event.NewExecutor(db, distlockSvc) go serveEventExecutor(&eventExecutor, &wg) agtSvr, err := scsvr.NewServer(services.NewService(&eventExecutor), &config.Cfg().RabbitMQ) From beaa925a6263bb9b49d95d957bc9fe25a52c1524 Mon Sep 17 00:00:00 2001 From: Sydonian <794346190@qq.com> Date: Wed, 14 Jun 2023 16:21:52 +0800 Subject: [PATCH 46/68] =?UTF-8?q?=E6=B6=89=E5=8F=8A=E5=88=B0=E5=A4=9A?= =?UTF-8?q?=E4=B8=AA=E8=A1=A8=E7=9A=84=E5=85=83=E6=95=B0=E6=8D=AE=E6=94=B9?= =?UTF-8?q?=E5=8A=A8=E5=A2=9E=E5=8A=A0=E4=BA=8B=E5=8A=A1=EF=BC=8C=E4=BF=9D?= =?UTF-8?q?=E8=AF=81=E6=95=B0=E6=8D=AE=E5=AE=8C=E6=95=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- internal/event/check_rep_count.go | 119 ++++++++++++++---------------- 1 file changed, 57 insertions(+), 62 deletions(-) diff --git a/internal/event/check_rep_count.go b/internal/event/check_rep_count.go index cdbecae..d7bc395 100644 --- a/internal/event/check_rep_count.go +++ b/internal/event/check_rep_count.go @@ -1,11 +1,9 @@ package event import ( - "database/sql" "fmt" "math" - "github.com/jmoiron/sqlx" "github.com/samber/lo" "gitlink.org.cn/cloudream/common/consts" "gitlink.org.cn/cloudream/common/pkg/distlock/reqbuilder" @@ -82,84 +80,81 @@ func (t *CheckRepCount) Execute(execCtx ExecuteContext) { func (t *CheckRepCount) checkOneRepCount(fileHash string, execCtx ExecuteContext) ([]int, error) { log := logger.WithType[CheckRepCount]("Event") + sqlCtx := execCtx.Args.DB.SQLCtx() var updatedNodeIDs []int - err := execCtx.Args.DB.DoTx(sql.LevelDefault, func(tx *sqlx.Tx) error { - // 计算所需的最少备份数： - // 1. ObjectRep中期望备份数的最大值 - // 2. 如果ObjectBlock存在对此文件的引用，则至少为1 - repMaxCnt, err := execCtx.Args.DB.ObjectRep().GetFileMaxRepCount(tx, fileHash) - if err != nil { - return fmt.Errorf("get file max rep count failed, err: %w", err) - } + // 计算所需的最少备份数： + // 1. ObjectRep中期望备份数的最大值 + // 2. 如果ObjectBlock存在对此文件的引用，则至少为1 + repMaxCnt, err := execCtx.Args.DB.ObjectRep().GetFileMaxRepCount(sqlCtx, fileHash) + if err != nil { + return nil, fmt.Errorf("get file max rep count failed, err: %w", err) + } - blkCnt, err := execCtx.Args.DB.ObjectBlock().CountBlockWithHash(tx, fileHash) - if err != nil { - return fmt.Errorf("count block with hash failed, err: %w", err) - } + blkCnt, err := execCtx.Args.DB.ObjectBlock().CountBlockWithHash(sqlCtx, fileHash) + if err != nil { + return nil, fmt.Errorf("count block with hash failed, err: %w", err) + } - needRepCount := mymath.Max(repMaxCnt, mymath.Min(1, blkCnt)) + needRepCount := mymath.Max(repMaxCnt, mymath.Min(1, blkCnt)) - repNodes, err := execCtx.Args.DB.Cache().GetCachingFileNodes(tx, fileHash) - if err != nil { - return fmt.Errorf("get caching file nodes failed, err: %w", err) - } + repNodes, err := execCtx.Args.DB.Cache().GetCachingFileNodes(sqlCtx, fileHash) + if err != nil { + return nil, fmt.Errorf("get caching file nodes failed, err: %w", err) + } - allNodes, err := execCtx.Args.DB.Node().GetAllNodes(tx) - if err != nil { - return fmt.Errorf("get all nodes failed, err: %w", err) - } + allNodes, err := execCtx.Args.DB.Node().GetAllNodes(sqlCtx) + if err != nil { + return nil, fmt.Errorf("get all nodes failed, err: %w", err) + } - var normalNodes, unavaiNodes []model.Node - for _, node := range repNodes { - if node.State == consts.NODE_STATE_NORMAL { - normalNodes = append(normalNodes, node) - } else if node.State == consts.NODE_STATE_UNAVAILABLE { - unavaiNodes = append(unavaiNodes, node) - } + var normalNodes, unavaiNodes []model.Node + for _, node := range repNodes { + if node.State == consts.NODE_STATE_NORMAL { + normalNodes = append(normalNodes, node) + } else if node.State == consts.NODE_STATE_UNAVAILABLE { + unavaiNodes = append(unavaiNodes, node) } + } - // 如果Available的备份数超过期望备份数，则让一些节点退出 - if len(normalNodes) > needRepCount { - delNodes := chooseDeleteAvaiRepNodes(allNodes, normalNodes, len(normalNodes)-needRepCount) - for _, node := range delNodes { - err := execCtx.Args.DB.Cache().SetTemp(tx, fileHash, node.NodeID) - if err != nil { - return fmt.Errorf("change cache state failed, err: %w", err) - } - updatedNodeIDs = append(updatedNodeIDs, node.NodeID) + // 如果Available的备份数超过期望备份数，则让一些节点退出 + if len(normalNodes) > needRepCount { + delNodes := chooseDeleteAvaiRepNodes(allNodes, normalNodes, len(normalNodes)-needRepCount) + for _, node := range delNodes { + err := execCtx.Args.DB.Cache().SetTemp(sqlCtx, fileHash, node.NodeID) + if err != nil { + return nil, fmt.Errorf("change cache state failed, err: %w", err) } - return nil + updatedNodeIDs = append(updatedNodeIDs, node.NodeID) } + return updatedNodeIDs, nil + } - // 因为总备份数不够，而需要增加的备份数 - add1 := mymath.Max(0, needRepCount-len(repNodes)) + // 因为总备份数不够，而需要增加的备份数 + add1 := mymath.Max(0, needRepCount-len(repNodes)) - // 因为Available的备份数占比过少，而需要增加的备份数 - minAvaiNodeCnt := int(math.Ceil(float64(config.Cfg().MinAvailableRepProportion) * float64(needRepCount))) - add2 := mymath.Max(0, minAvaiNodeCnt-len(normalNodes)) + // 因为Available的备份数占比过少，而需要增加的备份数 + minAvaiNodeCnt := int(math.Ceil(float64(config.Cfg().MinAvailableRepProportion) * float64(needRepCount))) + add2 := mymath.Max(0, minAvaiNodeCnt-len(normalNodes)) - // 最终需要增加的备份数，是以上两种情况的最大值 - finalAddCount := mymath.Max(add1, add2) + // 最终需要增加的备份数，是以上两种情况的最大值 + finalAddCount := mymath.Max(add1, add2) - if finalAddCount > 0 { - newNodes := chooseNewRepNodes(allNodes, repNodes, finalAddCount) - if len(newNodes) < finalAddCount { - log.WithField("FileHash", fileHash).Warnf("need %d more rep nodes, but get only %d nodes", finalAddCount, len(newNodes)) - // TODO 节点数不够，进行一个告警 - } + if finalAddCount > 0 { + newNodes := chooseNewRepNodes(allNodes, repNodes, finalAddCount) + if len(newNodes) < finalAddCount { + log.WithField("FileHash", fileHash).Warnf("need %d more rep nodes, but get only %d nodes", finalAddCount, len(newNodes)) + // TODO 节点数不够，进行一个告警 + } - for _, node := range newNodes { - err := execCtx.Args.DB.Cache().CreatePinned(tx, fileHash, node.NodeID) - if err != nil { - return fmt.Errorf("create cache failed, err: %w", err) - } - updatedNodeIDs = append(updatedNodeIDs, node.NodeID) + for _, node := range newNodes { + err := execCtx.Args.DB.Cache().CreatePinned(sqlCtx, fileHash, node.NodeID) + if err != nil { + return nil, fmt.Errorf("create cache failed, err: %w", err) } + updatedNodeIDs = append(updatedNodeIDs, node.NodeID) } - - return nil - }) + } return updatedNodeIDs, err } From d50f05ed21be95e8e929187b1377a24103580c21 Mon Sep 17 00:00:00 2001 From: Sydonian <794346190@qq.com> Date: Thu, 15 Jun 2023 17:11:24 +0800 Subject: [PATCH 47/68] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E8=B0=83=E8=AF=95?= =?UTF-8?q?=E5=8F=91=E7=8E=B0=E7=9A=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main.go | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/main.go b/main.go index f9a5e40..8c2b256 100644 --- a/main.go +++ b/main.go @@ -33,14 +33,15 @@ func main() { log.Fatalf("new db failed, err: %s", err.Error()) } + wg := sync.WaitGroup{} + wg.Add(3) + distlockSvc, err := distlocksvc.NewService(&config.Cfg().DistLock) if err != nil { log.Warnf("new distlock service failed, err: %s", err.Error()) os.Exit(1) } - - wg := sync.WaitGroup{} - wg.Add(2) + go serveDistLock(distlockSvc, &wg) eventExecutor := event.NewExecutor(db, distlockSvc) go serveEventExecutor(&eventExecutor, &wg) @@ -91,6 +92,20 @@ func serveScannerServer(server *scsvr.Server, wg *sync.WaitGroup) { wg.Done() } +func serveDistLock(svc *distlocksvc.Service, wg *sync.WaitGroup) { + log.Info("start serving distlock") + + err := svc.Serve() + + if err != nil { + log.Errorf("distlock stopped with error: %s", err.Error()) + } + + log.Info("distlock stopped") + + wg.Done() +} + func startTickEvent(tickExecutor *tickevent.Executor) { // TODO 可以考虑增加配置文件，配置这些任务间隔时间 From d013412d0b24495c8bc31538965107aced5f3787 Mon Sep 17 00:00:00 2001 From: Sydonian <794346190@qq.com> Date: Fri, 16 Jun 2023 15:22:35 +0800 Subject: [PATCH 48/68] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E8=B0=83=E8=AF=95?= =?UTF-8?q?=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Makefile | 2 +- internal/event/agent_check_cache.go | 1 + internal/event/agent_check_state.go | 1 + internal/event/agent_check_storage.go | 1 + internal/event/check_cache.go | 1 + internal/event/check_object.go | 1 + internal/event/check_rep_count.go | 1 + internal/tickevent/batch_all_agent_check_cache.go | 1 + internal/tickevent/batch_check_all_object.go | 1 + internal/tickevent/batch_check_all_rep_count.go | 1 + internal/tickevent/batch_check_all_storage.go | 1 + internal/tickevent/check_agent_state.go | 1 + internal/tickevent/check_cache.go | 1 + 13 files changed, 13 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index a8bb2e8..2cf60ba 100644 --- a/Makefile +++ b/Makefile @@ -6,7 +6,7 @@ ASSETS_DIR_NAME = "assets" BUILD_DIR = "../../build" build: - go build -o ${BUILD_DIR}/${OUTPUT_DIR_NAME}/${OUTPUT_BINARY_NAME} + go build -o ${BUILD_DIR}/${OUTPUT_DIR_NAME}/${OUTPUT_BINARY_NAME}${OUTPUT_BINARY_EXT} @if [ -d ${ASSETS_DIR_NAME} ] && [ -n "`ls -A ${ASSETS_DIR_NAME}`" ] ;then \ cp -r ${ASSETS_DIR_NAME}/* ${BUILD_DIR}/${OUTPUT_DIR_NAME}/; \ fi diff --git a/internal/event/agent_check_cache.go b/internal/event/agent_check_cache.go index 3176913..38e9c2b 100644 --- a/internal/event/agent_check_cache.go +++ b/internal/event/agent_check_cache.go @@ -47,6 +47,7 @@ func (t *AgentCheckCache) TryMerge(other Event) bool { func (t *AgentCheckCache) Execute(execCtx ExecuteContext) { log := logger.WithType[AgentCheckCache]("Event") log.Debugf("begin with %v", logger.FormatStruct(t)) + defer log.Debugf("end") // TODO unavailable的节点需不需要发送任务？ diff --git a/internal/event/agent_check_state.go b/internal/event/agent_check_state.go index 5e344cd..1471db3 100644 --- a/internal/event/agent_check_state.go +++ b/internal/event/agent_check_state.go @@ -37,6 +37,7 @@ func (t *AgentCheckState) TryMerge(other Event) bool { func (t *AgentCheckState) Execute(execCtx ExecuteContext) { log := logger.WithType[AgentCheckState]("Event") log.Debugf("begin with %v", logger.FormatStruct(t)) + defer log.Debugf("end") mutex, err := reqbuilder.NewBuilder(). Metadata(). diff --git a/internal/event/agent_check_storage.go b/internal/event/agent_check_storage.go index 652c6e5..1a6e5e4 100644 --- a/internal/event/agent_check_storage.go +++ b/internal/event/agent_check_storage.go @@ -47,6 +47,7 @@ func (t *AgentCheckStorage) TryMerge(other Event) bool { func (t *AgentCheckStorage) Execute(execCtx ExecuteContext) { log := logger.WithType[AgentCheckStorage]("Event") log.Debugf("begin with %v", logger.FormatStruct(t)) + defer log.Debugf("end") // 读取数据的地方就不加锁了，因为check任务会反复执行，单次失败问题不大 diff --git a/internal/event/check_cache.go b/internal/event/check_cache.go index c7acf6f..9e7fd44 100644 --- a/internal/event/check_cache.go +++ b/internal/event/check_cache.go @@ -36,6 +36,7 @@ func (t *CheckCache) TryMerge(other Event) bool { func (t *CheckCache) Execute(execCtx ExecuteContext) { log := logger.WithType[AgentCheckStorage]("Event") log.Debugf("begin with %v", logger.FormatStruct(t)) + defer log.Debugf("end") mutex, err := reqbuilder.NewBuilder(). Metadata(). diff --git a/internal/event/check_object.go b/internal/event/check_object.go index c6ddb3c..c610f36 100644 --- a/internal/event/check_object.go +++ b/internal/event/check_object.go @@ -30,6 +30,7 @@ func (t *CheckObject) TryMerge(other Event) bool { func (t *CheckObject) Execute(execCtx ExecuteContext) { log := logger.WithType[CheckObject]("Event") log.Debugf("begin with %v", logger.FormatStruct(t)) + defer log.Debugf("end") // 检查对象是否没有被引用的时候，需要读取StorageObject表 builder := reqbuilder.NewBuilder().Metadata().StorageObject().ReadAny() diff --git a/internal/event/check_rep_count.go b/internal/event/check_rep_count.go index d7bc395..644762a 100644 --- a/internal/event/check_rep_count.go +++ b/internal/event/check_rep_count.go @@ -39,6 +39,7 @@ func (t *CheckRepCount) TryMerge(other Event) bool { func (t *CheckRepCount) Execute(execCtx ExecuteContext) { log := logger.WithType[CheckRepCount]("Event") log.Debugf("begin with %v", logger.FormatStruct(t)) + defer log.Debugf("end") mutex, err := reqbuilder.NewBuilder(). Metadata(). diff --git a/internal/tickevent/batch_all_agent_check_cache.go b/internal/tickevent/batch_all_agent_check_cache.go index 931b999..8f51916 100644 --- a/internal/tickevent/batch_all_agent_check_cache.go +++ b/internal/tickevent/batch_all_agent_check_cache.go @@ -20,6 +20,7 @@ func NewBatchAllAgentCheckCache() *BatchAllAgentCheckCache { func (e *BatchAllAgentCheckCache) Execute(ctx ExecuteContext) { log := logger.WithType[BatchAllAgentCheckCache]("TickEvent") log.Debugf("begin") + defer log.Debugf("end") if e.nodeIDs == nil || len(e.nodeIDs) == 0 { nodes, err := ctx.Args.DB.Node().GetAllNodes(ctx.Args.DB.SQLCtx()) diff --git a/internal/tickevent/batch_check_all_object.go b/internal/tickevent/batch_check_all_object.go index 43d611b..f70b9e3 100644 --- a/internal/tickevent/batch_check_all_object.go +++ b/internal/tickevent/batch_check_all_object.go @@ -19,6 +19,7 @@ func NewBatchCheckAllObject() *BatchCheckAllObject { func (e *BatchCheckAllObject) Execute(ctx ExecuteContext) { log := logger.WithType[BatchCheckAllObject]("TickEvent") log.Debugf("begin") + defer log.Debugf("end") objectIDs, err := ctx.Args.DB.Object().BatchGetAllObjectIDs(ctx.Args.DB.SQLCtx(), e.lastCheckStart, CHECK_OBJECT_BATCH_SIZE) if err != nil { diff --git a/internal/tickevent/batch_check_all_rep_count.go b/internal/tickevent/batch_check_all_rep_count.go index cdb6902..2388efd 100644 --- a/internal/tickevent/batch_check_all_rep_count.go +++ b/internal/tickevent/batch_check_all_rep_count.go @@ -18,6 +18,7 @@ func NewBatchCheckAllRepCount() *BatchCheckAllRepCount { func (e *BatchCheckAllRepCount) Execute(ctx ExecuteContext) { log := logger.WithType[BatchCheckAllRepCount]("TickEvent") log.Debugf("begin") + defer log.Debugf("end") fileHashes, err := ctx.Args.DB.Cache().BatchGetAllFileHashes(ctx.Args.DB.SQLCtx(), e.lastCheckStart, CHECK_CACHE_BATCH_SIZE) if err != nil { diff --git a/internal/tickevent/batch_check_all_storage.go b/internal/tickevent/batch_check_all_storage.go index 81a94da..07015ae 100644 --- a/internal/tickevent/batch_check_all_storage.go +++ b/internal/tickevent/batch_check_all_storage.go @@ -18,6 +18,7 @@ func NewBatchCheckAllStorage() *BatchCheckAllStorage { func (e *BatchCheckAllStorage) Execute(ctx ExecuteContext) { log := logger.WithType[BatchCheckAllStorage]("TickEvent") log.Debugf("begin") + defer log.Debugf("end") storageIDs, err := ctx.Args.DB.Storage().BatchGetAllStorageIDs(ctx.Args.DB.SQLCtx(), e.lastCheckStart, CHECK_STORAGE_BATCH_SIZE) if err != nil { diff --git a/internal/tickevent/check_agent_state.go b/internal/tickevent/check_agent_state.go index 9adddea..4c34178 100644 --- a/internal/tickevent/check_agent_state.go +++ b/internal/tickevent/check_agent_state.go @@ -15,6 +15,7 @@ func NewCheckAgentState() *CheckAgentState { func (e *CheckAgentState) Execute(ctx ExecuteContext) { log := logger.WithType[CheckAgentState]("TickEvent") log.Debugf("begin") + defer log.Debugf("end") nodes, err := ctx.Args.DB.Node().GetAllNodes(ctx.Args.DB.SQLCtx()) if err != nil { diff --git a/internal/tickevent/check_cache.go b/internal/tickevent/check_cache.go index ed459da..3935489 100644 --- a/internal/tickevent/check_cache.go +++ b/internal/tickevent/check_cache.go @@ -15,6 +15,7 @@ func NewCheckCache() *CheckCache { func (e *CheckCache) Execute(ctx ExecuteContext) { log := logger.WithType[CheckCache]("TickEvent") log.Debugf("begin") + defer log.Debugf("end") nodes, err := ctx.Args.DB.Node().GetAllNodes(ctx.Args.DB.SQLCtx()) if err != nil { From bfca490deb014fb8107f1db6b288b15a5c515427 Mon Sep 17 00:00:00 2001 From: Sydonian <794346190@qq.com> Date: Sun, 25 Jun 2023 15:45:38 +0800 Subject: [PATCH 49/68] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E8=B0=83=E8=AF=95?= =?UTF-8?q?=E9=97=AE=E9=A2=98=EF=BC=9B=E4=BD=BF=E7=94=A8magefile=E4=BB=A3?= =?UTF-8?q?=E6=9B=BFmakefile?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Makefile | 15 --------------- internal/event/agent_check_state.go | 8 ++++---- magefiles/magefile.go | 22 ++++++++++++++++++++++ main.go | 14 ++++++++------ 4 files changed, 34 insertions(+), 25 deletions(-) delete mode 100644 Makefile create mode 100644 magefiles/magefile.go diff --git a/Makefile b/Makefile deleted file mode 100644 index 2cf60ba..0000000 --- a/Makefile +++ /dev/null @@ -1,15 +0,0 @@ -OUTPUT_BINARY_NAME = "scanner" -OUTPUT_DIR_NAME = "scanner" - - -ASSETS_DIR_NAME = "assets" -BUILD_DIR = "../../build" - -build: - go build -o ${BUILD_DIR}/${OUTPUT_DIR_NAME}/${OUTPUT_BINARY_NAME}${OUTPUT_BINARY_EXT} - @if [ -d ${ASSETS_DIR_NAME} ] && [ -n "`ls -A ${ASSETS_DIR_NAME}`" ] ;then \ - cp -r ${ASSETS_DIR_NAME}/* ${BUILD_DIR}/${OUTPUT_DIR_NAME}/; \ - fi - -clean: - rm -f ${BUILD_DIR}/${OUTPUT_DIR_NAME}/${OUTPUT_BINARY_NAME} \ No newline at end of file diff --git a/internal/event/agent_check_state.go b/internal/event/agent_check_state.go index 1471db3..1b17d4b 100644 --- a/internal/event/agent_check_state.go +++ b/internal/event/agent_check_state.go @@ -66,8 +66,8 @@ func (t *AgentCheckState) Execute(execCtx ExecuteContext) { // 检查上次上报时间，超时的设置为不可用 // TODO 没有上报过是否要特殊处理？ - if node.LastReportTime == nil && time.Since(*node.LastReportTime) > time.Duration(config.Cfg().NodeUnavailableSeconds)*time.Second { - err := execCtx.Args.DB.Node().ChangeState(execCtx.Args.DB.SQLCtx(), t.NodeID, consts.NODE_STATE_UNAVAILABLE) + if node.LastReportTime != nil && time.Since(*node.LastReportTime) > time.Duration(config.Cfg().NodeUnavailableSeconds)*time.Second { + err := execCtx.Args.DB.Node().UpdateState(execCtx.Args.DB.SQLCtx(), t.NodeID, consts.NODE_STATE_UNAVAILABLE) if err != nil { log.WithField("NodeID", t.NodeID).Warnf("set node state failed, err: %s", err.Error()) return @@ -104,7 +104,7 @@ func (t *AgentCheckState) Execute(execCtx ExecuteContext) { if getResp.Body.IPFSState != consts.IPFS_STATE_OK { log.WithField("NodeID", t.NodeID).Warnf("IPFS status is %s, set node state unavailable", getResp.Body.IPFSState) - err := execCtx.Args.DB.Node().ChangeState(execCtx.Args.DB.SQLCtx(), t.NodeID, consts.NODE_STATE_UNAVAILABLE) + err := execCtx.Args.DB.Node().UpdateState(execCtx.Args.DB.SQLCtx(), t.NodeID, consts.NODE_STATE_UNAVAILABLE) if err != nil { log.WithField("NodeID", t.NodeID).Warnf("change node state failed, err: %s", err.Error()) } @@ -112,7 +112,7 @@ func (t *AgentCheckState) Execute(execCtx ExecuteContext) { } // TODO 如果以后还有其他的状态，要判断哪些状态下能设置Normal - err = execCtx.Args.DB.Node().ChangeState(execCtx.Args.DB.SQLCtx(), t.NodeID, consts.NODE_STATE_NORMAL) + err = execCtx.Args.DB.Node().UpdateState(execCtx.Args.DB.SQLCtx(), t.NodeID, consts.NODE_STATE_NORMAL) if err != nil { log.WithField("NodeID", t.NodeID).Warnf("change node state failed, err: %s", err.Error()) } diff --git a/magefiles/magefile.go b/magefiles/magefile.go new file mode 100644 index 0000000..f146a71 --- /dev/null +++ b/magefiles/magefile.go @@ -0,0 +1,22 @@ +//go:build mage + +package main + +import ( + "magefiles" + "magefiles/utils" + + //mage:import + "magefiles/targets" +) + +var Default = Build + +func Build() error { + return utils.Build(magefiles.BuildArgs{ + OutputBinName: "scanner", + OutputDirName: "scanner", + AssetsDir: "assets", + PubArgs: targets.PubGoBuildArgs, + }) +} diff --git a/main.go b/main.go index 8c2b256..de7d256 100644 --- a/main.go +++ b/main.go @@ -109,15 +109,17 @@ func serveDistLock(svc *distlocksvc.Service, wg *sync.WaitGroup) { func startTickEvent(tickExecutor *tickevent.Executor) { // TODO 可以考虑增加配置文件，配置这些任务间隔时间 - tickExecutor.Start(tickevent.NewBatchAllAgentCheckCache(), 5*60*1000, tickevent.StartOption{RandomStartDelayMs: 60 * 1000}) + interval := 5 * 60 * 1000 - tickExecutor.Start(tickevent.NewBatchCheckAllObject(), 5*60*1000, tickevent.StartOption{RandomStartDelayMs: 60 * 1000}) + tickExecutor.Start(tickevent.NewBatchAllAgentCheckCache(), interval, tickevent.StartOption{RandomStartDelayMs: 60 * 1000}) - tickExecutor.Start(tickevent.NewBatchCheckAllRepCount(), 5*60*1000, tickevent.StartOption{RandomStartDelayMs: 60 * 1000}) + tickExecutor.Start(tickevent.NewBatchCheckAllObject(), interval, tickevent.StartOption{RandomStartDelayMs: 60 * 1000}) - tickExecutor.Start(tickevent.NewBatchCheckAllStorage(), 5*60*1000, tickevent.StartOption{RandomStartDelayMs: 60 * 1000}) + tickExecutor.Start(tickevent.NewBatchCheckAllRepCount(), interval, tickevent.StartOption{RandomStartDelayMs: 60 * 1000}) - //tickExecutor.Start(tickevent.NewCheckAgentState(), 5*60*1000, tickevent.StartOption{RandomFirstStartDelayMs: 60 * 1000}) + tickExecutor.Start(tickevent.NewBatchCheckAllStorage(), interval, tickevent.StartOption{RandomStartDelayMs: 60 * 1000}) - tickExecutor.Start(tickevent.NewCheckCache(), 5*60*1000, tickevent.StartOption{RandomStartDelayMs: 60 * 1000}) + tickExecutor.Start(tickevent.NewCheckAgentState(), 5*60*1000, tickevent.StartOption{RandomStartDelayMs: 60 * 1000}) + + tickExecutor.Start(tickevent.NewCheckCache(), interval, tickevent.StartOption{RandomStartDelayMs: 60 * 1000}) } From 99e4dab812ff621154749d5b7773134e2eb28c52 Mon Sep 17 00:00:00 2001 From: Sydonian <794346190@qq.com> Date: Wed, 19 Jul 2023 14:45:18 +0800 Subject: [PATCH 50/68] =?UTF-8?q?=E4=B8=8A=E4=BC=A0=E3=80=81=E6=9B=B4?= =?UTF-8?q?=E6=96=B0=E3=80=81=E8=B0=83=E5=BA=A6=E6=96=87=E4=BB=B6=E6=8E=A5?= =?UTF-8?q?=E5=8F=A3=E6=94=B9=E5=BC=82=E6=AD=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- magefiles/magefile.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/magefiles/magefile.go b/magefiles/magefile.go index f146a71..43f4e18 100644 --- a/magefiles/magefile.go +++ b/magefiles/magefile.go @@ -15,7 +15,7 @@ var Default = Build func Build() error { return utils.Build(magefiles.BuildArgs{ OutputBinName: "scanner", - OutputDirName: "scanner", + OutputDirPath: "scanner", AssetsDir: "assets", PubArgs: targets.PubGoBuildArgs, }) From a17da12dfe22b305a2835be54e1514ed33ed1bd0 Mon Sep 17 00:00:00 2001 From: Sydonian <794346190@qq.com> Date: Fri, 21 Jul 2023 10:43:08 +0800 Subject: [PATCH 51/68] =?UTF-8?q?=E4=BC=98=E5=8C=96rabbitm=E7=9B=B8?= =?UTF-8?q?=E5=85=B3=E4=BB=A3=E7=A0=81=EF=BC=8C=E6=94=AF=E6=8C=81=E8=AE=BE?= =?UTF-8?q?=E7=BD=AE=E8=B6=85=E6=97=B6=E6=97=B6=E9=97=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- internal/event/agent_check_cache.go | 12 +++++------- internal/event/agent_check_state.go | 13 +++++-------- internal/event/agent_check_storage.go | 12 +++++------- internal/services/event.go | 6 +++--- 4 files changed, 18 insertions(+), 25 deletions(-) diff --git a/internal/event/agent_check_cache.go b/internal/event/agent_check_cache.go index 38e9c2b..de84815 100644 --- a/internal/event/agent_check_cache.go +++ b/internal/event/agent_check_cache.go @@ -2,11 +2,13 @@ package event import ( "database/sql" + "time" "github.com/samber/lo" "gitlink.org.cn/cloudream/common/pkg/distlock/reqbuilder" "gitlink.org.cn/cloudream/common/pkg/logger" "gitlink.org.cn/cloudream/db/model" + "gitlink.org.cn/cloudream/rabbitmq" "gitlink.org.cn/cloudream/scanner/internal/config" agtcli "gitlink.org.cn/cloudream/rabbitmq/client/agent" @@ -132,18 +134,14 @@ func (t *AgentCheckCache) startCheck(execCtx ExecuteContext, isComplete bool, ca } defer agentClient.Close() - checkResp, err := agentClient.CheckIPFS(agtmsg.NewCheckIPFSBody(isComplete, caches)) + checkResp, err := agentClient.CheckIPFS(agtmsg.NewCheckIPFS(isComplete, caches), rabbitmq.RequestOption{Timeout: time.Minute}) if err != nil { - log.WithField("NodeID", t.NodeID).Warnf("request to agent failed, err: %s", err.Error()) - return - } - if checkResp.IsFailed() { - log.WithField("NodeID", t.NodeID).Warnf("agent operation failed, err: %s", err.Error()) + log.WithField("NodeID", t.NodeID).Warnf("checking ipfs: %s", err.Error()) return } // 根据返回结果修改数据库 - for _, entry := range checkResp.Body.Entries { + for _, entry := range checkResp.Entries { switch entry.Operation { case agtmsg.CHECK_IPFS_RESP_OP_DELETE_TEMP: err := execCtx.Args.DB.Cache().DeleteTemp(execCtx.Args.DB.SQLCtx(), entry.FileHash, t.NodeID) diff --git a/internal/event/agent_check_state.go b/internal/event/agent_check_state.go index 1b17d4b..5738f89 100644 --- a/internal/event/agent_check_state.go +++ b/internal/event/agent_check_state.go @@ -9,6 +9,7 @@ import ( "gitlink.org.cn/cloudream/common/pkg/distlock/reqbuilder" "gitlink.org.cn/cloudream/common/pkg/logger" "gitlink.org.cn/cloudream/db/model" + "gitlink.org.cn/cloudream/rabbitmq" agtcli "gitlink.org.cn/cloudream/rabbitmq/client/agent" agtmsg "gitlink.org.cn/cloudream/rabbitmq/message/agent" scevt "gitlink.org.cn/cloudream/rabbitmq/message/scanner/event" @@ -91,18 +92,14 @@ func (t *AgentCheckState) Execute(execCtx ExecuteContext) { } defer agentClient.Close() - getResp, err := agentClient.GetState(agtmsg.NewGetStateBody()) + getResp, err := agentClient.GetState(agtmsg.NewGetState(), rabbitmq.RequestOption{Timeout: time.Minute}) if err != nil { - log.WithField("NodeID", t.NodeID).Warnf("request to agent failed, err: %s", err.Error()) - } - if getResp.IsFailed() { - log.WithField("NodeID", t.NodeID).Warnf("agent operation failed, err: %s", err.Error()) - return + log.WithField("NodeID", t.NodeID).Warnf("getting state: %s", err.Error()) } // 根据返回结果修改节点状态 - if getResp.Body.IPFSState != consts.IPFS_STATE_OK { - log.WithField("NodeID", t.NodeID).Warnf("IPFS status is %s, set node state unavailable", getResp.Body.IPFSState) + if getResp.IPFSState != consts.IPFS_STATE_OK { + log.WithField("NodeID", t.NodeID).Warnf("IPFS status is %s, set node state unavailable", getResp.IPFSState) err := execCtx.Args.DB.Node().UpdateState(execCtx.Args.DB.SQLCtx(), t.NodeID, consts.NODE_STATE_UNAVAILABLE) if err != nil { diff --git a/internal/event/agent_check_storage.go b/internal/event/agent_check_storage.go index 1a6e5e4..9f981ec 100644 --- a/internal/event/agent_check_storage.go +++ b/internal/event/agent_check_storage.go @@ -2,12 +2,14 @@ package event import ( "database/sql" + "time" "github.com/samber/lo" "gitlink.org.cn/cloudream/common/consts" "gitlink.org.cn/cloudream/common/pkg/distlock/reqbuilder" "gitlink.org.cn/cloudream/common/pkg/logger" "gitlink.org.cn/cloudream/db/model" + "gitlink.org.cn/cloudream/rabbitmq" agtcli "gitlink.org.cn/cloudream/rabbitmq/client/agent" agtmsg "gitlink.org.cn/cloudream/rabbitmq/message/agent" scevt "gitlink.org.cn/cloudream/rabbitmq/message/scanner/event" @@ -149,18 +151,14 @@ func (t *AgentCheckStorage) startCheck(execCtx ExecuteContext, stg model.Storage } defer agentClient.Close() - checkResp, err := agentClient.CheckStorage(agtmsg.NewCheckStorageBody(stg.StorageID, stg.Directory, isComplete, objects)) + checkResp, err := agentClient.CheckStorage(agtmsg.NewCheckStorage(stg.StorageID, stg.Directory, isComplete, objects), rabbitmq.RequestOption{Timeout: time.Minute}) if err != nil { - log.WithField("NodeID", stg.NodeID).Warnf("request to agent failed, err: %s", err.Error()) - } - if checkResp.IsFailed() { - log.WithField("NodeID", stg.NodeID).Warnf("agent operation failed, err: %s", err.Error()) - return + log.WithField("NodeID", stg.NodeID).Warnf("checking storage: %s", err.Error()) } // 根据返回结果修改数据库 var chkObjIDs []int - for _, entry := range checkResp.Body.Entries { + for _, entry := range checkResp.Entries { switch entry.Operation { case agtmsg.CHECK_STORAGE_RESP_OP_DELETE: err := execCtx.Args.DB.StorageObject().Delete(execCtx.Args.DB.SQLCtx(), t.StorageID, entry.ObjectID, entry.UserID) diff --git a/internal/services/event.go b/internal/services/event.go index 5790c20..730d69b 100644 --- a/internal/services/event.go +++ b/internal/services/event.go @@ -9,7 +9,7 @@ import ( func (svc *Service) PostEvent(msg *scmsg.PostEvent) { - evtMsg, err := scevt.MapToMessage(msg.Body.Event) + evtMsg, err := scevt.MapToMessage(msg.Event) if err != nil { logger.Warnf("convert map to event message failed, err: %s", err.Error()) return @@ -22,7 +22,7 @@ func (svc *Service) PostEvent(msg *scmsg.PostEvent) { } svc.eventExecutor.Post(evt, event.ExecuteOption{ - IsEmergency: msg.Body.IsEmergency, - DontMerge: msg.Body.DontMerge, + IsEmergency: msg.IsEmergency, + DontMerge: msg.DontMerge, }) } From 3590d4077a856f532bb8cec63cc356c22f21587e Mon Sep 17 00:00:00 2001 From: Sydonian <794346190@qq.com> Date: Fri, 21 Jul 2023 14:46:39 +0800 Subject: [PATCH 52/68] =?UTF-8?q?=E5=B0=86=E5=90=84=E7=A7=8DID=E5=AD=97?= =?UTF-8?q?=E6=AE=B5=E6=94=B9=E6=88=90int64?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- internal/event/agent_check_cache.go | 2 +- internal/event/agent_check_state.go | 2 +- internal/event/agent_check_storage.go | 4 ++-- internal/event/check_cache.go | 2 +- internal/event/check_object.go | 2 +- internal/event/check_rep_count.go | 10 +++++----- internal/event/check_rep_count_test.go | 8 ++++---- internal/tickevent/batch_all_agent_check_cache.go | 4 ++-- internal/tickevent/batch_check_all_object.go | 4 +--- internal/tickevent/batch_check_all_storage.go | 3 +-- 10 files changed, 19 insertions(+), 22 deletions(-) diff --git a/internal/event/agent_check_cache.go b/internal/event/agent_check_cache.go index de84815..becdbcd 100644 --- a/internal/event/agent_check_cache.go +++ b/internal/event/agent_check_cache.go @@ -20,7 +20,7 @@ type AgentCheckCache struct { scevt.AgentCheckCache } -func NewAgentCheckCache(nodeID int, fileHashes []string) *AgentCheckCache { +func NewAgentCheckCache(nodeID int64, fileHashes []string) *AgentCheckCache { return &AgentCheckCache{ AgentCheckCache: scevt.NewAgentCheckCache(nodeID, fileHashes), } diff --git a/internal/event/agent_check_state.go b/internal/event/agent_check_state.go index 5738f89..3b6ea1b 100644 --- a/internal/event/agent_check_state.go +++ b/internal/event/agent_check_state.go @@ -20,7 +20,7 @@ type AgentCheckState struct { scevt.AgentCheckState } -func NewAgentCheckState(nodeID int) *AgentCheckState { +func NewAgentCheckState(nodeID int64) *AgentCheckState { return &AgentCheckState{ AgentCheckState: scevt.NewAgentCheckState(nodeID), } diff --git a/internal/event/agent_check_storage.go b/internal/event/agent_check_storage.go index 9f981ec..3b0c01f 100644 --- a/internal/event/agent_check_storage.go +++ b/internal/event/agent_check_storage.go @@ -20,7 +20,7 @@ type AgentCheckStorage struct { scevt.AgentCheckStorage } -func NewAgentCheckStorage(storageID int, objectIDs []int) *AgentCheckStorage { +func NewAgentCheckStorage(storageID int64, objectIDs []int64) *AgentCheckStorage { return &AgentCheckStorage{ AgentCheckStorage: scevt.NewAgentCheckStorage(storageID, objectIDs), } @@ -157,7 +157,7 @@ func (t *AgentCheckStorage) startCheck(execCtx ExecuteContext, stg model.Storage } // 根据返回结果修改数据库 - var chkObjIDs []int + var chkObjIDs []int64 for _, entry := range checkResp.Entries { switch entry.Operation { case agtmsg.CHECK_STORAGE_RESP_OP_DELETE: diff --git a/internal/event/check_cache.go b/internal/event/check_cache.go index 9e7fd44..83d5e10 100644 --- a/internal/event/check_cache.go +++ b/internal/event/check_cache.go @@ -15,7 +15,7 @@ type CheckCache struct { scevt.CheckCache } -func NewCheckCache(nodeID int) *CheckCache { +func NewCheckCache(nodeID int64) *CheckCache { return &CheckCache{ CheckCache: scevt.NewCheckCache(nodeID), } diff --git a/internal/event/check_object.go b/internal/event/check_object.go index c610f36..a40a753 100644 --- a/internal/event/check_object.go +++ b/internal/event/check_object.go @@ -11,7 +11,7 @@ type CheckObject struct { scevt.CheckObject } -func NewCheckObject(objIDs []int) *CheckObject { +func NewCheckObject(objIDs []int64) *CheckObject { return &CheckObject{ CheckObject: scevt.NewCheckObject(objIDs), } diff --git a/internal/event/check_rep_count.go b/internal/event/check_rep_count.go index 644762a..7410c8f 100644 --- a/internal/event/check_rep_count.go +++ b/internal/event/check_rep_count.go @@ -58,7 +58,7 @@ func (t *CheckRepCount) Execute(execCtx ExecuteContext) { } defer mutex.Unlock() - updatedNodeAndHashes := make(map[int][]string) + updatedNodeAndHashes := make(map[int64][]string) for _, fileHash := range t.FileHashes { updatedNodeIDs, err := t.checkOneRepCount(fileHash, execCtx) @@ -79,11 +79,11 @@ func (t *CheckRepCount) Execute(execCtx ExecuteContext) { } } -func (t *CheckRepCount) checkOneRepCount(fileHash string, execCtx ExecuteContext) ([]int, error) { +func (t *CheckRepCount) checkOneRepCount(fileHash string, execCtx ExecuteContext) ([]int64, error) { log := logger.WithType[CheckRepCount]("Event") sqlCtx := execCtx.Args.DB.SQLCtx() - var updatedNodeIDs []int + var updatedNodeIDs []int64 // 计算所需的最少备份数： // 1. ObjectRep中期望备份数的最大值 // 2. 如果ObjectBlock存在对此文件的引用，则至少为1 @@ -166,7 +166,7 @@ func chooseNewRepNodes(allNodes []model.Node, curRepNodes []model.Node, newCount node.State != consts.NODE_STATE_NORMAL }) - repNodeLocationIDs := make(map[int]bool) + repNodeLocationIDs := make(map[int64]bool) for _, node := range curRepNodes { repNodeLocationIDs[node.LocationID] = true } @@ -181,7 +181,7 @@ func chooseNewRepNodes(allNodes []model.Node, curRepNodes []model.Node, newCount func chooseDeleteAvaiRepNodes(allNodes []model.Node, curAvaiRepNodes []model.Node, delCount int) []model.Node { // 按照地域ID分组 - locationGroupedNodes := make(map[int][]model.Node) + locationGroupedNodes := make(map[int64][]model.Node) for _, node := range curAvaiRepNodes { nodes := locationGroupedNodes[node.LocationID] nodes = append(nodes, node) diff --git a/internal/event/check_rep_count_test.go b/internal/event/check_rep_count_test.go index 3c0beab..f513a6b 100644 --- a/internal/event/check_rep_count_test.go +++ b/internal/event/check_rep_count_test.go @@ -102,9 +102,9 @@ func Test_chooseNewRepNodes(t *testing.T) { for _, test := range testcases { Convey(test.title, t, func() { chooseNodes := chooseNewRepNodes(test.allNodes, test.curRepNodes, test.newCount) - chooseNodeIDs := lo.Map(chooseNodes, func(node model.Node, index int) int { return node.NodeID }) + chooseNodeIDs := lo.Map(chooseNodes, func(node model.Node, index int) int64 { return node.NodeID }) - sort.Sort(chooseNodeIDs, sort.Cmp[int]) + sort.Sort(chooseNodeIDs, sort.Cmp[int64]) So(chooseNodeIDs, ShouldResemble, test.wantNodeIDs) }) @@ -145,9 +145,9 @@ func Test_chooseDeleteAvaiRepNodes(t *testing.T) { for _, test := range testcases { Convey(test.title, t, func() { chooseNodes := chooseDeleteAvaiRepNodes(test.allNodes, test.curRepNodes, test.delCount) - chooseNodeLocationIDs := lo.Map(chooseNodes, func(node model.Node, index int) int { return node.LocationID }) + chooseNodeLocationIDs := lo.Map(chooseNodes, func(node model.Node, index int) int64 { return node.LocationID }) - sort.Sort(chooseNodeLocationIDs, sort.Cmp[int]) + sort.Sort(chooseNodeLocationIDs, sort.Cmp[int64]) So(chooseNodeLocationIDs, ShouldResemble, test.wantNodeLocationIDs) }) diff --git a/internal/tickevent/batch_all_agent_check_cache.go b/internal/tickevent/batch_all_agent_check_cache.go index 8f51916..a3020e3 100644 --- a/internal/tickevent/batch_all_agent_check_cache.go +++ b/internal/tickevent/batch_all_agent_check_cache.go @@ -10,7 +10,7 @@ import ( const AGENT_CHECK_CACHE_BATCH_SIZE = 2 type BatchAllAgentCheckCache struct { - nodeIDs []int + nodeIDs []int64 } func NewBatchAllAgentCheckCache() *BatchAllAgentCheckCache { @@ -29,7 +29,7 @@ func (e *BatchAllAgentCheckCache) Execute(ctx ExecuteContext) { return } - e.nodeIDs = lo.Map(nodes, func(node model.Node, index int) int { return node.NodeID }) + e.nodeIDs = lo.Map(nodes, func(node model.Node, index int) int64 { return node.NodeID }) log.Debugf("new check start, get all nodes") } diff --git a/internal/tickevent/batch_check_all_object.go b/internal/tickevent/batch_check_all_object.go index f70b9e3..73502be 100644 --- a/internal/tickevent/batch_check_all_object.go +++ b/internal/tickevent/batch_check_all_object.go @@ -1,7 +1,6 @@ package tickevent import ( - "github.com/samber/lo" "gitlink.org.cn/cloudream/common/pkg/logger" "gitlink.org.cn/cloudream/scanner/internal/event" ) @@ -27,8 +26,7 @@ func (e *BatchCheckAllObject) Execute(ctx ExecuteContext) { return } - // TODO 将ID字段换成int64类型 - ctx.Args.EventExecutor.Post(event.NewCheckObject(lo.Map(objectIDs, func(id int64, index int) int { return int(id) }))) + ctx.Args.EventExecutor.Post(event.NewCheckObject(objectIDs)) // 如果结果的长度小于预期的长度，则认为已经查询了所有，下次从头再来 if len(objectIDs) < CHECK_OBJECT_BATCH_SIZE { diff --git a/internal/tickevent/batch_check_all_storage.go b/internal/tickevent/batch_check_all_storage.go index 07015ae..b22aace 100644 --- a/internal/tickevent/batch_check_all_storage.go +++ b/internal/tickevent/batch_check_all_storage.go @@ -28,8 +28,7 @@ func (e *BatchCheckAllStorage) Execute(ctx ExecuteContext) { for _, stgID := range storageIDs { // 设置nil代表进行全量检查 - // TODO 将ID字段换成int64类型 - ctx.Args.EventExecutor.Post(event.NewAgentCheckStorage(int(stgID), nil)) + ctx.Args.EventExecutor.Post(event.NewAgentCheckStorage(stgID, nil)) } // 如果结果的长度小于预期的长度，则认为已经查询了所有，下次从头再来 From 398da4486c04874274cf13428c3dcf1cea51f542 Mon Sep 17 00:00:00 2001 From: Sydonian <794346190@qq.com> Date: Thu, 27 Jul 2023 17:01:48 +0800 Subject: [PATCH 53/68] =?UTF-8?q?=E5=A2=9E=E5=8A=A0agent=E5=9B=9E=E6=BA=90?= =?UTF-8?q?=E7=9A=84=E5=8A=9F=E8=83=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- internal/event/agent_check_storage.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/event/agent_check_storage.go b/internal/event/agent_check_storage.go index 3b0c01f..5604b11 100644 --- a/internal/event/agent_check_storage.go +++ b/internal/event/agent_check_storage.go @@ -151,7 +151,7 @@ func (t *AgentCheckStorage) startCheck(execCtx ExecuteContext, stg model.Storage } defer agentClient.Close() - checkResp, err := agentClient.CheckStorage(agtmsg.NewCheckStorage(stg.StorageID, stg.Directory, isComplete, objects), rabbitmq.RequestOption{Timeout: time.Minute}) + checkResp, err := agentClient.StorageCheck(agtmsg.NewStorageCheck(stg.StorageID, stg.Directory, isComplete, objects), rabbitmq.RequestOption{Timeout: time.Minute}) if err != nil { log.WithField("NodeID", stg.NodeID).Warnf("checking storage: %s", err.Error()) } From ce581dbc94415a6bd844a9362b39128f80bfabfa Mon Sep 17 00:00:00 2001 From: Sydonian <794346190@qq.com> Date: Fri, 28 Jul 2023 11:42:59 +0800 Subject: [PATCH 54/68] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E6=A3=80=E6=9F=A5?= =?UTF-8?q?=E8=8A=82=E7=82=B9=E7=8A=B6=E6=80=81=E7=9A=84=E9=80=BB=E8=BE=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- internal/event/agent_check_cache.go | 2 +- internal/event/agent_check_state.go | 47 +++++++++++++-------------- internal/event/agent_check_storage.go | 1 + 3 files changed, 24 insertions(+), 26 deletions(-) diff --git a/internal/event/agent_check_cache.go b/internal/event/agent_check_cache.go index becdbcd..e038f6e 100644 --- a/internal/event/agent_check_cache.go +++ b/internal/event/agent_check_cache.go @@ -120,7 +120,7 @@ func (t *AgentCheckCache) checkIncrement(execCtx ExecuteContext) { caches = append(caches, ch) } - t.startCheck(execCtx, true, caches) + t.startCheck(execCtx, false, caches) } func (t *AgentCheckCache) startCheck(execCtx ExecuteContext, isComplete bool, caches []model.Cache) { diff --git a/internal/event/agent_check_state.go b/internal/event/agent_check_state.go index 3b6ea1b..5f6137e 100644 --- a/internal/event/agent_check_state.go +++ b/internal/event/agent_check_state.go @@ -61,30 +61,6 @@ func (t *AgentCheckState) Execute(execCtx ExecuteContext) { return } - if node.State != consts.NODE_STATE_NORMAL { - return - } - - // 检查上次上报时间，超时的设置为不可用 - // TODO 没有上报过是否要特殊处理？ - if node.LastReportTime != nil && time.Since(*node.LastReportTime) > time.Duration(config.Cfg().NodeUnavailableSeconds)*time.Second { - err := execCtx.Args.DB.Node().UpdateState(execCtx.Args.DB.SQLCtx(), t.NodeID, consts.NODE_STATE_UNAVAILABLE) - if err != nil { - log.WithField("NodeID", t.NodeID).Warnf("set node state failed, err: %s", err.Error()) - return - } - - caches, err := execCtx.Args.DB.Cache().GetNodeCaches(execCtx.Args.DB.SQLCtx(), t.NodeID) - if err != nil { - log.WithField("NodeID", t.NodeID).Warnf("get node caches failed, err: %s", err.Error()) - return - } - - // 补充备份数 - execCtx.Executor.Post(NewCheckRepCount(lo.Map(caches, func(ch model.Cache, index int) string { return ch.FileHash }))) - return - } - agentClient, err := agtcli.NewClient(t.NodeID, &config.Cfg().RabbitMQ) if err != nil { log.WithField("NodeID", t.NodeID).Warnf("create agent client failed, err: %s", err.Error()) @@ -92,9 +68,30 @@ func (t *AgentCheckState) Execute(execCtx ExecuteContext) { } defer agentClient.Close() - getResp, err := agentClient.GetState(agtmsg.NewGetState(), rabbitmq.RequestOption{Timeout: time.Minute}) + getResp, err := agentClient.GetState(agtmsg.NewGetState(), rabbitmq.RequestOption{Timeout: time.Second * 30}) if err != nil { log.WithField("NodeID", t.NodeID).Warnf("getting state: %s", err.Error()) + + // 检查上次上报时间，超时的设置为不可用 + // TODO 没有上报过是否要特殊处理？ + if node.LastReportTime != nil && time.Since(*node.LastReportTime) > time.Duration(config.Cfg().NodeUnavailableSeconds)*time.Second { + err := execCtx.Args.DB.Node().UpdateState(execCtx.Args.DB.SQLCtx(), t.NodeID, consts.NODE_STATE_UNAVAILABLE) + if err != nil { + log.WithField("NodeID", t.NodeID).Warnf("set node state failed, err: %s", err.Error()) + return + } + + caches, err := execCtx.Args.DB.Cache().GetNodeCaches(execCtx.Args.DB.SQLCtx(), t.NodeID) + if err != nil { + log.WithField("NodeID", t.NodeID).Warnf("get node caches failed, err: %s", err.Error()) + return + } + + // 补充备份数 + execCtx.Executor.Post(NewCheckRepCount(lo.Map(caches, func(ch model.Cache, index int) string { return ch.FileHash }))) + return + } + return } // 根据返回结果修改节点状态 diff --git a/internal/event/agent_check_storage.go b/internal/event/agent_check_storage.go index 3b0c01f..13166f3 100644 --- a/internal/event/agent_check_storage.go +++ b/internal/event/agent_check_storage.go @@ -154,6 +154,7 @@ func (t *AgentCheckStorage) startCheck(execCtx ExecuteContext, stg model.Storage checkResp, err := agentClient.CheckStorage(agtmsg.NewCheckStorage(stg.StorageID, stg.Directory, isComplete, objects), rabbitmq.RequestOption{Timeout: time.Minute}) if err != nil { log.WithField("NodeID", stg.NodeID).Warnf("checking storage: %s", err.Error()) + return } // 根据返回结果修改数据库 From caf31146c2d36571691d514a14b8777a3ab403c2 Mon Sep 17 00:00:00 2001 From: Sydonian <794346190@qq.com> Date: Mon, 31 Jul 2023 15:26:53 +0800 Subject: [PATCH 55/68] =?UTF-8?q?client=E5=A2=9E=E5=8A=A0http=E6=A8=A1?= =?UTF-8?q?=E5=BC=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- internal/event/agent_check_state.go | 8 ++++---- internal/event/agent_check_storage.go | 2 +- internal/event/check_cache.go | 2 +- internal/event/check_rep_count.go | 6 +++--- internal/event/check_rep_count_test.go | 16 ++++++++-------- 5 files changed, 17 insertions(+), 17 deletions(-) diff --git a/internal/event/agent_check_state.go b/internal/event/agent_check_state.go index 5f6137e..bee38bd 100644 --- a/internal/event/agent_check_state.go +++ b/internal/event/agent_check_state.go @@ -75,7 +75,7 @@ func (t *AgentCheckState) Execute(execCtx ExecuteContext) { // 检查上次上报时间，超时的设置为不可用 // TODO 没有上报过是否要特殊处理？ if node.LastReportTime != nil && time.Since(*node.LastReportTime) > time.Duration(config.Cfg().NodeUnavailableSeconds)*time.Second { - err := execCtx.Args.DB.Node().UpdateState(execCtx.Args.DB.SQLCtx(), t.NodeID, consts.NODE_STATE_UNAVAILABLE) + err := execCtx.Args.DB.Node().UpdateState(execCtx.Args.DB.SQLCtx(), t.NodeID, consts.NodeStateUnavailable) if err != nil { log.WithField("NodeID", t.NodeID).Warnf("set node state failed, err: %s", err.Error()) return @@ -95,10 +95,10 @@ func (t *AgentCheckState) Execute(execCtx ExecuteContext) { } // 根据返回结果修改节点状态 - if getResp.IPFSState != consts.IPFS_STATE_OK { + if getResp.IPFSState != consts.IPFSStateOK { log.WithField("NodeID", t.NodeID).Warnf("IPFS status is %s, set node state unavailable", getResp.IPFSState) - err := execCtx.Args.DB.Node().UpdateState(execCtx.Args.DB.SQLCtx(), t.NodeID, consts.NODE_STATE_UNAVAILABLE) + err := execCtx.Args.DB.Node().UpdateState(execCtx.Args.DB.SQLCtx(), t.NodeID, consts.NodeStateUnavailable) if err != nil { log.WithField("NodeID", t.NodeID).Warnf("change node state failed, err: %s", err.Error()) } @@ -106,7 +106,7 @@ func (t *AgentCheckState) Execute(execCtx ExecuteContext) { } // TODO 如果以后还有其他的状态，要判断哪些状态下能设置Normal - err = execCtx.Args.DB.Node().UpdateState(execCtx.Args.DB.SQLCtx(), t.NodeID, consts.NODE_STATE_NORMAL) + err = execCtx.Args.DB.Node().UpdateState(execCtx.Args.DB.SQLCtx(), t.NodeID, consts.NodeStateNormal) if err != nil { log.WithField("NodeID", t.NodeID).Warnf("change node state failed, err: %s", err.Error()) } diff --git a/internal/event/agent_check_storage.go b/internal/event/agent_check_storage.go index ab5c450..89b2339 100644 --- a/internal/event/agent_check_storage.go +++ b/internal/event/agent_check_storage.go @@ -70,7 +70,7 @@ func (t *AgentCheckStorage) Execute(execCtx ExecuteContext) { } // TODO unavailable的节点需不需要发送任务？ - if node.State != consts.NODE_STATE_NORMAL { + if node.State != consts.NodeStateNormal { return } diff --git a/internal/event/check_cache.go b/internal/event/check_cache.go index 83d5e10..26ad80f 100644 --- a/internal/event/check_cache.go +++ b/internal/event/check_cache.go @@ -60,7 +60,7 @@ func (t *CheckCache) Execute(execCtx ExecuteContext) { return } - if node.State != consts.NODE_STATE_UNAVAILABLE { + if node.State != consts.NodeStateUnavailable { return } diff --git a/internal/event/check_rep_count.go b/internal/event/check_rep_count.go index 7410c8f..fc3f5fe 100644 --- a/internal/event/check_rep_count.go +++ b/internal/event/check_rep_count.go @@ -111,9 +111,9 @@ func (t *CheckRepCount) checkOneRepCount(fileHash string, execCtx ExecuteContext var normalNodes, unavaiNodes []model.Node for _, node := range repNodes { - if node.State == consts.NODE_STATE_NORMAL { + if node.State == consts.NodeStateNormal { normalNodes = append(normalNodes, node) - } else if node.State == consts.NODE_STATE_UNAVAILABLE { + } else if node.State == consts.NodeStateUnavailable { unavaiNodes = append(unavaiNodes, node) } } @@ -163,7 +163,7 @@ func (t *CheckRepCount) checkOneRepCount(fileHash string, execCtx ExecuteContext func chooseNewRepNodes(allNodes []model.Node, curRepNodes []model.Node, newCount int) []model.Node { noRepNodes := lo.Reject(allNodes, func(node model.Node, index int) bool { return lo.ContainsBy(curRepNodes, func(n model.Node) bool { return node.NodeID == n.NodeID }) || - node.State != consts.NODE_STATE_NORMAL + node.State != consts.NodeStateNormal }) repNodeLocationIDs := make(map[int64]bool) diff --git a/internal/event/check_rep_count_test.go b/internal/event/check_rep_count_test.go index f513a6b..9c690bc 100644 --- a/internal/event/check_rep_count_test.go +++ b/internal/event/check_rep_count_test.go @@ -24,22 +24,22 @@ func Test_chooseNewRepNodes(t *testing.T) { { NodeID: 1, LocationID: 1, - State: consts.NODE_STATE_NORMAL, + State: consts.NodeStateNormal, }, { NodeID: 2, LocationID: 1, - State: consts.NODE_STATE_NORMAL, + State: consts.NodeStateNormal, }, { NodeID: 3, LocationID: 2, - State: consts.NODE_STATE_NORMAL, + State: consts.NodeStateNormal, }, { NodeID: 4, LocationID: 3, - State: consts.NODE_STATE_NORMAL, + State: consts.NodeStateNormal, }, }, curRepNodes: []model.Node{ @@ -57,12 +57,12 @@ func Test_chooseNewRepNodes(t *testing.T) { { NodeID: 1, LocationID: 1, - State: consts.NODE_STATE_NORMAL, + State: consts.NodeStateNormal, }, { NodeID: 2, LocationID: 1, - State: consts.NODE_STATE_NORMAL, + State: consts.NodeStateNormal, }, }, curRepNodes: []model.Node{ @@ -80,12 +80,12 @@ func Test_chooseNewRepNodes(t *testing.T) { { NodeID: 1, LocationID: 1, - State: consts.NODE_STATE_UNAVAILABLE, + State: consts.NodeStateUnavailable, }, { NodeID: 2, LocationID: 1, - State: consts.NODE_STATE_NORMAL, + State: consts.NodeStateNormal, }, }, curRepNodes: []model.Node{ From ae16bc27e59ba49a68955f47e68699b153aae754 Mon Sep 17 00:00:00 2001 From: Sydonian <794346190@qq.com> Date: Tue, 1 Aug 2023 10:50:17 +0800 Subject: [PATCH 56/68] =?UTF-8?q?=E4=BC=98=E5=8C=96=E7=BC=96=E8=AF=91?= =?UTF-8?q?=E8=84=9A=E6=9C=AC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- magefiles/magefile.go | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/magefiles/magefile.go b/magefiles/magefile.go index 43f4e18..0f4ec2e 100644 --- a/magefiles/magefile.go +++ b/magefiles/magefile.go @@ -4,19 +4,17 @@ package main import ( "magefiles" - "magefiles/utils" //mage:import - "magefiles/targets" + _ "magefiles/targets" ) var Default = Build func Build() error { - return utils.Build(magefiles.BuildArgs{ - OutputBinName: "scanner", - OutputDirPath: "scanner", - AssetsDir: "assets", - PubArgs: targets.PubGoBuildArgs, + return magefiles.Build(magefiles.BuildArgs{ + OutputName: "scanner", + OutputDir: "../../build/scanner", + AssetsDir: "assets", }) } From 7e52b77ee91fa40815118a79cc8e5ac3b3a084f9 Mon Sep 17 00:00:00 2001 From: zkxx Date: Wed, 2 Aug 2023 10:51:48 +0800 Subject: [PATCH 57/68] =?UTF-8?q?=E8=B0=83=E8=AF=95=E5=8F=AF=E4=BB=A5?= =?UTF-8?q?=E8=BF=90=E8=A1=8C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- internal/event/check_rep_count.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/event/check_rep_count.go b/internal/event/check_rep_count.go index fc3f5fe..510c4bd 100644 --- a/internal/event/check_rep_count.go +++ b/internal/event/check_rep_count.go @@ -149,7 +149,7 @@ func (t *CheckRepCount) checkOneRepCount(fileHash string, execCtx ExecuteContext } for _, node := range newNodes { - err := execCtx.Args.DB.Cache().CreatePinned(sqlCtx, fileHash, node.NodeID) + err := execCtx.Args.DB.Cache().CreatePinned(sqlCtx, fileHash, node.NodeID, 0) if err != nil { return nil, fmt.Errorf("create cache failed, err: %w", err) } From 44de17b75953707017a8680af377e0ec6d63ba14 Mon Sep 17 00:00:00 2001 From: Sydonian Date: Fri, 11 Aug 2023 10:54:59 +0800 Subject: [PATCH 58/68] Initial commit --- README.md | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 README.md diff --git a/README.md b/README.md new file mode 100644 index 0000000..2ebec19 --- /dev/null +++ b/README.md @@ -0,0 +1,2 @@ +# storage-scanner + From 67126c4473c3f8699f207f37a2dcb26459dbbaa7 Mon Sep 17 00:00:00 2001 From: Sydonian <794346190@qq.com> Date: Mon, 14 Aug 2023 10:17:33 +0800 Subject: [PATCH 59/68] =?UTF-8?q?=E6=A8=A1=E5=9D=97=E6=94=B9=E5=90=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- go.mod | 4 ++-- internal/event/agent_check_cache.go | 2 +- internal/event/agent_check_state.go | 2 +- internal/event/agent_check_storage.go | 2 +- internal/event/check_rep_count.go | 2 +- internal/services/event.go | 2 +- internal/services/service.go | 2 +- internal/tickevent/batch_all_agent_check_cache.go | 2 +- internal/tickevent/batch_check_all_object.go | 2 +- internal/tickevent/batch_check_all_rep_count.go | 2 +- internal/tickevent/batch_check_all_storage.go | 2 +- internal/tickevent/check_agent_state.go | 2 +- internal/tickevent/check_cache.go | 2 +- internal/tickevent/tick_event.go | 2 +- main.go | 8 ++++---- 15 files changed, 19 insertions(+), 19 deletions(-) diff --git a/go.mod b/go.mod index cd5fcaa..73d6121 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ -module gitlink.org.cn/cloudream/scanner +module gitlink.org.cn/cloudream/storage-scanner -go 1.18 +go 1.20 require ( github.com/jmoiron/sqlx v1.3.5 diff --git a/internal/event/agent_check_cache.go b/internal/event/agent_check_cache.go index e038f6e..164d9f3 100644 --- a/internal/event/agent_check_cache.go +++ b/internal/event/agent_check_cache.go @@ -9,7 +9,7 @@ import ( "gitlink.org.cn/cloudream/common/pkg/logger" "gitlink.org.cn/cloudream/db/model" "gitlink.org.cn/cloudream/rabbitmq" - "gitlink.org.cn/cloudream/scanner/internal/config" + "gitlink.org.cn/cloudream/storage-scanner/internal/config" agtcli "gitlink.org.cn/cloudream/rabbitmq/client/agent" agtmsg "gitlink.org.cn/cloudream/rabbitmq/message/agent" diff --git a/internal/event/agent_check_state.go b/internal/event/agent_check_state.go index bee38bd..58a0708 100644 --- a/internal/event/agent_check_state.go +++ b/internal/event/agent_check_state.go @@ -13,7 +13,7 @@ import ( agtcli "gitlink.org.cn/cloudream/rabbitmq/client/agent" agtmsg "gitlink.org.cn/cloudream/rabbitmq/message/agent" scevt "gitlink.org.cn/cloudream/rabbitmq/message/scanner/event" - "gitlink.org.cn/cloudream/scanner/internal/config" + "gitlink.org.cn/cloudream/storage-scanner/internal/config" ) type AgentCheckState struct { diff --git a/internal/event/agent_check_storage.go b/internal/event/agent_check_storage.go index 89b2339..e470929 100644 --- a/internal/event/agent_check_storage.go +++ b/internal/event/agent_check_storage.go @@ -13,7 +13,7 @@ import ( agtcli "gitlink.org.cn/cloudream/rabbitmq/client/agent" agtmsg "gitlink.org.cn/cloudream/rabbitmq/message/agent" scevt "gitlink.org.cn/cloudream/rabbitmq/message/scanner/event" - "gitlink.org.cn/cloudream/scanner/internal/config" + "gitlink.org.cn/cloudream/storage-scanner/internal/config" ) type AgentCheckStorage struct { diff --git a/internal/event/check_rep_count.go b/internal/event/check_rep_count.go index 510c4bd..8aa6aef 100644 --- a/internal/event/check_rep_count.go +++ b/internal/event/check_rep_count.go @@ -10,7 +10,7 @@ import ( "gitlink.org.cn/cloudream/common/pkg/logger" mymath "gitlink.org.cn/cloudream/common/utils/math" mysort "gitlink.org.cn/cloudream/common/utils/sort" - "gitlink.org.cn/cloudream/scanner/internal/config" + "gitlink.org.cn/cloudream/storage-scanner/internal/config" "gitlink.org.cn/cloudream/db/model" scevt "gitlink.org.cn/cloudream/rabbitmq/message/scanner/event" diff --git a/internal/services/event.go b/internal/services/event.go index 730d69b..7268a40 100644 --- a/internal/services/event.go +++ b/internal/services/event.go @@ -4,7 +4,7 @@ import ( "gitlink.org.cn/cloudream/common/pkg/logger" scmsg "gitlink.org.cn/cloudream/rabbitmq/message/scanner" scevt "gitlink.org.cn/cloudream/rabbitmq/message/scanner/event" - "gitlink.org.cn/cloudream/scanner/internal/event" + "gitlink.org.cn/cloudream/storage-scanner/internal/event" ) func (svc *Service) PostEvent(msg *scmsg.PostEvent) { diff --git a/internal/services/service.go b/internal/services/service.go index 9c5af8a..efba3de 100644 --- a/internal/services/service.go +++ b/internal/services/service.go @@ -1,7 +1,7 @@ package services import ( - "gitlink.org.cn/cloudream/scanner/internal/event" + "gitlink.org.cn/cloudream/storage-scanner/internal/event" ) type Service struct { diff --git a/internal/tickevent/batch_all_agent_check_cache.go b/internal/tickevent/batch_all_agent_check_cache.go index a3020e3..5ea751d 100644 --- a/internal/tickevent/batch_all_agent_check_cache.go +++ b/internal/tickevent/batch_all_agent_check_cache.go @@ -4,7 +4,7 @@ import ( "github.com/samber/lo" "gitlink.org.cn/cloudream/common/pkg/logger" "gitlink.org.cn/cloudream/db/model" - "gitlink.org.cn/cloudream/scanner/internal/event" + "gitlink.org.cn/cloudream/storage-scanner/internal/event" ) const AGENT_CHECK_CACHE_BATCH_SIZE = 2 diff --git a/internal/tickevent/batch_check_all_object.go b/internal/tickevent/batch_check_all_object.go index 73502be..744b4c8 100644 --- a/internal/tickevent/batch_check_all_object.go +++ b/internal/tickevent/batch_check_all_object.go @@ -2,7 +2,7 @@ package tickevent import ( "gitlink.org.cn/cloudream/common/pkg/logger" - "gitlink.org.cn/cloudream/scanner/internal/event" + "gitlink.org.cn/cloudream/storage-scanner/internal/event" ) const CHECK_OBJECT_BATCH_SIZE = 100 diff --git a/internal/tickevent/batch_check_all_rep_count.go b/internal/tickevent/batch_check_all_rep_count.go index 2388efd..6643653 100644 --- a/internal/tickevent/batch_check_all_rep_count.go +++ b/internal/tickevent/batch_check_all_rep_count.go @@ -2,7 +2,7 @@ package tickevent import ( "gitlink.org.cn/cloudream/common/pkg/logger" - "gitlink.org.cn/cloudream/scanner/internal/event" + "gitlink.org.cn/cloudream/storage-scanner/internal/event" ) const CHECK_CACHE_BATCH_SIZE = 100 diff --git a/internal/tickevent/batch_check_all_storage.go b/internal/tickevent/batch_check_all_storage.go index b22aace..558b9ed 100644 --- a/internal/tickevent/batch_check_all_storage.go +++ b/internal/tickevent/batch_check_all_storage.go @@ -2,7 +2,7 @@ package tickevent import ( "gitlink.org.cn/cloudream/common/pkg/logger" - "gitlink.org.cn/cloudream/scanner/internal/event" + "gitlink.org.cn/cloudream/storage-scanner/internal/event" ) const CHECK_STORAGE_BATCH_SIZE = 5 diff --git a/internal/tickevent/check_agent_state.go b/internal/tickevent/check_agent_state.go index 4c34178..4bdc4e4 100644 --- a/internal/tickevent/check_agent_state.go +++ b/internal/tickevent/check_agent_state.go @@ -2,7 +2,7 @@ package tickevent import ( "gitlink.org.cn/cloudream/common/pkg/logger" - "gitlink.org.cn/cloudream/scanner/internal/event" + "gitlink.org.cn/cloudream/storage-scanner/internal/event" ) type CheckAgentState struct { diff --git a/internal/tickevent/check_cache.go b/internal/tickevent/check_cache.go index 3935489..5676f60 100644 --- a/internal/tickevent/check_cache.go +++ b/internal/tickevent/check_cache.go @@ -2,7 +2,7 @@ package tickevent import ( "gitlink.org.cn/cloudream/common/pkg/logger" - "gitlink.org.cn/cloudream/scanner/internal/event" + "gitlink.org.cn/cloudream/storage-scanner/internal/event" ) type CheckCache struct { diff --git a/internal/tickevent/tick_event.go b/internal/tickevent/tick_event.go index 4620624..00cdea1 100644 --- a/internal/tickevent/tick_event.go +++ b/internal/tickevent/tick_event.go @@ -3,7 +3,7 @@ package tickevent import ( tickevent "gitlink.org.cn/cloudream/common/pkg/tickevent" mydb "gitlink.org.cn/cloudream/db" - "gitlink.org.cn/cloudream/scanner/internal/event" + "gitlink.org.cn/cloudream/storage-scanner/internal/event" ) type ExecuteArgs struct { diff --git a/main.go b/main.go index de7d256..e2e983f 100644 --- a/main.go +++ b/main.go @@ -9,10 +9,10 @@ import ( log "gitlink.org.cn/cloudream/common/pkg/logger" "gitlink.org.cn/cloudream/db" scsvr "gitlink.org.cn/cloudream/rabbitmq/server/scanner" - "gitlink.org.cn/cloudream/scanner/internal/config" - "gitlink.org.cn/cloudream/scanner/internal/event" - "gitlink.org.cn/cloudream/scanner/internal/services" - "gitlink.org.cn/cloudream/scanner/internal/tickevent" + "gitlink.org.cn/cloudream/storage-scanner/internal/config" + "gitlink.org.cn/cloudream/storage-scanner/internal/event" + "gitlink.org.cn/cloudream/storage-scanner/internal/services" + "gitlink.org.cn/cloudream/storage-scanner/internal/tickevent" ) func main() { From 9854157d045af8d4d3f258edf06b12889a4aaa3c Mon Sep 17 00:00:00 2001 From: Sydonian <794346190@qq.com> Date: Mon, 14 Aug 2023 11:28:52 +0800 Subject: [PATCH 60/68] =?UTF-8?q?=E8=B0=83=E6=95=B4=E9=A1=B9=E7=9B=AE?= =?UTF-8?q?=E7=BB=93=E6=9E=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- go.mod | 45 +++++--- go.sum | 101 ++++++++++++++++-- internal/config/config.go | 4 +- internal/event/agent_check_cache.go | 12 +-- internal/event/agent_check_state.go | 12 +-- internal/event/agent_check_storage.go | 12 +-- internal/event/check_cache.go | 4 +- internal/event/check_object.go | 2 +- internal/event/check_rep_count.go | 4 +- internal/event/check_rep_count_test.go | 2 +- internal/event/event.go | 2 +- internal/services/event.go | 4 +- .../tickevent/batch_all_agent_check_cache.go | 2 +- internal/tickevent/tick_event.go | 2 +- main.go | 4 +- 15 files changed, 153 insertions(+), 59 deletions(-) diff --git a/go.mod b/go.mod index 73d6121..c068297 100644 --- a/go.mod +++ b/go.mod @@ -3,38 +3,53 @@ module gitlink.org.cn/cloudream/storage-scanner go 1.20 require ( - github.com/jmoiron/sqlx v1.3.5 github.com/samber/lo v1.38.1 github.com/smartystreets/goconvey v1.8.0 gitlink.org.cn/cloudream/common v0.0.0 - gitlink.org.cn/cloudream/db v0.0.0 - gitlink.org.cn/cloudream/rabbitmq v0.0.0 - golang.org/x/sync v0.1.0 + gitlink.org.cn/cloudream/storage-common v0.0.0 ) require ( + github.com/coreos/go-semver v0.3.0 // indirect + github.com/coreos/go-systemd/v22 v22.5.0 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang/protobuf v1.5.3 // indirect + github.com/google/uuid v1.3.0 // indirect github.com/gopherjs/gopherjs v1.17.2 // indirect + github.com/jmoiron/sqlx v1.3.5 // indirect + github.com/json-iterator/go v1.1.12 // indirect github.com/jtolds/gls v4.20.0+incompatible // indirect + github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect github.com/smartystreets/assertions v1.13.1 // indirect + go.etcd.io/etcd/api/v3 v3.5.9 // indirect + go.etcd.io/etcd/client/pkg/v3 v3.5.9 // indirect + go.etcd.io/etcd/client/v3 v3.5.9 // indirect + go.uber.org/atomic v1.10.0 // indirect + go.uber.org/multierr v1.9.0 // indirect + go.uber.org/zap v1.24.0 // indirect + golang.org/x/net v0.8.0 // indirect + golang.org/x/text v0.8.0 // indirect + google.golang.org/genproto v0.0.0-20230403163135-c38d8f061ccd // indirect + google.golang.org/grpc v1.54.0 // indirect + google.golang.org/protobuf v1.30.0 // indirect ) require ( github.com/antonfisher/nested-logrus-formatter v1.3.1 // indirect - github.com/go-sql-driver/mysql v1.6.0 // indirect - github.com/hashicorp/errwrap v1.0.0 // indirect + github.com/go-sql-driver/mysql v1.7.1 // indirect + github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/imdario/mergo v0.3.15 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect - github.com/sirupsen/logrus v1.9.0 // indirect - github.com/streadway/amqp v1.0.0 // indirect - github.com/zyedidia/generic v1.2.1 - golang.org/x/exp v0.0.0-20220303212507-bbda1eaf7a17 // indirect + github.com/sirupsen/logrus v1.9.2 // indirect + github.com/streadway/amqp v1.1.0 // indirect + github.com/zyedidia/generic v1.2.1 // indirect + golang.org/x/exp v0.0.0-20230519143937-03e91628a987 // indirect golang.org/x/sys v0.7.0 // indirect ) // 运行go mod tidy时需要将下面几行取消注释 -// replace gitlink.org.cn/cloudream/rabbitmq => ../rabbitmq -// -// replace gitlink.org.cn/cloudream/db => ../db -// -// replace gitlink.org.cn/cloudream/common => ../common +replace gitlink.org.cn/cloudream/common => ../../common + +replace gitlink.org.cn/cloudream/storage-common => ../storage-common diff --git a/go.sum b/go.sum index 8fc1746..cbc7d39 100644 --- a/go.sum +++ b/go.sum @@ -1,53 +1,132 @@ github.com/antonfisher/nested-logrus-formatter v1.3.1 h1:NFJIr+pzwv5QLHTPyKz9UMEoHck02Q9L0FP13b/xSbQ= github.com/antonfisher/nested-logrus-formatter v1.3.1/go.mod h1:6WTfyWFkBc9+zyBaKIqRrg/KwMqBbodBjgbHjDz7zjA= +github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A= +github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= +github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8iXXhfZs= +github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE= github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= +github.com/go-sql-driver/mysql v1.7.1 h1:lUIinVbN1DY0xBg0eMOzmmtGoHwWBbvnWubQUrtU8EI= +github.com/go-sql-driver/mysql v1.7.1/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI= +github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= +github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gopherjs/gopherjs v1.17.2 h1:fQnZVsXk8uxXIStYb0N4bGk7jeyTalG/wsZjQ25dO0g= github.com/gopherjs/gopherjs v1.17.2/go.mod h1:pRRIvn/QzFLrKfvEz3qUuEhtE/zLCWfreZ6J5gM2i+k= -github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= +github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= github.com/imdario/mergo v0.3.15 h1:M8XP7IuFNsqUx6VPK2P9OSmsYsI/YFaGil0uD21V3dM= github.com/imdario/mergo v0.3.15/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY= github.com/jmoiron/sqlx v1.3.5 h1:vFFPA71p1o5gAeqtEAwLU4dnX2napprKtHr7PYIcN3g= github.com/jmoiron/sqlx v1.3.5/go.mod h1:nRVWtLre0KfCLJvgxzCsLVMogSvQ1zNJtpYr2Ccp0mQ= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/lib/pq v1.2.0 h1:LXpIM/LZ5xGFhOpXAQUIMM1HdyqzVYM13zNdjCEEcA0= github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/mattn/go-sqlite3 v1.14.6 h1:dNPt6NO46WmLVt2DLNpwczCmdV5boIZ6g/tlDrlRUbg= github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/samber/lo v1.38.1 h1:j2XEAqXKb09Am4ebOg31SpvzUTTs6EN3VfgeLUhPdXM= github.com/samber/lo v1.38.1/go.mod h1:+m/ZKRl6ClXCE2Lgf3MsQlWfh4bn1bz6CXEOxnEXnEA= -github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= -github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/sirupsen/logrus v1.9.2 h1:oxx1eChJGI6Uks2ZC4W1zpLlVgqB8ner4EuQwV4Ik1Y= +github.com/sirupsen/logrus v1.9.2/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/smartystreets/assertions v1.13.1 h1:Ef7KhSmjZcK6AVf9YbJdvPYG9avaF0ZxudX+ThRdWfU= github.com/smartystreets/assertions v1.13.1/go.mod h1:cXr/IwVfSo/RbCSPhoAPv73p3hlSdrBH/b3SdnW/LMY= github.com/smartystreets/goconvey v1.8.0 h1:Oi49ha/2MURE0WexF052Z0m+BNSGirfjg5RL+JXWq3w= github.com/smartystreets/goconvey v1.8.0/go.mod h1:EdX8jtrTIj26jmjCOVNMVSIYAtgexqXKHOXW2Dx9JLg= -github.com/streadway/amqp v1.0.0 h1:kuuDrUJFZL1QYL9hUNuCxNObNzB0bV/ZG5jV3RWAQgo= -github.com/streadway/amqp v1.0.0/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= +github.com/streadway/amqp v1.1.0 h1:py12iX8XSyI7aN/3dUT8DFIDJazNJsVJdxNVEpnQTZM= +github.com/streadway/amqp v1.1.0/go.mod h1:WYSrTEYHOXHd0nwFeUXAe2G2hRnQT+deZJJf88uS9Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/zyedidia/generic v1.2.1 h1:Zv5KS/N2m0XZZiuLS82qheRG4X1o5gsWreGb0hR7XDc= github.com/zyedidia/generic v1.2.1/go.mod h1:ly2RBz4mnz1yeuVbQA/VFwGjK3mnHGRj1JuoG336Bis= -golang.org/x/exp v0.0.0-20220303212507-bbda1eaf7a17 h1:3MTrJm4PyNL9NBqvYDSj3DHl46qQakyfqfWo4jgfaEM= -golang.org/x/exp v0.0.0-20220303212507-bbda1eaf7a17/go.mod h1:lgLbSvA5ygNOMpwM/9anMpWVlVJ7Z+cHWq/eFuinpGE= -golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= -golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +go.etcd.io/etcd/api/v3 v3.5.9 h1:4wSsluwyTbGGmyjJktOf3wFQoTBIURXHnq9n/G/JQHs= +go.etcd.io/etcd/api/v3 v3.5.9/go.mod h1:uyAal843mC8uUVSLWz6eHa/d971iDGnCRpmKd2Z+X8k= +go.etcd.io/etcd/client/pkg/v3 v3.5.9 h1:oidDC4+YEuSIQbsR94rY9gur91UPL6DnxDCIYd2IGsE= +go.etcd.io/etcd/client/pkg/v3 v3.5.9/go.mod h1:y+CzeSmkMpWN2Jyu1npecjB9BBnABxGM4pN8cGuJeL4= +go.etcd.io/etcd/client/v3 v3.5.9 h1:r5xghnU7CwbUxD/fbUtRyJGaYNfDun8sp/gTr1hew6E= +go.etcd.io/etcd/client/v3 v3.5.9/go.mod h1:i/Eo5LrZ5IKqpbtpPDuaUnDOUv471oDg8cjQaUr2MbA= +go.uber.org/atomic v1.10.0 h1:9qC72Qh0+3MqyJbAn8YU5xVq1frD8bn3JtD2oXtafVQ= +go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= +go.uber.org/goleak v1.1.11 h1:wy28qYRKZgnJTxGxvye5/wgWr1EKjmUDGYox5mGlRlI= +go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI= +go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTVQ= +go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60= +go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/exp v0.0.0-20230519143937-03e91628a987 h1:3xJIFvzUFbu4ls0BTBYcgbCGhA63eAOEMxIHugyXJqA= +golang.org/x/exp v0.0.0-20230519143937-03e91628a987/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ= +golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU= golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.8.0 h1:57P1ETyNKtuIjB4SRd15iJxuhj8Gc416Y78H3qgMh68= +golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/genproto v0.0.0-20230403163135-c38d8f061ccd h1:sLpv7bNL1AsX3fdnWh9WVh7ejIzXdOc1RRHGeAmeStU= +google.golang.org/genproto v0.0.0-20230403163135-c38d8f061ccd/go.mod h1:UUQDJDOlWu4KYeJZffbWgBkS1YFobzKbLVfK69pe0Ak= +google.golang.org/grpc v1.54.0 h1:EhTqbhiYeixwWQtAEZAxmV9MGqcjEU2mFx52xCzNyag= +google.golang.org/grpc v1.54.0/go.mod h1:PUSEXI6iWghWaB6lXM4knEgpJNu2qUcKfDtNci3EC2g= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= +google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/config/config.go b/internal/config/config.go index d933cd4..02939b3 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -4,8 +4,8 @@ import ( "gitlink.org.cn/cloudream/common/pkg/distlock" log "gitlink.org.cn/cloudream/common/pkg/logger" c "gitlink.org.cn/cloudream/common/utils/config" - db "gitlink.org.cn/cloudream/db/config" - racfg "gitlink.org.cn/cloudream/rabbitmq/config" + db "gitlink.org.cn/cloudream/storage-common/pkgs/db/config" + racfg "gitlink.org.cn/cloudream/storage-common/pkgs/mq/config" ) type Config struct { diff --git a/internal/event/agent_check_cache.go b/internal/event/agent_check_cache.go index 164d9f3..047dfcc 100644 --- a/internal/event/agent_check_cache.go +++ b/internal/event/agent_check_cache.go @@ -7,13 +7,13 @@ import ( "github.com/samber/lo" "gitlink.org.cn/cloudream/common/pkg/distlock/reqbuilder" "gitlink.org.cn/cloudream/common/pkg/logger" - "gitlink.org.cn/cloudream/db/model" - "gitlink.org.cn/cloudream/rabbitmq" + "gitlink.org.cn/cloudream/common/pkg/mq" + "gitlink.org.cn/cloudream/storage-common/pkgs/db/model" "gitlink.org.cn/cloudream/storage-scanner/internal/config" - agtcli "gitlink.org.cn/cloudream/rabbitmq/client/agent" - agtmsg "gitlink.org.cn/cloudream/rabbitmq/message/agent" - scevt "gitlink.org.cn/cloudream/rabbitmq/message/scanner/event" + agtcli "gitlink.org.cn/cloudream/storage-common/pkgs/mq/client/agent" + agtmsg "gitlink.org.cn/cloudream/storage-common/pkgs/mq/message/agent" + scevt "gitlink.org.cn/cloudream/storage-common/pkgs/mq/message/scanner/event" ) type AgentCheckCache struct { @@ -134,7 +134,7 @@ func (t *AgentCheckCache) startCheck(execCtx ExecuteContext, isComplete bool, ca } defer agentClient.Close() - checkResp, err := agentClient.CheckIPFS(agtmsg.NewCheckIPFS(isComplete, caches), rabbitmq.RequestOption{Timeout: time.Minute}) + checkResp, err := agentClient.CheckIPFS(agtmsg.NewCheckIPFS(isComplete, caches), mq.RequestOption{Timeout: time.Minute}) if err != nil { log.WithField("NodeID", t.NodeID).Warnf("checking ipfs: %s", err.Error()) return diff --git a/internal/event/agent_check_state.go b/internal/event/agent_check_state.go index 58a0708..ec81ffe 100644 --- a/internal/event/agent_check_state.go +++ b/internal/event/agent_check_state.go @@ -8,11 +8,11 @@ import ( "gitlink.org.cn/cloudream/common/consts" "gitlink.org.cn/cloudream/common/pkg/distlock/reqbuilder" "gitlink.org.cn/cloudream/common/pkg/logger" - "gitlink.org.cn/cloudream/db/model" - "gitlink.org.cn/cloudream/rabbitmq" - agtcli "gitlink.org.cn/cloudream/rabbitmq/client/agent" - agtmsg "gitlink.org.cn/cloudream/rabbitmq/message/agent" - scevt "gitlink.org.cn/cloudream/rabbitmq/message/scanner/event" + "gitlink.org.cn/cloudream/common/pkg/mq" + "gitlink.org.cn/cloudream/storage-common/pkgs/db/model" + agtcli "gitlink.org.cn/cloudream/storage-common/pkgs/mq/client/agent" + agtmsg "gitlink.org.cn/cloudream/storage-common/pkgs/mq/message/agent" + scevt "gitlink.org.cn/cloudream/storage-common/pkgs/mq/message/scanner/event" "gitlink.org.cn/cloudream/storage-scanner/internal/config" ) @@ -68,7 +68,7 @@ func (t *AgentCheckState) Execute(execCtx ExecuteContext) { } defer agentClient.Close() - getResp, err := agentClient.GetState(agtmsg.NewGetState(), rabbitmq.RequestOption{Timeout: time.Second * 30}) + getResp, err := agentClient.GetState(agtmsg.NewGetState(), mq.RequestOption{Timeout: time.Second * 30}) if err != nil { log.WithField("NodeID", t.NodeID).Warnf("getting state: %s", err.Error()) diff --git a/internal/event/agent_check_storage.go b/internal/event/agent_check_storage.go index e470929..2f8dabc 100644 --- a/internal/event/agent_check_storage.go +++ b/internal/event/agent_check_storage.go @@ -8,11 +8,11 @@ import ( "gitlink.org.cn/cloudream/common/consts" "gitlink.org.cn/cloudream/common/pkg/distlock/reqbuilder" "gitlink.org.cn/cloudream/common/pkg/logger" - "gitlink.org.cn/cloudream/db/model" - "gitlink.org.cn/cloudream/rabbitmq" - agtcli "gitlink.org.cn/cloudream/rabbitmq/client/agent" - agtmsg "gitlink.org.cn/cloudream/rabbitmq/message/agent" - scevt "gitlink.org.cn/cloudream/rabbitmq/message/scanner/event" + "gitlink.org.cn/cloudream/common/pkg/mq" + "gitlink.org.cn/cloudream/storage-common/pkgs/db/model" + agtcli "gitlink.org.cn/cloudream/storage-common/pkgs/mq/client/agent" + agtmsg "gitlink.org.cn/cloudream/storage-common/pkgs/mq/message/agent" + scevt "gitlink.org.cn/cloudream/storage-common/pkgs/mq/message/scanner/event" "gitlink.org.cn/cloudream/storage-scanner/internal/config" ) @@ -151,7 +151,7 @@ func (t *AgentCheckStorage) startCheck(execCtx ExecuteContext, stg model.Storage } defer agentClient.Close() - checkResp, err := agentClient.StorageCheck(agtmsg.NewStorageCheck(stg.StorageID, stg.Directory, isComplete, objects), rabbitmq.RequestOption{Timeout: time.Minute}) + checkResp, err := agentClient.StorageCheck(agtmsg.NewStorageCheck(stg.StorageID, stg.Directory, isComplete, objects), mq.RequestOption{Timeout: time.Minute}) if err != nil { log.WithField("NodeID", stg.NodeID).Warnf("checking storage: %s", err.Error()) return diff --git a/internal/event/check_cache.go b/internal/event/check_cache.go index 26ad80f..0666b7b 100644 --- a/internal/event/check_cache.go +++ b/internal/event/check_cache.go @@ -7,8 +7,8 @@ import ( "gitlink.org.cn/cloudream/common/consts" "gitlink.org.cn/cloudream/common/pkg/distlock/reqbuilder" "gitlink.org.cn/cloudream/common/pkg/logger" - "gitlink.org.cn/cloudream/db/model" - scevt "gitlink.org.cn/cloudream/rabbitmq/message/scanner/event" + "gitlink.org.cn/cloudream/storage-common/pkgs/db/model" + scevt "gitlink.org.cn/cloudream/storage-common/pkgs/mq/message/scanner/event" ) type CheckCache struct { diff --git a/internal/event/check_object.go b/internal/event/check_object.go index a40a753..1a95355 100644 --- a/internal/event/check_object.go +++ b/internal/event/check_object.go @@ -4,7 +4,7 @@ import ( "github.com/samber/lo" "gitlink.org.cn/cloudream/common/pkg/distlock/reqbuilder" "gitlink.org.cn/cloudream/common/pkg/logger" - scevt "gitlink.org.cn/cloudream/rabbitmq/message/scanner/event" + scevt "gitlink.org.cn/cloudream/storage-common/pkgs/mq/message/scanner/event" ) type CheckObject struct { diff --git a/internal/event/check_rep_count.go b/internal/event/check_rep_count.go index 8aa6aef..c324b47 100644 --- a/internal/event/check_rep_count.go +++ b/internal/event/check_rep_count.go @@ -12,8 +12,8 @@ import ( mysort "gitlink.org.cn/cloudream/common/utils/sort" "gitlink.org.cn/cloudream/storage-scanner/internal/config" - "gitlink.org.cn/cloudream/db/model" - scevt "gitlink.org.cn/cloudream/rabbitmq/message/scanner/event" + "gitlink.org.cn/cloudream/storage-common/pkgs/db/model" + scevt "gitlink.org.cn/cloudream/storage-common/pkgs/mq/message/scanner/event" ) type CheckRepCount struct { diff --git a/internal/event/check_rep_count_test.go b/internal/event/check_rep_count_test.go index 9c690bc..b4b89f5 100644 --- a/internal/event/check_rep_count_test.go +++ b/internal/event/check_rep_count_test.go @@ -7,7 +7,7 @@ import ( . "github.com/smartystreets/goconvey/convey" "gitlink.org.cn/cloudream/common/consts" "gitlink.org.cn/cloudream/common/utils/sort" - "gitlink.org.cn/cloudream/db/model" + "gitlink.org.cn/cloudream/storage-common/pkgs/db/model" ) func Test_chooseNewRepNodes(t *testing.T) { diff --git a/internal/event/event.go b/internal/event/event.go index 7bf9076..a405669 100644 --- a/internal/event/event.go +++ b/internal/event/event.go @@ -7,7 +7,7 @@ import ( distlocksvc "gitlink.org.cn/cloudream/common/pkg/distlock/service" event "gitlink.org.cn/cloudream/common/pkg/event" "gitlink.org.cn/cloudream/common/pkg/typedispatcher" - mydb "gitlink.org.cn/cloudream/db" + mydb "gitlink.org.cn/cloudream/storage-common/pkgs/db" ) type ExecuteArgs struct { diff --git a/internal/services/event.go b/internal/services/event.go index 7268a40..90649e8 100644 --- a/internal/services/event.go +++ b/internal/services/event.go @@ -2,8 +2,8 @@ package services import ( "gitlink.org.cn/cloudream/common/pkg/logger" - scmsg "gitlink.org.cn/cloudream/rabbitmq/message/scanner" - scevt "gitlink.org.cn/cloudream/rabbitmq/message/scanner/event" + scmsg "gitlink.org.cn/cloudream/storage-common/pkgs/mq/message/scanner" + scevt "gitlink.org.cn/cloudream/storage-common/pkgs/mq/message/scanner/event" "gitlink.org.cn/cloudream/storage-scanner/internal/event" ) diff --git a/internal/tickevent/batch_all_agent_check_cache.go b/internal/tickevent/batch_all_agent_check_cache.go index 5ea751d..e567408 100644 --- a/internal/tickevent/batch_all_agent_check_cache.go +++ b/internal/tickevent/batch_all_agent_check_cache.go @@ -3,7 +3,7 @@ package tickevent import ( "github.com/samber/lo" "gitlink.org.cn/cloudream/common/pkg/logger" - "gitlink.org.cn/cloudream/db/model" + "gitlink.org.cn/cloudream/storage-common/pkgs/db/model" "gitlink.org.cn/cloudream/storage-scanner/internal/event" ) diff --git a/internal/tickevent/tick_event.go b/internal/tickevent/tick_event.go index 00cdea1..45815fa 100644 --- a/internal/tickevent/tick_event.go +++ b/internal/tickevent/tick_event.go @@ -2,7 +2,7 @@ package tickevent import ( tickevent "gitlink.org.cn/cloudream/common/pkg/tickevent" - mydb "gitlink.org.cn/cloudream/db" + mydb "gitlink.org.cn/cloudream/storage-common/pkgs/db" "gitlink.org.cn/cloudream/storage-scanner/internal/event" ) diff --git a/main.go b/main.go index e2e983f..e32c6fc 100644 --- a/main.go +++ b/main.go @@ -7,8 +7,8 @@ import ( distlocksvc "gitlink.org.cn/cloudream/common/pkg/distlock/service" log "gitlink.org.cn/cloudream/common/pkg/logger" - "gitlink.org.cn/cloudream/db" - scsvr "gitlink.org.cn/cloudream/rabbitmq/server/scanner" + "gitlink.org.cn/cloudream/storage-common/pkgs/db" + scsvr "gitlink.org.cn/cloudream/storage-common/pkgs/mq/server/scanner" "gitlink.org.cn/cloudream/storage-scanner/internal/config" "gitlink.org.cn/cloudream/storage-scanner/internal/event" "gitlink.org.cn/cloudream/storage-scanner/internal/services" From 35596ee98d40514bb4d047bd36b0f115a6469bcb Mon Sep 17 00:00:00 2001 From: Sydonian <794346190@qq.com> Date: Mon, 14 Aug 2023 14:54:47 +0800 Subject: [PATCH 61/68] =?UTF-8?q?=E6=8B=86=E5=88=86common=E5=BA=93?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- internal/event/agent_check_state.go | 2 +- internal/event/agent_check_storage.go | 2 +- internal/event/check_cache.go | 2 +- internal/event/check_rep_count.go | 2 +- internal/event/check_rep_count_test.go | 2 +- magefiles/magefile.go | 4 ++-- 6 files changed, 7 insertions(+), 7 deletions(-) diff --git a/internal/event/agent_check_state.go b/internal/event/agent_check_state.go index ec81ffe..77050aa 100644 --- a/internal/event/agent_check_state.go +++ b/internal/event/agent_check_state.go @@ -5,10 +5,10 @@ import ( "time" "github.com/samber/lo" - "gitlink.org.cn/cloudream/common/consts" "gitlink.org.cn/cloudream/common/pkg/distlock/reqbuilder" "gitlink.org.cn/cloudream/common/pkg/logger" "gitlink.org.cn/cloudream/common/pkg/mq" + "gitlink.org.cn/cloudream/storage-common/consts" "gitlink.org.cn/cloudream/storage-common/pkgs/db/model" agtcli "gitlink.org.cn/cloudream/storage-common/pkgs/mq/client/agent" agtmsg "gitlink.org.cn/cloudream/storage-common/pkgs/mq/message/agent" diff --git a/internal/event/agent_check_storage.go b/internal/event/agent_check_storage.go index 2f8dabc..9dc2981 100644 --- a/internal/event/agent_check_storage.go +++ b/internal/event/agent_check_storage.go @@ -5,10 +5,10 @@ import ( "time" "github.com/samber/lo" - "gitlink.org.cn/cloudream/common/consts" "gitlink.org.cn/cloudream/common/pkg/distlock/reqbuilder" "gitlink.org.cn/cloudream/common/pkg/logger" "gitlink.org.cn/cloudream/common/pkg/mq" + "gitlink.org.cn/cloudream/storage-common/consts" "gitlink.org.cn/cloudream/storage-common/pkgs/db/model" agtcli "gitlink.org.cn/cloudream/storage-common/pkgs/mq/client/agent" agtmsg "gitlink.org.cn/cloudream/storage-common/pkgs/mq/message/agent" diff --git a/internal/event/check_cache.go b/internal/event/check_cache.go index 0666b7b..4d7ee90 100644 --- a/internal/event/check_cache.go +++ b/internal/event/check_cache.go @@ -4,9 +4,9 @@ import ( "database/sql" "github.com/samber/lo" - "gitlink.org.cn/cloudream/common/consts" "gitlink.org.cn/cloudream/common/pkg/distlock/reqbuilder" "gitlink.org.cn/cloudream/common/pkg/logger" + "gitlink.org.cn/cloudream/storage-common/consts" "gitlink.org.cn/cloudream/storage-common/pkgs/db/model" scevt "gitlink.org.cn/cloudream/storage-common/pkgs/mq/message/scanner/event" ) diff --git a/internal/event/check_rep_count.go b/internal/event/check_rep_count.go index c324b47..5edb6b4 100644 --- a/internal/event/check_rep_count.go +++ b/internal/event/check_rep_count.go @@ -5,11 +5,11 @@ import ( "math" "github.com/samber/lo" - "gitlink.org.cn/cloudream/common/consts" "gitlink.org.cn/cloudream/common/pkg/distlock/reqbuilder" "gitlink.org.cn/cloudream/common/pkg/logger" mymath "gitlink.org.cn/cloudream/common/utils/math" mysort "gitlink.org.cn/cloudream/common/utils/sort" + "gitlink.org.cn/cloudream/storage-common/consts" "gitlink.org.cn/cloudream/storage-scanner/internal/config" "gitlink.org.cn/cloudream/storage-common/pkgs/db/model" diff --git a/internal/event/check_rep_count_test.go b/internal/event/check_rep_count_test.go index b4b89f5..7a8fe82 100644 --- a/internal/event/check_rep_count_test.go +++ b/internal/event/check_rep_count_test.go @@ -5,8 +5,8 @@ import ( "github.com/samber/lo" . "github.com/smartystreets/goconvey/convey" - "gitlink.org.cn/cloudream/common/consts" "gitlink.org.cn/cloudream/common/utils/sort" + "gitlink.org.cn/cloudream/storage-common/consts" "gitlink.org.cn/cloudream/storage-common/pkgs/db/model" ) diff --git a/magefiles/magefile.go b/magefiles/magefile.go index 0f4ec2e..84dba2c 100644 --- a/magefiles/magefile.go +++ b/magefiles/magefile.go @@ -3,10 +3,10 @@ package main import ( - "magefiles" + "gitlink.org.cn/cloudream/common/magefiles" //mage:import - _ "magefiles/targets" + _ "gitlink.org.cn/cloudream/common/magefiles/targets" ) var Default = Build From c22e417e3c7a0e4492095991e1939572ce9b35e0 Mon Sep 17 00:00:00 2001 From: Sydonian <794346190@qq.com> Date: Mon, 14 Aug 2023 15:40:27 +0800 Subject: [PATCH 62/68] =?UTF-8?q?=E4=BC=98=E5=8C=96=E7=BC=96=E8=AF=91?= =?UTF-8?q?=E8=84=9A=E6=9C=AC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- go.mod | 2 ++ go.sum | 5 +++++ magefiles/magefile.go | 2 +- 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index c068297..cd4c776 100644 --- a/go.mod +++ b/go.mod @@ -41,7 +41,9 @@ require ( github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/imdario/mergo v0.3.15 // indirect + github.com/magefile/mage v1.15.0 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect + github.com/otiai10/copy v1.12.0 // indirect github.com/sirupsen/logrus v1.9.2 // indirect github.com/streadway/amqp v1.1.0 // indirect github.com/zyedidia/generic v1.2.1 // indirect diff --git a/go.sum b/go.sum index cbc7d39..0b11fb9 100644 --- a/go.sum +++ b/go.sum @@ -41,6 +41,8 @@ github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/lib/pq v1.2.0 h1:LXpIM/LZ5xGFhOpXAQUIMM1HdyqzVYM13zNdjCEEcA0= github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/magefile/mage v1.15.0 h1:BvGheCMAsG3bWUDbZ8AyXXpCNwU9u5CB6sM+HNb9HYg= +github.com/magefile/mage v1.15.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A= github.com/mattn/go-sqlite3 v1.14.6 h1:dNPt6NO46WmLVt2DLNpwczCmdV5boIZ6g/tlDrlRUbg= github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= @@ -49,6 +51,9 @@ github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OH github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/otiai10/copy v1.12.0 h1:cLMgSQnXBs1eehF0Wy/FAGsgDTDmAqFR7rQylBb1nDY= +github.com/otiai10/copy v1.12.0/go.mod h1:rSaLseMUsZFFbsFGc7wCJnnkTAvdc5L6VWxPE4308Ww= +github.com/otiai10/mint v1.5.1 h1:XaPLeE+9vGbuyEHem1JNk3bYc7KKqyI/na0/mLd/Kks= github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= diff --git a/magefiles/magefile.go b/magefiles/magefile.go index 84dba2c..2f000f1 100644 --- a/magefiles/magefile.go +++ b/magefiles/magefile.go @@ -14,7 +14,7 @@ var Default = Build func Build() error { return magefiles.Build(magefiles.BuildArgs{ OutputName: "scanner", - OutputDir: "../../build/scanner", + OutputDir: "scanner", AssetsDir: "assets", }) } From f0bc81f4a9c0b3e2c1a3e6cc91f8278d41bae308 Mon Sep 17 00:00:00 2001 From: Sydonian <794346190@qq.com> Date: Mon, 14 Aug 2023 15:46:42 +0800 Subject: [PATCH 63/68] =?UTF-8?q?=E9=87=8D=E5=91=BD=E5=90=8D=E6=96=87?= =?UTF-8?q?=E4=BB=B6=E5=A4=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- internal/config/config.go | 4 ++-- internal/event/agent_check_cache.go | 6 +++--- internal/event/agent_check_state.go | 6 +++--- internal/event/agent_check_storage.go | 6 +++--- internal/event/check_cache.go | 4 ++-- internal/event/check_object.go | 4 ++-- internal/event/check_rep_count.go | 4 ++-- internal/event/event.go | 6 +++--- internal/services/event.go | 2 +- internal/tickevent/batch_all_agent_check_cache.go | 2 +- internal/tickevent/batch_check_all_object.go | 2 +- internal/tickevent/batch_check_all_rep_count.go | 2 +- internal/tickevent/batch_check_all_storage.go | 2 +- internal/tickevent/check_agent_state.go | 2 +- internal/tickevent/check_cache.go | 2 +- internal/tickevent/tick_event.go | 2 +- main.go | 4 ++-- 17 files changed, 30 insertions(+), 30 deletions(-) diff --git a/internal/config/config.go b/internal/config/config.go index 02939b3..e6acbce 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -1,8 +1,8 @@ package config import ( - "gitlink.org.cn/cloudream/common/pkg/distlock" - log "gitlink.org.cn/cloudream/common/pkg/logger" + "gitlink.org.cn/cloudream/common/pkgs/distlock" + log "gitlink.org.cn/cloudream/common/pkgs/logger" c "gitlink.org.cn/cloudream/common/utils/config" db "gitlink.org.cn/cloudream/storage-common/pkgs/db/config" racfg "gitlink.org.cn/cloudream/storage-common/pkgs/mq/config" diff --git a/internal/event/agent_check_cache.go b/internal/event/agent_check_cache.go index 047dfcc..24261eb 100644 --- a/internal/event/agent_check_cache.go +++ b/internal/event/agent_check_cache.go @@ -5,9 +5,9 @@ import ( "time" "github.com/samber/lo" - "gitlink.org.cn/cloudream/common/pkg/distlock/reqbuilder" - "gitlink.org.cn/cloudream/common/pkg/logger" - "gitlink.org.cn/cloudream/common/pkg/mq" + "gitlink.org.cn/cloudream/common/pkgs/distlock/reqbuilder" + "gitlink.org.cn/cloudream/common/pkgs/logger" + "gitlink.org.cn/cloudream/common/pkgs/mq" "gitlink.org.cn/cloudream/storage-common/pkgs/db/model" "gitlink.org.cn/cloudream/storage-scanner/internal/config" diff --git a/internal/event/agent_check_state.go b/internal/event/agent_check_state.go index 77050aa..257f9a3 100644 --- a/internal/event/agent_check_state.go +++ b/internal/event/agent_check_state.go @@ -5,9 +5,9 @@ import ( "time" "github.com/samber/lo" - "gitlink.org.cn/cloudream/common/pkg/distlock/reqbuilder" - "gitlink.org.cn/cloudream/common/pkg/logger" - "gitlink.org.cn/cloudream/common/pkg/mq" + "gitlink.org.cn/cloudream/common/pkgs/distlock/reqbuilder" + "gitlink.org.cn/cloudream/common/pkgs/logger" + "gitlink.org.cn/cloudream/common/pkgs/mq" "gitlink.org.cn/cloudream/storage-common/consts" "gitlink.org.cn/cloudream/storage-common/pkgs/db/model" agtcli "gitlink.org.cn/cloudream/storage-common/pkgs/mq/client/agent" diff --git a/internal/event/agent_check_storage.go b/internal/event/agent_check_storage.go index 9dc2981..8306b3d 100644 --- a/internal/event/agent_check_storage.go +++ b/internal/event/agent_check_storage.go @@ -5,9 +5,9 @@ import ( "time" "github.com/samber/lo" - "gitlink.org.cn/cloudream/common/pkg/distlock/reqbuilder" - "gitlink.org.cn/cloudream/common/pkg/logger" - "gitlink.org.cn/cloudream/common/pkg/mq" + "gitlink.org.cn/cloudream/common/pkgs/distlock/reqbuilder" + "gitlink.org.cn/cloudream/common/pkgs/logger" + "gitlink.org.cn/cloudream/common/pkgs/mq" "gitlink.org.cn/cloudream/storage-common/consts" "gitlink.org.cn/cloudream/storage-common/pkgs/db/model" agtcli "gitlink.org.cn/cloudream/storage-common/pkgs/mq/client/agent" diff --git a/internal/event/check_cache.go b/internal/event/check_cache.go index 4d7ee90..e4d0bcb 100644 --- a/internal/event/check_cache.go +++ b/internal/event/check_cache.go @@ -4,8 +4,8 @@ import ( "database/sql" "github.com/samber/lo" - "gitlink.org.cn/cloudream/common/pkg/distlock/reqbuilder" - "gitlink.org.cn/cloudream/common/pkg/logger" + "gitlink.org.cn/cloudream/common/pkgs/distlock/reqbuilder" + "gitlink.org.cn/cloudream/common/pkgs/logger" "gitlink.org.cn/cloudream/storage-common/consts" "gitlink.org.cn/cloudream/storage-common/pkgs/db/model" scevt "gitlink.org.cn/cloudream/storage-common/pkgs/mq/message/scanner/event" diff --git a/internal/event/check_object.go b/internal/event/check_object.go index 1a95355..0c88c24 100644 --- a/internal/event/check_object.go +++ b/internal/event/check_object.go @@ -2,8 +2,8 @@ package event import ( "github.com/samber/lo" - "gitlink.org.cn/cloudream/common/pkg/distlock/reqbuilder" - "gitlink.org.cn/cloudream/common/pkg/logger" + "gitlink.org.cn/cloudream/common/pkgs/distlock/reqbuilder" + "gitlink.org.cn/cloudream/common/pkgs/logger" scevt "gitlink.org.cn/cloudream/storage-common/pkgs/mq/message/scanner/event" ) diff --git a/internal/event/check_rep_count.go b/internal/event/check_rep_count.go index 5edb6b4..bfe7dff 100644 --- a/internal/event/check_rep_count.go +++ b/internal/event/check_rep_count.go @@ -5,8 +5,8 @@ import ( "math" "github.com/samber/lo" - "gitlink.org.cn/cloudream/common/pkg/distlock/reqbuilder" - "gitlink.org.cn/cloudream/common/pkg/logger" + "gitlink.org.cn/cloudream/common/pkgs/distlock/reqbuilder" + "gitlink.org.cn/cloudream/common/pkgs/logger" mymath "gitlink.org.cn/cloudream/common/utils/math" mysort "gitlink.org.cn/cloudream/common/utils/sort" "gitlink.org.cn/cloudream/storage-common/consts" diff --git a/internal/event/event.go b/internal/event/event.go index a405669..bbb25bd 100644 --- a/internal/event/event.go +++ b/internal/event/event.go @@ -4,9 +4,9 @@ import ( "fmt" "reflect" - distlocksvc "gitlink.org.cn/cloudream/common/pkg/distlock/service" - event "gitlink.org.cn/cloudream/common/pkg/event" - "gitlink.org.cn/cloudream/common/pkg/typedispatcher" + distlocksvc "gitlink.org.cn/cloudream/common/pkgs/distlock/service" + event "gitlink.org.cn/cloudream/common/pkgs/event" + "gitlink.org.cn/cloudream/common/pkgs/typedispatcher" mydb "gitlink.org.cn/cloudream/storage-common/pkgs/db" ) diff --git a/internal/services/event.go b/internal/services/event.go index 90649e8..c601a16 100644 --- a/internal/services/event.go +++ b/internal/services/event.go @@ -1,7 +1,7 @@ package services import ( - "gitlink.org.cn/cloudream/common/pkg/logger" + "gitlink.org.cn/cloudream/common/pkgs/logger" scmsg "gitlink.org.cn/cloudream/storage-common/pkgs/mq/message/scanner" scevt "gitlink.org.cn/cloudream/storage-common/pkgs/mq/message/scanner/event" "gitlink.org.cn/cloudream/storage-scanner/internal/event" diff --git a/internal/tickevent/batch_all_agent_check_cache.go b/internal/tickevent/batch_all_agent_check_cache.go index e567408..9d87297 100644 --- a/internal/tickevent/batch_all_agent_check_cache.go +++ b/internal/tickevent/batch_all_agent_check_cache.go @@ -2,7 +2,7 @@ package tickevent import ( "github.com/samber/lo" - "gitlink.org.cn/cloudream/common/pkg/logger" + "gitlink.org.cn/cloudream/common/pkgs/logger" "gitlink.org.cn/cloudream/storage-common/pkgs/db/model" "gitlink.org.cn/cloudream/storage-scanner/internal/event" ) diff --git a/internal/tickevent/batch_check_all_object.go b/internal/tickevent/batch_check_all_object.go index 744b4c8..043bc8a 100644 --- a/internal/tickevent/batch_check_all_object.go +++ b/internal/tickevent/batch_check_all_object.go @@ -1,7 +1,7 @@ package tickevent import ( - "gitlink.org.cn/cloudream/common/pkg/logger" + "gitlink.org.cn/cloudream/common/pkgs/logger" "gitlink.org.cn/cloudream/storage-scanner/internal/event" ) diff --git a/internal/tickevent/batch_check_all_rep_count.go b/internal/tickevent/batch_check_all_rep_count.go index 6643653..ef4e9d7 100644 --- a/internal/tickevent/batch_check_all_rep_count.go +++ b/internal/tickevent/batch_check_all_rep_count.go @@ -1,7 +1,7 @@ package tickevent import ( - "gitlink.org.cn/cloudream/common/pkg/logger" + "gitlink.org.cn/cloudream/common/pkgs/logger" "gitlink.org.cn/cloudream/storage-scanner/internal/event" ) diff --git a/internal/tickevent/batch_check_all_storage.go b/internal/tickevent/batch_check_all_storage.go index 558b9ed..bbd290d 100644 --- a/internal/tickevent/batch_check_all_storage.go +++ b/internal/tickevent/batch_check_all_storage.go @@ -1,7 +1,7 @@ package tickevent import ( - "gitlink.org.cn/cloudream/common/pkg/logger" + "gitlink.org.cn/cloudream/common/pkgs/logger" "gitlink.org.cn/cloudream/storage-scanner/internal/event" ) diff --git a/internal/tickevent/check_agent_state.go b/internal/tickevent/check_agent_state.go index 4bdc4e4..b12d2f7 100644 --- a/internal/tickevent/check_agent_state.go +++ b/internal/tickevent/check_agent_state.go @@ -1,7 +1,7 @@ package tickevent import ( - "gitlink.org.cn/cloudream/common/pkg/logger" + "gitlink.org.cn/cloudream/common/pkgs/logger" "gitlink.org.cn/cloudream/storage-scanner/internal/event" ) diff --git a/internal/tickevent/check_cache.go b/internal/tickevent/check_cache.go index 5676f60..a8e0245 100644 --- a/internal/tickevent/check_cache.go +++ b/internal/tickevent/check_cache.go @@ -1,7 +1,7 @@ package tickevent import ( - "gitlink.org.cn/cloudream/common/pkg/logger" + "gitlink.org.cn/cloudream/common/pkgs/logger" "gitlink.org.cn/cloudream/storage-scanner/internal/event" ) diff --git a/internal/tickevent/tick_event.go b/internal/tickevent/tick_event.go index 45815fa..8051fc9 100644 --- a/internal/tickevent/tick_event.go +++ b/internal/tickevent/tick_event.go @@ -1,7 +1,7 @@ package tickevent import ( - tickevent "gitlink.org.cn/cloudream/common/pkg/tickevent" + tickevent "gitlink.org.cn/cloudream/common/pkgs/tickevent" mydb "gitlink.org.cn/cloudream/storage-common/pkgs/db" "gitlink.org.cn/cloudream/storage-scanner/internal/event" ) diff --git a/main.go b/main.go index e32c6fc..1898eef 100644 --- a/main.go +++ b/main.go @@ -5,8 +5,8 @@ import ( "os" "sync" - distlocksvc "gitlink.org.cn/cloudream/common/pkg/distlock/service" - log "gitlink.org.cn/cloudream/common/pkg/logger" + distlocksvc "gitlink.org.cn/cloudream/common/pkgs/distlock/service" + log "gitlink.org.cn/cloudream/common/pkgs/logger" "gitlink.org.cn/cloudream/storage-common/pkgs/db" scsvr "gitlink.org.cn/cloudream/storage-common/pkgs/mq/server/scanner" "gitlink.org.cn/cloudream/storage-scanner/internal/config" From e4d20033b55d53787d20ce60e5439e804107fa20 Mon Sep 17 00:00:00 2001 From: Sydonian <794346190@qq.com> Date: Fri, 18 Aug 2023 16:35:41 +0800 Subject: [PATCH 64/68] =?UTF-8?q?=E7=BB=9F=E4=B8=80=E5=8D=95=E6=96=87?= =?UTF-8?q?=E4=BB=B6=E5=92=8C=E5=A4=9A=E6=96=87=E4=BB=B6=E9=80=BB=E8=BE=91?= =?UTF-8?q?=EF=BC=9B=E4=BC=98=E5=8C=96=E7=A8=8B=E5=BA=8F=E7=BB=93=E6=9E=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- internal/config/config.go | 4 +- internal/event/agent_check_cache.go | 13 ++- internal/event/agent_check_state.go | 9 +- internal/event/agent_check_storage.go | 85 +++++++++---------- internal/event/check_cache.go | 2 +- internal/event/check_object.go | 57 ------------- internal/event/check_package.go | 57 +++++++++++++ internal/event/check_rep_count.go | 2 +- internal/services/event.go | 6 +- internal/tickevent/batch_check_all_object.go | 39 --------- internal/tickevent/batch_check_all_package.go | 39 +++++++++ main.go | 38 ++++----- 12 files changed, 174 insertions(+), 177 deletions(-) delete mode 100644 internal/event/check_object.go create mode 100644 internal/event/check_package.go delete mode 100644 internal/tickevent/batch_check_all_object.go create mode 100644 internal/tickevent/batch_check_all_package.go diff --git a/internal/config/config.go b/internal/config/config.go index e6acbce..5ca8a39 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -5,7 +5,7 @@ import ( log "gitlink.org.cn/cloudream/common/pkgs/logger" c "gitlink.org.cn/cloudream/common/utils/config" db "gitlink.org.cn/cloudream/storage-common/pkgs/db/config" - racfg "gitlink.org.cn/cloudream/storage-common/pkgs/mq/config" + stgmq "gitlink.org.cn/cloudream/storage-common/pkgs/mq" ) type Config struct { @@ -14,7 +14,7 @@ type Config struct { Logger log.Config `json:"logger"` DB db.Config `json:"db"` - RabbitMQ racfg.Config `json:"rabbitMQ"` + RabbitMQ stgmq.Config `json:"rabbitMQ"` DistLock distlock.Config `json:"distlock"` } diff --git a/internal/event/agent_check_cache.go b/internal/event/agent_check_cache.go index 24261eb..70b3f90 100644 --- a/internal/event/agent_check_cache.go +++ b/internal/event/agent_check_cache.go @@ -11,9 +11,8 @@ import ( "gitlink.org.cn/cloudream/storage-common/pkgs/db/model" "gitlink.org.cn/cloudream/storage-scanner/internal/config" - agtcli "gitlink.org.cn/cloudream/storage-common/pkgs/mq/client/agent" - agtmsg "gitlink.org.cn/cloudream/storage-common/pkgs/mq/message/agent" - scevt "gitlink.org.cn/cloudream/storage-common/pkgs/mq/message/scanner/event" + agtmq "gitlink.org.cn/cloudream/storage-common/pkgs/mq/agent" + scevt "gitlink.org.cn/cloudream/storage-common/pkgs/mq/scanner/event" ) type AgentCheckCache struct { @@ -127,14 +126,14 @@ func (t *AgentCheckCache) startCheck(execCtx ExecuteContext, isComplete bool, ca log := logger.WithType[AgentCheckCache]("Event") // 然后向代理端发送移动文件的请求 - agentClient, err := agtcli.NewClient(t.NodeID, &config.Cfg().RabbitMQ) + agentClient, err := agtmq.NewClient(t.NodeID, &config.Cfg().RabbitMQ) if err != nil { log.WithField("NodeID", t.NodeID).Warnf("create agent client failed, err: %s", err.Error()) return } defer agentClient.Close() - checkResp, err := agentClient.CheckIPFS(agtmsg.NewCheckIPFS(isComplete, caches), mq.RequestOption{Timeout: time.Minute}) + checkResp, err := agentClient.CheckIPFS(agtmq.NewCheckIPFS(isComplete, caches), mq.RequestOption{Timeout: time.Minute}) if err != nil { log.WithField("NodeID", t.NodeID).Warnf("checking ipfs: %s", err.Error()) return @@ -143,7 +142,7 @@ func (t *AgentCheckCache) startCheck(execCtx ExecuteContext, isComplete bool, ca // 根据返回结果修改数据库 for _, entry := range checkResp.Entries { switch entry.Operation { - case agtmsg.CHECK_IPFS_RESP_OP_DELETE_TEMP: + case agtmq.CHECK_IPFS_RESP_OP_DELETE_TEMP: err := execCtx.Args.DB.Cache().DeleteTemp(execCtx.Args.DB.SQLCtx(), entry.FileHash, t.NodeID) if err != nil { log.WithField("FileHash", entry.FileHash). @@ -155,7 +154,7 @@ func (t *AgentCheckCache) startCheck(execCtx ExecuteContext, isComplete bool, ca WithField("NodeID", t.NodeID). Debugf("delete temp cache") - case agtmsg.CHECK_IPFS_RESP_OP_CREATE_TEMP: + case agtmq.CHECK_IPFS_RESP_OP_CREATE_TEMP: err := execCtx.Args.DB.Cache().CreateTemp(execCtx.Args.DB.SQLCtx(), entry.FileHash, t.NodeID) if err != nil { log.WithField("FileHash", entry.FileHash). diff --git a/internal/event/agent_check_state.go b/internal/event/agent_check_state.go index 257f9a3..e7fc582 100644 --- a/internal/event/agent_check_state.go +++ b/internal/event/agent_check_state.go @@ -10,9 +10,8 @@ import ( "gitlink.org.cn/cloudream/common/pkgs/mq" "gitlink.org.cn/cloudream/storage-common/consts" "gitlink.org.cn/cloudream/storage-common/pkgs/db/model" - agtcli "gitlink.org.cn/cloudream/storage-common/pkgs/mq/client/agent" - agtmsg "gitlink.org.cn/cloudream/storage-common/pkgs/mq/message/agent" - scevt "gitlink.org.cn/cloudream/storage-common/pkgs/mq/message/scanner/event" + agtmq "gitlink.org.cn/cloudream/storage-common/pkgs/mq/agent" + scevt "gitlink.org.cn/cloudream/storage-common/pkgs/mq/scanner/event" "gitlink.org.cn/cloudream/storage-scanner/internal/config" ) @@ -61,14 +60,14 @@ func (t *AgentCheckState) Execute(execCtx ExecuteContext) { return } - agentClient, err := agtcli.NewClient(t.NodeID, &config.Cfg().RabbitMQ) + agentClient, err := agtmq.NewClient(t.NodeID, &config.Cfg().RabbitMQ) if err != nil { log.WithField("NodeID", t.NodeID).Warnf("create agent client failed, err: %s", err.Error()) return } defer agentClient.Close() - getResp, err := agentClient.GetState(agtmsg.NewGetState(), mq.RequestOption{Timeout: time.Second * 30}) + getResp, err := agentClient.GetState(agtmq.NewGetState(), mq.RequestOption{Timeout: time.Second * 30}) if err != nil { log.WithField("NodeID", t.NodeID).Warnf("getting state: %s", err.Error()) diff --git a/internal/event/agent_check_storage.go b/internal/event/agent_check_storage.go index 8306b3d..ca89506 100644 --- a/internal/event/agent_check_storage.go +++ b/internal/event/agent_check_storage.go @@ -10,9 +10,8 @@ import ( "gitlink.org.cn/cloudream/common/pkgs/mq" "gitlink.org.cn/cloudream/storage-common/consts" "gitlink.org.cn/cloudream/storage-common/pkgs/db/model" - agtcli "gitlink.org.cn/cloudream/storage-common/pkgs/mq/client/agent" - agtmsg "gitlink.org.cn/cloudream/storage-common/pkgs/mq/message/agent" - scevt "gitlink.org.cn/cloudream/storage-common/pkgs/mq/message/scanner/event" + agtmq "gitlink.org.cn/cloudream/storage-common/pkgs/mq/agent" + scevt "gitlink.org.cn/cloudream/storage-common/pkgs/mq/scanner/event" "gitlink.org.cn/cloudream/storage-scanner/internal/config" ) @@ -20,9 +19,9 @@ type AgentCheckStorage struct { scevt.AgentCheckStorage } -func NewAgentCheckStorage(storageID int64, objectIDs []int64) *AgentCheckStorage { +func NewAgentCheckStorage(storageID int64, packageIDs []int64) *AgentCheckStorage { return &AgentCheckStorage{ - AgentCheckStorage: scevt.NewAgentCheckStorage(storageID, objectIDs), + AgentCheckStorage: scevt.NewAgentCheckStorage(storageID, packageIDs), } } @@ -36,11 +35,11 @@ func (t *AgentCheckStorage) TryMerge(other Event) bool { return false } - // ObjectIDs为nil时代表全量检查 - if event.ObjectIDs == nil { - t.ObjectIDs = nil - } else if t.ObjectIDs != nil { - t.ObjectIDs = lo.Union(t.ObjectIDs, event.ObjectIDs) + // PackageIDs为nil时代表全量检查 + if event.PackageIDs == nil { + t.PackageIDs = nil + } else if t.PackageIDs != nil { + t.PackageIDs = lo.Union(t.PackageIDs, event.PackageIDs) } return true @@ -74,7 +73,7 @@ func (t *AgentCheckStorage) Execute(execCtx ExecuteContext) { return } - if t.ObjectIDs == nil { + if t.PackageIDs == nil { t.checkComplete(execCtx, stg) } else { t.checkIncrement(execCtx, stg) @@ -87,10 +86,10 @@ func (t *AgentCheckStorage) checkComplete(execCtx ExecuteContext, stg model.Stor mutex, err := reqbuilder.NewBuilder(). Metadata(). // 全量模式下查询、修改Move记录 - StorageObject().WriteAny(). + StoragePackage().WriteAny(). Storage(). // 全量模式下删除对象文件 - WriteAnyObject(t.StorageID). + WriteAnyPackage(t.StorageID). MutexLock(execCtx.Args.DistLock) if err != nil { log.Warnf("acquire locks failed, err: %s", err.Error()) @@ -98,13 +97,13 @@ func (t *AgentCheckStorage) checkComplete(execCtx ExecuteContext, stg model.Stor } defer mutex.Unlock() - objects, err := execCtx.Args.DB.StorageObject().GetAllByStorageID(execCtx.Args.DB.SQLCtx(), t.StorageID) + packages, err := execCtx.Args.DB.StoragePackage().GetAllByStorageID(execCtx.Args.DB.SQLCtx(), t.StorageID) if err != nil { - log.WithField("StorageID", t.StorageID).Warnf("get storage objects failed, err: %s", err.Error()) + log.WithField("StorageID", t.StorageID).Warnf("get storage packages failed, err: %s", err.Error()) return } - t.startCheck(execCtx, stg, true, objects) + t.startCheck(execCtx, stg, true, packages) } func (t *AgentCheckStorage) checkIncrement(execCtx ExecuteContext, stg model.Storage) { @@ -113,10 +112,10 @@ func (t *AgentCheckStorage) checkIncrement(execCtx ExecuteContext, stg model.Sto mutex, err := reqbuilder.NewBuilder(). Metadata(). // 全量模式下查询、修改Move记录。因为可能有多个User Move相同的文件，所以只能用集合Write锁 - StorageObject().WriteAny(). + StoragePackage().WriteAny(). Storage(). // 全量模式下删除对象文件。因为可能有多个User Move相同的文件，所以只能用集合Write锁 - WriteAnyObject(t.StorageID). + WriteAnyPackage(t.StorageID). MutexLock(execCtx.Args.DistLock) if err != nil { log.Warnf("acquire locks failed, err: %s", err.Error()) @@ -124,34 +123,34 @@ func (t *AgentCheckStorage) checkIncrement(execCtx ExecuteContext, stg model.Sto } defer mutex.Unlock() - var objects []model.StorageObject - for _, objID := range t.ObjectIDs { - objs, err := execCtx.Args.DB.StorageObject().GetAllByStorageAndObjectID(execCtx.Args.DB.SQLCtx(), t.StorageID, objID) + var packages []model.StoragePackage + for _, objID := range t.PackageIDs { + objs, err := execCtx.Args.DB.StoragePackage().GetAllByStorageAndPackageID(execCtx.Args.DB.SQLCtx(), t.StorageID, objID) if err != nil { log.WithField("StorageID", t.StorageID). - WithField("ObjectID", objID). - Warnf("get storage object failed, err: %s", err.Error()) + WithField("PackageID", objID). + Warnf("get storage package failed, err: %s", err.Error()) return } - objects = append(objects, objs...) + packages = append(packages, objs...) } - t.startCheck(execCtx, stg, false, objects) + t.startCheck(execCtx, stg, false, packages) } -func (t *AgentCheckStorage) startCheck(execCtx ExecuteContext, stg model.Storage, isComplete bool, objects []model.StorageObject) { +func (t *AgentCheckStorage) startCheck(execCtx ExecuteContext, stg model.Storage, isComplete bool, packages []model.StoragePackage) { log := logger.WithType[AgentCheckStorage]("Event") // 投递任务 - agentClient, err := agtcli.NewClient(stg.NodeID, &config.Cfg().RabbitMQ) + agentClient, err := agtmq.NewClient(stg.NodeID, &config.Cfg().RabbitMQ) if err != nil { log.WithField("NodeID", stg.NodeID).Warnf("create agent client failed, err: %s", err.Error()) return } defer agentClient.Close() - checkResp, err := agentClient.StorageCheck(agtmsg.NewStorageCheck(stg.StorageID, stg.Directory, isComplete, objects), mq.RequestOption{Timeout: time.Minute}) + checkResp, err := agentClient.StorageCheck(agtmq.NewStorageCheck(stg.StorageID, stg.Directory, isComplete, packages), mq.RequestOption{Timeout: time.Minute}) if err != nil { log.WithField("NodeID", stg.NodeID).Warnf("checking storage: %s", err.Error()) return @@ -161,40 +160,40 @@ func (t *AgentCheckStorage) startCheck(execCtx ExecuteContext, stg model.Storage var chkObjIDs []int64 for _, entry := range checkResp.Entries { switch entry.Operation { - case agtmsg.CHECK_STORAGE_RESP_OP_DELETE: - err := execCtx.Args.DB.StorageObject().Delete(execCtx.Args.DB.SQLCtx(), t.StorageID, entry.ObjectID, entry.UserID) + case agtmq.CHECK_STORAGE_RESP_OP_DELETE: + err := execCtx.Args.DB.StoragePackage().Delete(execCtx.Args.DB.SQLCtx(), t.StorageID, entry.PackageID, entry.UserID) if err != nil { log.WithField("StorageID", t.StorageID). - WithField("ObjectID", entry.ObjectID). - Warnf("delete storage object failed, err: %s", err.Error()) + WithField("PackageID", entry.PackageID). + Warnf("delete storage package failed, err: %s", err.Error()) } - chkObjIDs = append(chkObjIDs, entry.ObjectID) + chkObjIDs = append(chkObjIDs, entry.PackageID) log.WithField("StorageID", t.StorageID). - WithField("ObjectID", entry.ObjectID). + WithField("PackageID", entry.PackageID). WithField("UserID", entry.UserID). - Debugf("delete storage object") + Debugf("delete storage package") - case agtmsg.CHECK_STORAGE_RESP_OP_SET_NORMAL: - err := execCtx.Args.DB.StorageObject().SetStateNormal(execCtx.Args.DB.SQLCtx(), t.StorageID, entry.ObjectID, entry.UserID) + case agtmq.CHECK_STORAGE_RESP_OP_SET_NORMAL: + err := execCtx.Args.DB.StoragePackage().SetStateNormal(execCtx.Args.DB.SQLCtx(), t.StorageID, entry.PackageID, entry.UserID) if err != nil { log.WithField("StorageID", t.StorageID). - WithField("ObjectID", entry.ObjectID). - Warnf("change storage object state failed, err: %s", err.Error()) + WithField("PackageID", entry.PackageID). + Warnf("change storage package state failed, err: %s", err.Error()) } log.WithField("StorageID", t.StorageID). - WithField("ObjectID", entry.ObjectID). + WithField("PackageID", entry.PackageID). WithField("UserID", entry.UserID). - Debugf("set storage object normal") + Debugf("set storage package normal") } } if len(chkObjIDs) > 0 { - execCtx.Executor.Post(NewCheckObject(chkObjIDs)) + execCtx.Executor.Post(NewCheckPackage(chkObjIDs)) } } func init() { - RegisterMessageConvertor(func(msg scevt.AgentCheckStorage) Event { return NewAgentCheckStorage(msg.StorageID, msg.ObjectIDs) }) + RegisterMessageConvertor(func(msg scevt.AgentCheckStorage) Event { return NewAgentCheckStorage(msg.StorageID, msg.PackageIDs) }) } diff --git a/internal/event/check_cache.go b/internal/event/check_cache.go index e4d0bcb..f6f6f0d 100644 --- a/internal/event/check_cache.go +++ b/internal/event/check_cache.go @@ -8,7 +8,7 @@ import ( "gitlink.org.cn/cloudream/common/pkgs/logger" "gitlink.org.cn/cloudream/storage-common/consts" "gitlink.org.cn/cloudream/storage-common/pkgs/db/model" - scevt "gitlink.org.cn/cloudream/storage-common/pkgs/mq/message/scanner/event" + scevt "gitlink.org.cn/cloudream/storage-common/pkgs/mq/scanner/event" ) type CheckCache struct { diff --git a/internal/event/check_object.go b/internal/event/check_object.go deleted file mode 100644 index 0c88c24..0000000 --- a/internal/event/check_object.go +++ /dev/null @@ -1,57 +0,0 @@ -package event - -import ( - "github.com/samber/lo" - "gitlink.org.cn/cloudream/common/pkgs/distlock/reqbuilder" - "gitlink.org.cn/cloudream/common/pkgs/logger" - scevt "gitlink.org.cn/cloudream/storage-common/pkgs/mq/message/scanner/event" -) - -type CheckObject struct { - scevt.CheckObject -} - -func NewCheckObject(objIDs []int64) *CheckObject { - return &CheckObject{ - CheckObject: scevt.NewCheckObject(objIDs), - } -} - -func (t *CheckObject) TryMerge(other Event) bool { - event, ok := other.(*CheckObject) - if !ok { - return false - } - - t.ObjectIDs = lo.Union(t.ObjectIDs, event.ObjectIDs) - return true -} - -func (t *CheckObject) Execute(execCtx ExecuteContext) { - log := logger.WithType[CheckObject]("Event") - log.Debugf("begin with %v", logger.FormatStruct(t)) - defer log.Debugf("end") - - // 检查对象是否没有被引用的时候，需要读取StorageObject表 - builder := reqbuilder.NewBuilder().Metadata().StorageObject().ReadAny() - for _, objID := range t.ObjectIDs { - builder.Metadata().Object().WriteOne(objID) - } - mutex, err := builder.MutexLock(execCtx.Args.DistLock) - if err != nil { - log.Warnf("acquire locks failed, err: %s", err.Error()) - return - } - defer mutex.Unlock() - - for _, objID := range t.ObjectIDs { - err := execCtx.Args.DB.Object().DeleteUnused(execCtx.Args.DB.SQLCtx(), objID) - if err != nil { - log.WithField("ObjectID", objID).Warnf("delete unused object failed, err: %s", err.Error()) - } - } -} - -func init() { - RegisterMessageConvertor(func(msg scevt.CheckObject) Event { return NewCheckObject(msg.ObjectIDs) }) -} diff --git a/internal/event/check_package.go b/internal/event/check_package.go new file mode 100644 index 0000000..969a56a --- /dev/null +++ b/internal/event/check_package.go @@ -0,0 +1,57 @@ +package event + +import ( + "github.com/samber/lo" + "gitlink.org.cn/cloudream/common/pkgs/distlock/reqbuilder" + "gitlink.org.cn/cloudream/common/pkgs/logger" + scevt "gitlink.org.cn/cloudream/storage-common/pkgs/mq/scanner/event" +) + +type CheckPackage struct { + scevt.CheckPackage +} + +func NewCheckPackage(objIDs []int64) *CheckPackage { + return &CheckPackage{ + CheckPackage: scevt.NewCheckPackage(objIDs), + } +} + +func (t *CheckPackage) TryMerge(other Event) bool { + event, ok := other.(*CheckPackage) + if !ok { + return false + } + + t.PackageIDs = lo.Union(t.PackageIDs, event.PackageIDs) + return true +} + +func (t *CheckPackage) Execute(execCtx ExecuteContext) { + log := logger.WithType[CheckPackage]("Event") + log.Debugf("begin with %v", logger.FormatStruct(t)) + defer log.Debugf("end") + + // 检查对象是否没有被引用的时候，需要读取StoragePackage表 + builder := reqbuilder.NewBuilder().Metadata().StoragePackage().ReadAny() + for _, objID := range t.PackageIDs { + builder.Metadata().Package().WriteOne(objID) + } + mutex, err := builder.MutexLock(execCtx.Args.DistLock) + if err != nil { + log.Warnf("acquire locks failed, err: %s", err.Error()) + return + } + defer mutex.Unlock() + + for _, objID := range t.PackageIDs { + err := execCtx.Args.DB.Package().DeleteUnused(execCtx.Args.DB.SQLCtx(), objID) + if err != nil { + log.WithField("PackageID", objID).Warnf("delete unused package failed, err: %s", err.Error()) + } + } +} + +func init() { + RegisterMessageConvertor(func(msg scevt.CheckPackage) Event { return NewCheckPackage(msg.PackageIDs) }) +} diff --git a/internal/event/check_rep_count.go b/internal/event/check_rep_count.go index bfe7dff..af6b302 100644 --- a/internal/event/check_rep_count.go +++ b/internal/event/check_rep_count.go @@ -13,7 +13,7 @@ import ( "gitlink.org.cn/cloudream/storage-scanner/internal/config" "gitlink.org.cn/cloudream/storage-common/pkgs/db/model" - scevt "gitlink.org.cn/cloudream/storage-common/pkgs/mq/message/scanner/event" + scevt "gitlink.org.cn/cloudream/storage-common/pkgs/mq/scanner/event" ) type CheckRepCount struct { diff --git a/internal/services/event.go b/internal/services/event.go index c601a16..0fc7a30 100644 --- a/internal/services/event.go +++ b/internal/services/event.go @@ -2,12 +2,12 @@ package services import ( "gitlink.org.cn/cloudream/common/pkgs/logger" - scmsg "gitlink.org.cn/cloudream/storage-common/pkgs/mq/message/scanner" - scevt "gitlink.org.cn/cloudream/storage-common/pkgs/mq/message/scanner/event" + scmq "gitlink.org.cn/cloudream/storage-common/pkgs/mq/scanner" + scevt "gitlink.org.cn/cloudream/storage-common/pkgs/mq/scanner/event" "gitlink.org.cn/cloudream/storage-scanner/internal/event" ) -func (svc *Service) PostEvent(msg *scmsg.PostEvent) { +func (svc *Service) PostEvent(msg *scmq.PostEvent) { evtMsg, err := scevt.MapToMessage(msg.Event) if err != nil { diff --git a/internal/tickevent/batch_check_all_object.go b/internal/tickevent/batch_check_all_object.go deleted file mode 100644 index 043bc8a..0000000 --- a/internal/tickevent/batch_check_all_object.go +++ /dev/null @@ -1,39 +0,0 @@ -package tickevent - -import ( - "gitlink.org.cn/cloudream/common/pkgs/logger" - "gitlink.org.cn/cloudream/storage-scanner/internal/event" -) - -const CHECK_OBJECT_BATCH_SIZE = 100 - -type BatchCheckAllObject struct { - lastCheckStart int -} - -func NewBatchCheckAllObject() *BatchCheckAllObject { - return &BatchCheckAllObject{} -} - -func (e *BatchCheckAllObject) Execute(ctx ExecuteContext) { - log := logger.WithType[BatchCheckAllObject]("TickEvent") - log.Debugf("begin") - defer log.Debugf("end") - - objectIDs, err := ctx.Args.DB.Object().BatchGetAllObjectIDs(ctx.Args.DB.SQLCtx(), e.lastCheckStart, CHECK_OBJECT_BATCH_SIZE) - if err != nil { - log.Warnf("batch get object ids failed, err: %s", err.Error()) - return - } - - ctx.Args.EventExecutor.Post(event.NewCheckObject(objectIDs)) - - // 如果结果的长度小于预期的长度，则认为已经查询了所有，下次从头再来 - if len(objectIDs) < CHECK_OBJECT_BATCH_SIZE { - e.lastCheckStart = 0 - log.Debugf("all object checked, next time will start check at 0") - - } else { - e.lastCheckStart += CHECK_OBJECT_BATCH_SIZE - } -} diff --git a/internal/tickevent/batch_check_all_package.go b/internal/tickevent/batch_check_all_package.go new file mode 100644 index 0000000..595bbac --- /dev/null +++ b/internal/tickevent/batch_check_all_package.go @@ -0,0 +1,39 @@ +package tickevent + +import ( + "gitlink.org.cn/cloudream/common/pkgs/logger" + "gitlink.org.cn/cloudream/storage-scanner/internal/event" +) + +const CheckPackageBatchSize = 100 + +type BatchCheckAllPackage struct { + lastCheckStart int +} + +func NewBatchCheckAllPackage() *BatchCheckAllPackage { + return &BatchCheckAllPackage{} +} + +func (e *BatchCheckAllPackage) Execute(ctx ExecuteContext) { + log := logger.WithType[BatchCheckAllPackage]("TickEvent") + log.Debugf("begin") + defer log.Debugf("end") + + packageIDs, err := ctx.Args.DB.Package().BatchGetAllPackageIDs(ctx.Args.DB.SQLCtx(), e.lastCheckStart, CheckPackageBatchSize) + if err != nil { + log.Warnf("batch get package ids failed, err: %s", err.Error()) + return + } + + ctx.Args.EventExecutor.Post(event.NewCheckPackage(packageIDs)) + + // 如果结果的长度小于预期的长度，则认为已经查询了所有，下次从头再来 + if len(packageIDs) < CheckPackageBatchSize { + e.lastCheckStart = 0 + log.Debugf("all package checked, next time will start check at 0") + + } else { + e.lastCheckStart += CheckPackageBatchSize + } +} diff --git a/main.go b/main.go index 1898eef..44a4407 100644 --- a/main.go +++ b/main.go @@ -6,9 +6,9 @@ import ( "sync" distlocksvc "gitlink.org.cn/cloudream/common/pkgs/distlock/service" - log "gitlink.org.cn/cloudream/common/pkgs/logger" + "gitlink.org.cn/cloudream/common/pkgs/logger" "gitlink.org.cn/cloudream/storage-common/pkgs/db" - scsvr "gitlink.org.cn/cloudream/storage-common/pkgs/mq/server/scanner" + scmq "gitlink.org.cn/cloudream/storage-common/pkgs/mq/scanner" "gitlink.org.cn/cloudream/storage-scanner/internal/config" "gitlink.org.cn/cloudream/storage-scanner/internal/event" "gitlink.org.cn/cloudream/storage-scanner/internal/services" @@ -22,7 +22,7 @@ func main() { os.Exit(1) } - err = log.Init(&config.Cfg().Logger) + err = logger.Init(&config.Cfg().Logger) if err != nil { fmt.Printf("init logger failed, err: %s", err.Error()) os.Exit(1) @@ -30,7 +30,7 @@ func main() { db, err := db.NewDB(&config.Cfg().DB) if err != nil { - log.Fatalf("new db failed, err: %s", err.Error()) + logger.Fatalf("new db failed, err: %s", err.Error()) } wg := sync.WaitGroup{} @@ -38,7 +38,7 @@ func main() { distlockSvc, err := distlocksvc.NewService(&config.Cfg().DistLock) if err != nil { - log.Warnf("new distlock service failed, err: %s", err.Error()) + logger.Warnf("new distlock service failed, err: %s", err.Error()) os.Exit(1) } go serveDistLock(distlockSvc, &wg) @@ -46,12 +46,12 @@ func main() { eventExecutor := event.NewExecutor(db, distlockSvc) go serveEventExecutor(&eventExecutor, &wg) - agtSvr, err := scsvr.NewServer(services.NewService(&eventExecutor), &config.Cfg().RabbitMQ) + agtSvr, err := scmq.NewServer(services.NewService(&eventExecutor), &config.Cfg().RabbitMQ) if err != nil { - log.Fatalf("new agent server failed, err: %s", err.Error()) + logger.Fatalf("new agent server failed, err: %s", err.Error()) } agtSvr.OnError = func(err error) { - log.Warnf("agent server err: %s", err.Error()) + logger.Warnf("agent server err: %s", err.Error()) } go serveScannerServer(agtSvr, &wg) @@ -65,43 +65,43 @@ func main() { } func serveEventExecutor(executor *event.Executor, wg *sync.WaitGroup) { - log.Info("start serving event executor") + logger.Info("start serving event executor") err := executor.Execute() if err != nil { - log.Errorf("event executor stopped with error: %s", err.Error()) + logger.Errorf("event executor stopped with error: %s", err.Error()) } - log.Info("event executor stopped") + logger.Info("event executor stopped") wg.Done() } -func serveScannerServer(server *scsvr.Server, wg *sync.WaitGroup) { - log.Info("start serving scanner server") +func serveScannerServer(server *scmq.Server, wg *sync.WaitGroup) { + logger.Info("start serving scanner server") err := server.Serve() if err != nil { - log.Errorf("scanner server stopped with error: %s", err.Error()) + logger.Errorf("scanner server stopped with error: %s", err.Error()) } - log.Info("scanner server stopped") + logger.Info("scanner server stopped") wg.Done() } func serveDistLock(svc *distlocksvc.Service, wg *sync.WaitGroup) { - log.Info("start serving distlock") + logger.Info("start serving distlock") err := svc.Serve() if err != nil { - log.Errorf("distlock stopped with error: %s", err.Error()) + logger.Errorf("distlock stopped with error: %s", err.Error()) } - log.Info("distlock stopped") + logger.Info("distlock stopped") wg.Done() } @@ -113,7 +113,7 @@ func startTickEvent(tickExecutor *tickevent.Executor) { tickExecutor.Start(tickevent.NewBatchAllAgentCheckCache(), interval, tickevent.StartOption{RandomStartDelayMs: 60 * 1000}) - tickExecutor.Start(tickevent.NewBatchCheckAllObject(), interval, tickevent.StartOption{RandomStartDelayMs: 60 * 1000}) + tickExecutor.Start(tickevent.NewBatchCheckAllPackage(), interval, tickevent.StartOption{RandomStartDelayMs: 60 * 1000}) tickExecutor.Start(tickevent.NewBatchCheckAllRepCount(), interval, tickevent.StartOption{RandomStartDelayMs: 60 * 1000}) From ce637e8758dcf0d706df0a186d354d3ac3bfea77 Mon Sep 17 00:00:00 2001 From: Sydonian <794346190@qq.com> Date: Tue, 22 Aug 2023 10:21:12 +0800 Subject: [PATCH 65/68] =?UTF-8?q?=E5=A2=9E=E5=8A=A0ClientPool=EF=BC=8C?= =?UTF-8?q?=E8=A7=A3=E9=99=A4=E5=AF=B9config=E7=9A=84=E4=BE=9D=E8=B5=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- go.mod | 25 +++++++++++ go.sum | 60 +++++++++++++++++++++++++-- internal/event/agent_check_cache.go | 4 +- internal/event/agent_check_state.go | 3 +- internal/event/agent_check_storage.go | 4 +- main.go | 3 ++ 6 files changed, 91 insertions(+), 8 deletions(-) diff --git a/go.mod b/go.mod index cd4c776..8f81a3b 100644 --- a/go.mod +++ b/go.mod @@ -37,18 +37,43 @@ require ( require ( github.com/antonfisher/nested-logrus-formatter v1.3.1 // indirect + github.com/benbjohnson/clock v1.3.0 // indirect + github.com/crackcomm/go-gitignore v0.0.0-20170627025303-887ab5e44cc3 // indirect + github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0 // indirect github.com/go-sql-driver/mysql v1.7.1 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/imdario/mergo v0.3.15 // indirect + github.com/ipfs/boxo v0.8.0 // indirect + github.com/ipfs/go-cid v0.4.0 // indirect + github.com/ipfs/go-ipfs-api v0.6.0 // indirect + github.com/klauspost/cpuid/v2 v2.2.3 // indirect + github.com/libp2p/go-buffer-pool v0.1.0 // indirect + github.com/libp2p/go-flow-metrics v0.1.0 // indirect + github.com/libp2p/go-libp2p v0.26.3 // indirect github.com/magefile/mage v1.15.0 // indirect + github.com/minio/sha256-simd v1.0.0 // indirect + github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect + github.com/mr-tron/base58 v1.2.0 // indirect + github.com/multiformats/go-base32 v0.1.0 // indirect + github.com/multiformats/go-base36 v0.2.0 // indirect + github.com/multiformats/go-multiaddr v0.8.0 // indirect + github.com/multiformats/go-multibase v0.1.1 // indirect + github.com/multiformats/go-multicodec v0.8.1 // indirect + github.com/multiformats/go-multihash v0.2.1 // indirect + github.com/multiformats/go-multistream v0.4.1 // indirect + github.com/multiformats/go-varint v0.0.7 // indirect github.com/otiai10/copy v1.12.0 // indirect github.com/sirupsen/logrus v1.9.2 // indirect + github.com/spaolacci/murmur3 v1.1.0 // indirect github.com/streadway/amqp v1.1.0 // indirect + github.com/whyrusleeping/tar-utils v0.0.0-20180509141711-8c6c8ba81d5c // indirect github.com/zyedidia/generic v1.2.1 // indirect + golang.org/x/crypto v0.6.0 // indirect golang.org/x/exp v0.0.0-20230519143937-03e91628a987 // indirect golang.org/x/sys v0.7.0 // indirect + lukechampine.com/blake3 v1.1.7 // indirect ) // 运行go mod tidy时需要将下面几行取消注释 diff --git a/go.sum b/go.sum index 0b11fb9..da7e466 100644 --- a/go.sum +++ b/go.sum @@ -1,13 +1,20 @@ github.com/antonfisher/nested-logrus-formatter v1.3.1 h1:NFJIr+pzwv5QLHTPyKz9UMEoHck02Q9L0FP13b/xSbQ= github.com/antonfisher/nested-logrus-formatter v1.3.1/go.mod h1:6WTfyWFkBc9+zyBaKIqRrg/KwMqBbodBjgbHjDz7zjA= github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A= +github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= +github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927 h1:SKI1/fuSdodxmNNyVBR8d7X/HuLnRpvvFO0AgyQk764= github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8iXXhfZs= github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= +github.com/crackcomm/go-gitignore v0.0.0-20170627025303-887ab5e44cc3 h1:HVTnpeuvF6Owjd5mniCL8DEXo7uYXdQEmOP4FJbV5tg= +github.com/crackcomm/go-gitignore v0.0.0-20170627025303-887ab5e44cc3/go.mod h1:p1d6YEZWvFzEh4KLyvBcVSnrfNDDvK2zfK/4x2v/4pE= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/decred/dcrd/crypto/blake256 v1.0.0 h1:/8DMNYp9SGi5f0w7uCm6d6M4OU2rGFK09Y2A4Xv7EE0= +github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0 h1:HbphB4TFFXpv7MNrT52FGrrgVXF1owhMVTHFZIlnvd4= +github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0/go.mod h1:DZGJHZMqrU4JJqFAWUS2UO1+lbSKsdiOoYi9Zzey7Fc= github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-sql-driver/mysql v1.7.1 h1:lUIinVbN1DY0xBg0eMOzmmtGoHwWBbvnWubQUrtU8EI= github.com/go-sql-driver/mysql v1.7.1/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI= @@ -31,6 +38,12 @@ github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+l github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= github.com/imdario/mergo v0.3.15 h1:M8XP7IuFNsqUx6VPK2P9OSmsYsI/YFaGil0uD21V3dM= github.com/imdario/mergo v0.3.15/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY= +github.com/ipfs/boxo v0.8.0 h1:UdjAJmHzQHo/j3g3b1bAcAXCj/GM6iTwvSlBDvPBNBs= +github.com/ipfs/boxo v0.8.0/go.mod h1:RIsi4CnTyQ7AUsNn5gXljJYZlQrHBMnJp94p73liFiA= +github.com/ipfs/go-cid v0.4.0 h1:a4pdZq0sx6ZSxbCizebnKiMCx/xI/aBBFlB73IgH4rA= +github.com/ipfs/go-cid v0.4.0/go.mod h1:uQHwDeX4c6CtyrFwdqyhpNcxVewur1M7l7fNU7LKwZk= +github.com/ipfs/go-ipfs-api v0.6.0 h1:JARgG0VTbjyVhO5ZfesnbXv9wTcMvoKRBLF1SzJqzmg= +github.com/ipfs/go-ipfs-api v0.6.0/go.mod h1:iDC2VMwN9LUpQV/GzEeZ2zNqd8NUdRmWcFM+K/6odf0= github.com/jmoiron/sqlx v1.3.5 h1:vFFPA71p1o5gAeqtEAwLU4dnX2napprKtHr7PYIcN3g= github.com/jmoiron/sqlx v1.3.5/go.mod h1:nRVWtLre0KfCLJvgxzCsLVMogSvQ1zNJtpYr2Ccp0mQ= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= @@ -39,22 +52,54 @@ github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7 github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.2.3 h1:sxCkb+qR91z4vsqw4vGGZlDgPz3G7gjaLyK3V8y70BU= +github.com/klauspost/cpuid/v2 v2.2.3/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= github.com/lib/pq v1.2.0 h1:LXpIM/LZ5xGFhOpXAQUIMM1HdyqzVYM13zNdjCEEcA0= github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/libp2p/go-buffer-pool v0.1.0 h1:oK4mSFcQz7cTQIfqbe4MIj9gLW+mnanjyFtc6cdF0Y8= +github.com/libp2p/go-buffer-pool v0.1.0/go.mod h1:N+vh8gMqimBzdKkSMVuydVDq+UV5QTWy5HSiZacSbPg= +github.com/libp2p/go-flow-metrics v0.1.0 h1:0iPhMI8PskQwzh57jB9WxIuIOQ0r+15PChFGkx3Q3WM= +github.com/libp2p/go-flow-metrics v0.1.0/go.mod h1:4Xi8MX8wj5aWNDAZttg6UPmc0ZrnFNsMtpsYUClFtro= +github.com/libp2p/go-libp2p v0.26.3 h1:6g/psubqwdaBqNNoidbRKSTBEYgaOuKBhHl8Q5tO+PM= +github.com/libp2p/go-libp2p v0.26.3/go.mod h1:x75BN32YbwuY0Awm2Uix4d4KOz+/4piInkp4Wr3yOo8= github.com/magefile/mage v1.15.0 h1:BvGheCMAsG3bWUDbZ8AyXXpCNwU9u5CB6sM+HNb9HYg= github.com/magefile/mage v1.15.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A= github.com/mattn/go-sqlite3 v1.14.6 h1:dNPt6NO46WmLVt2DLNpwczCmdV5boIZ6g/tlDrlRUbg= github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= +github.com/minio/sha256-simd v1.0.0 h1:v1ta+49hkWZyvaKwrQB8elexRqm6Y0aMLjCNsrYxo6g= +github.com/minio/sha256-simd v1.0.0/go.mod h1:OuYzVNI5vcoYIAmbIvHPl3N3jUzVedXbKy5RFepssQM= +github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o= +github.com/mr-tron/base58 v1.2.0/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= +github.com/multiformats/go-base32 v0.1.0 h1:pVx9xoSPqEIQG8o+UbAe7DNi51oej1NtK+aGkbLYxPE= +github.com/multiformats/go-base32 v0.1.0/go.mod h1:Kj3tFY6zNr+ABYMqeUNeGvkIC/UYgtWibDcT0rExnbI= +github.com/multiformats/go-base36 v0.2.0 h1:lFsAbNOGeKtuKozrtBsAkSVhv1p9D0/qedU9rQyccr0= +github.com/multiformats/go-base36 v0.2.0/go.mod h1:qvnKE++v+2MWCfePClUEjE78Z7P2a1UV0xHgWc0hkp4= +github.com/multiformats/go-multiaddr v0.8.0 h1:aqjksEcqK+iD/Foe1RRFsGZh8+XFiGo7FgUCZlpv3LU= +github.com/multiformats/go-multiaddr v0.8.0/go.mod h1:Fs50eBDWvZu+l3/9S6xAE7ZYj6yhxlvaVZjakWN7xRs= +github.com/multiformats/go-multibase v0.1.1 h1:3ASCDsuLX8+j4kx58qnJ4YFq/JWTJpCyDW27ztsVTOI= +github.com/multiformats/go-multibase v0.1.1/go.mod h1:ZEjHE+IsUrgp5mhlEAYjMtZwK1k4haNkcaPg9aoe1a8= +github.com/multiformats/go-multicodec v0.8.1 h1:ycepHwavHafh3grIbR1jIXnKCsFm0fqsfEOsJ8NtKE8= +github.com/multiformats/go-multicodec v0.8.1/go.mod h1:L3QTQvMIaVBkXOXXtVmYE+LI16i14xuaojr/H7Ai54k= +github.com/multiformats/go-multihash v0.2.1 h1:aem8ZT0VA2nCHHk7bPJ1BjUbHNciqZC/d16Vve9l108= +github.com/multiformats/go-multihash v0.2.1/go.mod h1:WxoMcYG85AZVQUyRyo9s4wULvW5qrI9vb2Lt6evduFc= +github.com/multiformats/go-multistream v0.4.1 h1:rFy0Iiyn3YT0asivDUIR05leAdwZq3de4741sbiSdfo= +github.com/multiformats/go-multistream v0.4.1/go.mod h1:Mz5eykRVAjJWckE2U78c6xqdtyNUEhKSM0Lwar2p77Q= +github.com/multiformats/go-varint v0.0.7 h1:sWSGR+f/eu5ABZA2ZpYKBILXTTs9JWpdEM/nEGOHFS8= +github.com/multiformats/go-varint v0.0.7/go.mod h1:r8PUYw/fD/SjBCiKOoDlGF6QawOELpZAu9eioSos/OU= github.com/otiai10/copy v1.12.0 h1:cLMgSQnXBs1eehF0Wy/FAGsgDTDmAqFR7rQylBb1nDY= github.com/otiai10/copy v1.12.0/go.mod h1:rSaLseMUsZFFbsFGc7wCJnnkTAvdc5L6VWxPE4308Ww= github.com/otiai10/mint v1.5.1 h1:XaPLeE+9vGbuyEHem1JNk3bYc7KKqyI/na0/mLd/Kks= -github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/samber/lo v1.38.1 h1:j2XEAqXKb09Am4ebOg31SpvzUTTs6EN3VfgeLUhPdXM= @@ -65,12 +110,16 @@ github.com/smartystreets/assertions v1.13.1 h1:Ef7KhSmjZcK6AVf9YbJdvPYG9avaF0Zxu github.com/smartystreets/assertions v1.13.1/go.mod h1:cXr/IwVfSo/RbCSPhoAPv73p3hlSdrBH/b3SdnW/LMY= github.com/smartystreets/goconvey v1.8.0 h1:Oi49ha/2MURE0WexF052Z0m+BNSGirfjg5RL+JXWq3w= github.com/smartystreets/goconvey v1.8.0/go.mod h1:EdX8jtrTIj26jmjCOVNMVSIYAtgexqXKHOXW2Dx9JLg= +github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= +github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/streadway/amqp v1.1.0 h1:py12iX8XSyI7aN/3dUT8DFIDJazNJsVJdxNVEpnQTZM= github.com/streadway/amqp v1.1.0/go.mod h1:WYSrTEYHOXHd0nwFeUXAe2G2hRnQT+deZJJf88uS9Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= +github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= +github.com/whyrusleeping/tar-utils v0.0.0-20180509141711-8c6c8ba81d5c h1:GGsyl0dZ2jJgVT+VvWBf/cNijrHRhkrTjkmp5wg7li0= +github.com/whyrusleeping/tar-utils v0.0.0-20180509141711-8c6c8ba81d5c/go.mod h1:xxcJeBb7SIUl/Wzkz1eVKJE/CB34YNrqX2TQI6jY9zs= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/zyedidia/generic v1.2.1 h1:Zv5KS/N2m0XZZiuLS82qheRG4X1o5gsWreGb0hR7XDc= @@ -83,7 +132,7 @@ go.etcd.io/etcd/client/v3 v3.5.9 h1:r5xghnU7CwbUxD/fbUtRyJGaYNfDun8sp/gTr1hew6E= go.etcd.io/etcd/client/v3 v3.5.9/go.mod h1:i/Eo5LrZ5IKqpbtpPDuaUnDOUv471oDg8cjQaUr2MbA= go.uber.org/atomic v1.10.0 h1:9qC72Qh0+3MqyJbAn8YU5xVq1frD8bn3JtD2oXtafVQ= go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= -go.uber.org/goleak v1.1.11 h1:wy28qYRKZgnJTxGxvye5/wgWr1EKjmUDGYox5mGlRlI= +go.uber.org/goleak v1.1.12 h1:gZAh5/EyT/HQwlpkCy6wTpqfH9H8Lz8zbm3dZh+OyzA= go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI= go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTVQ= go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60= @@ -91,6 +140,8 @@ go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.6.0 h1:qfktjS5LUO+fFKeJXZ+ikTRijMmljikvG68fpMMruSc= +golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= golang.org/x/exp v0.0.0-20230519143937-03e91628a987 h1:3xJIFvzUFbu4ls0BTBYcgbCGhA63eAOEMxIHugyXJqA= golang.org/x/exp v0.0.0-20230519143937-03e91628a987/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= @@ -107,6 +158,7 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU= golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -135,3 +187,5 @@ gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +lukechampine.com/blake3 v1.1.7 h1:GgRMhmdsuK8+ii6UZFDL8Nb+VyMwadAgcJyfYHxG6n0= +lukechampine.com/blake3 v1.1.7/go.mod h1:tkKEOtDkNtklkXtLNEOGNq5tcV90tJiA1vAA12R78LA= diff --git a/internal/event/agent_check_cache.go b/internal/event/agent_check_cache.go index 70b3f90..3a52b12 100644 --- a/internal/event/agent_check_cache.go +++ b/internal/event/agent_check_cache.go @@ -8,8 +8,8 @@ import ( "gitlink.org.cn/cloudream/common/pkgs/distlock/reqbuilder" "gitlink.org.cn/cloudream/common/pkgs/logger" "gitlink.org.cn/cloudream/common/pkgs/mq" + "gitlink.org.cn/cloudream/storage-common/globals" "gitlink.org.cn/cloudream/storage-common/pkgs/db/model" - "gitlink.org.cn/cloudream/storage-scanner/internal/config" agtmq "gitlink.org.cn/cloudream/storage-common/pkgs/mq/agent" scevt "gitlink.org.cn/cloudream/storage-common/pkgs/mq/scanner/event" @@ -126,7 +126,7 @@ func (t *AgentCheckCache) startCheck(execCtx ExecuteContext, isComplete bool, ca log := logger.WithType[AgentCheckCache]("Event") // 然后向代理端发送移动文件的请求 - agentClient, err := agtmq.NewClient(t.NodeID, &config.Cfg().RabbitMQ) + agentClient, err := globals.AgentMQPool.Acquire(t.NodeID) if err != nil { log.WithField("NodeID", t.NodeID).Warnf("create agent client failed, err: %s", err.Error()) return diff --git a/internal/event/agent_check_state.go b/internal/event/agent_check_state.go index e7fc582..2392554 100644 --- a/internal/event/agent_check_state.go +++ b/internal/event/agent_check_state.go @@ -9,6 +9,7 @@ import ( "gitlink.org.cn/cloudream/common/pkgs/logger" "gitlink.org.cn/cloudream/common/pkgs/mq" "gitlink.org.cn/cloudream/storage-common/consts" + "gitlink.org.cn/cloudream/storage-common/globals" "gitlink.org.cn/cloudream/storage-common/pkgs/db/model" agtmq "gitlink.org.cn/cloudream/storage-common/pkgs/mq/agent" scevt "gitlink.org.cn/cloudream/storage-common/pkgs/mq/scanner/event" @@ -60,7 +61,7 @@ func (t *AgentCheckState) Execute(execCtx ExecuteContext) { return } - agentClient, err := agtmq.NewClient(t.NodeID, &config.Cfg().RabbitMQ) + agentClient, err := globals.AgentMQPool.Acquire(t.NodeID) if err != nil { log.WithField("NodeID", t.NodeID).Warnf("create agent client failed, err: %s", err.Error()) return diff --git a/internal/event/agent_check_storage.go b/internal/event/agent_check_storage.go index ca89506..b6d9950 100644 --- a/internal/event/agent_check_storage.go +++ b/internal/event/agent_check_storage.go @@ -9,10 +9,10 @@ import ( "gitlink.org.cn/cloudream/common/pkgs/logger" "gitlink.org.cn/cloudream/common/pkgs/mq" "gitlink.org.cn/cloudream/storage-common/consts" + "gitlink.org.cn/cloudream/storage-common/globals" "gitlink.org.cn/cloudream/storage-common/pkgs/db/model" agtmq "gitlink.org.cn/cloudream/storage-common/pkgs/mq/agent" scevt "gitlink.org.cn/cloudream/storage-common/pkgs/mq/scanner/event" - "gitlink.org.cn/cloudream/storage-scanner/internal/config" ) type AgentCheckStorage struct { @@ -143,7 +143,7 @@ func (t *AgentCheckStorage) startCheck(execCtx ExecuteContext, stg model.Storage log := logger.WithType[AgentCheckStorage]("Event") // 投递任务 - agentClient, err := agtmq.NewClient(stg.NodeID, &config.Cfg().RabbitMQ) + agentClient, err := globals.AgentMQPool.Acquire(stg.NodeID) if err != nil { log.WithField("NodeID", stg.NodeID).Warnf("create agent client failed, err: %s", err.Error()) return diff --git a/main.go b/main.go index 44a4407..7296dde 100644 --- a/main.go +++ b/main.go @@ -7,6 +7,7 @@ import ( distlocksvc "gitlink.org.cn/cloudream/common/pkgs/distlock/service" "gitlink.org.cn/cloudream/common/pkgs/logger" + "gitlink.org.cn/cloudream/storage-common/globals" "gitlink.org.cn/cloudream/storage-common/pkgs/db" scmq "gitlink.org.cn/cloudream/storage-common/pkgs/mq/scanner" "gitlink.org.cn/cloudream/storage-scanner/internal/config" @@ -33,6 +34,8 @@ func main() { logger.Fatalf("new db failed, err: %s", err.Error()) } + globals.InitMQPool(&config.Cfg().RabbitMQ) + wg := sync.WaitGroup{} wg.Add(3) From 2089e2f5145e49a8d7214f35409d4110bf8bd80b Mon Sep 17 00:00:00 2001 From: Sydonian <794346190@qq.com> Date: Tue, 22 Aug 2023 16:53:51 +0800 Subject: [PATCH 66/68] =?UTF-8?q?=E6=8B=86=E5=88=86=E5=92=8C=E4=BF=AE?= =?UTF-8?q?=E5=A4=8D=E9=94=81=E6=9C=8D=E5=8A=A1=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- internal/event/agent_check_cache.go | 2 +- internal/event/agent_check_state.go | 2 +- internal/event/agent_check_storage.go | 2 +- internal/event/check_cache.go | 2 +- internal/event/check_package.go | 2 +- internal/event/check_rep_count.go | 2 +- main.go | 6 +++--- 7 files changed, 9 insertions(+), 9 deletions(-) diff --git a/internal/event/agent_check_cache.go b/internal/event/agent_check_cache.go index 3a52b12..a015a1e 100644 --- a/internal/event/agent_check_cache.go +++ b/internal/event/agent_check_cache.go @@ -5,11 +5,11 @@ import ( "time" "github.com/samber/lo" - "gitlink.org.cn/cloudream/common/pkgs/distlock/reqbuilder" "gitlink.org.cn/cloudream/common/pkgs/logger" "gitlink.org.cn/cloudream/common/pkgs/mq" "gitlink.org.cn/cloudream/storage-common/globals" "gitlink.org.cn/cloudream/storage-common/pkgs/db/model" + "gitlink.org.cn/cloudream/storage-common/pkgs/distlock/reqbuilder" agtmq "gitlink.org.cn/cloudream/storage-common/pkgs/mq/agent" scevt "gitlink.org.cn/cloudream/storage-common/pkgs/mq/scanner/event" diff --git a/internal/event/agent_check_state.go b/internal/event/agent_check_state.go index 2392554..fc23e31 100644 --- a/internal/event/agent_check_state.go +++ b/internal/event/agent_check_state.go @@ -5,12 +5,12 @@ import ( "time" "github.com/samber/lo" - "gitlink.org.cn/cloudream/common/pkgs/distlock/reqbuilder" "gitlink.org.cn/cloudream/common/pkgs/logger" "gitlink.org.cn/cloudream/common/pkgs/mq" "gitlink.org.cn/cloudream/storage-common/consts" "gitlink.org.cn/cloudream/storage-common/globals" "gitlink.org.cn/cloudream/storage-common/pkgs/db/model" + "gitlink.org.cn/cloudream/storage-common/pkgs/distlock/reqbuilder" agtmq "gitlink.org.cn/cloudream/storage-common/pkgs/mq/agent" scevt "gitlink.org.cn/cloudream/storage-common/pkgs/mq/scanner/event" "gitlink.org.cn/cloudream/storage-scanner/internal/config" diff --git a/internal/event/agent_check_storage.go b/internal/event/agent_check_storage.go index b6d9950..c355f11 100644 --- a/internal/event/agent_check_storage.go +++ b/internal/event/agent_check_storage.go @@ -5,12 +5,12 @@ import ( "time" "github.com/samber/lo" - "gitlink.org.cn/cloudream/common/pkgs/distlock/reqbuilder" "gitlink.org.cn/cloudream/common/pkgs/logger" "gitlink.org.cn/cloudream/common/pkgs/mq" "gitlink.org.cn/cloudream/storage-common/consts" "gitlink.org.cn/cloudream/storage-common/globals" "gitlink.org.cn/cloudream/storage-common/pkgs/db/model" + "gitlink.org.cn/cloudream/storage-common/pkgs/distlock/reqbuilder" agtmq "gitlink.org.cn/cloudream/storage-common/pkgs/mq/agent" scevt "gitlink.org.cn/cloudream/storage-common/pkgs/mq/scanner/event" ) diff --git a/internal/event/check_cache.go b/internal/event/check_cache.go index f6f6f0d..784dd76 100644 --- a/internal/event/check_cache.go +++ b/internal/event/check_cache.go @@ -4,10 +4,10 @@ import ( "database/sql" "github.com/samber/lo" - "gitlink.org.cn/cloudream/common/pkgs/distlock/reqbuilder" "gitlink.org.cn/cloudream/common/pkgs/logger" "gitlink.org.cn/cloudream/storage-common/consts" "gitlink.org.cn/cloudream/storage-common/pkgs/db/model" + "gitlink.org.cn/cloudream/storage-common/pkgs/distlock/reqbuilder" scevt "gitlink.org.cn/cloudream/storage-common/pkgs/mq/scanner/event" ) diff --git a/internal/event/check_package.go b/internal/event/check_package.go index 969a56a..3d04ea1 100644 --- a/internal/event/check_package.go +++ b/internal/event/check_package.go @@ -2,8 +2,8 @@ package event import ( "github.com/samber/lo" - "gitlink.org.cn/cloudream/common/pkgs/distlock/reqbuilder" "gitlink.org.cn/cloudream/common/pkgs/logger" + "gitlink.org.cn/cloudream/storage-common/pkgs/distlock/reqbuilder" scevt "gitlink.org.cn/cloudream/storage-common/pkgs/mq/scanner/event" ) diff --git a/internal/event/check_rep_count.go b/internal/event/check_rep_count.go index af6b302..09ac5e1 100644 --- a/internal/event/check_rep_count.go +++ b/internal/event/check_rep_count.go @@ -5,11 +5,11 @@ import ( "math" "github.com/samber/lo" - "gitlink.org.cn/cloudream/common/pkgs/distlock/reqbuilder" "gitlink.org.cn/cloudream/common/pkgs/logger" mymath "gitlink.org.cn/cloudream/common/utils/math" mysort "gitlink.org.cn/cloudream/common/utils/sort" "gitlink.org.cn/cloudream/storage-common/consts" + "gitlink.org.cn/cloudream/storage-common/pkgs/distlock/reqbuilder" "gitlink.org.cn/cloudream/storage-scanner/internal/config" "gitlink.org.cn/cloudream/storage-common/pkgs/db/model" diff --git a/main.go b/main.go index 7296dde..1c2f68a 100644 --- a/main.go +++ b/main.go @@ -5,10 +5,10 @@ import ( "os" "sync" - distlocksvc "gitlink.org.cn/cloudream/common/pkgs/distlock/service" "gitlink.org.cn/cloudream/common/pkgs/logger" "gitlink.org.cn/cloudream/storage-common/globals" "gitlink.org.cn/cloudream/storage-common/pkgs/db" + "gitlink.org.cn/cloudream/storage-common/pkgs/distlock" scmq "gitlink.org.cn/cloudream/storage-common/pkgs/mq/scanner" "gitlink.org.cn/cloudream/storage-scanner/internal/config" "gitlink.org.cn/cloudream/storage-scanner/internal/event" @@ -39,7 +39,7 @@ func main() { wg := sync.WaitGroup{} wg.Add(3) - distlockSvc, err := distlocksvc.NewService(&config.Cfg().DistLock) + distlockSvc, err := distlock.NewService(&config.Cfg().DistLock) if err != nil { logger.Warnf("new distlock service failed, err: %s", err.Error()) os.Exit(1) @@ -95,7 +95,7 @@ func serveScannerServer(server *scmq.Server, wg *sync.WaitGroup) { wg.Done() } -func serveDistLock(svc *distlocksvc.Service, wg *sync.WaitGroup) { +func serveDistLock(svc *distlock.Service, wg *sync.WaitGroup) { logger.Info("start serving distlock") err := svc.Serve() From daaab972ac34d6d2df9807a3dc353444b76bbaed Mon Sep 17 00:00:00 2001 From: Sydonian <794346190@qq.com> Date: Wed, 23 Aug 2023 16:13:18 +0800 Subject: [PATCH 67/68] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E7=BC=93=E5=AD=98?= =?UTF-8?q?=E7=A7=BB=E5=8A=A8=E5=8A=9F=E8=83=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- internal/event/agent_check_cache.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/event/agent_check_cache.go b/internal/event/agent_check_cache.go index a015a1e..d631dfd 100644 --- a/internal/event/agent_check_cache.go +++ b/internal/event/agent_check_cache.go @@ -133,7 +133,7 @@ func (t *AgentCheckCache) startCheck(execCtx ExecuteContext, isComplete bool, ca } defer agentClient.Close() - checkResp, err := agentClient.CheckIPFS(agtmq.NewCheckIPFS(isComplete, caches), mq.RequestOption{Timeout: time.Minute}) + checkResp, err := agentClient.CheckCache(agtmq.NewCheckCache(isComplete, caches), mq.RequestOption{Timeout: time.Minute}) if err != nil { log.WithField("NodeID", t.NodeID).Warnf("checking ipfs: %s", err.Error()) return From 0c08a5ecefb8c4a71594a15f0046553a68bcf980 Mon Sep 17 00:00:00 2001 From: Sydonian <794346190@qq.com> Date: Tue, 29 Aug 2023 15:31:28 +0800 Subject: [PATCH 68/68] =?UTF-8?q?=E8=B0=83=E6=95=B4=E7=BB=93=E6=9E=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md => scanner/README.md | 0 go.mod => scanner/go.mod | 0 go.sum => scanner/go.sum | 0 {internal => scanner/internal}/config/config.go | 0 {internal => scanner/internal}/event/agent_check_cache.go | 0 {internal => scanner/internal}/event/agent_check_state.go | 0 {internal => scanner/internal}/event/agent_check_storage.go | 0 {internal => scanner/internal}/event/check_cache.go | 0 {internal => scanner/internal}/event/check_package.go | 0 {internal => scanner/internal}/event/check_rep_count.go | 0 {internal => scanner/internal}/event/check_rep_count_test.go | 0 {internal => scanner/internal}/event/event.go | 0 {internal => scanner/internal}/services/event.go | 0 {internal => scanner/internal}/services/service.go | 0 .../internal}/tickevent/batch_all_agent_check_cache.go | 0 .../internal}/tickevent/batch_check_all_package.go | 0 .../internal}/tickevent/batch_check_all_rep_count.go | 0 .../internal}/tickevent/batch_check_all_storage.go | 0 {internal => scanner/internal}/tickevent/check_agent_state.go | 0 {internal => scanner/internal}/tickevent/check_cache.go | 0 {internal => scanner/internal}/tickevent/tick_event.go | 0 {magefiles => scanner/magefiles}/magefile.go | 0 main.go => scanner/main.go | 0 23 files changed, 0 insertions(+), 0 deletions(-) rename README.md => scanner/README.md (100%) rename go.mod => scanner/go.mod (100%) rename go.sum => scanner/go.sum (100%) rename {internal => scanner/internal}/config/config.go (100%) rename {internal => scanner/internal}/event/agent_check_cache.go (100%) rename {internal => scanner/internal}/event/agent_check_state.go (100%) rename {internal => scanner/internal}/event/agent_check_storage.go (100%) rename {internal => scanner/internal}/event/check_cache.go (100%) rename {internal => scanner/internal}/event/check_package.go (100%) rename {internal => scanner/internal}/event/check_rep_count.go (100%) rename {internal => scanner/internal}/event/check_rep_count_test.go (100%) rename {internal => scanner/internal}/event/event.go (100%) rename {internal => scanner/internal}/services/event.go (100%) rename {internal => scanner/internal}/services/service.go (100%) rename {internal => scanner/internal}/tickevent/batch_all_agent_check_cache.go (100%) rename {internal => scanner/internal}/tickevent/batch_check_all_package.go (100%) rename {internal => scanner/internal}/tickevent/batch_check_all_rep_count.go (100%) rename {internal => scanner/internal}/tickevent/batch_check_all_storage.go (100%) rename {internal => scanner/internal}/tickevent/check_agent_state.go (100%) rename {internal => scanner/internal}/tickevent/check_cache.go (100%) rename {internal => scanner/internal}/tickevent/tick_event.go (100%) rename {magefiles => scanner/magefiles}/magefile.go (100%) rename main.go => scanner/main.go (100%) diff --git a/README.md b/scanner/README.md similarity index 100% rename from README.md rename to scanner/README.md diff --git a/go.mod b/scanner/go.mod similarity index 100% rename from go.mod rename to scanner/go.mod diff --git a/go.sum b/scanner/go.sum similarity index 100% rename from go.sum rename to scanner/go.sum diff --git a/internal/config/config.go b/scanner/internal/config/config.go similarity index 100% rename from internal/config/config.go rename to scanner/internal/config/config.go diff --git a/internal/event/agent_check_cache.go b/scanner/internal/event/agent_check_cache.go similarity index 100% rename from internal/event/agent_check_cache.go rename to scanner/internal/event/agent_check_cache.go diff --git a/internal/event/agent_check_state.go b/scanner/internal/event/agent_check_state.go similarity index 100% rename from internal/event/agent_check_state.go rename to scanner/internal/event/agent_check_state.go diff --git a/internal/event/agent_check_storage.go b/scanner/internal/event/agent_check_storage.go similarity index 100% rename from internal/event/agent_check_storage.go rename to scanner/internal/event/agent_check_storage.go diff --git a/internal/event/check_cache.go b/scanner/internal/event/check_cache.go similarity index 100% rename from internal/event/check_cache.go rename to scanner/internal/event/check_cache.go diff --git a/internal/event/check_package.go b/scanner/internal/event/check_package.go similarity index 100% rename from internal/event/check_package.go rename to scanner/internal/event/check_package.go diff --git a/internal/event/check_rep_count.go b/scanner/internal/event/check_rep_count.go similarity index 100% rename from internal/event/check_rep_count.go rename to scanner/internal/event/check_rep_count.go diff --git a/internal/event/check_rep_count_test.go b/scanner/internal/event/check_rep_count_test.go similarity index 100% rename from internal/event/check_rep_count_test.go rename to scanner/internal/event/check_rep_count_test.go diff --git a/internal/event/event.go b/scanner/internal/event/event.go similarity index 100% rename from internal/event/event.go rename to scanner/internal/event/event.go diff --git a/internal/services/event.go b/scanner/internal/services/event.go similarity index 100% rename from internal/services/event.go rename to scanner/internal/services/event.go diff --git a/internal/services/service.go b/scanner/internal/services/service.go similarity index 100% rename from internal/services/service.go rename to scanner/internal/services/service.go diff --git a/internal/tickevent/batch_all_agent_check_cache.go b/scanner/internal/tickevent/batch_all_agent_check_cache.go similarity index 100% rename from internal/tickevent/batch_all_agent_check_cache.go rename to scanner/internal/tickevent/batch_all_agent_check_cache.go diff --git a/internal/tickevent/batch_check_all_package.go b/scanner/internal/tickevent/batch_check_all_package.go similarity index 100% rename from internal/tickevent/batch_check_all_package.go rename to scanner/internal/tickevent/batch_check_all_package.go diff --git a/internal/tickevent/batch_check_all_rep_count.go b/scanner/internal/tickevent/batch_check_all_rep_count.go similarity index 100% rename from internal/tickevent/batch_check_all_rep_count.go rename to scanner/internal/tickevent/batch_check_all_rep_count.go diff --git a/internal/tickevent/batch_check_all_storage.go b/scanner/internal/tickevent/batch_check_all_storage.go similarity index 100% rename from internal/tickevent/batch_check_all_storage.go rename to scanner/internal/tickevent/batch_check_all_storage.go diff --git a/internal/tickevent/check_agent_state.go b/scanner/internal/tickevent/check_agent_state.go similarity index 100% rename from internal/tickevent/check_agent_state.go rename to scanner/internal/tickevent/check_agent_state.go diff --git a/internal/tickevent/check_cache.go b/scanner/internal/tickevent/check_cache.go similarity index 100% rename from internal/tickevent/check_cache.go rename to scanner/internal/tickevent/check_cache.go diff --git a/internal/tickevent/tick_event.go b/scanner/internal/tickevent/tick_event.go similarity index 100% rename from internal/tickevent/tick_event.go rename to scanner/internal/tickevent/tick_event.go diff --git a/magefiles/magefile.go b/scanner/magefiles/magefile.go similarity index 100% rename from magefiles/magefile.go rename to scanner/magefiles/magefile.go diff --git a/main.go b/scanner/main.go similarity index 100% rename from main.go rename to scanner/main.go