From 00a37d6de79da1c6c5e5cfbb6b5d876bd1736e5e Mon Sep 17 00:00:00 2001 From: Yening Qin <710leo@gmail.com> Date: Sat, 6 Apr 2024 22:02:07 +0800 Subject: [PATCH] feat: Integration ibex (#1904) * Ibex integrate (#1876) --------- Co-authored-by: Deke Wang <94156972+wdkcc@users.noreply.github.com> --- alert/aconf/conf.go | 7 - alert/alert.go | 23 ++- alert/dispatch/dispatch.go | 6 +- alert/sender/callback.go | 195 ++++++++++++++++---------- center/center.go | 15 +- center/router/router.go | 4 +- center/router/router_funcs.go | 30 +--- center/router/router_mw.go | 16 +++ center/router/router_notify_config.go | 4 - center/router/router_task.go | 144 ++----------------- center/router/router_task_tpl.go | 8 ++ cmd/edge/edge.go | 12 +- conf/conf.go | 12 ++ etc/config.toml | 4 + etc/edge/edge.toml | 6 +- go.mod | 1 + go.sum | 2 + memsto/notify_config.go | 30 ---- memsto/task_tpl_cache.go | 109 ++++++++++++++ models/migrate/migrate.go | 22 +++ models/task_tpl.go | 108 ++++++++++++++ 21 files changed, 471 insertions(+), 287 deletions(-) create mode 100644 memsto/task_tpl_cache.go diff --git a/alert/aconf/conf.go b/alert/aconf/conf.go index 97282dd6..11e97eda 100644 --- a/alert/aconf/conf.go +++ b/alert/aconf/conf.go @@ -46,13 +46,6 @@ type RedisPub struct { ChannelKey string } -type Ibex struct { - Address string - BasicAuthUser string - BasicAuthPass string - Timeout int64 -} - func (a *Alert) PreCheck(configDir string) { if a.Alerting.TemplatesDir == "" { a.Alerting.TemplatesDir = path.Join(configDir, "template") diff --git a/alert/alert.go b/alert/alert.go index 01b1013b..4f6c25ec 100644 --- a/alert/alert.go +++ b/alert/alert.go @@ -24,7 +24,10 @@ import ( "github.com/ccfos/nightingale/v6/prom" "github.com/ccfos/nightingale/v6/pushgw/pconf" "github.com/ccfos/nightingale/v6/pushgw/writer" + "github.com/ccfos/nightingale/v6/storage" "github.com/ccfos/nightingale/v6/tdengine" + + "github.com/flashcatcloud/ibex/src/cmd/ibex" ) func Initialize(configDir string, cryptoKey string) (func(), error) { @@ -40,6 +43,14 @@ func Initialize(configDir string, cryptoKey string) (func(), error) { ctx := ctx.NewContext(context.Background(), nil, false, config.CenterApi) + var redis storage.Redis + if config.Redis.Address != "" { + redis, err = storage.NewRedis(config.Redis) + if err != nil { + return nil, err + } + } + syncStats := memsto.NewSyncStats() alertStats := astats.NewSyncStats() @@ -52,16 +63,22 @@ func Initialize(configDir string, cryptoKey string) (func(), error) { dsCache := memsto.NewDatasourceCache(ctx, syncStats) userCache := memsto.NewUserCache(ctx, syncStats) userGroupCache := memsto.NewUserGroupCache(ctx, syncStats) + taskTplsCache := memsto.NewTaskTplCache(ctx) promClients := prom.NewPromClient(ctx) tdengineClients := tdengine.NewTdengineClient(ctx, config.Alert.Heartbeat) externalProcessors := process.NewExternalProcessors() - Start(config.Alert, config.Pushgw, syncStats, alertStats, externalProcessors, targetCache, busiGroupCache, alertMuteCache, alertRuleCache, notifyConfigCache, dsCache, ctx, promClients, tdengineClients, userCache, userGroupCache) + Start(config.Alert, config.Pushgw, syncStats, alertStats, externalProcessors, targetCache, busiGroupCache, alertMuteCache, alertRuleCache, notifyConfigCache, taskTplsCache, dsCache, ctx, promClients, tdengineClients, userCache, userGroupCache) r := httpx.GinEngine(config.Global.RunMode, config.HTTP) rt := router.New(config.HTTP, config.Alert, alertMuteCache, targetCache, busiGroupCache, alertStats, ctx, externalProcessors) + + if config.Ibex.Enable { + ibex.ServerStart(false, nil, redis, config.HTTP.APIForService.BasicAuth, config.Alert.Heartbeat, &config.CenterApi, r, nil, config.Ibex, config.HTTP.Port) + } + rt.Config(r) dumper.ConfigRouter(r) @@ -74,7 +91,7 @@ func Initialize(configDir string, cryptoKey string) (func(), error) { } func Start(alertc aconf.Alert, pushgwc pconf.Pushgw, syncStats *memsto.Stats, alertStats *astats.Stats, externalProcessors *process.ExternalProcessorsType, targetCache *memsto.TargetCacheType, busiGroupCache *memsto.BusiGroupCacheType, - alertMuteCache *memsto.AlertMuteCacheType, alertRuleCache *memsto.AlertRuleCacheType, notifyConfigCache *memsto.NotifyConfigCacheType, datasourceCache *memsto.DatasourceCacheType, ctx *ctx.Context, + alertMuteCache *memsto.AlertMuteCacheType, alertRuleCache *memsto.AlertRuleCacheType, notifyConfigCache *memsto.NotifyConfigCacheType, taskTplsCache *memsto.TaskTplCache, datasourceCache *memsto.DatasourceCacheType, ctx *ctx.Context, promClients *prom.PromClientMap, tdendgineClients *tdengine.TdengineClientMap, userCache *memsto.UserCacheType, userGroupCache *memsto.UserGroupCacheType) { alertSubscribeCache := memsto.NewAlertSubscribeCache(ctx, syncStats) recordingRuleCache := memsto.NewRecordingRuleCache(ctx, syncStats) @@ -90,7 +107,7 @@ func Start(alertc aconf.Alert, pushgwc pconf.Pushgw, syncStats *memsto.Stats, al eval.NewScheduler(alertc, externalProcessors, alertRuleCache, targetCache, targetsOfAlertRulesCache, busiGroupCache, alertMuteCache, datasourceCache, promClients, tdendgineClients, naming, ctx, alertStats) - dp := dispatch.NewDispatch(alertRuleCache, userCache, userGroupCache, alertSubscribeCache, targetCache, notifyConfigCache, alertc.Alerting, ctx, alertStats) + dp := dispatch.NewDispatch(alertRuleCache, userCache, userGroupCache, alertSubscribeCache, targetCache, notifyConfigCache, taskTplsCache, alertc.Alerting, ctx, alertStats) consumer := dispatch.NewConsumer(alertc.Alerting, ctx, dp) go dp.ReloadTpls() diff --git a/alert/dispatch/dispatch.go b/alert/dispatch/dispatch.go index f5c12b23..19f44576 100644 --- a/alert/dispatch/dispatch.go +++ b/alert/dispatch/dispatch.go @@ -26,6 +26,7 @@ type Dispatch struct { alertSubscribeCache *memsto.AlertSubscribeCacheType targetCache *memsto.TargetCacheType notifyConfigCache *memsto.NotifyConfigCacheType + taskTplsCache *memsto.TaskTplCache alerting aconf.Alerting @@ -43,7 +44,7 @@ type Dispatch struct { // 创建一个 Notify 实例 func NewDispatch(alertRuleCache *memsto.AlertRuleCacheType, userCache *memsto.UserCacheType, userGroupCache *memsto.UserGroupCacheType, alertSubscribeCache *memsto.AlertSubscribeCacheType, targetCache *memsto.TargetCacheType, notifyConfigCache *memsto.NotifyConfigCacheType, - alerting aconf.Alerting, ctx *ctx.Context, astats *astats.Stats) *Dispatch { + taskTplsCache *memsto.TaskTplCache, alerting aconf.Alerting, ctx *ctx.Context, astats *astats.Stats) *Dispatch { notify := &Dispatch{ alertRuleCache: alertRuleCache, userCache: userCache, @@ -51,6 +52,7 @@ func NewDispatch(alertRuleCache *memsto.AlertRuleCacheType, userCache *memsto.Us alertSubscribeCache: alertSubscribeCache, targetCache: targetCache, notifyConfigCache: notifyConfigCache, + taskTplsCache: taskTplsCache, alerting: alerting, @@ -241,7 +243,7 @@ func (e *Dispatch) Send(rule *models.AlertRule, event *models.AlertCurEvent, not } // handle event callbacks - sender.SendCallbacks(e.ctx, notifyTarget.ToCallbackList(), event, e.targetCache, e.userCache, e.notifyConfigCache.GetIbex(), e.Astats) + sender.SendCallbacks(e.ctx, notifyTarget.ToCallbackList(), event, e.targetCache, e.userCache, e.taskTplsCache, e.Astats) // handle global webhooks sender.SendWebhooks(notifyTarget.ToWebhookList(), event, e.Astats) diff --git a/alert/sender/callback.go b/alert/sender/callback.go index 1755dd87..2cf1aca2 100644 --- a/alert/sender/callback.go +++ b/alert/sender/callback.go @@ -2,23 +2,24 @@ package sender import ( "encoding/json" + "fmt" "strconv" "strings" "time" - "github.com/ccfos/nightingale/v6/alert/aconf" "github.com/ccfos/nightingale/v6/alert/astats" "github.com/ccfos/nightingale/v6/memsto" "github.com/ccfos/nightingale/v6/models" "github.com/ccfos/nightingale/v6/pkg/ctx" - "github.com/ccfos/nightingale/v6/pkg/ibex" "github.com/ccfos/nightingale/v6/pkg/poster" + imodels "github.com/flashcatcloud/ibex/src/models" + "github.com/flashcatcloud/ibex/src/storage" "github.com/toolkits/pkg/logger" ) func SendCallbacks(ctx *ctx.Context, urls []string, event *models.AlertCurEvent, targetCache *memsto.TargetCacheType, userCache *memsto.UserCacheType, - ibexConf aconf.Ibex, stats *astats.Stats) { + taskTplCache *memsto.TaskTplCache, stats *astats.Stats) { for _, url := range urls { if url == "" { continue @@ -26,7 +27,7 @@ func SendCallbacks(ctx *ctx.Context, urls []string, event *models.AlertCurEvent, if strings.HasPrefix(url, "${ibex}") { if !event.IsRecovered { - handleIbex(ctx, url, event, targetCache, userCache, ibexConf) + handleIbex(ctx, url, event, targetCache, userCache, taskTplCache) } continue } @@ -46,27 +47,13 @@ func SendCallbacks(ctx *ctx.Context, urls []string, event *models.AlertCurEvent, } } -type TaskForm struct { - Title string `json:"title"` - Account string `json:"account"` - Batch int `json:"batch"` - Tolerance int `json:"tolerance"` - Timeout int `json:"timeout"` - Pause string `json:"pause"` - Script string `json:"script"` - Args string `json:"args"` - Stdin string `json:"stdin"` - Action string `json:"action"` - Creator string `json:"creator"` - Hosts []string `json:"hosts"` -} - type TaskCreateReply struct { Err string `json:"err"` Dat int64 `json:"dat"` // task.id } -func handleIbex(ctx *ctx.Context, url string, event *models.AlertCurEvent, targetCache *memsto.TargetCacheType, userCache *memsto.UserCacheType, ibexConf aconf.Ibex) { +func handleIbex(ctx *ctx.Context, url string, event *models.AlertCurEvent, targetCache *memsto.TargetCacheType, userCache *memsto.UserCacheType, + taskTplCache *memsto.TaskTplCache) { arr := strings.Split(url, "/") var idstr string @@ -96,12 +83,7 @@ func handleIbex(ctx *ctx.Context, url string, event *models.AlertCurEvent, targe return } - tpl, err := models.TaskTplGetById(ctx, id) - if err != nil { - logger.Errorf("event_callback_ibex: failed to get tpl: %v", err) - return - } - + tpl := taskTplCache.Get(id) if tpl == nil { logger.Errorf("event_callback_ibex: no such tpl(%d)", id) return @@ -145,61 +127,43 @@ func handleIbex(ctx *ctx.Context, url string, event *models.AlertCurEvent, targe } // call ibex - in := TaskForm{ - Title: tpl.Title + " FH: " + host, - Account: tpl.Account, - Batch: tpl.Batch, - Tolerance: tpl.Tolerance, - Timeout: tpl.Timeout, - Pause: tpl.Pause, - Script: tpl.Script, - Args: tpl.Args, - Stdin: string(tags), - Action: "start", - Creator: tpl.UpdateBy, - Hosts: []string{host}, - } - - var res TaskCreateReply - err = ibex.New( - ibexConf.Address, - ibexConf.BasicAuthUser, - ibexConf.BasicAuthPass, - ibexConf.Timeout, - ). - Path("/ibex/v1/tasks"). - In(in). - Out(&res). - POST() + in := models.TaskForm{ + Title: tpl.Title + " FH: " + host, + Account: tpl.Account, + Batch: tpl.Batch, + Tolerance: tpl.Tolerance, + Timeout: tpl.Timeout, + Pause: tpl.Pause, + Script: tpl.Script, + Args: tpl.Args, + Stdin: string(tags), + Action: "start", + Creator: tpl.UpdateBy, + Hosts: []string{host}, + AlertTriggered: true, + } + id, err = TaskAdd(in, tpl.UpdateBy, ctx.IsCenter) if err != nil { logger.Errorf("event_callback_ibex: call ibex fail: %v", err) return } - if res.Err != "" { - logger.Errorf("event_callback_ibex: call ibex response error: %v", res.Err) - return - } - // write db record := models.TaskRecord{ - Id: res.Dat, - EventId: event.Id, - GroupId: tpl.GroupId, - IbexAddress: ibexConf.Address, - IbexAuthUser: ibexConf.BasicAuthUser, - IbexAuthPass: ibexConf.BasicAuthPass, - Title: in.Title, - Account: in.Account, - Batch: in.Batch, - Tolerance: in.Tolerance, - Timeout: in.Timeout, - Pause: in.Pause, - Script: in.Script, - Args: in.Args, - CreateAt: time.Now().Unix(), - CreateBy: in.Creator, + Id: id, + EventId: event.Id, + GroupId: tpl.GroupId, + Title: in.Title, + Account: in.Account, + Batch: in.Batch, + Tolerance: in.Tolerance, + Timeout: in.Timeout, + Pause: in.Pause, + Script: in.Script, + Args: in.Args, + CreateAt: time.Now().Unix(), + CreateBy: in.Creator, } if err = record.Add(ctx); err != nil { @@ -220,3 +184,88 @@ func canDoIbex(username string, tpl *models.TaskTpl, host string, targetCache *m return target.GroupId == tpl.GroupId, nil } + +func TaskAdd(f models.TaskForm, authUser string, isCenter bool) (int64, error) { + hosts := cleanHosts(f.Hosts) + if len(hosts) == 0 { + return 0, fmt.Errorf("arg(hosts) empty") + } + + taskMeta := &imodels.TaskMeta{ + Title: f.Title, + Account: f.Account, + Batch: f.Batch, + Tolerance: f.Tolerance, + Timeout: f.Timeout, + Pause: f.Pause, + Script: f.Script, + Args: f.Args, + Stdin: f.Stdin, + Creator: f.Creator, + } + + err := taskMeta.CleanFields() + if err != nil { + return 0, err + } + + taskMeta.HandleFH(hosts[0]) + + // 任务类型分为"告警规则触发"和"n9e center用户下发"两种； + // 边缘机房"告警规则触发"的任务不需要规划，并且它可能是失联的，无法使用db资源，所以放入redis缓存中，直接下发给agentd执行 + if !isCenter && f.AlertTriggered { + if err := taskMeta.Create(); err != nil { + // 当网络不连通时，生成唯一的id，防止边缘机房中不同任务的id相同； + // 方法是，redis自增id去防止同一个机房的不同n9e edge生成的id相同； + // 但没法防止不同边缘机房生成同样的id，所以，生成id的数据不会上报存入数据库，只用于闭环执行。 + taskMeta.Id, err = storage.IdGet() + if err != nil { + return 0, err + } + } + + taskHost := imodels.TaskHost{ + Id: taskMeta.Id, + Host: hosts[0], + Status: "running", + } + if err = taskHost.Create(); err != nil { + logger.Warningf("task_add_fail: authUser=%s title=%s err=%s", authUser, taskMeta.Title, err.Error()) + } + + // 缓存任务元信息和待下发的任务 + err = taskMeta.Cache(hosts[0]) + if err != nil { + return 0, err + } + + } else { + // 如果是中心机房，还是保持之前的逻辑 + err = taskMeta.Save(hosts, f.Action) + if err != nil { + return 0, err + } + } + + logger.Infof("task_add_succ: authUser=%s title=%s", authUser, taskMeta.Title) + return taskMeta.Id, nil +} + +func cleanHosts(formHosts []string) []string { + cnt := len(formHosts) + arr := make([]string, 0, cnt) + for i := 0; i < cnt; i++ { + item := strings.TrimSpace(formHosts[i]) + if item == "" { + continue + } + + if strings.HasPrefix(item, "#") { + continue + } + + arr = append(arr, item) + } + + return arr +} diff --git a/center/center.go b/center/center.go index 4f9df6d6..de109681 100644 --- a/center/center.go +++ b/center/center.go @@ -7,10 +7,12 @@ import ( "github.com/ccfos/nightingale/v6/alert" "github.com/ccfos/nightingale/v6/alert/astats" "github.com/ccfos/nightingale/v6/alert/process" + alertrt "github.com/ccfos/nightingale/v6/alert/router" "github.com/ccfos/nightingale/v6/center/cconf" "github.com/ccfos/nightingale/v6/center/cconf/rsa" "github.com/ccfos/nightingale/v6/center/cstats" "github.com/ccfos/nightingale/v6/center/metas" + centerrt "github.com/ccfos/nightingale/v6/center/router" "github.com/ccfos/nightingale/v6/center/sso" "github.com/ccfos/nightingale/v6/conf" "github.com/ccfos/nightingale/v6/dumper" @@ -25,13 +27,12 @@ import ( "github.com/ccfos/nightingale/v6/pkg/version" "github.com/ccfos/nightingale/v6/prom" "github.com/ccfos/nightingale/v6/pushgw/idents" + pushgwrt "github.com/ccfos/nightingale/v6/pushgw/router" "github.com/ccfos/nightingale/v6/pushgw/writer" "github.com/ccfos/nightingale/v6/storage" "github.com/ccfos/nightingale/v6/tdengine" - alertrt "github.com/ccfos/nightingale/v6/alert/router" - centerrt "github.com/ccfos/nightingale/v6/center/router" - pushgwrt "github.com/ccfos/nightingale/v6/pushgw/router" + "github.com/flashcatcloud/ibex/src/cmd/ibex" ) func Initialize(configDir string, cryptoKey string) (func(), error) { @@ -90,12 +91,13 @@ func Initialize(configDir string, cryptoKey string) (func(), error) { notifyConfigCache := memsto.NewNotifyConfigCache(ctx, configCache) userCache := memsto.NewUserCache(ctx, syncStats) userGroupCache := memsto.NewUserGroupCache(ctx, syncStats) + taskTplCache := memsto.NewTaskTplCache(ctx) promClients := prom.NewPromClient(ctx) tdengineClients := tdengine.NewTdengineClient(ctx, config.Alert.Heartbeat) externalProcessors := process.NewExternalProcessors() - alert.Start(config.Alert, config.Pushgw, syncStats, alertStats, externalProcessors, targetCache, busiGroupCache, alertMuteCache, alertRuleCache, notifyConfigCache, dsCache, ctx, promClients, tdengineClients, userCache, userGroupCache) + alert.Start(config.Alert, config.Pushgw, syncStats, alertStats, externalProcessors, targetCache, busiGroupCache, alertMuteCache, alertRuleCache, notifyConfigCache, taskTplCache, dsCache, ctx, promClients, tdengineClients, userCache, userGroupCache) writers := writer.NewWriters(config.Pushgw) @@ -113,6 +115,11 @@ func Initialize(configDir string, cryptoKey string) (func(), error) { pushgwRouter.Config(r) dumper.ConfigRouter(r) + if config.Ibex.Enable { + migrate.MigrateIbexTables(db) + ibex.ServerStart(true, db, redis, config.HTTP.APIForService.BasicAuth, config.Alert.Heartbeat, &config.CenterApi, r, centerRouter, config.Ibex, config.HTTP.Port) + } + httpClean := httpx.Init(config.HTTP, r) return func() { diff --git a/center/router/router.go b/center/router/router.go index 5693f752..3c0c5eb2 100644 --- a/center/router/router.go +++ b/center/router/router.go @@ -356,8 +356,6 @@ func (rt *Router) Config(r *gin.Engine) { pages.GET("/busi-groups/tasks", rt.auth(), rt.user(), rt.perm("/job-tasks"), rt.taskGetsByGids) pages.GET("/busi-group/:id/tasks", rt.auth(), rt.user(), rt.perm("/job-tasks"), rt.bgro(), rt.taskGets) pages.POST("/busi-group/:id/tasks", rt.auth(), rt.user(), rt.perm("/job-tasks/add"), rt.bgrw(), rt.taskAdd) - pages.GET("/busi-group/:id/task/*url", rt.auth(), rt.user(), rt.perm("/job-tasks"), rt.taskProxy) - pages.PUT("/busi-group/:id/task/*url", rt.auth(), rt.user(), rt.perm("/job-tasks/put"), rt.bgrw(), rt.taskProxy) pages.GET("/servers", rt.auth(), rt.admin(), rt.serversGet) pages.GET("/server-clusters", rt.auth(), rt.admin(), rt.serverClustersGet) @@ -488,6 +486,8 @@ func (rt *Router) Config(r *gin.Engine) { service.GET("/alert-his-event/:eid", rt.alertHisEventGet) service.GET("/task-tpl/:tid", rt.taskTplGetByService) + service.GET("/task-tpls", rt.taskTplGetsByService) + service.GET("/task-tpl/statistics", rt.taskTplStatistics) service.GET("/config/:id", rt.configGet) service.GET("/configs", rt.configsGet) diff --git a/center/router/router_funcs.go b/center/router/router_funcs.go index fe978b6e..c73943da 100644 --- a/center/router/router_funcs.go +++ b/center/router/router_funcs.go @@ -1,17 +1,14 @@ package router import ( - "fmt" "net/http" "strconv" "strings" - "github.com/ccfos/nightingale/v6/alert/aconf" "github.com/ccfos/nightingale/v6/models" "github.com/ccfos/nightingale/v6/pkg/ctx" - "github.com/ccfos/nightingale/v6/pkg/ibex" - "github.com/gin-gonic/gin" + "github.com/gin-gonic/gin" "github.com/toolkits/pkg/ginx" ) @@ -135,31 +132,6 @@ type TaskCreateReply struct { Dat int64 `json:"dat"` // task.id } -// return task.id, error -func TaskCreate(v interface{}, ibexc aconf.Ibex) (int64, error) { - var res TaskCreateReply - err := ibex.New( - ibexc.Address, - ibexc.BasicAuthUser, - ibexc.BasicAuthPass, - ibexc.Timeout, - ). - Path("/ibex/v1/tasks"). - In(v). - Out(&res). - POST() - - if err != nil { - return 0, err - } - - if res.Err != "" { - return 0, fmt.Errorf("response.err: %v", res.Err) - } - - return res.Dat, nil -} - func Username(c *gin.Context) string { username := c.GetString(gin.AuthUserKey) if username == "" { diff --git a/center/router/router_mw.go b/center/router/router_mw.go index b0b0bcb0..e365212e 100644 --- a/center/router/router_mw.go +++ b/center/router/router_mw.go @@ -92,6 +92,10 @@ func (rt *Router) jwtAuth() gin.HandlerFunc { } } +func (rt *Router) Auth() gin.HandlerFunc { + return rt.auth() +} + func (rt *Router) auth() gin.HandlerFunc { if rt.HTTP.ProxyAuth.Enable { return rt.proxyAuth() @@ -120,6 +124,10 @@ func (rt *Router) jwtMock() gin.HandlerFunc { } } +func (rt *Router) User() gin.HandlerFunc { + return rt.user() +} + func (rt *Router) user() gin.HandlerFunc { return func(c *gin.Context) { userid := c.MustGet("userid").(int64) @@ -174,6 +182,10 @@ func (rt *Router) bgro() gin.HandlerFunc { } // bgrw 逐步要被干掉，不安全 +func (rt *Router) Bgrw() gin.HandlerFunc { + return rt.bgrw() +} + func (rt *Router) bgrw() gin.HandlerFunc { return func(c *gin.Context) { me := c.MustGet("user").(*models.User) @@ -233,6 +245,10 @@ func (rt *Router) bgroCheck(c *gin.Context, bgid int64) { c.Set("busi_group", bg) } +func (rt *Router) Perm(operation string) gin.HandlerFunc { + return rt.perm(operation) +} + func (rt *Router) perm(operation string) gin.HandlerFunc { return func(c *gin.Context) { me := c.MustGet("user").(*models.User) diff --git a/center/router/router_notify_config.go b/center/router/router_notify_config.go index f64cb41a..d9a73427 100644 --- a/center/router/router_notify_config.go +++ b/center/router/router_notify_config.go @@ -169,10 +169,6 @@ func (rt *Router) notifyConfigPut(c *gin.Context) { var smtp aconf.SMTPConfig err := toml.Unmarshal([]byte(text), &smtp) ginx.Dangerous(err) - case models.IBEX: - var ibex aconf.Ibex - err := toml.Unmarshal([]byte(f.Cval), &ibex) - ginx.Dangerous(err) default: ginx.Bomb(200, "key %s can not modify", f.Ckey) } diff --git a/center/router/router_task.go b/center/router/router_task.go index bacf0908..12b7357c 100644 --- a/center/router/router_task.go +++ b/center/router/router_task.go @@ -1,13 +1,9 @@ package router import ( - "fmt" - "net/http" - "net/http/httputil" - "net/url" - "strings" "time" + "github.com/ccfos/nightingale/v6/alert/sender" "github.com/ccfos/nightingale/v6/models" "github.com/gin-gonic/gin" @@ -96,71 +92,6 @@ type taskForm struct { Hosts []string `json:"hosts" binding:"required"` } -func (f *taskForm) Verify() error { - if f.Batch < 0 { - return fmt.Errorf("arg(batch) should be nonnegative") - } - - if f.Tolerance < 0 { - return fmt.Errorf("arg(tolerance) should be nonnegative") - } - - if f.Timeout < 0 { - return fmt.Errorf("arg(timeout) should be nonnegative") - } - - if f.Timeout > 3600*24 { - return fmt.Errorf("arg(timeout) longer than one day") - } - - if f.Timeout == 0 { - f.Timeout = 30 - } - - f.Pause = strings.Replace(f.Pause, "，", ",", -1) - f.Pause = strings.Replace(f.Pause, " ", "", -1) - f.Args = strings.Replace(f.Args, "，", ",", -1) - - if f.Title == "" { - return fmt.Errorf("arg(title) is required") - } - - if str.Dangerous(f.Title) { - return fmt.Errorf("arg(title) is dangerous") - } - - if f.Script == "" { - return fmt.Errorf("arg(script) is required") - } - f.Script = strings.Replace(f.Script, "\r\n", "\n", -1) - - if str.Dangerous(f.Args) { - return fmt.Errorf("arg(args) is dangerous") - } - - if str.Dangerous(f.Pause) { - return fmt.Errorf("arg(pause) is dangerous") - } - - if len(f.Hosts) == 0 { - return fmt.Errorf("arg(hosts) empty") - } - - if f.Action != "start" && f.Action != "pause" { - return fmt.Errorf("arg(action) invalid") - } - - return nil -} - -func (f *taskForm) HandleFH(fh string) { - i := strings.Index(f.Title, " FH: ") - if i > 0 { - f.Title = f.Title[:i] - } - f.Title = f.Title + " FH: " + fh -} - func (rt *Router) taskRecordAdd(c *gin.Context) { var f *models.TaskRecord ginx.BindJSON(c, &f) @@ -168,7 +99,7 @@ func (rt *Router) taskRecordAdd(c *gin.Context) { } func (rt *Router) taskAdd(c *gin.Context) { - var f taskForm + var f models.TaskForm ginx.BindJSON(c, &f) bgid := ginx.UrlParamInt64(c, "id") @@ -184,7 +115,7 @@ func (rt *Router) taskAdd(c *gin.Context) { rt.checkTargetPerm(c, f.Hosts) // call ibex - taskId, err := TaskCreate(f, rt.NotifyConfigCache.GetIbex()) + taskId, err := sender.TaskAdd(f, user.Username, rt.Ctx.IsCenter) ginx.Dangerous(err) if taskId <= 0 { @@ -193,65 +124,20 @@ func (rt *Router) taskAdd(c *gin.Context) { // write db record := models.TaskRecord{ - Id: taskId, - GroupId: bgid, - IbexAddress: rt.NotifyConfigCache.GetIbex().Address, - IbexAuthUser: rt.NotifyConfigCache.GetIbex().BasicAuthUser, - IbexAuthPass: rt.NotifyConfigCache.GetIbex().BasicAuthPass, - Title: f.Title, - Account: f.Account, - Batch: f.Batch, - Tolerance: f.Tolerance, - Timeout: f.Timeout, - Pause: f.Pause, - Script: f.Script, - Args: f.Args, - CreateAt: time.Now().Unix(), - CreateBy: f.Creator, + Id: taskId, + GroupId: bgid, + Title: f.Title, + Account: f.Account, + Batch: f.Batch, + Tolerance: f.Tolerance, + Timeout: f.Timeout, + Pause: f.Pause, + Script: f.Script, + Args: f.Args, + CreateAt: time.Now().Unix(), + CreateBy: f.Creator, } err = record.Add(rt.Ctx) ginx.NewRender(c).Data(taskId, err) } - -func (rt *Router) taskProxy(c *gin.Context) { - target, err := url.Parse(rt.NotifyConfigCache.GetIbex().Address) - if err != nil { - ginx.NewRender(c).Message("invalid ibex address: %s", rt.NotifyConfigCache.GetIbex().Address) - return - } - - director := func(req *http.Request) { - req.URL.Scheme = target.Scheme - req.URL.Host = target.Host - - // fe request e.g. /api/n9e/busi-group/:id/task/*url - index := strings.Index(req.URL.Path, "/task/") - if index == -1 { - panic("url path invalid") - } - - req.URL.Path = "/ibex/v1" + req.URL.Path[index:] - - if target.RawQuery == "" || req.URL.RawQuery == "" { - req.URL.RawQuery = target.RawQuery + req.URL.RawQuery - } else { - req.URL.RawQuery = target.RawQuery + "&" + req.URL.RawQuery - } - - if rt.NotifyConfigCache.GetIbex().BasicAuthUser != "" { - req.SetBasicAuth(rt.NotifyConfigCache.GetIbex().BasicAuthUser, rt.NotifyConfigCache.GetIbex().BasicAuthPass) - } - } - - errFunc := func(w http.ResponseWriter, r *http.Request, err error) { - ginx.NewRender(c, http.StatusBadGateway).Message(err) - } - - proxy := &httputil.ReverseProxy{ - Director: director, - ErrorHandler: errFunc, - } - - proxy.ServeHTTP(c.Writer, c.Request) -} diff --git a/center/router/router_task_tpl.go b/center/router/router_task_tpl.go index f062a8ce..c0d44f6d 100644 --- a/center/router/router_task_tpl.go +++ b/center/router/router_task_tpl.go @@ -91,6 +91,14 @@ func (rt *Router) taskTplGetByService(c *gin.Context) { ginx.NewRender(c).Data(tpl, err) } +func (rt *Router) taskTplGetsByService(c *gin.Context) { + ginx.NewRender(c).Data(models.TaskTplGetAll(rt.Ctx)) +} + +func (rt *Router) taskTplStatistics(c *gin.Context) { + ginx.NewRender(c).Data(models.TaskTplStatistics(rt.Ctx)) +} + type taskTplForm struct { Title string `json:"title" binding:"required"` Batch int `json:"batch"` diff --git a/cmd/edge/edge.go b/cmd/edge/edge.go index 2b52db40..9d1b349d 100644 --- a/cmd/edge/edge.go +++ b/cmd/edge/edge.go @@ -8,6 +8,7 @@ import ( "github.com/ccfos/nightingale/v6/alert" "github.com/ccfos/nightingale/v6/alert/astats" "github.com/ccfos/nightingale/v6/alert/process" + alertrt "github.com/ccfos/nightingale/v6/alert/router" "github.com/ccfos/nightingale/v6/center/metas" "github.com/ccfos/nightingale/v6/conf" "github.com/ccfos/nightingale/v6/dumper" @@ -17,12 +18,12 @@ import ( "github.com/ccfos/nightingale/v6/pkg/logx" "github.com/ccfos/nightingale/v6/prom" "github.com/ccfos/nightingale/v6/pushgw/idents" + pushgwrt "github.com/ccfos/nightingale/v6/pushgw/router" "github.com/ccfos/nightingale/v6/pushgw/writer" "github.com/ccfos/nightingale/v6/storage" "github.com/ccfos/nightingale/v6/tdengine" - alertrt "github.com/ccfos/nightingale/v6/alert/router" - pushgwrt "github.com/ccfos/nightingale/v6/pushgw/router" + "github.com/flashcatcloud/ibex/src/cmd/ibex" ) func Initialize(configDir string, cryptoKey string) (func(), error) { @@ -69,17 +70,22 @@ func Initialize(configDir string, cryptoKey string) (func(), error) { notifyConfigCache := memsto.NewNotifyConfigCache(ctx, configCache) userCache := memsto.NewUserCache(ctx, syncStats) userGroupCache := memsto.NewUserGroupCache(ctx, syncStats) + taskTplsCache := memsto.NewTaskTplCache(ctx) promClients := prom.NewPromClient(ctx) tdengineClients := tdengine.NewTdengineClient(ctx, config.Alert.Heartbeat) externalProcessors := process.NewExternalProcessors() alert.Start(config.Alert, config.Pushgw, syncStats, alertStats, externalProcessors, targetCache, busiGroupCache, alertMuteCache, - alertRuleCache, notifyConfigCache, dsCache, ctx, promClients, tdengineClients, userCache, userGroupCache) + alertRuleCache, notifyConfigCache, taskTplsCache, dsCache, ctx, promClients, tdengineClients, userCache, userGroupCache) alertrtRouter := alertrt.New(config.HTTP, config.Alert, alertMuteCache, targetCache, busiGroupCache, alertStats, ctx, externalProcessors) alertrtRouter.Config(r) + + if config.Ibex.Enable { + ibex.ServerStart(false, nil, redis, config.HTTP.APIForService.BasicAuth, config.Alert.Heartbeat, &config.CenterApi, r, nil, config.Ibex, config.HTTP.Port) + } } dumper.ConfigRouter(r) diff --git a/conf/conf.go b/conf/conf.go index 5f7730da..5c34b8bc 100644 --- a/conf/conf.go +++ b/conf/conf.go @@ -27,6 +27,7 @@ type ConfigType struct { Pushgw pconf.Pushgw Alert aconf.Alert Center cconf.Center + Ibex Ibex } type CenterApi struct { @@ -40,6 +41,17 @@ type GlobalConfig struct { RunMode string } +type Ibex struct { + Enable bool + RPCListen string + Output Output +} + +type Output struct { + ComeFrom string + AgtdPort int +} + func InitConfig(configDir, cryptoKey string) (*ConfigType, error) { var config = new(ConfigType) diff --git a/etc/config.toml b/etc/config.toml index ccd1740b..5598ee66 100644 --- a/etc/config.toml +++ b/etc/config.toml @@ -173,3 +173,7 @@ MaxIdleConnsPerHost = 100 # Regex = "([^:]+)(?::\\d+)?" # Replacement = "$1:80" # TargetLabel = "__address__" + +[Ibex] +Enable = false +RPCListen = "0.0.0.0:20090" \ No newline at end of file diff --git a/etc/edge/edge.toml b/etc/edge/edge.toml index 1f09f50e..a22c75e3 100644 --- a/etc/edge/edge.toml +++ b/etc/edge/edge.toml @@ -116,6 +116,10 @@ MaxIdleConnsPerHost = 100 # Replacement = "$1:80" # TargetLabel = "__address__" +[Ibex] +Enable = false +RPCListen = "0.0.0.0:20090" + [Redis] # address, ip:port or ip1:port,ip2:port for cluster and sentinel(SentinelAddrs) Address = "127.0.0.1:6379" @@ -129,4 +133,4 @@ RedisType = "standalone" # Mastername for sentinel type # MasterName = "mymaster" # SentinelUsername = "" -# SentinelPassword = "" \ No newline at end of file +# SentinelPassword = "" diff --git a/go.mod b/go.mod index 3a6021d7..e40350af 100644 --- a/go.mod +++ b/go.mod @@ -8,6 +8,7 @@ require ( github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc github.com/dgrijalva/jwt-go v3.2.0+incompatible github.com/expr-lang/expr v1.16.1 + github.com/flashcatcloud/ibex v1.3.0 github.com/gin-contrib/pprof v1.4.0 github.com/gin-gonic/gin v1.9.1 github.com/go-ldap/ldap/v3 v3.4.4 diff --git a/go.sum b/go.sum index afaf84b4..cf0ab914 100644 --- a/go.sum +++ b/go.sum @@ -38,6 +38,8 @@ github.com/fatih/camelcase v1.0.0 h1:hxNvNX/xYBp0ovncs8WyWZrOrpBNub/JfaMvbURyft8 github.com/fatih/camelcase v1.0.0/go.mod h1:yN2Sb0lFhZJUdVvtELVWefmrXpuZESvPmqwoZc+/fpc= github.com/fatih/structs v1.1.0 h1:Q7juDM0QtcnhCpeyLGQKyg4TOIghuNXrkL32pHAUMxo= github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= +github.com/flashcatcloud/ibex v1.3.0 h1:pmapZfuQE3ZZKtOgAxUUxFbQeySG2LxiYbayjnCGiZg= +github.com/flashcatcloud/ibex v1.3.0/go.mod h1:T8hbMUySK2q6cXUaYp0AUVeKkU9Od2LjzwmB5lmTRBM= github.com/gabriel-vasile/mimetype v1.4.2 h1:w5qFW6JKBz9Y393Y4q372O9A7cUSequkh1Q7OhCmWKU= github.com/gabriel-vasile/mimetype v1.4.2/go.mod h1:zApsH/mKG4w07erKIaJPFiX0Tsq9BFQgN3qGY5GnNgA= github.com/garyburd/redigo v1.6.2/go.mod h1:NR3MbYisc3/PwhQ00EMzDiPmrwpPxAn5GI05/YaO1SY= diff --git a/memsto/notify_config.go b/memsto/notify_config.go index 69ad9e42..fc78310a 100644 --- a/memsto/notify_config.go +++ b/memsto/notify_config.go @@ -22,7 +22,6 @@ type NotifyConfigCacheType struct { webhooks []*models.Webhook smtp aconf.SMTPConfig script models.NotifyScript - ibex aconf.Ibex sync.RWMutex } @@ -131,29 +130,6 @@ func (w *NotifyConfigCacheType) syncNotifyConfigs() error { dumper.PutSyncRecord("notify_script", start.Unix(), time.Since(start).Milliseconds(), 1, "success, notify_script:\n"+cval) - start = time.Now() - cval, err = models.ConfigsGet(w.ctx, models.IBEX) - if err != nil { - dumper.PutSyncRecord("ibex", start.Unix(), -1, -1, "failed to query configs.ibex_server: "+err.Error()) - return err - } - - if strings.TrimSpace(cval) != "" { - err = toml.Unmarshal([]byte(cval), &w.ibex) - if err != nil { - dumper.PutSyncRecord("ibex", start.Unix(), -1, -1, "failed to unmarshal configs.ibex_server: "+err.Error()) - logger.Errorf("failed to unmarshal ibex:%s error:%v", cval, err) - } - } else { - err = toml.Unmarshal([]byte(DefaultIbex), &w.ibex) - if err != nil { - dumper.PutSyncRecord("ibex", start.Unix(), -1, -1, "failed to unmarshal configs.ibex_server: "+err.Error()) - logger.Errorf("failed to unmarshal ibex:%s error:%v", cval, err) - } - } - - dumper.PutSyncRecord("ibex", start.Unix(), time.Since(start).Milliseconds(), 1, "success, ibex_server config:\n"+cval) - return nil } @@ -178,9 +154,3 @@ func (w *NotifyConfigCacheType) GetNotifyScript() models.NotifyScript { return w.script } - -func (w *NotifyConfigCacheType) GetIbex() aconf.Ibex { - w.RWMutex.RLock() - defer w.RWMutex.RUnlock() - return w.ibex -} diff --git a/memsto/task_tpl_cache.go b/memsto/task_tpl_cache.go new file mode 100644 index 00000000..6ece12ed --- /dev/null +++ b/memsto/task_tpl_cache.go @@ -0,0 +1,109 @@ +package memsto + +import ( + "fmt" + "sync" + "time" + + "github.com/ccfos/nightingale/v6/dumper" + "github.com/ccfos/nightingale/v6/models" + "github.com/ccfos/nightingale/v6/pkg/ctx" + + "github.com/pkg/errors" + "github.com/toolkits/pkg/logger" +) + +type TaskTplCache struct { + statTotal int64 + statLastUpdated int64 + ctx *ctx.Context + tpls map[int64]*models.TaskTpl + sync.RWMutex +} + +func NewTaskTplCache(ctx *ctx.Context) *TaskTplCache { + ttc := &TaskTplCache{ + statTotal: -1, + statLastUpdated: -1, + ctx: ctx, + tpls: make(map[int64]*models.TaskTpl), + } + + ttc.SyncTaskTpl() + return ttc +} + +func (ttc *TaskTplCache) Set(tpls map[int64]*models.TaskTpl, total, lastUpdated int64) { + ttc.Lock() + ttc.tpls = tpls + ttc.Unlock() + + ttc.statTotal = total + ttc.statLastUpdated = lastUpdated +} + +func (ttc *TaskTplCache) Get(id int64) *models.TaskTpl { + ttc.Lock() + defer ttc.Unlock() + + return ttc.tpls[id] +} + +func (ttc *TaskTplCache) SyncTaskTpl() { + if err := ttc.syncTaskTpl(); err != nil { + fmt.Println("failed to sync task tpls:", err) + exit(1) + } + go ttc.loopSyncTaskTpl() +} + +func (ttc *TaskTplCache) syncTaskTpl() error { + start := time.Now() + stat, err := models.TaskTplStatistics(ttc.ctx) + if err != nil { + dumper.PutSyncRecord("task_tpls", start.Unix(), -1, -1, "failed to query statistics: "+err.Error()) + return errors.WithMessage(err, "failed to exec TaskTplStatistics") + } + + if !ttc.StatChange(stat.Total, stat.LastUpdated) { + dumper.PutSyncRecord("task_tpls", start.Unix(), -1, -1, "not changed") + return nil + } + + lst, err := models.TaskTplGetAll(ttc.ctx) + if err != nil { + dumper.PutSyncRecord("task_tpls", start.Unix(), -1, -1, "failed to query records: "+err.Error()) + return errors.WithMessage(err, "failed to exec TaskTplGetAll") + } + + m := make(map[int64]*models.TaskTpl, len(lst)) + for _, tpl := range lst { + m[tpl.Id] = tpl + } + + ttc.Set(m, stat.Total, stat.LastUpdated) + + ms := time.Since(start).Milliseconds() + logger.Infof("timer: sync task tpls done, cost: %dms, number: %d", ms, len(m)) + dumper.PutSyncRecord("task_tpls", start.Unix(), ms, len(m), "success") + + return nil +} + +func (ttc *TaskTplCache) loopSyncTaskTpl() { + d := time.Duration(9) * time.Second + for { + time.Sleep(d) + if err := ttc.syncTaskTpl(); err != nil { + logger.Warning("failed to sync task tpl:", err) + } + } +} + +func (ttc *TaskTplCache) StatChange(total int64, lastUpdated int64) bool { + if ttc.statTotal == total && ttc.statLastUpdated == lastUpdated { + return false + } + + return true +} diff --git a/models/migrate/migrate.go b/models/migrate/migrate.go index 4a99c0d3..ba5c179d 100644 --- a/models/migrate/migrate.go +++ b/models/migrate/migrate.go @@ -1,8 +1,12 @@ package migrate import ( + "fmt" + "github.com/ccfos/nightingale/v6/models" "github.com/ccfos/nightingale/v6/pkg/ormx" + + imodels "github.com/flashcatcloud/ibex/src/models" "github.com/toolkits/pkg/logger" "gorm.io/gorm" ) @@ -12,6 +16,24 @@ func Migrate(db *gorm.DB) { MigrateEsIndexPatternTable(db) } +func MigrateIbexTables(db *gorm.DB) { + dts := []interface{}{&imodels.TaskMeta{}, &imodels.TaskScheduler{}, &imodels.TaskSchedulerHealth{}, &imodels.TaskHostDoing{}, &imodels.TaskAction{}} + for _, dt := range dts { + err := db.AutoMigrate(dt) + if err != nil { + logger.Errorf("failed to migrate table:%v %v", dt, err) + } + } + + for i := 0; i < 100; i++ { + tableName := fmt.Sprintf("task_host_%d", i) + err := db.Table(tableName).AutoMigrate(&imodels.TaskHost{}) + if err != nil { + logger.Errorf("failed to migrate table:%s %v", tableName, err) + } + } +} + func MigrateTables(db *gorm.DB) error { dts := []interface{}{&RecordingRule{}, &AlertRule{}, &AlertSubscribe{}, &AlertMute{}, &TaskRecord{}, &ChartShare{}, &Target{}, &Configs{}, &Datasource{}, &NotifyTpl{}, diff --git a/models/task_tpl.go b/models/task_tpl.go index feef5835..d3ff1025 100644 --- a/models/task_tpl.go +++ b/models/task_tpl.go @@ -60,6 +60,33 @@ func TaskTplTotal(ctx *ctx.Context, bgids []int64, query string) (int64, error) return Count(session) } +func TaskTplStatistics(ctx *ctx.Context) (*Statistics, error) { + if !ctx.IsCenter { + return poster.GetByUrls[*Statistics](ctx, "/v1/n9e/task-tpl/statistics") + } + + session := DB(ctx).Model(&TaskTpl{}).Select("count(*) as total", "max(update_at) as last_updated") + + var stats []*Statistics + err := session.Find(&stats).Error + if err != nil { + return nil, err + } + + return stats[0], nil +} + +func TaskTplGetAll(ctx *ctx.Context) ([]*TaskTpl, error) { + if !ctx.IsCenter { + return poster.GetByUrls[[]*TaskTpl](ctx, "/v1/n9e/task-tpls") + } + + lst := make([]*TaskTpl, 0) + err := DB(ctx).Find(&lst).Error + return lst, err + +} + func TaskTplGets(ctx *ctx.Context, bgids []int64, query string, limit, offset int) ([]TaskTpl, error) { session := DB(ctx).Order("title").Limit(limit).Offset(offset) if len(bgids) > 0 { @@ -316,3 +343,84 @@ func (t *TaskTpl) UpdateGroup(ctx *ctx.Context, groupId int64, updateBy string) "update_at": time.Now().Unix(), }).Error } + +type TaskForm struct { + Title string `json:"title"` + Account string `json:"account"` + Batch int `json:"batch"` + Tolerance int `json:"tolerance"` + Timeout int `json:"timeout"` + Pause string `json:"pause"` + Script string `json:"script"` + Args string `json:"args"` + Stdin string `json:"stdin"` + Action string `json:"action"` + Creator string `json:"creator"` + Hosts []string `json:"hosts"` + AlertTriggered bool `json:"alert_triggered"` +} + +func (f *TaskForm) Verify() error { + if f.Batch < 0 { + return fmt.Errorf("arg(batch) should be nonnegative") + } + + if f.Tolerance < 0 { + return fmt.Errorf("arg(tolerance) should be nonnegative") + } + + if f.Timeout < 0 { + return fmt.Errorf("arg(timeout) should be nonnegative") + } + + if f.Timeout > 3600*24 { + return fmt.Errorf("arg(timeout) longer than one day") + } + + if f.Timeout == 0 { + f.Timeout = 30 + } + + f.Pause = strings.Replace(f.Pause, "，", ",", -1) + f.Pause = strings.Replace(f.Pause, " ", "", -1) + f.Args = strings.Replace(f.Args, "，", ",", -1) + + if f.Title == "" { + return fmt.Errorf("arg(title) is required") + } + + if str.Dangerous(f.Title) { + return fmt.Errorf("arg(title) is dangerous") + } + + if f.Script == "" { + return fmt.Errorf("arg(script) is required") + } + f.Script = strings.Replace(f.Script, "\r\n", "\n", -1) + + if str.Dangerous(f.Args) { + return fmt.Errorf("arg(args) is dangerous") + } + + if str.Dangerous(f.Pause) { + return fmt.Errorf("arg(pause) is dangerous") + } + + if len(f.Hosts) == 0 { + return fmt.Errorf("arg(hosts) empty") + } + + if f.Action != "start" && f.Action != "pause" { + return fmt.Errorf("arg(action) invalid") + } + + return nil +} + +func (f *TaskForm) HandleFH(fh string) { + i := strings.Index(f.Title, " FH: ") + if i > 0 { + f.Title = f.Title[:i] + } + f.Title = f.Title + " FH: " + fh +}