nightingale/timer/alert_rule.go

package timer

import (
	"fmt"
	"math/rand"
	"time"

	"github.com/didi/nightingale/v5/cache"
	"github.com/didi/nightingale/v5/models"

	"github.com/toolkits/pkg/logger"
)

func SyncAlertRules() {
	if err := syncAlertRules(); err != nil {
		fmt.Println(err)
		exit(1)
	}

	go loopSyncAlertRules()
}

func loopSyncAlertRules() {
	randtime := rand.Intn(9000)
	fmt.Printf("timer: sync alert rules: random sleep %dms\n", randtime)
	time.Sleep(time.Duration(randtime) * time.Millisecond)

	interval := time.Duration(9) * time.Second

	for {
		time.Sleep(interval)
		if err := syncAlertRules(); err != nil {
			logger.Warning(err)
		}
	}
}

func syncAlertRules() error {
	start := time.Now()

	// 上次同步的时候同步了多少条rule，数据库中最近的更新时间，如果这俩信息都没变，说明DB中数据没变
	// 数据库中数据没变，那就不用再做操作了
	lastMaxUpdateTs := cache.AlertRulesByMetric.MaxUpdateTs
	ruleNum := cache.AlertRulesByMetric.RuleNum

	statistic, err := models.GetAlertRuleStatistic()
	if err != nil {
		return fmt.Errorf("sync alertRules getAlertRuleStatistics err: %v", err)
	}

	if statistic.Count == ruleNum && statistic.MaxUpdateAt == lastMaxUpdateTs {
		lastMaxUpdateStr := time.Unix(lastMaxUpdateTs, 0).Format("2006-01-02 15:04:05")
		logger.Debugf("[no_change_not_sync][LastUpdateAt:%+v][ruleNum:%+v]:", lastMaxUpdateStr, ruleNum)
		return nil
	}

	// 数据库中的记录和上次拉取的数据相比，发生变化，重新从数据库拉取最新数据
	logger.Debugf("[alert_rule_change_start_sync][last_num:%d this_num:%d][last_max_update_ts:%d this_max_update_ts:%d]:",
		ruleNum,
		statistic.Count,
		lastMaxUpdateTs,
		statistic.MaxUpdateAt)

	alertRules, err := models.AllAlertRules()
	alertRulesMap := make(map[int64]*models.AlertRule)

	if err != nil {
		return fmt.Errorf("sync alertRules [type=all] err: %v", err)
	}

	metricAlertRulesMap := make(map[string][]*models.AlertRule)
	for i := range alertRules {
		if err := alertRules[i].Decode(); err != nil {
			// 单个rule无法decode，直接忽略继续处理别的，等后面用户修复好了，数据库last_update信息变化，这里自然能感知
			logger.Warningf("syncAlertRule %v err:%v", alertRules[i], err)
			continue
		}

		alertRulesMap[alertRules[i].Id] = alertRules[i]
		if alertRules[i].Type == models.PUSH {
			metricAlertRulesMap[alertRules[i].FirstMetric] = append(metricAlertRulesMap[alertRules[i].FirstMetric], alertRules[i])
		}
	}

	cache.AlertRules.SetAll(alertRulesMap)
	cache.AlertRulesByMetric.SetAll(metricAlertRulesMap, statistic.MaxUpdateAt, statistic.Count, start.UnixNano())
	logger.Infof("[timer] sync alert rules done, found %d records, cost: %dms", statistic.Count, time.Since(start).Milliseconds())

	return nil
}