Files
grafana/pkg/setting/setting_unified_alerting.go
Sofia Papagiannaki f6f3a54742 Alerting: tune rule evaluation via configuration (#35623)
* Alerting: Configure max evaluation retries

* Alerting: Enforce minimum rule evaluation interval

* Alerting: Disable rule evaluation from configuration

* Update docs

* Alerting: Configure rule evaluation timeout

* Move options on unified_alerting config section

* Apply suggestions from code review

Co-authored-by: gotjosh <josue@grafana.com>
2021-09-28 13:00:16 +03:00

156 lines
6.6 KiB
Go

package setting
import (
"strings"
"time"
"github.com/grafana/grafana-plugin-sdk-go/backend/gtime"
"github.com/prometheus/alertmanager/cluster"
"gopkg.in/ini.v1"
)
const (
alertmanagerDefaultClusterAddr = "0.0.0.0:9094"
alertmanagerDefaultPeerTimeout = 15 * time.Second
alertmanagerDefaultGossipInterval = cluster.DefaultGossipInterval
alertmanagerDefaultPushPullInterval = cluster.DefaultPushPullInterval
alertmanagerDefaultConfigPollInterval = 60 * time.Second
// To start, the alertmanager needs at least one route defined.
// TODO: we should move this to Grafana settings and define this as the default.
alertmanagerDefaultConfiguration = `{
"alertmanager_config": {
"route": {
"receiver": "grafana-default-email"
},
"receivers": [{
"name": "grafana-default-email",
"grafana_managed_receiver_configs": [{
"uid": "",
"name": "email receiver",
"type": "email",
"isDefault": true,
"settings": {
"addresses": "<example@email.com>"
}
}]
}]
}
}
`
evaluatorDefaultEvaluationTimeout = 30 * time.Second
schedulerDefaultAdminConfigPollInterval = 60 * time.Second
schedulereDefaultExecuteAlerts = true
schedulerDefaultMaxAttempts = 3
schedulerDefaultLegacyMinInterval = 1
schedulerDefaultMinInterval = 10 * time.Second
)
type UnifiedAlertingSettings struct {
AdminConfigPollInterval time.Duration
AlertmanagerConfigPollInterval time.Duration
HAListenAddr string
HAAdvertiseAddr string
HAPeers []string
HAPeerTimeout time.Duration
HAGossipInterval time.Duration
HAPushPullInterval time.Duration
MaxAttempts int64
MinInterval time.Duration
EvaluationTimeout time.Duration
ExecuteAlerts bool
DefaultConfiguration string
}
// ReadUnifiedAlertingSettings reads both the `unified_alerting` and `alerting` sections of the configuration while preferring configuration the `alerting` section.
// It first reads the `unified_alerting` section, then looks for non-defaults on the `alerting` section and prefers those.
func (cfg *Cfg) ReadUnifiedAlertingSettings(iniFile *ini.File) error {
uaCfg := UnifiedAlertingSettings{}
ua := iniFile.Section("unified_alerting")
var err error
uaCfg.AdminConfigPollInterval, err = gtime.ParseDuration(valueAsString(ua, "admin_config_poll_interval", (schedulerDefaultAdminConfigPollInterval).String()))
if err != nil {
return err
}
uaCfg.AlertmanagerConfigPollInterval, err = gtime.ParseDuration(valueAsString(ua, "alertmanager_config_poll_interval", (alertmanagerDefaultConfigPollInterval).String()))
if err != nil {
return err
}
uaCfg.HAPeerTimeout, err = gtime.ParseDuration(valueAsString(ua, "ha_peer_timeout", (alertmanagerDefaultPeerTimeout).String()))
if err != nil {
return err
}
uaCfg.HAGossipInterval, err = gtime.ParseDuration(valueAsString(ua, "ha_gossip_interval", (alertmanagerDefaultGossipInterval).String()))
if err != nil {
return err
}
uaCfg.HAPushPullInterval, err = gtime.ParseDuration(valueAsString(ua, "ha_push_pull_interval", (alertmanagerDefaultPushPullInterval).String()))
if err != nil {
return err
}
uaCfg.HAListenAddr = ua.Key("ha_listen_address").MustString(alertmanagerDefaultClusterAddr)
uaCfg.HAAdvertiseAddr = ua.Key("ha_advertise_address").MustString("")
peers := ua.Key("ha_peers").MustString("")
uaCfg.HAPeers = make([]string, 0)
if peers != "" {
for _, peer := range strings.Split(peers, ",") {
peer = strings.TrimSpace(peer)
uaCfg.HAPeers = append(uaCfg.HAPeers, peer)
}
}
// TODO load from ini file
uaCfg.DefaultConfiguration = alertmanagerDefaultConfiguration
alerting := iniFile.Section("alerting")
uaExecuteAlerts := ua.Key("execute_alerts").MustBool(schedulereDefaultExecuteAlerts)
if uaExecuteAlerts { // unified option equals the default (true)
legacyExecuteAlerts := alerting.Key("execute_alerts").MustBool(schedulereDefaultExecuteAlerts)
if !legacyExecuteAlerts {
cfg.Logger.Warn("falling back to legacy setting of 'execute_alerts'; please use the configuration option in the `unified_alerting` section if Grafana 8 alerts are enabled.")
}
uaExecuteAlerts = legacyExecuteAlerts
}
uaCfg.ExecuteAlerts = uaExecuteAlerts
// if the unified alerting options equal the defaults, apply the respective legacy one
uaEvaluationTimeout, err := gtime.ParseDuration(valueAsString(ua, "evaluation_timeout", evaluatorDefaultEvaluationTimeout.String()))
if err != nil || uaEvaluationTimeout == evaluatorDefaultEvaluationTimeout { // unified option is invalid duration or equals the default
legaceEvaluationTimeout := time.Duration(alerting.Key("evaluation_timeout_seconds").MustInt64(int64(evaluatorDefaultEvaluationTimeout.Seconds()))) * time.Second
if legaceEvaluationTimeout != evaluatorDefaultEvaluationTimeout {
cfg.Logger.Warn("falling back to legacy setting of 'evaluation_timeout_seconds'; please use the configuration option in the `unified_alerting` section if Grafana 8 alerts are enabled.")
}
uaEvaluationTimeout = legaceEvaluationTimeout
}
uaCfg.EvaluationTimeout = uaEvaluationTimeout
uaMaxAttempts := ua.Key("max_attempts").MustInt64(schedulerDefaultMaxAttempts)
if uaMaxAttempts == schedulerDefaultMaxAttempts { // unified option or equals the default
legacyMaxAttempts := alerting.Key("max_attempts").MustInt64(schedulerDefaultMaxAttempts)
if legacyMaxAttempts != schedulerDefaultMaxAttempts {
cfg.Logger.Warn("falling back to legacy setting of 'max_attempts'; please use the configuration option in the `unified_alerting` section if Grafana 8 alerts are enabled.")
}
uaMaxAttempts = legacyMaxAttempts
}
uaCfg.MaxAttempts = uaMaxAttempts
uaMinInterval, err := gtime.ParseDuration(valueAsString(ua, "min_interval", schedulerDefaultMinInterval.String()))
if err != nil || uaMinInterval == schedulerDefaultMinInterval { // unified option is invalid duration or equals the default
// if the legacy option is invalid, fallback to 10 (unified alerting min interval default)
legacyMinInterval := time.Duration(alerting.Key("min_interval_seconds").MustInt64(int64(schedulerDefaultMinInterval.Seconds()))) * time.Second
if legacyMinInterval != schedulerDefaultLegacyMinInterval {
cfg.Logger.Warn("falling back to legacy setting of 'min_interval_seconds'; please use the configuration option in the `unified_alerting` section if Grafana 8 alerts are enabled.")
}
uaMinInterval = legacyMinInterval
}
uaCfg.MinInterval = uaMinInterval
cfg.UnifiedAlerting = uaCfg
return nil
}
func GetAlertmanagerDefaultConfiguration() string {
return alertmanagerDefaultConfiguration
}