diff --git a/pkg/services/ngalert/state/manager.go b/pkg/services/ngalert/state/manager.go index 1d77c273947..27781b1f533 100644 --- a/pkg/services/ngalert/state/manager.go +++ b/pkg/services/ngalert/state/manager.go @@ -17,6 +17,8 @@ import ( "github.com/grafana/grafana/pkg/services/ngalert/store" ) +var ResendDelay = 30 * time.Second + type Manager struct { log log.Logger metrics *metrics.Metrics @@ -33,7 +35,7 @@ func NewManager(logger log.Logger, metrics *metrics.Metrics, ruleStore store.Rul manager := &Manager{ cache: newCache(logger, metrics), quit: make(chan struct{}), - ResendDelay: 1 * time.Minute, // TODO: make this configurable + ResendDelay: ResendDelay, // TODO: make this configurable log: logger, metrics: metrics, ruleStore: ruleStore, diff --git a/pkg/services/ngalert/state/manager_test.go b/pkg/services/ngalert/state/manager_test.go index 465e7ab539d..0251f1ce86b 100644 --- a/pkg/services/ngalert/state/manager_test.go +++ b/pkg/services/ngalert/state/manager_test.go @@ -152,7 +152,7 @@ func TestProcessEvalResults(t *testing.T) { }, }, StartsAt: evaluationTime, - EndsAt: evaluationTime.Add(20 * time.Second), + EndsAt: evaluationTime.Add(state.ResendDelay * 3), LastEvaluationTime: evaluationTime, EvaluationDuration: evaluationDuration, Annotations: map[string]string{"annotation": "test"}, @@ -274,7 +274,7 @@ func TestProcessEvalResults(t *testing.T) { }, }, StartsAt: evaluationTime.Add(1 * time.Minute), - EndsAt: evaluationTime.Add(1 * time.Minute).Add(time.Duration(20) * time.Second), + EndsAt: evaluationTime.Add(1 * time.Minute).Add(state.ResendDelay * 3), LastEvaluationTime: evaluationTime.Add(1 * time.Minute), EvaluationDuration: evaluationDuration, Annotations: map[string]string{"annotation": "test"}, @@ -350,7 +350,7 @@ func TestProcessEvalResults(t *testing.T) { }, }, StartsAt: evaluationTime.Add(80 * time.Second), - EndsAt: evaluationTime.Add(80 * time.Second).Add(1 * time.Minute), + EndsAt: evaluationTime.Add(80 * time.Second).Add(state.ResendDelay * 3), LastEvaluationTime: evaluationTime.Add(80 * time.Second), EvaluationDuration: evaluationDuration, Annotations: map[string]string{"annotation": "test"}, @@ -413,7 +413,7 @@ func TestProcessEvalResults(t *testing.T) { }, }, StartsAt: evaluationTime.Add(10 * time.Second), - EndsAt: evaluationTime.Add(10 * time.Second).Add(1 * time.Minute), + EndsAt: evaluationTime.Add(10 * time.Second).Add(state.ResendDelay * 3), LastEvaluationTime: evaluationTime.Add(10 * time.Second), EvaluationDuration: evaluationDuration, Annotations: map[string]string{"annotation": "test"}, @@ -476,7 +476,7 @@ func TestProcessEvalResults(t *testing.T) { }, }, StartsAt: evaluationTime, - EndsAt: evaluationTime.Add(1 * time.Minute), + EndsAt: evaluationTime.Add(state.ResendDelay * 3), LastEvaluationTime: evaluationTime.Add(10 * time.Second), EvaluationDuration: evaluationDuration, Annotations: map[string]string{"annotation": "test"}, @@ -539,7 +539,7 @@ func TestProcessEvalResults(t *testing.T) { }, }, StartsAt: evaluationTime.Add(10 * time.Second), - EndsAt: evaluationTime.Add(10 * time.Second).Add(20 * time.Second), + EndsAt: evaluationTime.Add(10 * time.Second).Add(state.ResendDelay * 3), LastEvaluationTime: evaluationTime.Add(10 * time.Second), EvaluationDuration: evaluationDuration, Annotations: map[string]string{"annotation": "test"}, @@ -602,7 +602,7 @@ func TestProcessEvalResults(t *testing.T) { }, }, StartsAt: evaluationTime.Add(10 * time.Second), - EndsAt: evaluationTime.Add(10 * time.Second).Add(20 * time.Second), + EndsAt: evaluationTime.Add(10 * time.Second).Add(state.ResendDelay * 3), LastEvaluationTime: evaluationTime.Add(10 * time.Second), EvaluationDuration: evaluationDuration, Annotations: map[string]string{"annotation": "test"}, @@ -665,7 +665,7 @@ func TestProcessEvalResults(t *testing.T) { }, }, StartsAt: evaluationTime.Add(10 * time.Second), - EndsAt: evaluationTime.Add(10 * time.Second).Add(20 * time.Second), + EndsAt: evaluationTime.Add(10 * time.Second).Add(state.ResendDelay * 3), LastEvaluationTime: evaluationTime.Add(10 * time.Second), EvaluationDuration: evaluationDuration, Annotations: map[string]string{"annotation": "test"}, @@ -729,7 +729,7 @@ func TestProcessEvalResults(t *testing.T) { }, }, StartsAt: evaluationTime.Add(10 * time.Second), - EndsAt: evaluationTime.Add(10 * time.Second).Add(1 * time.Minute), + EndsAt: evaluationTime.Add(10 * time.Second).Add(state.ResendDelay * 3), LastEvaluationTime: evaluationTime.Add(10 * time.Second), EvaluationDuration: evaluationDuration, Annotations: map[string]string{"annotation": "test"}, @@ -793,7 +793,7 @@ func TestProcessEvalResults(t *testing.T) { }, }, StartsAt: evaluationTime.Add(10 * time.Second), - EndsAt: evaluationTime.Add(10 * time.Second).Add(1 * time.Minute), + EndsAt: evaluationTime.Add(10 * time.Second).Add(state.ResendDelay * 3), LastEvaluationTime: evaluationTime.Add(10 * time.Second), EvaluationDuration: evaluationDuration, Annotations: map[string]string{"annotation": "test"}, diff --git a/pkg/services/ngalert/state/state.go b/pkg/services/ngalert/state/state.go index 85f8f6f7b8f..85bd2e0081c 100644 --- a/pkg/services/ngalert/state/state.go +++ b/pkg/services/ngalert/state/state.go @@ -151,12 +151,15 @@ func (a *State) TrimResults(alertRule *ngModels.AlertRule) { a.Results = newResults } +// setEndsAt sets the ending timestamp of the alert. +// The internal Alertmanager will use this time to know when it should automatically resolve the alert +// in case it hasn't received additional alerts. Under regular operations the scheduler will continue to send the +// alert with an updated EndsAt, if the alert is resolved then a last alert is sent with EndsAt = last evaluation time. func (a *State) setEndsAt(alertRule *ngModels.AlertRule, result eval.Result) { - if int64(alertRule.For.Seconds()) > alertRule.IntervalSeconds { - // For is set and longer than IntervalSeconds - a.EndsAt = result.EvaluatedAt.Add(alertRule.For) - } else { - // For is not set or is less than or equal to IntervalSeconds - a.EndsAt = result.EvaluatedAt.Add(time.Duration(alertRule.IntervalSeconds*2) * time.Second) + ends := ResendDelay + if alertRule.IntervalSeconds > int64(ResendDelay.Seconds()) { + ends = time.Duration(alertRule.IntervalSeconds) } + + a.EndsAt = result.EvaluatedAt.Add(ends * 3) } diff --git a/pkg/services/ngalert/state/state_test.go b/pkg/services/ngalert/state/state_test.go index 89c0f663632..a22fa98cb39 100644 --- a/pkg/services/ngalert/state/state_test.go +++ b/pkg/services/ngalert/state/state_test.go @@ -4,9 +4,8 @@ import ( "testing" "time" - ngmodels "github.com/grafana/grafana/pkg/services/ngalert/models" - "github.com/grafana/grafana/pkg/services/ngalert/eval" + ngmodels "github.com/grafana/grafana/pkg/services/ngalert/models" "github.com/stretchr/testify/assert" ) @@ -114,87 +113,79 @@ func TestSetEndsAt(t *testing.T) { testCases := []struct { name string expected time.Time - testState *State testRule *ngmodels.AlertRule testResult eval.Result }{ { - name: "For: unset Interval: 10s EndsAt should be evaluation time + 2X IntervalSeconds", - expected: evaluationTime.Add(20 * time.Second), - testState: &State{}, + name: "less than resend delay: for=unset,interval=10s - endsAt = resendDelay * 3", + expected: evaluationTime.Add(ResendDelay * 3), testRule: &ngmodels.AlertRule{ IntervalSeconds: 10, }, - testResult: eval.Result{ - EvaluatedAt: evaluationTime, - }, }, { - name: "For: 0s Interval: 10s EndsAt should be evaluation time + 2X IntervalSeconds", - expected: evaluationTime.Add(20 * time.Second), - testState: &State{}, + name: "less than resend delay: for=0s,interval=10s - endsAt = resendDelay * 3", + expected: evaluationTime.Add(ResendDelay * 3), testRule: &ngmodels.AlertRule{ For: 0 * time.Second, IntervalSeconds: 10, }, - testResult: eval.Result{ - EvaluatedAt: evaluationTime, - }, }, { - name: "For: 1s Interval: 10s EndsAt should be evaluation time + 2X IntervalSeconds", - expected: evaluationTime.Add(20 * time.Second), - testState: &State{}, - testRule: &ngmodels.AlertRule{ - For: 0 * time.Second, - IntervalSeconds: 10, - }, - testResult: eval.Result{ - EvaluatedAt: evaluationTime, - }, - }, - { - name: "For: 10s Interval: 10s EndsAt should be evaluation time + 2X IntervalSeconds", - expected: evaluationTime.Add(20 * time.Second), - testState: &State{}, + name: "less than resend delay: for=10s,interval=10s - endsAt = resendDelay * 3", + expected: evaluationTime.Add(ResendDelay * 3), testRule: &ngmodels.AlertRule{ For: 10 * time.Second, IntervalSeconds: 10, }, - testResult: eval.Result{ - EvaluatedAt: evaluationTime, + }, + { + name: "less than resend delay: for=10s,interval=20s - endsAt = resendDelay * 3", + expected: evaluationTime.Add(ResendDelay * 3), + testRule: &ngmodels.AlertRule{ + For: 10 * time.Second, + IntervalSeconds: 20, }, }, { - name: "For: 11s Interval: 10s EndsAt should be evaluation time + For duration", - expected: evaluationTime.Add(11 * time.Second), - testState: &State{}, + name: "more than resend delay: for=unset,interval=1m - endsAt = interval * 3", + expected: evaluationTime.Add(60 * 3), testRule: &ngmodels.AlertRule{ - For: 11 * time.Second, - IntervalSeconds: 10, - }, - testResult: eval.Result{ - EvaluatedAt: evaluationTime, + IntervalSeconds: 60, }, }, { - name: "For: 20s Interval: 10s EndsAt should be evaluation time + For duration", - expected: evaluationTime.Add(20 * time.Second), - testState: &State{}, + name: "more than resend delay: for=0s,interval=1m - endsAt = resendDelay * 3", + expected: evaluationTime.Add(60 * 3), testRule: &ngmodels.AlertRule{ - For: 20 * time.Second, - IntervalSeconds: 10, + For: 0 * time.Second, + IntervalSeconds: 60, }, - testResult: eval.Result{ - EvaluatedAt: evaluationTime, + }, + { + name: "more than resend delay: for=1m,interval=5m - endsAt = interval * 3", + expected: evaluationTime.Add(300 * 3), + testRule: &ngmodels.AlertRule{ + For: 60 * time.Second, + IntervalSeconds: 300, + }, + }, + { + name: "more than resend delay: for=5m,interval=1m - endsAt = interval * 3", + expected: evaluationTime.Add(60 * 3), + testRule: &ngmodels.AlertRule{ + For: 300 * time.Second, + IntervalSeconds: 60, }, }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - tc.testState.setEndsAt(tc.testRule, tc.testResult) - assert.Equal(t, tc.expected, tc.testState.EndsAt) + s := &State{} + r := eval.Result{EvaluatedAt: evaluationTime} + s.setEndsAt(tc.testRule, r) + assert.Equal(t, tc.expected, s.EndsAt) }) } }