From b1a8c676894cd6ac1fcbdd4f164ed9511bd980d5 Mon Sep 17 00:00:00 2001 From: David Parrott Date: Tue, 4 May 2021 10:08:12 -0700 Subject: [PATCH] Alerting return evaluation errors to /rules (#33663) * Set and return errors produced by evaluation results * test fixup --- pkg/services/ngalert/api/api_prometheus.go | 7 ++++++- pkg/services/ngalert/state/state.go | 21 ++++----------------- pkg/services/ngalert/tests/manager_test.go | 16 ++++++++-------- 3 files changed, 18 insertions(+), 26 deletions(-) diff --git a/pkg/services/ngalert/api/api_prometheus.go b/pkg/services/ngalert/api/api_prometheus.go index 178ae3de788..b972a93c912 100644 --- a/pkg/services/ngalert/api/api_prometheus.go +++ b/pkg/services/ngalert/api/api_prometheus.go @@ -100,7 +100,7 @@ func (srv PrometheusSrv) RouteGetRuleStatuses(c *models.ReqContext) response.Res alertingRule := apimodels.AlertingRule{ State: "inactive", Name: rule.Title, - Query: queryStr, // TODO: don't escape <>& etc + Query: queryStr, Duration: rule.For.Seconds(), Annotations: rule.Annotations, } @@ -144,6 +144,11 @@ func (srv PrometheusSrv) RouteGetRuleStatuses(c *models.ReqContext) response.Res case eval.NoData: newRule.Health = "nodata" } + + if alertState.Error != nil { + newRule.LastError = alertState.Error.Error() + newRule.Health = "error" + } alertingRule.Alerts = append(alertingRule.Alerts, alert) } diff --git a/pkg/services/ngalert/state/state.go b/pkg/services/ngalert/state/state.go index 6db1de1ae82..7793db0649e 100644 --- a/pkg/services/ngalert/state/state.go +++ b/pkg/services/ngalert/state/state.go @@ -20,6 +20,7 @@ type State struct { LastEvaluationTime time.Time EvaluationDuration time.Duration Annotations map[string]string + Error error } type Evaluation struct { @@ -31,19 +32,13 @@ func resultNormal(alertState *State, result eval.Result) *State { newState := alertState if alertState.State != eval.Normal { newState.EndsAt = result.EvaluatedAt + newState.StartsAt = result.EvaluatedAt } + newState.Error = result.Error // should be nil since state is not error newState.State = eval.Normal return newState } -// addAnnotation adds an annotation to the state. -func (a *State) addAnnotation(key, value string) { - if a.Annotations == nil { - a.Annotations = make(map[string]string) - } - a.Annotations[key] = value -} - func (a *State) resultAlerting(alertRule *ngModels.AlertRule, result eval.Result) *State { switch a.State { case eval.Alerting: @@ -59,19 +54,16 @@ func (a *State) resultAlerting(alertRule *ngModels.AlertRule, result eval.Result a.State = eval.Alerting a.StartsAt = result.EvaluatedAt a.EndsAt = result.EvaluatedAt.Add(alertRule.For) - a.addAnnotation("alerting_at", result.EvaluatedAt.String()) } default: a.StartsAt = result.EvaluatedAt if !(alertRule.For > 0) { a.EndsAt = result.EvaluatedAt.Add(time.Duration(alertRule.IntervalSeconds*2) * time.Second) a.State = eval.Alerting - a.addAnnotation("alerting_at", result.EvaluatedAt.String()) } else { a.EndsAt = result.EvaluatedAt.Add(alertRule.For) if result.EvaluatedAt.Sub(a.StartsAt) > alertRule.For { a.State = eval.Alerting - a.addAnnotation("alerting_at", result.EvaluatedAt.String()) } else { a.State = eval.Pending } @@ -81,6 +73,7 @@ func (a *State) resultAlerting(alertRule *ngModels.AlertRule, result eval.Result } func (a *State) resultError(alertRule *ngModels.AlertRule, result eval.Result) *State { + a.Error = result.Error if a.StartsAt.IsZero() { a.StartsAt = result.EvaluatedAt } @@ -89,9 +82,6 @@ func (a *State) resultError(alertRule *ngModels.AlertRule, result eval.Result) * } else { a.EndsAt = result.EvaluatedAt.Add(alertRule.For) } - if a.State != eval.Error { - a.addAnnotation("last_error", result.EvaluatedAt.String()) - } switch alertRule.ExecErrState { case ngModels.AlertingErrState: @@ -110,9 +100,6 @@ func (a *State) resultNoData(alertRule *ngModels.AlertRule, result eval.Result) } else { a.EndsAt = result.EvaluatedAt.Add(alertRule.For) } - if a.State != eval.NoData { - a.addAnnotation("no_data", result.EvaluatedAt.String()) - } switch alertRule.NoDataState { case ngModels.Alerting: diff --git a/pkg/services/ngalert/tests/manager_test.go b/pkg/services/ngalert/tests/manager_test.go index 5a7341790cf..dc91dcbaabe 100644 --- a/pkg/services/ngalert/tests/manager_test.go +++ b/pkg/services/ngalert/tests/manager_test.go @@ -268,7 +268,7 @@ func TestProcessEvalResults(t *testing.T) { EndsAt: evaluationTime.Add(1 * time.Minute).Add(time.Duration(20) * time.Second), LastEvaluationTime: evaluationTime.Add(1 * time.Minute), EvaluationDuration: evaluationDuration, - Annotations: map[string]string{"annotation": "test", "alerting_at": "2021-03-25 00:01:00 +0000 UTC"}, + Annotations: map[string]string{"annotation": "test"}, }, }, }, @@ -341,7 +341,7 @@ func TestProcessEvalResults(t *testing.T) { EndsAt: evaluationTime.Add(80 * time.Second).Add(1 * time.Minute), LastEvaluationTime: evaluationTime.Add(80 * time.Second), EvaluationDuration: evaluationDuration, - Annotations: map[string]string{"annotation": "test", "alerting_at": "2021-03-25 00:01:20 +0000 UTC"}, + Annotations: map[string]string{"annotation": "test"}, }, }, }, @@ -463,7 +463,7 @@ func TestProcessEvalResults(t *testing.T) { EndsAt: evaluationTime.Add(10 * time.Second).Add(20 * time.Second), LastEvaluationTime: evaluationTime.Add(10 * time.Second), EvaluationDuration: evaluationDuration, - Annotations: map[string]string{"annotation": "test", "no_data": "2021-03-25 00:00:10 +0000 UTC"}, + Annotations: map[string]string{"annotation": "test"}, }, }, }, @@ -524,7 +524,7 @@ func TestProcessEvalResults(t *testing.T) { EndsAt: evaluationTime.Add(10 * time.Second).Add(20 * time.Second), LastEvaluationTime: evaluationTime.Add(10 * time.Second), EvaluationDuration: evaluationDuration, - Annotations: map[string]string{"annotation": "test", "no_data": "2021-03-25 00:00:10 +0000 UTC"}, + Annotations: map[string]string{"annotation": "test"}, }, }, }, @@ -585,7 +585,7 @@ func TestProcessEvalResults(t *testing.T) { EndsAt: evaluationTime.Add(10 * time.Second).Add(20 * time.Second), LastEvaluationTime: evaluationTime.Add(10 * time.Second), EvaluationDuration: evaluationDuration, - Annotations: map[string]string{"annotation": "test", "no_data": "2021-03-25 00:00:10 +0000 UTC"}, + Annotations: map[string]string{"annotation": "test"}, }, }, }, @@ -647,7 +647,7 @@ func TestProcessEvalResults(t *testing.T) { EndsAt: evaluationTime.Add(10 * time.Second).Add(1 * time.Minute), LastEvaluationTime: evaluationTime.Add(10 * time.Second), EvaluationDuration: evaluationDuration, - Annotations: map[string]string{"annotation": "test", "no_data": "2021-03-25 00:00:10 +0000 UTC"}, + Annotations: map[string]string{"annotation": "test"}, }, }, }, @@ -709,7 +709,7 @@ func TestProcessEvalResults(t *testing.T) { EndsAt: evaluationTime.Add(10 * time.Second).Add(1 * time.Minute), LastEvaluationTime: evaluationTime.Add(10 * time.Second), EvaluationDuration: evaluationDuration, - Annotations: map[string]string{"annotation": "test", "no_data": "2021-03-25 00:00:10 +0000 UTC"}, + Annotations: map[string]string{"annotation": "test"}, }, }, }, @@ -771,7 +771,7 @@ func TestProcessEvalResults(t *testing.T) { EndsAt: evaluationTime.Add(10 * time.Second).Add(1 * time.Minute), LastEvaluationTime: evaluationTime.Add(10 * time.Second), EvaluationDuration: evaluationDuration, - Annotations: map[string]string{"annotation": "test", "no_data": "2021-03-25 00:00:10 +0000 UTC"}, + Annotations: map[string]string{"annotation": "test"}, }, }, },