From b47e7d12e6627c86aeab57f2e2ff3e39f6b13b8a Mon Sep 17 00:00:00 2001 From: Kyle Brandt Date: Fri, 28 May 2021 11:04:20 -0400 Subject: [PATCH] Alerting: Extract values from MD expr alerts (#34757) When using mulit-dimensional Grafana managed alerts (e.g. SSE math) extract refIds values and labels so they can be shown in the notification and dashboards. --- pkg/services/ngalert/eval/eval.go | 61 ++++++++++++++++++- pkg/services/ngalert/eval/extract_md.go | 59 ++++++++++++------ .../api/alerting/api_alertmanager_test.go | 4 +- .../alerting/api_notification_channel_test.go | 3 +- 4 files changed, 102 insertions(+), 25 deletions(-) diff --git a/pkg/services/ngalert/eval/eval.go b/pkg/services/ngalert/eval/eval.go index 593a5bd4da2..d775a034762 100644 --- a/pkg/services/ngalert/eval/eval.go +++ b/pkg/services/ngalert/eval/eval.go @@ -8,6 +8,7 @@ import ( "sort" "time" + "github.com/grafana/grafana/pkg/expr/classic" "github.com/grafana/grafana/pkg/services/ngalert/models" "github.com/grafana/grafana/pkg/setting" @@ -146,6 +147,12 @@ func GetExprRequest(ctx AlertExecCtx, data []models.AlertQuery, now time.Time) ( return req, nil } +type NumberValueCapture struct { + Var string // RefID + Labels data.Labels + Value *float64 +} + func executeCondition(ctx AlertExecCtx, c *models.Condition, now time.Time, dataService *tsdb.Service) ExecutionResults { result := ExecutionResults{} @@ -155,11 +162,59 @@ func executeCondition(ctx AlertExecCtx, c *models.Condition, now time.Time, data return ExecutionResults{Error: err} } + // eval captures for the '__value__' label. + captures := make([]NumberValueCapture, 0, len(execResp.Responses)) + + captureVal := func(refID string, labels data.Labels, value *float64) { + captures = append(captures, NumberValueCapture{ + Var: refID, + Value: value, + Labels: labels.Copy(), + }) + } + for refID, res := range execResp.Responses { - if refID != c.Condition { - continue + // for each frame within each response, the response can contain several data types including time-series data. + // For now, we favour simplicity and only care about single scalar values. + for _, frame := range res.Frames { + if len(frame.Fields) != 1 || frame.Fields[0].Type() != data.FieldTypeNullableFloat64 { + continue + } + var v *float64 + if frame.Fields[0].Len() == 1 { + v = frame.At(0, 0).(*float64) // type checked above + } + captureVal(frame.RefID, frame.Fields[0].Labels, v) + } + + if refID == c.Condition { + result.Results = res.Frames + } + } + + // add capture values as data frame metadata to each result (frame) that has matching labels. + for _, frame := range result.Results { + // classic conditions already have metadata set and only have one value, there's no need to add anything in this case. + if frame.Meta != nil && frame.Meta.Custom != nil { + if _, ok := frame.Meta.Custom.([]classic.EvalMatch); ok { + continue // do not overwrite EvalMatch from classic condition. + } + } + + frame.SetMeta(&data.FrameMeta{}) // overwrite metadata + + if len(frame.Fields) == 1 { + theseLabels := frame.Fields[0].Labels + for _, cap := range captures { + // matching labels are equal labels, or when one set of labels includes the labels of the other. + if theseLabels.Equals(cap.Labels) || theseLabels.Contains(cap.Labels) || cap.Labels.Contains(theseLabels) { + if frame.Meta.Custom == nil { + frame.Meta.Custom = []NumberValueCapture{} + } + frame.Meta.Custom = append(frame.Meta.Custom.([]NumberValueCapture), cap) + } + } } - result.Results = res.Frames } return result diff --git a/pkg/services/ngalert/eval/extract_md.go b/pkg/services/ngalert/eval/extract_md.go index 8516d1e1daf..fd515c74bcb 100644 --- a/pkg/services/ngalert/eval/extract_md.go +++ b/pkg/services/ngalert/eval/extract_md.go @@ -17,30 +17,51 @@ func extractEvalString(frame *data.Frame) (s string) { return } - evalMatches, ok := frame.Meta.Custom.([]classic.EvalMatch) - if !ok { - return + if evalMatches, ok := frame.Meta.Custom.([]classic.EvalMatch); ok { + sb := strings.Builder{} + + for i, m := range evalMatches { + sb.WriteString("[ ") + sb.WriteString(fmt.Sprintf("metric='%s' ", m.Metric)) + sb.WriteString(fmt.Sprintf("labels={%s} ", m.Labels)) + + valString := "null" + if m.Value != nil { + valString = fmt.Sprintf("%v", *m.Value) + } + + sb.WriteString(fmt.Sprintf("value=%v ", valString)) + + sb.WriteString("]") + if i < len(evalMatches)-1 { + sb.WriteString(", ") + } + } + return sb.String() } - sb := strings.Builder{} + if caps, ok := frame.Meta.Custom.([]NumberValueCapture); ok { + sb := strings.Builder{} - for i, m := range evalMatches { - sb.WriteString("[ ") - sb.WriteString(fmt.Sprintf("metric='%s' ", m.Metric)) - sb.WriteString(fmt.Sprintf("labels={%s} ", m.Labels)) + for i, c := range caps { + sb.WriteString("[ ") + sb.WriteString(fmt.Sprintf("var='%s' ", c.Var)) + sb.WriteString(fmt.Sprintf("labels={%s} ", c.Labels)) - valString := "null" - if m.Value != nil { - valString = fmt.Sprintf("%v", *m.Value) - } - - sb.WriteString(fmt.Sprintf("value=%v ", valString)) - - sb.WriteString("]") - if i < len(evalMatches)-1 { - sb.WriteString(", ") + valString := "null" + if c.Value != nil { + valString = fmt.Sprintf("%v", *c.Value) + } + + sb.WriteString(fmt.Sprintf("value=%v ", valString)) + + sb.WriteString("]") + if i < len(caps)-1 { + sb.WriteString(", ") + } } + return sb.String() } - return sb.String() + return "" } diff --git a/pkg/tests/api/alerting/api_alertmanager_test.go b/pkg/tests/api/alerting/api_alertmanager_test.go index 15c9b56c25c..f3a23f2defb 100644 --- a/pkg/tests/api/alerting/api_alertmanager_test.go +++ b/pkg/tests/api/alerting/api_alertmanager_test.go @@ -1613,7 +1613,7 @@ func TestEval(t *testing.T) { "Alerting" ], [ - "" + "[ var='A' labels={} value=1 ]" ] ] } @@ -1674,7 +1674,7 @@ func TestEval(t *testing.T) { "Normal" ], [ - "" + "[ var='A' labels={} value=0 ]" ] ] } diff --git a/pkg/tests/api/alerting/api_notification_channel_test.go b/pkg/tests/api/alerting/api_notification_channel_test.go index 11806c824ab..ba9a57d3a38 100644 --- a/pkg/tests/api/alerting/api_notification_channel_test.go +++ b/pkg/tests/api/alerting/api_notification_channel_test.go @@ -1454,7 +1454,7 @@ var expNotifications = map[string][]string{ "startsAt": "%s", "endsAt": "0001-01-01T00:00:00Z", "generatorURL": "", - "fingerprint": "929467973978d053", + "fingerprint": "7611eef9e67f6e50", "silenceURL": "http://localhost:3000/alerting/silence/new?alertmanager=grafana&matchers=alertname%%3DWebhookAlert", "dashboardURL": "", "panelURL": "" @@ -1618,6 +1618,7 @@ var expNotifications = map[string][]string{ { "labels": { "__alert_rule_uid__": "UID_AlertmanagerAlert", + "__value__": "[ var='A' labels={} value=1 ]", "alertname": "AlertmanagerAlert" }, "annotations": {},