opensource/grafana
1
0
Fork 0
mirror of https://github.com/grafana/grafana.git synced 2025-08-03 06:06:33 +08:00
Code Issues Packages Projects Releases Wiki Activity

[Alerting]: Add alerting endpoint for Query Evaluation (#33174)

Browse Source 
* [Alerting]: Add alerting endpoint for Query Evaluation

* Fix passing down now parameter

* Add validations and test

* Fix eval queries and expressions test

* Add eval tests
This commit is contained in:
Sofia Papagiannaki
2021-04-21 22:44:50 +03:00
committed by GitHub
parent ed3f5e6ca3
commit b2288f7ef9
10 changed files with 4315 additions and 3765 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
pkg/expr/translate/translate.go
View File

@ -34,7 +34,7 @@ func DashboardAlertConditions(rawDCondJSON []byte, orgID int64) (*ngmodels.Condi
return nil, err
}
backendReq, err := eval.GetQueryDataRequest(eval.AlertExecCtx{ExpressionsEnabled: true}, ngCond, time.Unix(500, 0))
backendReq, err := eval.GetQueryDataRequest(eval.AlertExecCtx{ExpressionsEnabled: true}, ngCond.Data, time.Unix(500, 0))
if err != nil {
return nil, err

19
pkg/services/ngalert/api/api_testing.go
View File

@ -11,6 +11,7 @@ import (
"github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/services/datasources"
apimodels "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
"github.com/grafana/grafana/pkg/services/ngalert/eval"
"github.com/grafana/grafana/pkg/setting"
"github.com/grafana/grafana/pkg/tsdb"
"github.com/grafana/grafana/pkg/util"
@ -77,3 +78,21 @@ func (srv TestingApiSrv) RouteTestRuleConfig(c *models.ReqContext, body apimodel
nil,
)
}
func (srv TestingApiSrv) RouteEvalQueries(c *models.ReqContext, cmd apimodels.EvalQueriesPayload) response.Response {
now := cmd.Now
if now.IsZero() {
now = timeNow()
}
if err := validateQueriesAndExpressions(cmd.Data, c.SignedInUser, c.SkipCache, srv.DatasourceCache); err != nil {
return response.Error(http.StatusBadRequest, "invalid queries or expressions", err)
}
evaluator := eval.Evaluator{Cfg: srv.Cfg}
evalResults, err := evaluator.QueriesAndExpressionsEval(c.SignedInUser.OrgId, cmd.Data, now, srv.DataService)
if err != nil {
return response.Error(http.StatusBadRequest, "Failed to evaluate queries and expressions", err)
}
return response.JSONStreaming(http.StatusOK, evalResults)
}

2
pkg/services/ngalert/api/generated_base_api_testing.go
View File

@ -18,12 +18,14 @@ import (
)
type TestingApiService interface {
RouteEvalQueries(*models.ReqContext, apimodels.EvalQueriesPayload) response.Response
RouteTestReceiverConfig(*models.ReqContext, apimodels.ExtendedReceiver) response.Response
RouteTestRuleConfig(*models.ReqContext, apimodels.TestRulePayload) response.Response
}
func (api *API) RegisterTestingApiEndpoints(srv TestingApiService) {
api.RouteRegister.Group("", func(group routing.RouteRegister) {
group.Post(toMacaronPath("/api/v1/eval"), binding.Bind(apimodels.EvalQueriesPayload{}), routing.Wrap(srv.RouteEvalQueries))
group.Post(toMacaronPath("/api/v1/receiver/test/{Recipient}"), binding.Bind(apimodels.ExtendedReceiver{}), routing.Wrap(srv.RouteTestReceiverConfig))
group.Post(toMacaronPath("/api/v1/rule/test/{Recipient}"), binding.Bind(apimodels.TestRulePayload{}), routing.Wrap(srv.RouteTestRuleConfig))
}, middleware.ReqSignedIn)

46
pkg/services/ngalert/api/test-data/test.http
View File

@ -25,6 +25,52 @@ content-type: application/json
}
}
###
POST http://admin:admin@localhost:3000/api/v1/eval
content-type: application/json
{
"data": [
{
"refId": "A",
"queryType": "",
"relativeTimeRange": {
"from": 18000,
"to": 10800
},
"model": {
"datasourceUid": "000000004",
"intervalMs": 1000,
"maxDataPoints": 100,
"orgId": 0,
"refId": "A",
"scenarioId": "csv_metric_values",
"stringInput": "1,20,90,30,5,0"
}
},
{
"refId": "B",
"queryType": "",
"relativeTimeRange": {
"from": 18000,
"to": 10800
},
"model": {
"datasourceUid": "-100",
"expression": "$A",
"intervalMs": 2000,
"maxDataPoints": 200,
"orgId": 0,
"reducer": "mean",
"refId": "B",
"type": "reduce"
}
}
],
"now": "2021-04-11T14:38:14Z"
}
###
POST http://admin:admin@localhost:3000/api/v1/rule/test/{{lokiDatasourceID}}
content-type: application/json

31
pkg/services/ngalert/api/tooling/definitions/testing.go
View File

@ -3,6 +3,9 @@ package definitions
import (
"encoding/json"
"fmt"
"time"
"github.com/grafana/grafana-plugin-sdk-go/backend"
"github.com/grafana/grafana/pkg/services/ngalert/models"
"github.com/prometheus/alertmanager/config"
@ -37,6 +40,19 @@ import (
// Responses:
// 200: TestRuleResponse
// swagger:route Post /api/v1/eval testing RouteEvalQueries
//
// Test rule
//
// Consumes:
// - application/json
//
// Produces:
// - application/json
//
// Responses:
// 200: EvalQueriesResponse
// swagger:parameters RouteTestReceiverConfig
type TestReceiverRequest struct {
// in:body
@ -57,6 +73,18 @@ type TestRulePayload struct {
GrafanaManagedCondition *models.EvalAlertConditionCommand `json:"grafana_condition,omitempty"`
}
// swagger:parameters RouteEvalQueries
type EvalQueriesRequest struct {
// in:body
Body EvalQueriesPayload
}
// swagger:model
type EvalQueriesPayload struct {
Data []models.AlertQuery `json:"data"`
Now time.Time `json:"now"`
}
func (p *TestRulePayload) UnmarshalJSON(b []byte) error {
type plain TestRulePayload
if err := json.Unmarshal(b, (*plain)(p)); err != nil {
@ -96,6 +124,9 @@ type TestRuleResponse struct {
GrafanaAlertInstances AlertInstancesResponse `json:"grafana_alert_instances"`
}
// swagger:model
type EvalQueriesResponse = backend.QueryDataResponse
// swagger:model
type AlertInstancesResponse struct {
// Instances is an array of arrow encoded dataframes

7477
pkg/services/ngalert/api/tooling/post.json
View File

File diff suppressed because it is too large Load Diff

74
pkg/services/ngalert/api/tooling/spec.json
View File

@ -1,14 +1,4 @@
{
"consumes": [
"application/json"
],
"produces": [
"application/json"
],
"schemes": [
"http",
"https"
],
"swagger": "2.0",
"info": {
"description": "Package definitions includes the types required for generating or consuming an OpenAPI\nspec for the Unified Alerting API.\nDocumentation of the API.",
@ -717,6 +707,38 @@
}
}
},
"/api/v1/eval": {
"post": {
"description": "Test rule",
"consumes": [
"application/json"
],
"produces": [
"application/json"
],
"tags": [
"testing"
],
"operationId": "RouteEvalQueries",
"parameters": [
{
"name": "Body",
"in": "body",
"schema": {
"$ref": "#/definitions/EvalQueriesPayload"
}
}
],
"responses": {
"200": {
"description": "EvalQueriesResponse",
"schema": {
"$ref": "#/definitions/EvalQueriesResponse"
}
}
}
}
},
"/api/v1/receiver/test/{Recipient}": {
"post": {
"description": "Test receiver",
@ -1332,6 +1354,27 @@
},
"x-go-package": "github.com/grafana/grafana/pkg/services/ngalert/models"
},
"EvalQueriesPayload": {
"type": "object",
"properties": {
"data": {
"type": "array",
"items": {
"$ref": "#/definitions/AlertQuery"
},
"x-go-name": "Data"
},
"now": {
"type": "string",
"format": "date-time",
"x-go-name": "Now"
}
},
"x-go-package": "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
},
"EvalQueriesResponse": {
"$ref": "#/definitions/EvalQueriesResponse"
},
"ExtendedReceiver": {
"type": "object",
"properties": {
@ -1369,7 +1412,7 @@
"$ref": "#/definitions/ResponseDetails"
},
"GettableAlert": {
"$ref": "#/definitions/GettableAlert"
"$ref": "#/definitions/gettableAlert"
},
"GettableAlerts": {
"$ref": "#/definitions/GettableAlerts"
@ -1638,7 +1681,7 @@
"x-go-package": "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
},
"GettableSilence": {
"$ref": "#/definitions/gettableSilence"
"$ref": "#/definitions/GettableSilence"
},
"GettableSilences": {
"$ref": "#/definitions/GettableSilences"
@ -3204,7 +3247,7 @@
"description": "alerts",
"type": "array",
"items": {
"$ref": "#/definitions/gettableAlert"
"$ref": "#/definitions/GettableAlert"
},
"x-go-name": "Alerts"
},
@ -3775,10 +3818,5 @@
"x-go-name": "VersionInfo",
"x-go-package": "github.com/prometheus/alertmanager/api/v2/models"
}
},
"securityDefinitions": {
"basic": {
"type": "basic"
}
}
}

33
pkg/services/ngalert/api/util.go
View File

@ -194,6 +194,33 @@ func validateCondition(c ngmodels.Condition, user *models.SignedInUser, skipCach
return nil
}
func validateQueriesAndExpressions(data []ngmodels.AlertQuery, user *models.SignedInUser, skipCache bool, datasourceCache datasources.CacheService) error {
if len(data) == 0 {
return nil
}
for _, query := range data {
datasourceUID, err := query.GetDatasource()
if err != nil {
return err
}
isExpression, err := query.IsExpression()
if err != nil {
return err
}
if isExpression {
continue
}
_, err = datasourceCache.GetDatasourceByUID(datasourceUID, user, skipCache)
if err != nil {
return fmt.Errorf("failed to get datasource: %s: %w", datasourceUID, err)
}
}
return nil
}
func conditionEval(c *models.ReqContext, cmd ngmodels.EvalAlertConditionCommand, datasourceCache datasources.CacheService, dataService *tsdb.Service, cfg *setting.Cfg) response.Response {
evalCond := ngmodels.Condition{
Condition: cmd.Condition,
@ -210,14 +237,14 @@ func conditionEval(c *models.ReqContext, cmd ngmodels.EvalAlertConditionCommand,
}
evaluator := eval.Evaluator{Cfg: cfg}
evalResults, err := evaluator.ConditionEval(&evalCond, timeNow(), dataService)
evalResults, err := evaluator.ConditionEval(&evalCond, now, dataService)
if err != nil {
return response.Error(400, "Failed to evaluate conditions", err)
return response.Error(http.StatusBadRequest, "Failed to evaluate conditions", err)
}
frame := evalResults.AsDataFrame()
return response.JSONStreaming(200, util.DynMap{
return response.JSONStreaming(http.StatusOK, util.DynMap{
"instances": []*data.Frame{&frame},
})
}

58
pkg/services/ngalert/eval/eval.go
View File

@ -103,12 +103,7 @@ type AlertExecCtx struct {
}
// GetQueryDataRequest validates the condition and creates a backend.QueryDataRequest from it.
func GetQueryDataRequest(ctx AlertExecCtx, c *models.Condition, now time.Time) (*backend.QueryDataRequest, error) {
if !c.IsValid() {
return nil, fmt.Errorf("invalid conditions")
// TODO: Things probably
}
func GetQueryDataRequest(ctx AlertExecCtx, data []models.AlertQuery, now time.Time) (*backend.QueryDataRequest, error) {
queryDataReq := &backend.QueryDataRequest{
PluginContext: backend.PluginContext{
OrgID: ctx.OrgID,
@ -116,8 +111,8 @@ func GetQueryDataRequest(ctx AlertExecCtx, c *models.Condition, now time.Time) (
Queries: []backend.DataQuery{},
}
for i := range c.Data {
q := c.Data[i]
for i := range data {
q := data[i]
model, err := q.GetModel()
if err != nil {
return nil, fmt.Errorf("failed to get query model: %w", err)
@ -144,25 +139,16 @@ func GetQueryDataRequest(ctx AlertExecCtx, c *models.Condition, now time.Time) (
return queryDataReq, nil
}
// execute runs the Condition's expressions or queries.
func execute(ctx AlertExecCtx, c *models.Condition, now time.Time, dataService *tsdb.Service) (*ExecutionResults, error) {
func executeCondition(ctx AlertExecCtx, c *models.Condition, now time.Time, dataService *tsdb.Service) (*ExecutionResults, error) {
result := ExecutionResults{}
queryDataReq, err := GetQueryDataRequest(ctx, c, now)
execResp, err := executeQueriesAndExpressions(ctx, c.Data, now, dataService)
if err != nil {
return &result, err
}
exprService := expr.Service{
Cfg: &setting.Cfg{ExpressionsEnabled: ctx.ExpressionsEnabled},
DataService: dataService,
}
pbRes, err := exprService.TransformData(ctx.Ctx, queryDataReq)
if err != nil {
return &result, err
}
for refID, res := range pbRes.Responses {
for refID, res := range execResp.Responses {
if refID != c.Condition {
continue
}
@ -178,6 +164,19 @@ func execute(ctx AlertExecCtx, c *models.Condition, now time.Time, dataService *
return &result, nil
}
func executeQueriesAndExpressions(ctx AlertExecCtx, data []models.AlertQuery, now time.Time, dataService *tsdb.Service) (*backend.QueryDataResponse, error) {
queryDataReq, err := GetQueryDataRequest(ctx, data, now)
if err != nil {
return nil, err
}
exprService := expr.Service{
Cfg: &setting.Cfg{ExpressionsEnabled: ctx.ExpressionsEnabled},
DataService: dataService,
}
return exprService.TransformData(ctx.Ctx, queryDataReq)
}
// evaluateExecutionResult takes the ExecutionResult, and returns a frame where
// each column is a string type that holds a string representing its State.
func evaluateExecutionResult(results *ExecutionResults, ts time.Time) (Results, error) {
@ -275,7 +274,7 @@ func (e *Evaluator) ConditionEval(condition *models.Condition, now time.Time, da
alertExecCtx := AlertExecCtx{OrgID: condition.OrgID, Ctx: alertCtx, ExpressionsEnabled: e.Cfg.ExpressionsEnabled}
execResult, err := execute(alertExecCtx, condition, now, dataService)
execResult, err := executeCondition(alertExecCtx, condition, now, dataService)
if err != nil {
return nil, fmt.Errorf("failed to execute conditions: %w", err)
}
@ -286,3 +285,18 @@ func (e *Evaluator) ConditionEval(condition *models.Condition, now time.Time, da
}
return evalResults, nil
}
// QueriesAndExpressionsEval executes queries and expressions and returns the result.
func (e *Evaluator) QueriesAndExpressionsEval(orgID int64, data []models.AlertQuery, now time.Time, dataService *tsdb.Service) (*backend.QueryDataResponse, error) {
alertCtx, cancelFn := context.WithTimeout(context.Background(), alertingEvaluationTimeout)
defer cancelFn()
alertExecCtx := AlertExecCtx{OrgID: orgID, Ctx: alertCtx, ExpressionsEnabled: e.Cfg.ExpressionsEnabled}
execResult, err := executeQueriesAndExpressions(alertExecCtx, data, now, dataService)
if err != nil {
return nil, fmt.Errorf("failed to execute conditions: %w", err)
}
return execResult, nil
}

338
pkg/tests/api/alerting/api_alertmanager_test.go
View File

@ -797,6 +797,344 @@ func TestAlertRuleCRUD(t *testing.T) {
require.JSONEq(t, `{"message":"rule group deleted"}`, string(b))
})
}
// test eval conditions
testCases := []struct {
desc string
payload string
expectedStatusCode int
expectedResponse string
}{
{
desc: "alerting condition",
payload: `
{
"grafana_condition": {
"condition": "A",
"data": [
{
"refId": "A",
"relativeTimeRange": {
"from": 18000,
"to": 10800
},
"model": {
"datasourceUid": "-100",
"type":"math",
"expression":"1 < 2"
}
}
],
"now": "2021-04-11T14:38:14Z"
}
}
`,
expectedStatusCode: http.StatusOK,
expectedResponse: `{
"instances": [
{
"schema": {
"name": "evaluation results",
"fields": [
{
"name": "State",
"type": "string",
"typeInfo": {
"frame": "string"
}
}
]
},
"data": {
"values": [
[
"Alerting"
]
]
}
}
]
}`,
},
{
desc: "normal condition",
payload: `
{
"grafana_condition": {
"condition": "A",
"data": [
{
"refId": "A",
"relativeTimeRange": {
"from": 18000,
"to": 10800
},
"model": {
"datasourceUid": "-100",
"type":"math",
"expression":"1 > 2"
}
}
],
"now": "2021-04-11T14:38:14Z"
}
}
`,
expectedStatusCode: http.StatusOK,
expectedResponse: `{
"instances": [
{
"schema": {
"name": "evaluation results",
"fields": [
{
"name": "State",
"type": "string",
"typeInfo": {
"frame": "string"
}
}
]
},
"data": {
"values": [
[
"Normal"
]
]
}
}
]
}`,
},
{
desc: "condition not found in any query or expression",
payload: `
{
"grafana_condition": {
"condition": "B",
"data": [
{
"refId": "A",
"relativeTimeRange": {
"from": 18000,
"to": 10800
},
"model": {
"datasourceUid": "-100",
"type":"math",
"expression":"1 > 2"
}
}
],
"now": "2021-04-11T14:38:14Z"
}
}
`,
expectedStatusCode: http.StatusBadRequest,
expectedResponse: `{"error":"condition B not found in any query or expression","message":"invalid condition"}`,
},
{
desc: "unknown query datasource",
payload: `
{
"grafana_condition": {
"condition": "A",
"data": [
{
"refId": "A",
"relativeTimeRange": {
"from": 18000,
"to": 10800
},
"model": {
"datasourceUid": "unknown"
}
}
],
"now": "2021-04-11T14:38:14Z"
}
}
`,
expectedStatusCode: http.StatusBadRequest,
expectedResponse: `{"error":"failed to get datasource: unknown: data source not found","message":"invalid condition"}`,
},
}
for _, tc := range testCases {
t.Run(tc.desc, func(t *testing.T) {
u := fmt.Sprintf("http://%s/api/v1/rule/test/grafana", grafanaListedAddr)
r := strings.NewReader(tc.payload)
// nolint:gosec
resp, err := http.Post(u, "application/json", r)
require.NoError(t, err)
t.Cleanup(func() {
err := resp.Body.Close()
require.NoError(t, err)
})
b, err := ioutil.ReadAll(resp.Body)
require.NoError(t, err)
assert.Equal(t, tc.expectedStatusCode, resp.StatusCode)
require.JSONEq(t, tc.expectedResponse, string(b))
})
}
// test eval queries and expressions
testCases = []struct {
desc string
payload string
expectedStatusCode int
expectedResponse string
}{
{
desc: "alerting condition",
payload: `
{
"data": [
{
"refId": "A",
"relativeTimeRange": {
"from": 18000,
"to": 10800
},
"model": {
"datasourceUid": "-100",
"type":"math",
"expression":"1 < 2"
}
}
],
"now": "2021-04-11T14:38:14Z"
}
`,
expectedStatusCode: http.StatusOK,
expectedResponse: `{
"results": {
"A": {
"frames": [
{
"schema": {
"refId": "A",
"fields": [
{
"name": "A",
"type": "number",
"typeInfo": {
"frame": "float64",
"nullable": true
}
}
]
},
"data": {
"values": [
[
1
]
]
}
}
]
}
}
}`,
},
{
desc: "normal condition",
payload: `
{
"data": [
{
"refId": "A",
"relativeTimeRange": {
"from": 18000,
"to": 10800
},
"model": {
"datasourceUid": "-100",
"type":"math",
"expression":"1 > 2"
}
}
],
"now": "2021-04-11T14:38:14Z"
}
`,
expectedStatusCode: http.StatusOK,
expectedResponse: `{
"results": {
"A": {
"frames": [
{
"schema": {
"refId": "A",
"fields": [
{
"name": "A",
"type": "number",
"typeInfo": {
"frame": "float64",
"nullable": true
}
}
]
},
"data": {
"values": [
[
0
]
]
}
}
]
}
}
}`,
},
{
desc: "unknown query datasource",
payload: `
{
"data": [
{
"refId": "A",
"relativeTimeRange": {
"from": 18000,
"to": 10800
},
"model": {
"datasourceUid": "unknown"
}
}
],
"now": "2021-04-11T14:38:14Z"
}
`,
expectedStatusCode: http.StatusBadRequest,
expectedResponse: `{"error":"failed to get datasource: unknown: data source not found","message":"invalid queries or expressions"}`,
},
}
for _, tc := range testCases {
t.Run(tc.desc, func(t *testing.T) {
u := fmt.Sprintf("http://%s/api/v1/eval", grafanaListedAddr)
r := strings.NewReader(tc.payload)
// nolint:gosec
resp, err := http.Post(u, "application/json", r)
require.NoError(t, err)
t.Cleanup(func() {
err := resp.Body.Close()
require.NoError(t, err)
})
b, err := ioutil.ReadAll(resp.Body)
require.NoError(t, err)
assert.Equal(t, tc.expectedStatusCode, resp.StatusCode)
require.JSONEq(t, tc.expectedResponse, string(b))
})
}
}
// createFolder creates a folder for storing our alerts under. Grafana uses folders as a replacement for alert namespaces to match its permission model.