From 8c31e25926eb25acfcf33e4c245ebebb7c94c908 Mon Sep 17 00:00:00 2001 From: Sofia Papagiannaki Date: Mon, 18 Jan 2021 20:57:17 +0200 Subject: [PATCH] AlertingNG: Save alert instances (#30223) * AlertingNG: Save alert instances Co-authored-by: Kyle Brandt * Rename alert instance fields/columns * Include definition title in listing alert instances * Delete instances when deleting defintion Co-authored-by: Kyle Brandt --- pkg/services/ngalert/api.go | 4 + pkg/services/ngalert/database.go | 11 +- pkg/services/ngalert/database_mig.go | 25 +++ pkg/services/ngalert/database_test.go | 28 ++- pkg/services/ngalert/instance.go | 96 +++++++++++ pkg/services/ngalert/instance_api.go | 17 ++ pkg/services/ngalert/instance_database.go | 115 ++++++++++++ .../ngalert/instance_database_test.go | 163 ++++++++++++++++++ pkg/services/ngalert/instance_labels.go | 105 +++++++++++ pkg/services/ngalert/ngalert.go | 2 + pkg/services/ngalert/schedule.go | 12 +- pkg/services/ngalert/schedule_test.go | 2 +- pkg/services/sqlstore/migrator/dialect.go | 7 + .../sqlstore/migrator/mysql_dialect.go | 25 +++ .../sqlstore/migrator/postgres_dialect.go | 37 ++++ .../sqlstore/migrator/sqlite_dialect.go | 38 ++++ 16 files changed, 675 insertions(+), 12 deletions(-) create mode 100644 pkg/services/ngalert/instance.go create mode 100644 pkg/services/ngalert/instance_api.go create mode 100644 pkg/services/ngalert/instance_database.go create mode 100644 pkg/services/ngalert/instance_database_test.go create mode 100644 pkg/services/ngalert/instance_labels.go diff --git a/pkg/services/ngalert/api.go b/pkg/services/ngalert/api.go index 12427dc70f1..90d1d17353c 100644 --- a/pkg/services/ngalert/api.go +++ b/pkg/services/ngalert/api.go @@ -27,6 +27,10 @@ func (ng *AlertNG) registerAPIEndpoints() { schedulerRouter.Post("/pause", routing.Wrap(ng.pauseScheduler)) schedulerRouter.Post("/unpause", routing.Wrap(ng.unpauseScheduler)) }, middleware.ReqOrgAdmin) + + ng.RouteRegister.Group("/api/alert-instances", func(alertInstances routing.RouteRegister) { + alertInstances.Get("", middleware.ReqSignedIn, routing.Wrap(ng.listAlertInstancesEndpoint)) + }) } // conditionEvalEndpoint handles POST /api/alert-definitions/eval. diff --git a/pkg/services/ngalert/database.go b/pkg/services/ngalert/database.go index 19fa92a929a..bde570c4d5e 100644 --- a/pkg/services/ngalert/database.go +++ b/pkg/services/ngalert/database.go @@ -26,12 +26,7 @@ func getAlertDefinitionByUID(sess *sqlstore.DBSession, alertDefinitionUID string // It returns models.ErrAlertDefinitionNotFound if no alert definition is found for the provided ID. func (ng *AlertNG) deleteAlertDefinitionByUID(cmd *deleteAlertDefinitionByUIDCommand) error { return ng.SQLStore.WithTransactionalDbSession(context.Background(), func(sess *sqlstore.DBSession) error { - res, err := sess.Exec("DELETE FROM alert_definition WHERE uid = ? AND org_id = ?", cmd.UID, cmd.OrgID) - if err != nil { - return err - } - - _, err = res.RowsAffected() + _, err := sess.Exec("DELETE FROM alert_definition WHERE uid = ? AND org_id = ?", cmd.UID, cmd.OrgID) if err != nil { return err } @@ -41,6 +36,10 @@ func (ng *AlertNG) deleteAlertDefinitionByUID(cmd *deleteAlertDefinitionByUIDCom return err } + _, err = sess.Exec("DELETE FROM alert_instance WHERE def_org_id = ? AND def_uid = ?", cmd.OrgID, cmd.UID) + if err != nil { + return err + } return nil }) } diff --git a/pkg/services/ngalert/database_mig.go b/pkg/services/ngalert/database_mig.go index 6d515a911d5..e7b868ba0d5 100644 --- a/pkg/services/ngalert/database_mig.go +++ b/pkg/services/ngalert/database_mig.go @@ -68,3 +68,28 @@ func addAlertDefinitionVersionMigrations(mg *migrator.Migrator) { mg.AddMigration("alter alert_definition_version table data column to mediumtext in mysql", migrator.NewRawSQLMigration(""). Mysql("ALTER TABLE alert_definition_version MODIFY data MEDIUMTEXT;")) } + +func alertInstanceMigration(mg *migrator.Migrator) { + alertInstance := migrator.Table{ + Name: "alert_instance", + Columns: []*migrator.Column{ + {Name: "def_org_id", Type: migrator.DB_BigInt, Nullable: false}, + {Name: "def_uid", Type: migrator.DB_NVarchar, Length: 40, Nullable: false, Default: "0"}, + {Name: "labels", Type: migrator.DB_Text, Nullable: false}, + {Name: "labels_hash", Type: migrator.DB_NVarchar, Length: 190, Nullable: false}, + {Name: "current_state", Type: migrator.DB_NVarchar, Length: 190, Nullable: false}, + {Name: "current_state_since", Type: migrator.DB_BigInt, Nullable: false}, + {Name: "last_eval_time", Type: migrator.DB_BigInt, Nullable: false}, + }, + PrimaryKeys: []string{"def_org_id", "def_uid", "labels_hash"}, + Indices: []*migrator.Index{ + {Cols: []string{"def_org_id", "def_uid", "current_state"}, Type: migrator.IndexType}, + {Cols: []string{"def_org_id", "current_state"}, Type: migrator.IndexType}, + }, + } + + // create table + mg.AddMigration("create alert_instance table", migrator.NewAddTableMigration(alertInstance)) + mg.AddMigration("add index in alert_instance table on def_org_id, def_uid and current_state columns", migrator.NewAddIndexMigration(alertInstance, alertInstance.Indices[0])) + mg.AddMigration("add index in alert_instance table on def_org_id, current_state columns", migrator.NewAddIndexMigration(alertInstance, alertInstance.Indices[1])) +} diff --git a/pkg/services/ngalert/database_test.go b/pkg/services/ngalert/database_test.go index 8a43f60e492..9b0bd54a17a 100644 --- a/pkg/services/ngalert/database_test.go +++ b/pkg/services/ngalert/database_test.go @@ -242,7 +242,8 @@ func TestUpdatingAlertDefinition(t *testing.T) { assert.Equal(t, previousAlertDefinition.UID, q.Result.UID) default: require.NoError(t, err) - assert.Equal(t, int64(1), q.Result.ID) + assert.Equal(t, previousAlertDefinition.ID, q.Result.ID) + assert.Equal(t, previousAlertDefinition.UID, q.Result.UID) assert.True(t, q.Result.Updated.After(lastUpdated)) assert.Equal(t, tc.expectedUpdated, q.Result.Updated) assert.Equal(t, previousAlertDefinition.Version+1, q.Result.Version) @@ -292,8 +293,31 @@ func TestDeletingAlertDefinition(t *testing.T) { OrgID: 1, } - err := ng.deleteAlertDefinitionByUID(&q) + // save an instance for the definition + saveCmd := &saveAlertInstanceCommand{ + DefinitionOrgID: alertDefinition.OrgID, + DefinitionUID: alertDefinition.UID, + State: InstanceStateFiring, + Labels: InstanceLabels{"test": "testValue"}, + } + err := ng.saveAlertInstance(saveCmd) require.NoError(t, err) + listCommand := &listAlertInstancesQuery{ + DefinitionOrgID: alertDefinition.OrgID, + DefinitionUID: alertDefinition.UID, + } + err = ng.listAlertInstances(listCommand) + require.NoError(t, err) + require.Len(t, listCommand.Result, 1) + + err = ng.deleteAlertDefinitionByUID(&q) + require.NoError(t, err) + + // assert that alert instance is deleted + err = ng.listAlertInstances(listCommand) + require.NoError(t, err) + + require.Len(t, listCommand.Result, 0) }) } diff --git a/pkg/services/ngalert/instance.go b/pkg/services/ngalert/instance.go new file mode 100644 index 00000000000..69a5008d5a9 --- /dev/null +++ b/pkg/services/ngalert/instance.go @@ -0,0 +1,96 @@ +package ngalert + +import ( + "fmt" + "time" +) + +// AlertInstance represents a single alert instance. +type AlertInstance struct { + DefinitionOrgID int64 `xorm:"def_org_id"` + DefinitionUID string `xorm:"def_uid"` + Labels InstanceLabels + LabelsHash string + CurrentState InstanceStateType + CurrentStateSince time.Time + LastEvalTime time.Time +} + +// InstanceStateType is an enum for instance states. +type InstanceStateType string + +const ( + // InstanceStateFiring is for a firing alert. + InstanceStateFiring InstanceStateType = "Alerting" + // InstanceStateNormal is for a normal alert. + InstanceStateNormal InstanceStateType = "Normal" +) + +// IsValid checks that the value of InstanceStateType is a valid +// string. +func (i InstanceStateType) IsValid() bool { + return i == InstanceStateFiring || + i == InstanceStateNormal +} + +// saveAlertInstanceCommand is the query for saving a new alert instance. +type saveAlertInstanceCommand struct { + DefinitionOrgID int64 + DefinitionUID string + Labels InstanceLabels + State InstanceStateType + LastEvalTime time.Time +} + +// getAlertDefinitionByIDQuery is the query for retrieving/deleting an alert definition by ID. +// nolint:unused +type getAlertInstanceQuery struct { + DefinitionOrgID int64 + DefinitionUID string + Labels InstanceLabels + + Result *AlertInstance +} + +// listAlertInstancesCommand is the query list alert Instances. +type listAlertInstancesQuery struct { + DefinitionOrgID int64 `json:"-"` + DefinitionUID string + State InstanceStateType + + Result []*listAlertInstancesQueryResult +} + +// listAlertInstancesQueryResult represents the result of listAlertInstancesQuery. +type listAlertInstancesQueryResult struct { + DefinitionOrgID int64 `xorm:"def_org_id"` + DefinitionUID string `xorm:"def_uid"` + DefinitionTitle string `xorm:"def_title"` + Labels InstanceLabels + LabelsHash string + CurrentState InstanceStateType + CurrentStateSince time.Time + LastEvalTime time.Time +} + +// validateAlertInstance validates that the alert instance contains an alert definition id, +// and state. +func validateAlertInstance(alertInstance *AlertInstance) error { + if alertInstance == nil { + return fmt.Errorf("alert instance is invalid because it is nil") + } + + if alertInstance.DefinitionOrgID == 0 { + return fmt.Errorf("alert instance is invalid due to missing alert definition organisation") + } + + if alertInstance.DefinitionUID == "" { + return fmt.Errorf("alert instance is invalid due to missing alert definition uid") + } + + if !alertInstance.CurrentState.IsValid() { + return fmt.Errorf("alert instance is invalid because the state '%v' is invalid", alertInstance.CurrentState) + } + + return nil +} diff --git a/pkg/services/ngalert/instance_api.go b/pkg/services/ngalert/instance_api.go new file mode 100644 index 00000000000..26788458fb7 --- /dev/null +++ b/pkg/services/ngalert/instance_api.go @@ -0,0 +1,17 @@ +package ngalert + +import ( + "github.com/grafana/grafana/pkg/api/response" + "github.com/grafana/grafana/pkg/models" +) + +// listAlertInstancesEndpoint handles GET /api/alert-instances. +func (ng *AlertNG) listAlertInstancesEndpoint(c *models.ReqContext) response.Response { + cmd := listAlertInstancesQuery{DefinitionOrgID: c.SignedInUser.OrgId} + + if err := ng.listAlertInstances(&cmd); err != nil { + return response.Error(500, "Failed to list alert instances", err) + } + + return response.JSON(200, cmd.Result) +} diff --git a/pkg/services/ngalert/instance_database.go b/pkg/services/ngalert/instance_database.go new file mode 100644 index 00000000000..32d87979c0f --- /dev/null +++ b/pkg/services/ngalert/instance_database.go @@ -0,0 +1,115 @@ +package ngalert + +import ( + "context" + "fmt" + "strings" + "time" + + "github.com/grafana/grafana/pkg/services/sqlstore" +) + +// getAlertInstance is a handler for retrieving an alert instance based on OrgId, AlertDefintionID, and +// the hash of the labels. +// nolint:unused +func (ng *AlertNG) getAlertInstance(cmd *getAlertInstanceQuery) error { + return ng.SQLStore.WithDbSession(context.Background(), func(sess *sqlstore.DBSession) error { + instance := AlertInstance{} + s := strings.Builder{} + s.WriteString(`SELECT * FROM alert_instance + WHERE + def_org_id=? AND + def_uid=? AND + labels_hash=? + `) + + _, hash, err := cmd.Labels.StringAndHash() + if err != nil { + return err + } + + params := append(make([]interface{}, 0), cmd.DefinitionOrgID, cmd.DefinitionUID, hash) + + has, err := sess.SQL(s.String(), params...).Get(&instance) + if !has { + return fmt.Errorf("instance not found for labels %v (hash: %v), alert definition %v (org %v)", cmd.Labels, hash, cmd.DefinitionUID, cmd.DefinitionOrgID) + } + if err != nil { + return err + } + + cmd.Result = &instance + return nil + }) +} + +// listAlertInstances is a handler for retrieving alert instances within specific organisation +// based on various filters. +func (ng *AlertNG) listAlertInstances(cmd *listAlertInstancesQuery) error { + return ng.SQLStore.WithDbSession(context.Background(), func(sess *sqlstore.DBSession) error { + alertInstances := make([]*listAlertInstancesQueryResult, 0) + + s := strings.Builder{} + params := make([]interface{}, 0) + + addToQuery := func(stmt string, p ...interface{}) { + s.WriteString(stmt) + params = append(params, p...) + } + + addToQuery("SELECT alert_instance.*, alert_definition.title AS def_title FROM alert_instance LEFT JOIN alert_definition ON alert_instance.def_org_id = alert_definition.org_id AND alert_instance.def_uid = alert_definition.uid WHERE def_org_id = ?", cmd.DefinitionOrgID) + + if cmd.DefinitionUID != "" { + addToQuery(` AND def_uid = ?`, cmd.DefinitionUID) + } + + if cmd.State != "" { + addToQuery(` AND current_state = ?`, cmd.State) + } + + if err := sess.SQL(s.String(), params...).Find(&alertInstances); err != nil { + return err + } + + cmd.Result = alertInstances + return nil + }) +} + +// saveAlertDefinition is a handler for saving a new alert definition. +// nolint:unused +func (ng *AlertNG) saveAlertInstance(cmd *saveAlertInstanceCommand) error { + return ng.SQLStore.WithDbSession(context.Background(), func(sess *sqlstore.DBSession) error { + labelTupleJSON, labelsHash, err := cmd.Labels.StringAndHash() + if err != nil { + return err + } + + alertInstance := &AlertInstance{ + DefinitionOrgID: cmd.DefinitionOrgID, + DefinitionUID: cmd.DefinitionUID, + Labels: cmd.Labels, + LabelsHash: labelsHash, + CurrentState: cmd.State, + CurrentStateSince: time.Now(), + LastEvalTime: cmd.LastEvalTime, + } + + if err := validateAlertInstance(alertInstance); err != nil { + return err + } + + params := append(make([]interface{}, 0), alertInstance.DefinitionOrgID, alertInstance.DefinitionUID, labelTupleJSON, alertInstance.LabelsHash, alertInstance.CurrentState, alertInstance.CurrentStateSince.Unix(), alertInstance.LastEvalTime.Unix()) + + upsertSQL := ng.SQLStore.Dialect.UpsertSQL( + "alert_instance", + []string{"def_org_id", "def_uid", "labels_hash"}, + []string{"def_org_id", "def_uid", "labels", "labels_hash", "current_state", "current_state_since", "last_eval_time"}) + _, err = sess.SQL(upsertSQL, params...).Query() + if err != nil { + return err + } + + return nil + }) +} diff --git a/pkg/services/ngalert/instance_database_test.go b/pkg/services/ngalert/instance_database_test.go new file mode 100644 index 00000000000..8c522dd1ce9 --- /dev/null +++ b/pkg/services/ngalert/instance_database_test.go @@ -0,0 +1,163 @@ +// +build integration + +package ngalert + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestAlertInstanceOperations(t *testing.T) { + ng := setupTestEnv(t) + + alertDefinition1 := createTestAlertDefinition(t, ng, 60) + orgID := alertDefinition1.OrgID + + alertDefinition2 := createTestAlertDefinition(t, ng, 60) + require.Equal(t, orgID, alertDefinition2.OrgID) + + alertDefinition3 := createTestAlertDefinition(t, ng, 60) + require.Equal(t, orgID, alertDefinition3.OrgID) + + alertDefinition4 := createTestAlertDefinition(t, ng, 60) + require.Equal(t, orgID, alertDefinition4.OrgID) + + t.Run("can save and read new alert instance", func(t *testing.T) { + saveCmd := &saveAlertInstanceCommand{ + DefinitionOrgID: alertDefinition1.OrgID, + DefinitionUID: alertDefinition1.UID, + State: InstanceStateFiring, + Labels: InstanceLabels{"test": "testValue"}, + } + err := ng.saveAlertInstance(saveCmd) + require.NoError(t, err) + + getCmd := &getAlertInstanceQuery{ + DefinitionOrgID: saveCmd.DefinitionOrgID, + DefinitionUID: saveCmd.DefinitionUID, + Labels: InstanceLabels{"test": "testValue"}, + } + + err = ng.getAlertInstance(getCmd) + require.NoError(t, err) + + require.Equal(t, saveCmd.Labels, getCmd.Result.Labels) + require.Equal(t, alertDefinition1.OrgID, getCmd.Result.DefinitionOrgID) + require.Equal(t, alertDefinition1.UID, getCmd.Result.DefinitionUID) + }) + + t.Run("can save and read new alert instance with no labels", func(t *testing.T) { + saveCmd := &saveAlertInstanceCommand{ + DefinitionOrgID: alertDefinition2.OrgID, + DefinitionUID: alertDefinition2.UID, + State: InstanceStateNormal, + } + err := ng.saveAlertInstance(saveCmd) + require.NoError(t, err) + + getCmd := &getAlertInstanceQuery{ + DefinitionOrgID: saveCmd.DefinitionOrgID, + DefinitionUID: saveCmd.DefinitionUID, + } + + err = ng.getAlertInstance(getCmd) + require.NoError(t, err) + + require.Equal(t, alertDefinition2.OrgID, getCmd.Result.DefinitionOrgID) + require.Equal(t, alertDefinition2.UID, getCmd.Result.DefinitionUID) + require.Equal(t, saveCmd.Labels, getCmd.Result.Labels) + }) + + t.Run("can save two instances with same org_id, uid and different labels", func(t *testing.T) { + saveCmdOne := &saveAlertInstanceCommand{ + DefinitionOrgID: alertDefinition3.OrgID, + DefinitionUID: alertDefinition3.UID, + State: InstanceStateFiring, + Labels: InstanceLabels{"test": "testValue"}, + } + + err := ng.saveAlertInstance(saveCmdOne) + require.NoError(t, err) + + saveCmdTwo := &saveAlertInstanceCommand{ + DefinitionOrgID: saveCmdOne.DefinitionOrgID, + DefinitionUID: saveCmdOne.DefinitionUID, + State: InstanceStateFiring, + Labels: InstanceLabels{"test": "meow"}, + } + err = ng.saveAlertInstance(saveCmdTwo) + require.NoError(t, err) + + listCommand := &listAlertInstancesQuery{ + DefinitionOrgID: saveCmdOne.DefinitionOrgID, + DefinitionUID: saveCmdOne.DefinitionUID, + } + + err = ng.listAlertInstances(listCommand) + require.NoError(t, err) + + require.Len(t, listCommand.Result, 2) + }) + + t.Run("can list all added instances in org", func(t *testing.T) { + listCommand := &listAlertInstancesQuery{ + DefinitionOrgID: orgID, + } + + err := ng.listAlertInstances(listCommand) + require.NoError(t, err) + + require.Len(t, listCommand.Result, 4) + }) + + t.Run("can list all added instances in org filtered by current state", func(t *testing.T) { + listCommand := &listAlertInstancesQuery{ + DefinitionOrgID: orgID, + State: InstanceStateNormal, + } + + err := ng.listAlertInstances(listCommand) + require.NoError(t, err) + + require.Len(t, listCommand.Result, 1) + }) + + t.Run("update instance with same org_id, uid and different labels", func(t *testing.T) { + saveCmdOne := &saveAlertInstanceCommand{ + DefinitionOrgID: alertDefinition4.OrgID, + DefinitionUID: alertDefinition4.UID, + State: InstanceStateFiring, + Labels: InstanceLabels{"test": "testValue"}, + } + + err := ng.saveAlertInstance(saveCmdOne) + require.NoError(t, err) + + saveCmdTwo := &saveAlertInstanceCommand{ + DefinitionOrgID: saveCmdOne.DefinitionOrgID, + DefinitionUID: saveCmdOne.DefinitionUID, + State: InstanceStateNormal, + Labels: InstanceLabels{"test": "testValue"}, + } + err = ng.saveAlertInstance(saveCmdTwo) + require.NoError(t, err) + + listCommand := &listAlertInstancesQuery{ + DefinitionOrgID: alertDefinition4.OrgID, + DefinitionUID: alertDefinition4.UID, + } + + err = ng.listAlertInstances(listCommand) + require.NoError(t, err) + + require.Len(t, listCommand.Result, 1) + + require.Equal(t, saveCmdTwo.DefinitionOrgID, listCommand.Result[0].DefinitionOrgID) + require.Equal(t, saveCmdTwo.DefinitionUID, listCommand.Result[0].DefinitionUID) + require.Equal(t, saveCmdTwo.Labels, listCommand.Result[0].Labels) + require.Equal(t, saveCmdTwo.State, listCommand.Result[0].CurrentState) + require.NotEmpty(t, listCommand.Result[0].DefinitionTitle) + require.Equal(t, alertDefinition4.Title, listCommand.Result[0].DefinitionTitle) + }) +} diff --git a/pkg/services/ngalert/instance_labels.go b/pkg/services/ngalert/instance_labels.go new file mode 100644 index 00000000000..93a2d8f1d5f --- /dev/null +++ b/pkg/services/ngalert/instance_labels.go @@ -0,0 +1,105 @@ +package ngalert + +import ( + // nolint:gosec + "crypto/sha1" + "encoding/json" + "fmt" + "sort" + + "github.com/grafana/grafana-plugin-sdk-go/data" +) + +// InstanceLabels is an extension to data.Labels with methods +// for database serialization. +type InstanceLabels data.Labels + +// FromDB loads labels stored in the database as json tuples into InstanceLabels. +// FromDB is part of the xorm Conversion interface. +func (il *InstanceLabels) FromDB(b []byte) error { + tl := &tupleLabels{} + err := json.Unmarshal(b, tl) + if err != nil { + return err + } + labels, err := tupleLablesToLabels(*tl) + if err != nil { + return err + } + *il = labels + return nil +} + +// ToDB is not implemented as serialization is handled with manual SQL queries). +// ToDB is part of the xorm Conversion interface. +func (il *InstanceLabels) ToDB() ([]byte, error) { + // Currently handled manually in sql command, needed to fulfill the xorm + // converter interface it seems + return []byte{}, fmt.Errorf("database serialization of alerting ng Instance labels is not implemented") +} + +// StringAndHash returns a the json representation of the labels as tuples +// sorted by key. It also returns the a hash of that representation. +func (il *InstanceLabels) StringAndHash() (string, string, error) { + tl := labelsToTupleLabels(*il) + + b, err := json.Marshal(tl) + if err != nil { + return "", "", fmt.Errorf("can not gereate key for alert instance due to failure to encode labels: %w", err) + } + + h := sha1.New() + if _, err := h.Write(b); err != nil { + return "", "", err + } + + return string(b), fmt.Sprintf("%x", h.Sum(nil)), nil +} + +// The following is based on SDK code, copied for now + +// tupleLables is an alternative representation of Labels (map[string]string) that can be sorted +// and then marshalled into a consistent string that can be used a map key. All tupleLabel objects +// in tupleLabels should have unique first elements (keys). +type tupleLabels []tupleLabel + +// tupleLabel is an element of tupleLabels and should be in the form of [2]{"key", "value"}. +type tupleLabel [2]string + +// Sort tupleLabels by each elements first property (key). +func (t *tupleLabels) sortBtKey() { + if t == nil { + return + } + sort.Slice((*t)[:], func(i, j int) bool { + return (*t)[i][0] < (*t)[j][0] + }) +} + +// labelsToTupleLabels converts Labels (map[string]string) to tupleLabels. +func labelsToTupleLabels(l InstanceLabels) tupleLabels { + if l == nil { + return nil + } + t := make(tupleLabels, 0, len(l)) + for k, v := range l { + t = append(t, tupleLabel{k, v}) + } + t.sortBtKey() + return t +} + +// tupleLabelsToLabels converts tupleLabels to Labels (map[string]string), erroring if there are duplicate keys. +func tupleLablesToLabels(tuples tupleLabels) (InstanceLabels, error) { + if tuples == nil { + return nil, nil + } + labels := make(map[string]string) + for _, tuple := range tuples { + if key, ok := labels[tuple[0]]; ok { + return nil, fmt.Errorf("duplicate key '%v' in lables: %v", key, tuples) + } + labels[tuple[0]] = tuple[1] + } + return labels, nil +} diff --git a/pkg/services/ngalert/ngalert.go b/pkg/services/ngalert/ngalert.go index ff21b6c42dc..d744aebdcf6 100644 --- a/pkg/services/ngalert/ngalert.go +++ b/pkg/services/ngalert/ngalert.go @@ -74,6 +74,8 @@ func (ng *AlertNG) AddMigration(mg *migrator.Migrator) { } addAlertDefinitionMigrations(mg) addAlertDefinitionVersionMigrations(mg) + // Create alert_instance table + alertInstanceMigration(mg) } // LoadAlertCondition returns a Condition object for the given alertDefinitionID. diff --git a/pkg/services/ngalert/schedule.go b/pkg/services/ngalert/schedule.go index 10253a52eed..00b592aa8ca 100644 --- a/pkg/services/ngalert/schedule.go +++ b/pkg/services/ngalert/schedule.go @@ -39,7 +39,7 @@ func (ng *AlertNG) definitionRoutine(grafanaCtx context.Context, key alertDefini return err } alertDefinition = q.Result - ng.schedule.log.Debug("new alert definition version fetched", "key", key, "version", alertDefinition.Version) + ng.schedule.log.Debug("new alert definition version fetched", "title", alertDefinition.Title, "key", key, "version", alertDefinition.Version) } condition := eval.Condition{ @@ -50,11 +50,17 @@ func (ng *AlertNG) definitionRoutine(grafanaCtx context.Context, key alertDefini results, err := eval.ConditionEval(&condition, ctx.now) end = timeNow() if err != nil { - ng.schedule.log.Error("failed to evaluate alert definition", "key", key, "attempt", attempt, "now", ctx.now, "duration", end.Sub(start), "error", err) + // consider saving alert instance on error + ng.schedule.log.Error("failed to evaluate alert definition", "title", alertDefinition.Title, "key", key, "attempt", attempt, "now", ctx.now, "duration", end.Sub(start), "error", err) return err } for _, r := range results { - ng.schedule.log.Info("alert definition result", "key", key, "attempt", attempt, "now", ctx.now, "duration", end.Sub(start), "instance", r.Instance, "state", r.State.String()) + ng.schedule.log.Debug("alert definition result", "title", alertDefinition.Title, "key", key, "attempt", attempt, "now", ctx.now, "duration", end.Sub(start), "instance", r.Instance, "state", r.State.String()) + cmd := saveAlertInstanceCommand{DefinitionOrgID: key.orgID, DefinitionUID: key.definitionUID, State: InstanceStateType(r.State.String()), Labels: InstanceLabels(r.Instance), LastEvalTime: ctx.now} + err := ng.saveAlertInstance(&cmd) + if err != nil { + ng.schedule.log.Error("failed saving alert instance", "title", alertDefinition.Title, "key", key, "attempt", attempt, "now", ctx.now, "instance", r.Instance, "state", r.State.String(), "error", err) + } } return nil } diff --git a/pkg/services/ngalert/schedule_test.go b/pkg/services/ngalert/schedule_test.go index 07039051e3c..5fd8ced24b9 100644 --- a/pkg/services/ngalert/schedule_test.go +++ b/pkg/services/ngalert/schedule_test.go @@ -184,5 +184,5 @@ func concatenate(keys []alertDefinitionKey) string { for _, k := range keys { s = append(s, k.String()) } - return fmt.Sprintf("[%s]", strings.TrimLeft(strings.Join(s, ","), ",")) + return fmt.Sprintf("[%s]", strings.Join(s, ",")) } diff --git a/pkg/services/sqlstore/migrator/dialect.go b/pkg/services/sqlstore/migrator/dialect.go index 206a7c35234..e151d25f083 100644 --- a/pkg/services/sqlstore/migrator/dialect.go +++ b/pkg/services/sqlstore/migrator/dialect.go @@ -34,6 +34,8 @@ type Dialect interface { IndexCheckSQL(tableName, indexName string) (string, []interface{}) ColumnCheckSQL(tableName, columnName string) (string, []interface{}) + // UpsertSQL returns the upsert sql statement for a dialect + UpsertSQL(tableName string, keyCols, updateCols []string) string ColString(*Column) string ColStringNoPk(*Column) string @@ -281,3 +283,8 @@ func (b *BaseDialect) NoOpSQL() string { func (b *BaseDialect) TruncateDBTables() error { return nil } + +//UpsertSQL returns empty string +func (b *BaseDialect) UpsertSQL(tableName string, keyCols, updateCols []string) string { + return "" +} diff --git a/pkg/services/sqlstore/migrator/mysql_dialect.go b/pkg/services/sqlstore/migrator/mysql_dialect.go index fecb9e45361..2a8393da984 100644 --- a/pkg/services/sqlstore/migrator/mysql_dialect.go +++ b/pkg/services/sqlstore/migrator/mysql_dialect.go @@ -199,3 +199,28 @@ func (db *MySQLDialect) ErrorMessage(err error) string { func (db *MySQLDialect) IsDeadlock(err error) bool { return db.isThisError(err, mysqlerr.ER_LOCK_DEADLOCK) } + +// UpsertSQL returns the upsert sql statement for PostgreSQL dialect +func (db *MySQLDialect) UpsertSQL(tableName string, keyCols, updateCols []string) string { + columnsStr := strings.Builder{} + colPlaceHoldersStr := strings.Builder{} + setStr := strings.Builder{} + + separator := ", " + for i, c := range updateCols { + if i == len(updateCols)-1 { + separator = "" + } + columnsStr.WriteString(fmt.Sprintf("%s%s", db.Quote(c), separator)) + colPlaceHoldersStr.WriteString(fmt.Sprintf("?%s", separator)) + setStr.WriteString(fmt.Sprintf("%s=VALUES(%s)%s", db.Quote(c), db.Quote(c), separator)) + } + + s := fmt.Sprintf(`INSERT INTO %s (%s) VALUES (%s) ON DUPLICATE KEY UPDATE %s`, + tableName, + columnsStr.String(), + colPlaceHoldersStr.String(), + setStr.String(), + ) + return s +} diff --git a/pkg/services/sqlstore/migrator/postgres_dialect.go b/pkg/services/sqlstore/migrator/postgres_dialect.go index 2cee888d432..b698d5007f5 100644 --- a/pkg/services/sqlstore/migrator/postgres_dialect.go +++ b/pkg/services/sqlstore/migrator/postgres_dialect.go @@ -215,3 +215,40 @@ func (db *PostgresDialect) PostInsertId(table string, sess *xorm.Session) error } return nil } + +// UpsertSQL returns the upsert sql statement for PostgreSQL dialect +func (db *PostgresDialect) UpsertSQL(tableName string, keyCols, updateCols []string) string { + columnsStr := strings.Builder{} + onConflictStr := strings.Builder{} + colPlaceHoldersStr := strings.Builder{} + setStr := strings.Builder{} + + const separator = ", " + separatorVar := separator + for i, c := range updateCols { + if i == len(updateCols)-1 { + separatorVar = "" + } + + columnsStr.WriteString(fmt.Sprintf("%s%s", db.Quote(c), separatorVar)) + colPlaceHoldersStr.WriteString(fmt.Sprintf("?%s", separatorVar)) + setStr.WriteString(fmt.Sprintf("%s=excluded.%s%s", db.Quote(c), db.Quote(c), separatorVar)) + } + + separatorVar = separator + for i, c := range keyCols { + if i == len(keyCols)-1 { + separatorVar = "" + } + onConflictStr.WriteString(fmt.Sprintf("%s%s", db.Quote(c), separatorVar)) + } + + s := fmt.Sprintf(`INSERT INTO %s (%s) VALUES (%s) ON CONFLICT(%s) DO UPDATE SET %s`, + tableName, + columnsStr.String(), + colPlaceHoldersStr.String(), + onConflictStr.String(), + setStr.String(), + ) + return s +} diff --git a/pkg/services/sqlstore/migrator/sqlite_dialect.go b/pkg/services/sqlstore/migrator/sqlite_dialect.go index 19c3169738f..cb5ca9301d0 100644 --- a/pkg/services/sqlstore/migrator/sqlite_dialect.go +++ b/pkg/services/sqlstore/migrator/sqlite_dialect.go @@ -3,6 +3,7 @@ package migrator import ( "errors" "fmt" + "strings" "github.com/grafana/grafana/pkg/util/errutil" "github.com/mattn/go-sqlite3" @@ -147,3 +148,40 @@ func (db *SQLite3) IsUniqueConstraintViolation(err error) bool { func (db *SQLite3) IsDeadlock(err error) bool { return false // No deadlock } + +// UpsertSQL returns the upsert sql statement for SQLite dialect +func (db *SQLite3) UpsertSQL(tableName string, keyCols, updateCols []string) string { + columnsStr := strings.Builder{} + onConflictStr := strings.Builder{} + colPlaceHoldersStr := strings.Builder{} + setStr := strings.Builder{} + + const separator = ", " + separatorVar := separator + for i, c := range updateCols { + if i == len(updateCols)-1 { + separatorVar = "" + } + + columnsStr.WriteString(fmt.Sprintf("%s%s", db.Quote(c), separatorVar)) + colPlaceHoldersStr.WriteString(fmt.Sprintf("?%s", separatorVar)) + setStr.WriteString(fmt.Sprintf("%s=excluded.%s%s", db.Quote(c), db.Quote(c), separatorVar)) + } + + separatorVar = separator + for i, c := range keyCols { + if i == len(keyCols)-1 { + separatorVar = "" + } + onConflictStr.WriteString(fmt.Sprintf("%s%s", db.Quote(c), separatorVar)) + } + + s := fmt.Sprintf(`INSERT INTO %s (%s) VALUES (%s) ON CONFLICT(%s) DO UPDATE SET %s`, + tableName, + columnsStr.String(), + colPlaceHoldersStr.String(), + onConflictStr.String(), + setStr.String(), + ) + return s +}