AlertingNG: Save alert instances (#30223)

* AlertingNG: Save alert instances

Co-authored-by: Kyle Brandt <kyle@grafana.com>

* Rename alert instance fields/columns

* Include definition title in listing alert instances

* Delete instances when deleting defintion

Co-authored-by: Kyle Brandt <kyle@grafana.com>

pkg/services/ngalert/api.go

@@ -27,6 +27,10 @@ func (ng *AlertNG) registerAPIEndpoints() {
 		schedulerRouter.Post("/pause", routing.Wrap(ng.pauseScheduler))
 		schedulerRouter.Post("/unpause", routing.Wrap(ng.unpauseScheduler))
 	}, middleware.ReqOrgAdmin)
+
+	ng.RouteRegister.Group("/api/alert-instances", func(alertInstances routing.RouteRegister) {
+		alertInstances.Get("", middleware.ReqSignedIn, routing.Wrap(ng.listAlertInstancesEndpoint))
+	})
 }
 
 // conditionEvalEndpoint handles POST /api/alert-definitions/eval.

pkg/services/ngalert/database.go

@@ -26,12 +26,7 @@ func getAlertDefinitionByUID(sess *sqlstore.DBSession, alertDefinitionUID string
 // It returns models.ErrAlertDefinitionNotFound if no alert definition is found for the provided ID.
 func (ng *AlertNG) deleteAlertDefinitionByUID(cmd *deleteAlertDefinitionByUIDCommand) error {
 	return ng.SQLStore.WithTransactionalDbSession(context.Background(), func(sess *sqlstore.DBSession) error {
-		res, err := sess.Exec("DELETE FROM alert_definition WHERE uid = ? AND org_id = ?", cmd.UID, cmd.OrgID)
-		if err != nil {
-			return err
-		}
-
-		_, err = res.RowsAffected()
+		_, err := sess.Exec("DELETE FROM alert_definition WHERE uid = ? AND org_id = ?", cmd.UID, cmd.OrgID)
 		if err != nil {
 			return err
 		}
@@ -41,6 +36,10 @@ func (ng *AlertNG) deleteAlertDefinitionByUID(cmd *deleteAlertDefinitionByUIDCom
 			return err
 		}
 
+		_, err = sess.Exec("DELETE FROM alert_instance WHERE def_org_id = ? AND def_uid = ?", cmd.OrgID, cmd.UID)
+		if err != nil {
+			return err
+		}
 		return nil
 	})
 }

pkg/services/ngalert/database_mig.go

@@ -68,3 +68,28 @@ func addAlertDefinitionVersionMigrations(mg *migrator.Migrator) {
 	mg.AddMigration("alter alert_definition_version table data column to mediumtext in mysql", migrator.NewRawSQLMigration("").
 		Mysql("ALTER TABLE alert_definition_version MODIFY data MEDIUMTEXT;"))
 }
+
+func alertInstanceMigration(mg *migrator.Migrator) {
+	alertInstance := migrator.Table{
+		Name: "alert_instance",
+		Columns: []*migrator.Column{
+			{Name: "def_org_id", Type: migrator.DB_BigInt, Nullable: false},
+			{Name: "def_uid", Type: migrator.DB_NVarchar, Length: 40, Nullable: false, Default: "0"},
+			{Name: "labels", Type: migrator.DB_Text, Nullable: false},
+			{Name: "labels_hash", Type: migrator.DB_NVarchar, Length: 190, Nullable: false},
+			{Name: "current_state", Type: migrator.DB_NVarchar, Length: 190, Nullable: false},
+			{Name: "current_state_since", Type: migrator.DB_BigInt, Nullable: false},
+			{Name: "last_eval_time", Type: migrator.DB_BigInt, Nullable: false},
+		},
+		PrimaryKeys: []string{"def_org_id", "def_uid", "labels_hash"},
+		Indices: []*migrator.Index{
+			{Cols: []string{"def_org_id", "def_uid", "current_state"}, Type: migrator.IndexType},
+			{Cols: []string{"def_org_id", "current_state"}, Type: migrator.IndexType},
+		},
+	}
+
+	// create table
+	mg.AddMigration("create alert_instance table", migrator.NewAddTableMigration(alertInstance))
+	mg.AddMigration("add index in alert_instance table on def_org_id, def_uid and current_state columns", migrator.NewAddIndexMigration(alertInstance, alertInstance.Indices[0]))
+	mg.AddMigration("add index in alert_instance table on def_org_id, current_state columns", migrator.NewAddIndexMigration(alertInstance, alertInstance.Indices[1]))
+}

pkg/services/ngalert/database_test.go

@@ -242,7 +242,8 @@ func TestUpdatingAlertDefinition(t *testing.T) {
 					assert.Equal(t, previousAlertDefinition.UID, q.Result.UID)
 				default:
 					require.NoError(t, err)
-					assert.Equal(t, int64(1), q.Result.ID)
+					assert.Equal(t, previousAlertDefinition.ID, q.Result.ID)
+					assert.Equal(t, previousAlertDefinition.UID, q.Result.UID)
 					assert.True(t, q.Result.Updated.After(lastUpdated))
 					assert.Equal(t, tc.expectedUpdated, q.Result.Updated)
 					assert.Equal(t, previousAlertDefinition.Version+1, q.Result.Version)
@@ -292,8 +293,31 @@ func TestDeletingAlertDefinition(t *testing.T) {
 			OrgID: 1,
 		}
 
-		err := ng.deleteAlertDefinitionByUID(&q)
+		// save an instance for the definition
+		saveCmd := &saveAlertInstanceCommand{
+			DefinitionOrgID: alertDefinition.OrgID,
+			DefinitionUID:   alertDefinition.UID,
+			State:           InstanceStateFiring,
+			Labels:          InstanceLabels{"test": "testValue"},
+		}
+		err := ng.saveAlertInstance(saveCmd)
 		require.NoError(t, err)
+		listCommand := &listAlertInstancesQuery{
+			DefinitionOrgID: alertDefinition.OrgID,
+			DefinitionUID:   alertDefinition.UID,
+		}
+		err = ng.listAlertInstances(listCommand)
+		require.NoError(t, err)
+		require.Len(t, listCommand.Result, 1)
+
+		err = ng.deleteAlertDefinitionByUID(&q)
+		require.NoError(t, err)
+
+		// assert that alert instance is deleted
+		err = ng.listAlertInstances(listCommand)
+		require.NoError(t, err)
+
+		require.Len(t, listCommand.Result, 0)
 	})
 }
 

pkg/services/ngalert/instance.go

@@ -0,0 +1,96 @@
+package ngalert
+
+import (
+	"fmt"
+	"time"
+)
+
+// AlertInstance represents a single alert instance.
+type AlertInstance struct {
+	DefinitionOrgID   int64  `xorm:"def_org_id"`
+	DefinitionUID     string `xorm:"def_uid"`
+	Labels            InstanceLabels
+	LabelsHash        string
+	CurrentState      InstanceStateType
+	CurrentStateSince time.Time
+	LastEvalTime      time.Time
+}
+
+// InstanceStateType is an enum for instance states.
+type InstanceStateType string
+
+const (
+	// InstanceStateFiring is for a firing alert.
+	InstanceStateFiring InstanceStateType = "Alerting"
+	// InstanceStateNormal is for a normal alert.
+	InstanceStateNormal InstanceStateType = "Normal"
+)
+
+// IsValid checks that the value of InstanceStateType is a valid
+// string.
+func (i InstanceStateType) IsValid() bool {
+	return i == InstanceStateFiring ||
+		i == InstanceStateNormal
+}
+
+// saveAlertInstanceCommand is the query for saving a new alert instance.
+type saveAlertInstanceCommand struct {
+	DefinitionOrgID int64
+	DefinitionUID   string
+	Labels          InstanceLabels
+	State           InstanceStateType
+	LastEvalTime    time.Time
+}
+
+// getAlertDefinitionByIDQuery is the query for retrieving/deleting an alert definition by ID.
+// nolint:unused
+type getAlertInstanceQuery struct {
+	DefinitionOrgID int64
+	DefinitionUID   string
+	Labels          InstanceLabels
+
+	Result *AlertInstance
+}
+
+// listAlertInstancesCommand is the query list alert Instances.
+type listAlertInstancesQuery struct {
+	DefinitionOrgID int64 `json:"-"`
+	DefinitionUID   string
+	State           InstanceStateType
+
+	Result []*listAlertInstancesQueryResult
+}
+
+// listAlertInstancesQueryResult represents the result of listAlertInstancesQuery.
+type listAlertInstancesQueryResult struct {
+	DefinitionOrgID   int64  `xorm:"def_org_id"`
+	DefinitionUID     string `xorm:"def_uid"`
+	DefinitionTitle   string `xorm:"def_title"`
+	Labels            InstanceLabels
+	LabelsHash        string
+	CurrentState      InstanceStateType
+	CurrentStateSince time.Time
+	LastEvalTime      time.Time
+}
+
+// validateAlertInstance validates that the alert instance contains an alert definition id,
+// and state.
+func validateAlertInstance(alertInstance *AlertInstance) error {
+	if alertInstance == nil {
+		return fmt.Errorf("alert instance is invalid because it is nil")
+	}
+
+	if alertInstance.DefinitionOrgID == 0 {
+		return fmt.Errorf("alert instance is invalid due to missing alert definition organisation")
+	}
+
+	if alertInstance.DefinitionUID == "" {
+		return fmt.Errorf("alert instance is invalid due to missing alert definition uid")
+	}
+
+	if !alertInstance.CurrentState.IsValid() {
+		return fmt.Errorf("alert instance is invalid because the state '%v' is invalid", alertInstance.CurrentState)
+	}
+
+	return nil
+}

pkg/services/ngalert/instance_api.go

@@ -0,0 +1,17 @@
+package ngalert
+
+import (
+	"github.com/grafana/grafana/pkg/api/response"
+	"github.com/grafana/grafana/pkg/models"
+)
+
+// listAlertInstancesEndpoint handles GET /api/alert-instances.
+func (ng *AlertNG) listAlertInstancesEndpoint(c *models.ReqContext) response.Response {
+	cmd := listAlertInstancesQuery{DefinitionOrgID: c.SignedInUser.OrgId}
+
+	if err := ng.listAlertInstances(&cmd); err != nil {
+		return response.Error(500, "Failed to list alert instances", err)
+	}
+
+	return response.JSON(200, cmd.Result)
+}

pkg/services/ngalert/instance_database.go

@@ -0,0 +1,115 @@
+package ngalert
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"time"
+
+	"github.com/grafana/grafana/pkg/services/sqlstore"
+)
+
+// getAlertInstance is a handler for retrieving an alert instance based on OrgId, AlertDefintionID, and
+// the hash of the labels.
+// nolint:unused
+func (ng *AlertNG) getAlertInstance(cmd *getAlertInstanceQuery) error {
+	return ng.SQLStore.WithDbSession(context.Background(), func(sess *sqlstore.DBSession) error {
+		instance := AlertInstance{}
+		s := strings.Builder{}
+		s.WriteString(`SELECT * FROM alert_instance
+			WHERE
+				def_org_id=? AND
+				def_uid=? AND
+				labels_hash=?
+		`)
+
+		_, hash, err := cmd.Labels.StringAndHash()
+		if err != nil {
+			return err
+		}
+
+		params := append(make([]interface{}, 0), cmd.DefinitionOrgID, cmd.DefinitionUID, hash)
+
+		has, err := sess.SQL(s.String(), params...).Get(&instance)
+		if !has {
+			return fmt.Errorf("instance not found for labels %v (hash: %v), alert definition %v (org %v)", cmd.Labels, hash, cmd.DefinitionUID, cmd.DefinitionOrgID)
+		}
+		if err != nil {
+			return err
+		}
+
+		cmd.Result = &instance
+		return nil
+	})
+}
+
+// listAlertInstances is a handler for retrieving alert instances within specific organisation
+// based on various filters.
+func (ng *AlertNG) listAlertInstances(cmd *listAlertInstancesQuery) error {
+	return ng.SQLStore.WithDbSession(context.Background(), func(sess *sqlstore.DBSession) error {
+		alertInstances := make([]*listAlertInstancesQueryResult, 0)
+
+		s := strings.Builder{}
+		params := make([]interface{}, 0)
+
+		addToQuery := func(stmt string, p ...interface{}) {
+			s.WriteString(stmt)
+			params = append(params, p...)
+		}
+
+		addToQuery("SELECT alert_instance.*, alert_definition.title AS def_title FROM alert_instance LEFT JOIN alert_definition ON alert_instance.def_org_id = alert_definition.org_id AND alert_instance.def_uid = alert_definition.uid WHERE def_org_id = ?", cmd.DefinitionOrgID)
+
+		if cmd.DefinitionUID != "" {
+			addToQuery(` AND def_uid = ?`, cmd.DefinitionUID)
+		}
+
+		if cmd.State != "" {
+			addToQuery(` AND current_state = ?`, cmd.State)
+		}
+
+		if err := sess.SQL(s.String(), params...).Find(&alertInstances); err != nil {
+			return err
+		}
+
+		cmd.Result = alertInstances
+		return nil
+	})
+}
+
+// saveAlertDefinition is a handler for saving a new alert definition.
+// nolint:unused
+func (ng *AlertNG) saveAlertInstance(cmd *saveAlertInstanceCommand) error {
+	return ng.SQLStore.WithDbSession(context.Background(), func(sess *sqlstore.DBSession) error {
+		labelTupleJSON, labelsHash, err := cmd.Labels.StringAndHash()
+		if err != nil {
+			return err
+		}
+
+		alertInstance := &AlertInstance{
+			DefinitionOrgID:   cmd.DefinitionOrgID,
+			DefinitionUID:     cmd.DefinitionUID,
+			Labels:            cmd.Labels,
+			LabelsHash:        labelsHash,
+			CurrentState:      cmd.State,
+			CurrentStateSince: time.Now(),
+			LastEvalTime:      cmd.LastEvalTime,
+		}
+
+		if err := validateAlertInstance(alertInstance); err != nil {
+			return err
+		}
+
+		params := append(make([]interface{}, 0), alertInstance.DefinitionOrgID, alertInstance.DefinitionUID, labelTupleJSON, alertInstance.LabelsHash, alertInstance.CurrentState, alertInstance.CurrentStateSince.Unix(), alertInstance.LastEvalTime.Unix())
+
+		upsertSQL := ng.SQLStore.Dialect.UpsertSQL(
+			"alert_instance",
+			[]string{"def_org_id", "def_uid", "labels_hash"},
+			[]string{"def_org_id", "def_uid", "labels", "labels_hash", "current_state", "current_state_since", "last_eval_time"})
+		_, err = sess.SQL(upsertSQL, params...).Query()
+		if err != nil {
+			return err
+		}
+
+		return nil
+	})
+}

pkg/services/ngalert/instance_database_test.go

@@ -0,0 +1,163 @@
+// +build integration
+
+package ngalert
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestAlertInstanceOperations(t *testing.T) {
+	ng := setupTestEnv(t)
+
+	alertDefinition1 := createTestAlertDefinition(t, ng, 60)
+	orgID := alertDefinition1.OrgID
+
+	alertDefinition2 := createTestAlertDefinition(t, ng, 60)
+	require.Equal(t, orgID, alertDefinition2.OrgID)
+
+	alertDefinition3 := createTestAlertDefinition(t, ng, 60)
+	require.Equal(t, orgID, alertDefinition3.OrgID)
+
+	alertDefinition4 := createTestAlertDefinition(t, ng, 60)
+	require.Equal(t, orgID, alertDefinition4.OrgID)
+
+	t.Run("can save and read new alert instance", func(t *testing.T) {
+		saveCmd := &saveAlertInstanceCommand{
+			DefinitionOrgID: alertDefinition1.OrgID,
+			DefinitionUID:   alertDefinition1.UID,
+			State:           InstanceStateFiring,
+			Labels:          InstanceLabels{"test": "testValue"},
+		}
+		err := ng.saveAlertInstance(saveCmd)
+		require.NoError(t, err)
+
+		getCmd := &getAlertInstanceQuery{
+			DefinitionOrgID: saveCmd.DefinitionOrgID,
+			DefinitionUID:   saveCmd.DefinitionUID,
+			Labels:          InstanceLabels{"test": "testValue"},
+		}
+
+		err = ng.getAlertInstance(getCmd)
+		require.NoError(t, err)
+
+		require.Equal(t, saveCmd.Labels, getCmd.Result.Labels)
+		require.Equal(t, alertDefinition1.OrgID, getCmd.Result.DefinitionOrgID)
+		require.Equal(t, alertDefinition1.UID, getCmd.Result.DefinitionUID)
+	})
+
+	t.Run("can save and read new alert instance with no labels", func(t *testing.T) {
+		saveCmd := &saveAlertInstanceCommand{
+			DefinitionOrgID: alertDefinition2.OrgID,
+			DefinitionUID:   alertDefinition2.UID,
+			State:           InstanceStateNormal,
+		}
+		err := ng.saveAlertInstance(saveCmd)
+		require.NoError(t, err)
+
+		getCmd := &getAlertInstanceQuery{
+			DefinitionOrgID: saveCmd.DefinitionOrgID,
+			DefinitionUID:   saveCmd.DefinitionUID,
+		}
+
+		err = ng.getAlertInstance(getCmd)
+		require.NoError(t, err)
+
+		require.Equal(t, alertDefinition2.OrgID, getCmd.Result.DefinitionOrgID)
+		require.Equal(t, alertDefinition2.UID, getCmd.Result.DefinitionUID)
+		require.Equal(t, saveCmd.Labels, getCmd.Result.Labels)
+	})
+
+	t.Run("can save two instances with same org_id, uid and different labels", func(t *testing.T) {
+		saveCmdOne := &saveAlertInstanceCommand{
+			DefinitionOrgID: alertDefinition3.OrgID,
+			DefinitionUID:   alertDefinition3.UID,
+			State:           InstanceStateFiring,
+			Labels:          InstanceLabels{"test": "testValue"},
+		}
+
+		err := ng.saveAlertInstance(saveCmdOne)
+		require.NoError(t, err)
+
+		saveCmdTwo := &saveAlertInstanceCommand{
+			DefinitionOrgID: saveCmdOne.DefinitionOrgID,
+			DefinitionUID:   saveCmdOne.DefinitionUID,
+			State:           InstanceStateFiring,
+			Labels:          InstanceLabels{"test": "meow"},
+		}
+		err = ng.saveAlertInstance(saveCmdTwo)
+		require.NoError(t, err)
+
+		listCommand := &listAlertInstancesQuery{
+			DefinitionOrgID: saveCmdOne.DefinitionOrgID,
+			DefinitionUID:   saveCmdOne.DefinitionUID,
+		}
+
+		err = ng.listAlertInstances(listCommand)
+		require.NoError(t, err)
+
+		require.Len(t, listCommand.Result, 2)
+	})
+
+	t.Run("can list all added instances in org", func(t *testing.T) {
+		listCommand := &listAlertInstancesQuery{
+			DefinitionOrgID: orgID,
+		}
+
+		err := ng.listAlertInstances(listCommand)
+		require.NoError(t, err)
+
+		require.Len(t, listCommand.Result, 4)
+	})
+
+	t.Run("can list all added instances in org filtered by current state", func(t *testing.T) {
+		listCommand := &listAlertInstancesQuery{
+			DefinitionOrgID: orgID,
+			State:           InstanceStateNormal,
+		}
+
+		err := ng.listAlertInstances(listCommand)
+		require.NoError(t, err)
+
+		require.Len(t, listCommand.Result, 1)
+	})
+
+	t.Run("update instance with same org_id, uid and different labels", func(t *testing.T) {
+		saveCmdOne := &saveAlertInstanceCommand{
+			DefinitionOrgID: alertDefinition4.OrgID,
+			DefinitionUID:   alertDefinition4.UID,
+			State:           InstanceStateFiring,
+			Labels:          InstanceLabels{"test": "testValue"},
+		}
+
+		err := ng.saveAlertInstance(saveCmdOne)
+		require.NoError(t, err)
+
+		saveCmdTwo := &saveAlertInstanceCommand{
+			DefinitionOrgID: saveCmdOne.DefinitionOrgID,
+			DefinitionUID:   saveCmdOne.DefinitionUID,
+			State:           InstanceStateNormal,
+			Labels:          InstanceLabels{"test": "testValue"},
+		}
+		err = ng.saveAlertInstance(saveCmdTwo)
+		require.NoError(t, err)
+
+		listCommand := &listAlertInstancesQuery{
+			DefinitionOrgID: alertDefinition4.OrgID,
+			DefinitionUID:   alertDefinition4.UID,
+		}
+
+		err = ng.listAlertInstances(listCommand)
+		require.NoError(t, err)
+
+		require.Len(t, listCommand.Result, 1)
+
+		require.Equal(t, saveCmdTwo.DefinitionOrgID, listCommand.Result[0].DefinitionOrgID)
+		require.Equal(t, saveCmdTwo.DefinitionUID, listCommand.Result[0].DefinitionUID)
+		require.Equal(t, saveCmdTwo.Labels, listCommand.Result[0].Labels)
+		require.Equal(t, saveCmdTwo.State, listCommand.Result[0].CurrentState)
+		require.NotEmpty(t, listCommand.Result[0].DefinitionTitle)
+		require.Equal(t, alertDefinition4.Title, listCommand.Result[0].DefinitionTitle)
+	})
+}

pkg/services/ngalert/instance_labels.go

@@ -0,0 +1,105 @@
+package ngalert
+
+import (
+	// nolint:gosec
+	"crypto/sha1"
+	"encoding/json"
+	"fmt"
+	"sort"
+
+	"github.com/grafana/grafana-plugin-sdk-go/data"
+)
+
+// InstanceLabels is an extension to data.Labels with methods
+// for database serialization.
+type InstanceLabels data.Labels
+
+// FromDB loads labels stored in the database as json tuples into InstanceLabels.
+// FromDB is part of the xorm Conversion interface.
+func (il *InstanceLabels) FromDB(b []byte) error {
+	tl := &tupleLabels{}
+	err := json.Unmarshal(b, tl)
+	if err != nil {
+		return err
+	}
+	labels, err := tupleLablesToLabels(*tl)
+	if err != nil {
+		return err
+	}
+	*il = labels
+	return nil
+}
+
+// ToDB is not implemented as serialization is handled with manual SQL queries).
+// ToDB is part of the xorm Conversion interface.
+func (il *InstanceLabels) ToDB() ([]byte, error) {
+	// Currently handled manually in sql command, needed to fulfill the xorm
+	// converter interface it seems
+	return []byte{}, fmt.Errorf("database serialization of alerting ng Instance labels is not implemented")
+}
+
+// StringAndHash returns a the json representation of the labels as tuples
+// sorted by key. It also returns the a hash of that representation.
+func (il *InstanceLabels) StringAndHash() (string, string, error) {
+	tl := labelsToTupleLabels(*il)
+
+	b, err := json.Marshal(tl)
+	if err != nil {
+		return "", "", fmt.Errorf("can not gereate key for alert instance due to failure to encode labels: %w", err)
+	}
+
+	h := sha1.New()
+	if _, err := h.Write(b); err != nil {
+		return "", "", err
+	}
+
+	return string(b), fmt.Sprintf("%x", h.Sum(nil)), nil
+}
+
+// The following is based on SDK code, copied for now
+
+// tupleLables is an alternative representation of Labels (map[string]string) that can be sorted
+// and then marshalled into a consistent string that can be used a map key. All tupleLabel objects
+// in tupleLabels should have unique first elements (keys).
+type tupleLabels []tupleLabel
+
+// tupleLabel is an element of tupleLabels and should be in the form of [2]{"key", "value"}.
+type tupleLabel [2]string
+
+// Sort tupleLabels by each elements first property (key).
+func (t *tupleLabels) sortBtKey() {
+	if t == nil {
+		return
+	}
+	sort.Slice((*t)[:], func(i, j int) bool {
+		return (*t)[i][0] < (*t)[j][0]
+	})
+}
+
+// labelsToTupleLabels converts Labels (map[string]string) to tupleLabels.
+func labelsToTupleLabels(l InstanceLabels) tupleLabels {
+	if l == nil {
+		return nil
+	}
+	t := make(tupleLabels, 0, len(l))
+	for k, v := range l {
+		t = append(t, tupleLabel{k, v})
+	}
+	t.sortBtKey()
+	return t
+}
+
+// tupleLabelsToLabels converts tupleLabels to Labels (map[string]string), erroring if there are duplicate keys.
+func tupleLablesToLabels(tuples tupleLabels) (InstanceLabels, error) {
+	if tuples == nil {
+		return nil, nil
+	}
+	labels := make(map[string]string)
+	for _, tuple := range tuples {
+		if key, ok := labels[tuple[0]]; ok {
+			return nil, fmt.Errorf("duplicate key '%v' in lables: %v", key, tuples)
+		}
+		labels[tuple[0]] = tuple[1]
+	}
+	return labels, nil
+}

pkg/services/ngalert/ngalert.go

@@ -74,6 +74,8 @@ func (ng *AlertNG) AddMigration(mg *migrator.Migrator) {
 	}
 	addAlertDefinitionMigrations(mg)
 	addAlertDefinitionVersionMigrations(mg)
+	// Create alert_instance table
+	alertInstanceMigration(mg)
 }
 
 // LoadAlertCondition returns a Condition object for the given alertDefinitionID.

pkg/services/ngalert/schedule.go

@@ -39,7 +39,7 @@ func (ng *AlertNG) definitionRoutine(grafanaCtx context.Context, key alertDefini
 						return err
 					}
 					alertDefinition = q.Result
-					ng.schedule.log.Debug("new alert definition version fetched", "key", key, "version", alertDefinition.Version)
+					ng.schedule.log.Debug("new alert definition version fetched", "title", alertDefinition.Title, "key", key, "version", alertDefinition.Version)
 				}
 
 				condition := eval.Condition{
@@ -50,11 +50,17 @@ func (ng *AlertNG) definitionRoutine(grafanaCtx context.Context, key alertDefini
 				results, err := eval.ConditionEval(&condition, ctx.now)
 				end = timeNow()
 				if err != nil {
-					ng.schedule.log.Error("failed to evaluate alert definition", "key", key, "attempt", attempt, "now", ctx.now, "duration", end.Sub(start), "error", err)
+					// consider saving alert instance on error
+					ng.schedule.log.Error("failed to evaluate alert definition", "title", alertDefinition.Title, "key", key, "attempt", attempt, "now", ctx.now, "duration", end.Sub(start), "error", err)
 					return err
 				}
 				for _, r := range results {
-					ng.schedule.log.Info("alert definition result", "key", key, "attempt", attempt, "now", ctx.now, "duration", end.Sub(start), "instance", r.Instance, "state", r.State.String())
+					ng.schedule.log.Debug("alert definition result", "title", alertDefinition.Title, "key", key, "attempt", attempt, "now", ctx.now, "duration", end.Sub(start), "instance", r.Instance, "state", r.State.String())
+					cmd := saveAlertInstanceCommand{DefinitionOrgID: key.orgID, DefinitionUID: key.definitionUID, State: InstanceStateType(r.State.String()), Labels: InstanceLabels(r.Instance), LastEvalTime: ctx.now}
+					err := ng.saveAlertInstance(&cmd)
+					if err != nil {
+						ng.schedule.log.Error("failed saving alert instance", "title", alertDefinition.Title, "key", key, "attempt", attempt, "now", ctx.now, "instance", r.Instance, "state", r.State.String(), "error", err)
+					}
 				}
 				return nil
 			}

pkg/services/ngalert/schedule_test.go

@@ -184,5 +184,5 @@ func concatenate(keys []alertDefinitionKey) string {
 	for _, k := range keys {
 		s = append(s, k.String())
 	}
-	return fmt.Sprintf("[%s]", strings.TrimLeft(strings.Join(s, ","), ","))
+	return fmt.Sprintf("[%s]", strings.Join(s, ","))
 }

pkg/services/sqlstore/migrator/dialect.go

@@ -34,6 +34,8 @@ type Dialect interface {
 
 	IndexCheckSQL(tableName, indexName string) (string, []interface{})
 	ColumnCheckSQL(tableName, columnName string) (string, []interface{})
+	// UpsertSQL returns the upsert sql statement for a dialect
+	UpsertSQL(tableName string, keyCols, updateCols []string) string
 
 	ColString(*Column) string
 	ColStringNoPk(*Column) string
@@ -281,3 +283,8 @@ func (b *BaseDialect) NoOpSQL() string {
 func (b *BaseDialect) TruncateDBTables() error {
 	return nil
 }
+
+//UpsertSQL returns empty string
+func (b *BaseDialect) UpsertSQL(tableName string, keyCols, updateCols []string) string {
+	return ""
+}

pkg/services/sqlstore/migrator/mysql_dialect.go

@@ -199,3 +199,28 @@ func (db *MySQLDialect) ErrorMessage(err error) string {
 func (db *MySQLDialect) IsDeadlock(err error) bool {
 	return db.isThisError(err, mysqlerr.ER_LOCK_DEADLOCK)
 }
+
+// UpsertSQL returns the upsert sql statement for PostgreSQL dialect
+func (db *MySQLDialect) UpsertSQL(tableName string, keyCols, updateCols []string) string {
+	columnsStr := strings.Builder{}
+	colPlaceHoldersStr := strings.Builder{}
+	setStr := strings.Builder{}
+
+	separator := ", "
+	for i, c := range updateCols {
+		if i == len(updateCols)-1 {
+			separator = ""
+		}
+		columnsStr.WriteString(fmt.Sprintf("%s%s", db.Quote(c), separator))
+		colPlaceHoldersStr.WriteString(fmt.Sprintf("?%s", separator))
+		setStr.WriteString(fmt.Sprintf("%s=VALUES(%s)%s", db.Quote(c), db.Quote(c), separator))
+	}
+
+	s := fmt.Sprintf(`INSERT INTO %s (%s) VALUES (%s) ON DUPLICATE KEY UPDATE %s`,
+		tableName,
+		columnsStr.String(),
+		colPlaceHoldersStr.String(),
+		setStr.String(),
+	)
+	return s
+}

pkg/services/sqlstore/migrator/postgres_dialect.go

@@ -215,3 +215,40 @@ func (db *PostgresDialect) PostInsertId(table string, sess *xorm.Session) error
 	}
 	return nil
 }
+
+// UpsertSQL returns the upsert sql statement for PostgreSQL dialect
+func (db *PostgresDialect) UpsertSQL(tableName string, keyCols, updateCols []string) string {
+	columnsStr := strings.Builder{}
+	onConflictStr := strings.Builder{}
+	colPlaceHoldersStr := strings.Builder{}
+	setStr := strings.Builder{}
+
+	const separator = ", "
+	separatorVar := separator
+	for i, c := range updateCols {
+		if i == len(updateCols)-1 {
+			separatorVar = ""
+		}
+
+		columnsStr.WriteString(fmt.Sprintf("%s%s", db.Quote(c), separatorVar))
+		colPlaceHoldersStr.WriteString(fmt.Sprintf("?%s", separatorVar))
+		setStr.WriteString(fmt.Sprintf("%s=excluded.%s%s", db.Quote(c), db.Quote(c), separatorVar))
+	}
+
+	separatorVar = separator
+	for i, c := range keyCols {
+		if i == len(keyCols)-1 {
+			separatorVar = ""
+		}
+		onConflictStr.WriteString(fmt.Sprintf("%s%s", db.Quote(c), separatorVar))
+	}
+
+	s := fmt.Sprintf(`INSERT INTO %s (%s) VALUES (%s) ON CONFLICT(%s) DO UPDATE SET %s`,
+		tableName,
+		columnsStr.String(),
+		colPlaceHoldersStr.String(),
+		onConflictStr.String(),
+		setStr.String(),
+	)
+	return s
+}

pkg/services/sqlstore/migrator/sqlite_dialect.go

@@ -3,6 +3,7 @@ package migrator
 import (
 	"errors"
 	"fmt"
+	"strings"
 
 	"github.com/grafana/grafana/pkg/util/errutil"
 	"github.com/mattn/go-sqlite3"
@@ -147,3 +148,40 @@ func (db *SQLite3) IsUniqueConstraintViolation(err error) bool {
 func (db *SQLite3) IsDeadlock(err error) bool {
 	return false // No deadlock
 }
+
+// UpsertSQL returns the upsert sql statement for SQLite dialect
+func (db *SQLite3) UpsertSQL(tableName string, keyCols, updateCols []string) string {
+	columnsStr := strings.Builder{}
+	onConflictStr := strings.Builder{}
+	colPlaceHoldersStr := strings.Builder{}
+	setStr := strings.Builder{}
+
+	const separator = ", "
+	separatorVar := separator
+	for i, c := range updateCols {
+		if i == len(updateCols)-1 {
+			separatorVar = ""
+		}
+
+		columnsStr.WriteString(fmt.Sprintf("%s%s", db.Quote(c), separatorVar))
+		colPlaceHoldersStr.WriteString(fmt.Sprintf("?%s", separatorVar))
+		setStr.WriteString(fmt.Sprintf("%s=excluded.%s%s", db.Quote(c), db.Quote(c), separatorVar))
+	}
+
+	separatorVar = separator
+	for i, c := range keyCols {
+		if i == len(keyCols)-1 {
+			separatorVar = ""
+		}
+		onConflictStr.WriteString(fmt.Sprintf("%s%s", db.Quote(c), separatorVar))
+	}
+
+	s := fmt.Sprintf(`INSERT INTO %s (%s) VALUES (%s) ON CONFLICT(%s) DO UPDATE SET %s`,
+		tableName,
+		columnsStr.String(),
+		colPlaceHoldersStr.String(),
+		onConflictStr.String(),
+		setStr.String(),
+	)
+	return s
+}