From e068be4c26dc2d969ca4b0cc70bb00e2ee4d85a1 Mon Sep 17 00:00:00 2001
From: John Baublitz <john.baublitz@threatstack.com>
Date: Sat, 12 May 2018 21:11:58 -0400
Subject: [PATCH 01/39] Feature for repeated alerting in grafana

---
 pkg/api/dtos/alerting.go                      |  3 +++
 pkg/models/alert.go                           |  9 ++++++++
 pkg/services/alerting/extractor.go            |  2 ++
 pkg/services/alerting/notifiers/base.go       |  5 ++++-
 pkg/services/alerting/result_handler.go       |  1 +
 pkg/services/alerting/rule.go                 |  6 +++++
 pkg/services/sqlstore/alert.go                | 22 ++++++++++++++++++-
 pkg/services/sqlstore/migrations/alert_mig.go |  3 +++
 .../app/features/alerting/alert_tab_ctrl.ts   |  3 +++
 .../features/alerting/partials/alert_tab.html |  3 +++
 10 files changed, 55 insertions(+), 2 deletions(-)

diff --git a/pkg/api/dtos/alerting.go b/pkg/api/dtos/alerting.go
index d30f2697f3f..64dd619a4eb 100644
--- a/pkg/api/dtos/alerting.go
+++ b/pkg/api/dtos/alerting.go
@@ -21,6 +21,9 @@ type AlertRule struct {
 	ExecutionError string           `json:"executionError"`
 	Url            string           `json:"url"`
 	CanEdit        bool             `json:"canEdit"`
+	NotifyOnce     bool             `json:"notifyOnce"`
+	NotifyEval     uint64           `json:"notifyEval"`
+	NotifyFreq     uint64           `json:"notifyFrequency"`
 }
 
 type AlertNotification struct {
diff --git a/pkg/models/alert.go b/pkg/models/alert.go
index fba2aa63df9..56ceeb2cbf7 100644
--- a/pkg/models/alert.go
+++ b/pkg/models/alert.go
@@ -72,6 +72,9 @@ type Alert struct {
 	Silenced       bool
 	ExecutionError string
 	Frequency      int64
+	NotifyOnce     bool
+	NotifyFreq     uint64
+	NotifyEval     uint64
 
 	EvalData     *simplejson.Json
 	NewStateDate time.Time
@@ -95,6 +98,8 @@ func (this *Alert) ContainsUpdates(other *Alert) bool {
 	result := false
 	result = result || this.Name != other.Name
 	result = result || this.Message != other.Message
+	result = result || this.NotifyOnce != other.NotifyOnce
+	result = result || (!other.NotifyOnce && this.NotifyFreq != other.NotifyFreq)
 
 	if this.Settings != nil && other.Settings != nil {
 		json1, err1 := this.Settings.Encode()
@@ -159,6 +164,10 @@ type SetAlertStateCommand struct {
 	Timestamp time.Time
 }
 
+type IncAlertEvalCommand struct {
+	AlertId int64
+}
+
 //Queries
 type GetAlertsQuery struct {
 	OrgId        int64
diff --git a/pkg/services/alerting/extractor.go b/pkg/services/alerting/extractor.go
index e1c1bfacb2e..f820e546a93 100644
--- a/pkg/services/alerting/extractor.go
+++ b/pkg/services/alerting/extractor.go
@@ -122,6 +122,8 @@ func (e *DashAlertExtractor) getAlertFromPanels(jsonWithPanels *simplejson.Json,
 			Handler:     jsonAlert.Get("handler").MustInt64(),
 			Message:     jsonAlert.Get("message").MustString(),
 			Frequency:   frequency,
+			NotifyOnce:  jsonAlert.Get("notifyOnce").MustBool(),
+			NotifyFreq:  jsonAlert.Get("notifyFrequency").MustUint64(),
 		}
 
 		for _, condition := range jsonAlert.Get("conditions").MustArray() {
diff --git a/pkg/services/alerting/notifiers/base.go b/pkg/services/alerting/notifiers/base.go
index 51676efdfd5..498bae3a6e6 100644
--- a/pkg/services/alerting/notifiers/base.go
+++ b/pkg/services/alerting/notifiers/base.go
@@ -32,7 +32,10 @@ func NewNotifierBase(id int64, isDefault bool, name, notifierType string, model
 
 func defaultShouldNotify(context *alerting.EvalContext) bool {
 	// Only notify on state change.
-	if context.PrevAlertState == context.Rule.State {
+	if context.PrevAlertState == context.Rule.State && context.Rule.NotifyOnce {
+		return false
+	}
+	if !context.Rule.NotifyOnce && context.Rule.NotifyEval != 0 {
 		return false
 	}
 	// Do not notify when we become OK for the first time.
diff --git a/pkg/services/alerting/result_handler.go b/pkg/services/alerting/result_handler.go
index c57b28c7c3e..56d299001f0 100644
--- a/pkg/services/alerting/result_handler.go
+++ b/pkg/services/alerting/result_handler.go
@@ -88,6 +88,7 @@ func (handler *DefaultResultHandler) Handle(evalContext *EvalContext) error {
 		}
 	}
 
+	bus.Dispatch(&m.IncAlertEvalCommand{AlertId: evalContext.Rule.Id})
 	handler.notifier.SendIfNeeded(evalContext)
 
 	return nil
diff --git a/pkg/services/alerting/rule.go b/pkg/services/alerting/rule.go
index 018d138dbe4..0003fe791e3 100644
--- a/pkg/services/alerting/rule.go
+++ b/pkg/services/alerting/rule.go
@@ -23,6 +23,9 @@ type Rule struct {
 	State               m.AlertStateType
 	Conditions          []Condition
 	Notifications       []int64
+	NotifyOnce          bool
+	NotifyFreq          uint64
+	NotifyEval          uint64
 }
 
 type ValidationError struct {
@@ -97,6 +100,9 @@ func NewRuleFromDBAlert(ruleDef *m.Alert) (*Rule, error) {
 	model.Name = ruleDef.Name
 	model.Message = ruleDef.Message
 	model.Frequency = ruleDef.Frequency
+	model.NotifyOnce = ruleDef.NotifyOnce
+	model.NotifyFreq = ruleDef.NotifyFreq
+	model.NotifyEval = ruleDef.NotifyEval
 	model.State = ruleDef.State
 	model.NoDataState = m.NoDataOption(ruleDef.Settings.Get("noDataState").MustString("no_data"))
 	model.ExecutionErrorState = m.ExecutionErrorOption(ruleDef.Settings.Get("executionErrorState").MustString("alerting"))
diff --git a/pkg/services/sqlstore/alert.go b/pkg/services/sqlstore/alert.go
index 58ec7e2857a..9ab28be84ee 100644
--- a/pkg/services/sqlstore/alert.go
+++ b/pkg/services/sqlstore/alert.go
@@ -22,6 +22,7 @@ func init() {
 	bus.AddHandler("sql", GetAlertStatesForDashboard)
 	bus.AddHandler("sql", PauseAlert)
 	bus.AddHandler("sql", PauseAllAlerts)
+	bus.AddHandler("sql", IncAlertEval)
 }
 
 func GetAlertById(query *m.GetAlertByIdQuery) error {
@@ -188,7 +189,7 @@ func updateAlerts(existingAlerts []*m.Alert, cmd *m.SaveAlertsCommand, sess *DBS
 			if alertToUpdate.ContainsUpdates(alert) {
 				alert.Updated = timeNow()
 				alert.State = alertToUpdate.State
-				sess.MustCols("message")
+				sess.MustCols("message", "notify_freq", "notify_once")
 				_, err := sess.Id(alert.Id).Update(alert)
 				if err != nil {
 					return err
@@ -343,3 +344,22 @@ func GetAlertStatesForDashboard(query *m.GetAlertStatesForDashboardQuery) error
 
 	return err
 }
+
+func IncAlertEval(cmd *m.IncAlertEvalCommand) error {
+	return inTransaction(func(sess *DBSession) error {
+		alert := m.Alert{}
+
+		if _, err := sess.Id(cmd.AlertId).Get(&alert); err != nil {
+			return err
+		}
+
+		alert.NotifyEval = (alert.NotifyEval + 1) % alert.NotifyFreq
+
+		sess.MustCols("notify_eval")
+		if _, err := sess.Id(cmd.AlertId).Update(alert); err != nil {
+			return err
+		}
+
+		return nil
+	})
+}
diff --git a/pkg/services/sqlstore/migrations/alert_mig.go b/pkg/services/sqlstore/migrations/alert_mig.go
index 2a364d5f464..3452e5710cc 100644
--- a/pkg/services/sqlstore/migrations/alert_mig.go
+++ b/pkg/services/sqlstore/migrations/alert_mig.go
@@ -29,6 +29,9 @@ func addAlertMigrations(mg *Migrator) {
 			{Name: "state_changes", Type: DB_Int, Nullable: false},
 			{Name: "created", Type: DB_DateTime, Nullable: false},
 			{Name: "updated", Type: DB_DateTime, Nullable: false},
+			{Name: "notify_once", Type: DB_Bool, Nullable: false},
+			{Name: "notify_freq", Type: DB_Int, Nullable: false},
+			{Name: "notify_eval", Type: DB_Int, Nullable: false},
 		},
 		Indices: []*Index{
 			{Cols: []string{"org_id", "id"}, Type: IndexType},
diff --git a/public/app/features/alerting/alert_tab_ctrl.ts b/public/app/features/alerting/alert_tab_ctrl.ts
index 79baa1e3f5a..f0d965ae81e 100644
--- a/public/app/features/alerting/alert_tab_ctrl.ts
+++ b/public/app/features/alerting/alert_tab_ctrl.ts
@@ -167,6 +167,9 @@ export class AlertTabCtrl {
     alert.noDataState = alert.noDataState || 'no_data';
     alert.executionErrorState = alert.executionErrorState || 'alerting';
     alert.frequency = alert.frequency || '60s';
+    alert.notifyFrequency = alert.notifyFrequency || 10;
+    alert.notifyOnce = alert.notifyOnce == null ? true : alert.notifyOnce;
+    alert.frequency = alert.frequency || '60s';
     alert.handler = alert.handler || 1;
     alert.notifications = alert.notifications || [];
 
diff --git a/public/app/features/alerting/partials/alert_tab.html b/public/app/features/alerting/partials/alert_tab.html
index cb101672aa4..084aeb2036a 100644
--- a/public/app/features/alerting/partials/alert_tab.html
+++ b/public/app/features/alerting/partials/alert_tab.html
@@ -31,6 +31,9 @@
 					<input type="text" class="gf-form-input width-20" ng-model="ctrl.alert.name">
 					<span class="gf-form-label">Evaluate every</span>
 					<input class="gf-form-input max-width-5" type="text" ng-model="ctrl.alert.frequency"></input>
+					<a class="gf-form-label" ng-click="ctrl.alert.notifyOnce = !ctrl.alert.notifyOnce;">{{ ctrl.alert.notifyOnce ? 'Notify on state change' : 'Notify every' }}</a>
+					<input class="gf-form-input max-width-5" type="number" ng-model="ctrl.alert.notifyFrequency" ng-hide="ctrl.alert.notifyOnce"></input>
+					<span class="gf-form-label" ng-hide="ctrl.alert.notifyOnce">evaluations</span>
 				</div>
 			</div>
 

From 3cb0e27e1c474e5d203eb32428b2f39ee5fb3216 Mon Sep 17 00:00:00 2001
From: John Baublitz <john.baublitz@threatstack.com>
Date: Sat, 19 May 2018 16:21:00 -0400
Subject: [PATCH 02/39] Revert changes post code review and move them to
 notification page

---
 pkg/api/dtos/alerting.go                      | 25 +++----
 pkg/models/alert.go                           |  9 ---
 pkg/models/alert_notifications.go             | 71 ++++++++++++++-----
 pkg/services/alerting/eval_context.go         | 15 ++++
 pkg/services/alerting/extractor.go            |  2 -
 pkg/services/alerting/interfaces.go           |  2 +
 pkg/services/alerting/notifier.go             | 12 +++-
 .../alerting/notifiers/alertmanager.go        |  2 +-
 pkg/services/alerting/notifiers/base.go       | 25 +++++--
 pkg/services/alerting/notifiers/dingding.go   |  2 +-
 pkg/services/alerting/notifiers/discord.go    |  2 +-
 pkg/services/alerting/notifiers/email.go      |  2 +-
 pkg/services/alerting/notifiers/hipchat.go    |  2 +-
 pkg/services/alerting/notifiers/kafka.go      |  2 +-
 pkg/services/alerting/notifiers/line.go       |  2 +-
 pkg/services/alerting/notifiers/opsgenie.go   |  2 +-
 pkg/services/alerting/notifiers/pagerduty.go  |  2 +-
 pkg/services/alerting/notifiers/pushover.go   |  2 +-
 pkg/services/alerting/notifiers/sensu.go      |  2 +-
 pkg/services/alerting/notifiers/slack.go      |  2 +-
 pkg/services/alerting/notifiers/teams.go      |  2 +-
 pkg/services/alerting/notifiers/telegram.go   |  2 +-
 pkg/services/alerting/notifiers/threema.go    |  2 +-
 pkg/services/alerting/notifiers/victorops.go  |  2 +-
 pkg/services/alerting/notifiers/webhook.go    |  2 +-
 pkg/services/alerting/result_handler.go       |  1 -
 pkg/services/alerting/rule.go                 |  6 --
 pkg/services/sqlstore/alert.go                | 22 +-----
 pkg/services/sqlstore/alert_notification.go   | 58 +++++++++++++--
 pkg/services/sqlstore/migrations/alert_mig.go | 27 ++++++-
 .../app/features/alerting/alert_tab_ctrl.ts   |  3 -
 .../alerting/notification_edit_ctrl.ts        |  2 +
 .../features/alerting/partials/alert_tab.html |  3 -
 .../alerting/partials/notification_edit.html  |  5 ++
 34 files changed, 215 insertions(+), 107 deletions(-)

diff --git a/pkg/api/dtos/alerting.go b/pkg/api/dtos/alerting.go
index 64dd619a4eb..5e0196c20d1 100644
--- a/pkg/api/dtos/alerting.go
+++ b/pkg/api/dtos/alerting.go
@@ -21,18 +21,17 @@ type AlertRule struct {
 	ExecutionError string           `json:"executionError"`
 	Url            string           `json:"url"`
 	CanEdit        bool             `json:"canEdit"`
-	NotifyOnce     bool             `json:"notifyOnce"`
-	NotifyEval     uint64           `json:"notifyEval"`
-	NotifyFreq     uint64           `json:"notifyFrequency"`
 }
 
 type AlertNotification struct {
-	Id        int64     `json:"id"`
-	Name      string    `json:"name"`
-	Type      string    `json:"type"`
-	IsDefault bool      `json:"isDefault"`
-	Created   time.Time `json:"created"`
-	Updated   time.Time `json:"updated"`
+	Id         int64     `json:"id"`
+	Name       string    `json:"name"`
+	Type       string    `json:"type"`
+	IsDefault  bool      `json:"isDefault"`
+	NotifyOnce bool      `json:"notifyOnce"`
+	Frequency  bool      `json:"frequency"`
+	Created    time.Time `json:"created"`
+	Updated    time.Time `json:"updated"`
 }
 
 type AlertTestCommand struct {
@@ -62,9 +61,11 @@ type EvalMatch struct {
 }
 
 type NotificationTestCommand struct {
-	Name     string           `json:"name"`
-	Type     string           `json:"type"`
-	Settings *simplejson.Json `json:"settings"`
+	Name       string           `json:"name"`
+	Type       string           `json:"type"`
+	NotifyOnce bool             `json:"notifyOnce"`
+	Frequency  time.Duration    `json:"frequency"`
+	Settings   *simplejson.Json `json:"settings"`
 }
 
 type PauseAlertCommand struct {
diff --git a/pkg/models/alert.go b/pkg/models/alert.go
index 56ceeb2cbf7..fba2aa63df9 100644
--- a/pkg/models/alert.go
+++ b/pkg/models/alert.go
@@ -72,9 +72,6 @@ type Alert struct {
 	Silenced       bool
 	ExecutionError string
 	Frequency      int64
-	NotifyOnce     bool
-	NotifyFreq     uint64
-	NotifyEval     uint64
 
 	EvalData     *simplejson.Json
 	NewStateDate time.Time
@@ -98,8 +95,6 @@ func (this *Alert) ContainsUpdates(other *Alert) bool {
 	result := false
 	result = result || this.Name != other.Name
 	result = result || this.Message != other.Message
-	result = result || this.NotifyOnce != other.NotifyOnce
-	result = result || (!other.NotifyOnce && this.NotifyFreq != other.NotifyFreq)
 
 	if this.Settings != nil && other.Settings != nil {
 		json1, err1 := this.Settings.Encode()
@@ -164,10 +159,6 @@ type SetAlertStateCommand struct {
 	Timestamp time.Time
 }
 
-type IncAlertEvalCommand struct {
-	AlertId int64
-}
-
 //Queries
 type GetAlertsQuery struct {
 	OrgId        int64
diff --git a/pkg/models/alert_notifications.go b/pkg/models/alert_notifications.go
index 87b515f370c..cba62a51527 100644
--- a/pkg/models/alert_notifications.go
+++ b/pkg/models/alert_notifications.go
@@ -7,32 +7,38 @@ import (
 )
 
 type AlertNotification struct {
-	Id        int64            `json:"id"`
-	OrgId     int64            `json:"-"`
-	Name      string           `json:"name"`
-	Type      string           `json:"type"`
-	IsDefault bool             `json:"isDefault"`
-	Settings  *simplejson.Json `json:"settings"`
-	Created   time.Time        `json:"created"`
-	Updated   time.Time        `json:"updated"`
+	Id         int64            `json:"id"`
+	OrgId      int64            `json:"-"`
+	Name       string           `json:"name"`
+	Type       string           `json:"type"`
+	NotifyOnce bool             `json:"notifyOnce"`
+	Frequency  time.Duration    `json:"frequency"`
+	IsDefault  bool             `json:"isDefault"`
+	Settings   *simplejson.Json `json:"settings"`
+	Created    time.Time        `json:"created"`
+	Updated    time.Time        `json:"updated"`
 }
 
 type CreateAlertNotificationCommand struct {
-	Name      string           `json:"name"  binding:"Required"`
-	Type      string           `json:"type"  binding:"Required"`
-	IsDefault bool             `json:"isDefault"`
-	Settings  *simplejson.Json `json:"settings"`
+	Name       string           `json:"name"  binding:"Required"`
+	Type       string           `json:"type"  binding:"Required"`
+	NotifyOnce bool             `json:"notifyOnce"  binding:"Required"`
+	Frequency  time.Duration    `json:"frequency"`
+	IsDefault  bool             `json:"isDefault"`
+	Settings   *simplejson.Json `json:"settings"`
 
 	OrgId  int64 `json:"-"`
 	Result *AlertNotification
 }
 
 type UpdateAlertNotificationCommand struct {
-	Id        int64            `json:"id"  binding:"Required"`
-	Name      string           `json:"name"  binding:"Required"`
-	Type      string           `json:"type"  binding:"Required"`
-	IsDefault bool             `json:"isDefault"`
-	Settings  *simplejson.Json `json:"settings"  binding:"Required"`
+	Id         int64            `json:"id"  binding:"Required"`
+	Name       string           `json:"name"  binding:"Required"`
+	Type       string           `json:"type"  binding:"Required"`
+	NotifyOnce string           `json:"notifyOnce"  binding:"Required"`
+	Frequency  string           `json:"frequency"`
+	IsDefault  bool             `json:"isDefault"`
+	Settings   *simplejson.Json `json:"settings"  binding:"Required"`
 
 	OrgId  int64 `json:"-"`
 	Result *AlertNotification
@@ -63,3 +69,34 @@ type GetAllAlertNotificationsQuery struct {
 
 	Result []*AlertNotification
 }
+
+type NotificationJournal struct {
+	Id         int64
+	OrgId      int64
+	AlertId    int64
+	NotifierId int64
+	SentAt     time.Time
+	Success    bool
+}
+
+type RecordNotificationJournalCommand struct {
+	OrgId      int64
+	AlertId    int64
+	NotifierId int64
+	SentAt     time.Time
+	Success    bool
+}
+
+type GetLatestNotificationQuery struct {
+	OrgId      int64
+	AlertId    int64
+	NotifierId int64
+
+	Result *NotificationJournal
+}
+
+type CleanNotificationJournalCommand struct {
+	OrgId      int64
+	AlertId    int64
+	NotifierId int64
+}
diff --git a/pkg/services/alerting/eval_context.go b/pkg/services/alerting/eval_context.go
index d0441d379b7..b451d188a64 100644
--- a/pkg/services/alerting/eval_context.go
+++ b/pkg/services/alerting/eval_context.go
@@ -143,3 +143,18 @@ func (c *EvalContext) GetNewState() m.AlertStateType {
 
 	return m.AlertStateOK
 }
+
+func (c *EvalContext) LastNotify(notifierId int64) *time.Time {
+	cmd := &m.GetLatestNotificationQuery{
+		OrgId:      c.Rule.OrgId,
+		AlertId:    c.Rule.Id,
+		NotifierId: notifierId,
+	}
+	if err := bus.Dispatch(cmd); err != nil {
+		c.log.Warn("Could not determine last time alert",
+			c.Rule.Name, "notified")
+		return nil
+	}
+
+	return &cmd.Result.SentAt
+}
diff --git a/pkg/services/alerting/extractor.go b/pkg/services/alerting/extractor.go
index f820e546a93..e1c1bfacb2e 100644
--- a/pkg/services/alerting/extractor.go
+++ b/pkg/services/alerting/extractor.go
@@ -122,8 +122,6 @@ func (e *DashAlertExtractor) getAlertFromPanels(jsonWithPanels *simplejson.Json,
 			Handler:     jsonAlert.Get("handler").MustInt64(),
 			Message:     jsonAlert.Get("message").MustString(),
 			Frequency:   frequency,
-			NotifyOnce:  jsonAlert.Get("notifyOnce").MustBool(),
-			NotifyFreq:  jsonAlert.Get("notifyFrequency").MustUint64(),
 		}
 
 		for _, condition := range jsonAlert.Get("conditions").MustArray() {
diff --git a/pkg/services/alerting/interfaces.go b/pkg/services/alerting/interfaces.go
index 18f969ba1b9..8842b35fba2 100644
--- a/pkg/services/alerting/interfaces.go
+++ b/pkg/services/alerting/interfaces.go
@@ -19,6 +19,8 @@ type Notifier interface {
 
 	GetNotifierId() int64
 	GetIsDefault() bool
+	GetNotifyOnce() bool
+	GetFrequency() time.Duration
 }
 
 type NotifierSlice []Notifier
diff --git a/pkg/services/alerting/notifier.go b/pkg/services/alerting/notifier.go
index 2ea68cf5085..53923a420fe 100644
--- a/pkg/services/alerting/notifier.go
+++ b/pkg/services/alerting/notifier.go
@@ -66,7 +66,17 @@ func (n *notificationService) sendNotifications(context *EvalContext, notifiers
 		not := notifier //avoid updating scope variable in go routine
 		n.log.Debug("Sending notification", "type", not.GetType(), "id", not.GetNotifierId(), "isDefault", not.GetIsDefault())
 		metrics.M_Alerting_Notification_Sent.WithLabelValues(not.GetType()).Inc()
-		g.Go(func() error { return not.Notify(context) })
+		g.Go(func() error {
+			success := not.Notify(context) == nil
+			cmd := &m.RecordNotificationJournalCommand{
+				OrgId:      context.Rule.OrgId,
+				AlertId:    context.Rule.Id,
+				NotifierId: not.GetNotifierId(),
+				SentAt:     time.Now(),
+				Success:    success,
+			}
+			return bus.Dispatch(cmd)
+		})
 	}
 
 	return g.Wait()
diff --git a/pkg/services/alerting/notifiers/alertmanager.go b/pkg/services/alerting/notifiers/alertmanager.go
index d449167de13..3eeb25986e0 100644
--- a/pkg/services/alerting/notifiers/alertmanager.go
+++ b/pkg/services/alerting/notifiers/alertmanager.go
@@ -33,7 +33,7 @@ func NewAlertmanagerNotifier(model *m.AlertNotification) (alerting.Notifier, err
 	}
 
 	return &AlertmanagerNotifier{
-		NotifierBase: NewNotifierBase(model.Id, model.IsDefault, model.Name, model.Type, model.Settings),
+		NotifierBase: NewNotifierBase(model.Id, model.IsDefault, model.Name, model.Type, model.NotifyOnce, model.Frequency, model.Settings),
 		Url:          url,
 		log:          log.New("alerting.notifier.prometheus-alertmanager"),
 	}, nil
diff --git a/pkg/services/alerting/notifiers/base.go b/pkg/services/alerting/notifiers/base.go
index 498bae3a6e6..e9e32020c6d 100644
--- a/pkg/services/alerting/notifiers/base.go
+++ b/pkg/services/alerting/notifiers/base.go
@@ -1,6 +1,8 @@
 package notifiers
 
 import (
+	"time"
+
 	"github.com/grafana/grafana/pkg/components/simplejson"
 	m "github.com/grafana/grafana/pkg/models"
 	"github.com/grafana/grafana/pkg/services/alerting"
@@ -12,9 +14,11 @@ type NotifierBase struct {
 	Id          int64
 	IsDeault    bool
 	UploadImage bool
+	NotifyOnce  bool
+	Frequency   time.Duration
 }
 
-func NewNotifierBase(id int64, isDefault bool, name, notifierType string, model *simplejson.Json) NotifierBase {
+func NewNotifierBase(id int64, isDefault bool, name, notifierType string, notifyOnce bool, frequency time.Duration, model *simplejson.Json) NotifierBase {
 	uploadImage := true
 	value, exist := model.CheckGet("uploadImage")
 	if exist {
@@ -27,15 +31,17 @@ func NewNotifierBase(id int64, isDefault bool, name, notifierType string, model
 		IsDeault:    isDefault,
 		Type:        notifierType,
 		UploadImage: uploadImage,
+		NotifyOnce:  notifyOnce,
+		Frequency:   frequency,
 	}
 }
 
-func defaultShouldNotify(context *alerting.EvalContext) bool {
+func defaultShouldNotify(context *alerting.EvalContext, notifyOnce bool, frequency time.Duration, lastNotify *time.Time) bool {
 	// Only notify on state change.
-	if context.PrevAlertState == context.Rule.State && context.Rule.NotifyOnce {
+	if context.PrevAlertState == context.Rule.State && notifyOnce {
 		return false
 	}
-	if !context.Rule.NotifyOnce && context.Rule.NotifyEval != 0 {
+	if !notifyOnce && lastNotify != nil && lastNotify.Add(frequency).After(time.Now()) {
 		return false
 	}
 	// Do not notify when we become OK for the first time.
@@ -46,7 +52,8 @@ func defaultShouldNotify(context *alerting.EvalContext) bool {
 }
 
 func (n *NotifierBase) ShouldNotify(context *alerting.EvalContext) bool {
-	return defaultShouldNotify(context)
+	lastNotify := context.LastNotify(n.Id)
+	return defaultShouldNotify(context, n.NotifyOnce, n.Frequency, lastNotify)
 }
 
 func (n *NotifierBase) GetType() string {
@@ -64,3 +71,11 @@ func (n *NotifierBase) GetNotifierId() int64 {
 func (n *NotifierBase) GetIsDefault() bool {
 	return n.IsDeault
 }
+
+func (n *NotifierBase) GetNotifyOnce() bool {
+	return n.NotifyOnce
+}
+
+func (n *NotifierBase) GetFrequency() time.Duration {
+	return n.Frequency
+}
diff --git a/pkg/services/alerting/notifiers/dingding.go b/pkg/services/alerting/notifiers/dingding.go
index 14eacef5831..78446c56f88 100644
--- a/pkg/services/alerting/notifiers/dingding.go
+++ b/pkg/services/alerting/notifiers/dingding.go
@@ -32,7 +32,7 @@ func NewDingDingNotifier(model *m.AlertNotification) (alerting.Notifier, error)
 	}
 
 	return &DingDingNotifier{
-		NotifierBase: NewNotifierBase(model.Id, model.IsDefault, model.Name, model.Type, model.Settings),
+		NotifierBase: NewNotifierBase(model.Id, model.IsDefault, model.Name, model.Type, model.NotifyOnce, model.Frequency, model.Settings),
 		Url:          url,
 		log:          log.New("alerting.notifier.dingding"),
 	}, nil
diff --git a/pkg/services/alerting/notifiers/discord.go b/pkg/services/alerting/notifiers/discord.go
index 3ffa7484870..693ed31e206 100644
--- a/pkg/services/alerting/notifiers/discord.go
+++ b/pkg/services/alerting/notifiers/discord.go
@@ -39,7 +39,7 @@ func NewDiscordNotifier(model *m.AlertNotification) (alerting.Notifier, error) {
 	}
 
 	return &DiscordNotifier{
-		NotifierBase: NewNotifierBase(model.Id, model.IsDefault, model.Name, model.Type, model.Settings),
+		NotifierBase: NewNotifierBase(model.Id, model.IsDefault, model.Name, model.Type, model.NotifyOnce, model.Frequency, model.Settings),
 		WebhookURL:   url,
 		log:          log.New("alerting.notifier.discord"),
 	}, nil
diff --git a/pkg/services/alerting/notifiers/email.go b/pkg/services/alerting/notifiers/email.go
index 562ffbe1269..234a4f8e756 100644
--- a/pkg/services/alerting/notifiers/email.go
+++ b/pkg/services/alerting/notifiers/email.go
@@ -52,7 +52,7 @@ func NewEmailNotifier(model *m.AlertNotification) (alerting.Notifier, error) {
 	})
 
 	return &EmailNotifier{
-		NotifierBase: NewNotifierBase(model.Id, model.IsDefault, model.Name, model.Type, model.Settings),
+		NotifierBase: NewNotifierBase(model.Id, model.IsDefault, model.Name, model.Type, model.NotifyOnce, model.Frequency, model.Settings),
 		Addresses:    addresses,
 		log:          log.New("alerting.notifier.email"),
 	}, nil
diff --git a/pkg/services/alerting/notifiers/hipchat.go b/pkg/services/alerting/notifiers/hipchat.go
index 58e1b7bd71e..4eb5b78811e 100644
--- a/pkg/services/alerting/notifiers/hipchat.go
+++ b/pkg/services/alerting/notifiers/hipchat.go
@@ -59,7 +59,7 @@ func NewHipChatNotifier(model *models.AlertNotification) (alerting.Notifier, err
 	roomId := model.Settings.Get("roomid").MustString()
 
 	return &HipChatNotifier{
-		NotifierBase: NewNotifierBase(model.Id, model.IsDefault, model.Name, model.Type, model.Settings),
+		NotifierBase: NewNotifierBase(model.Id, model.IsDefault, model.Name, model.Type, model.NotifyOnce, model.Frequency, model.Settings),
 		Url:          url,
 		ApiKey:       apikey,
 		RoomId:       roomId,
diff --git a/pkg/services/alerting/notifiers/kafka.go b/pkg/services/alerting/notifiers/kafka.go
index 92f6489106b..0dab556d5e1 100644
--- a/pkg/services/alerting/notifiers/kafka.go
+++ b/pkg/services/alerting/notifiers/kafka.go
@@ -43,7 +43,7 @@ func NewKafkaNotifier(model *m.AlertNotification) (alerting.Notifier, error) {
 	}
 
 	return &KafkaNotifier{
-		NotifierBase: NewNotifierBase(model.Id, model.IsDefault, model.Name, model.Type, model.Settings),
+		NotifierBase: NewNotifierBase(model.Id, model.IsDefault, model.Name, model.Type, model.NotifyOnce, model.Frequency, model.Settings),
 		Endpoint:     endpoint,
 		Topic:        topic,
 		log:          log.New("alerting.notifier.kafka"),
diff --git a/pkg/services/alerting/notifiers/line.go b/pkg/services/alerting/notifiers/line.go
index 4814662f3a9..0ee252e6447 100644
--- a/pkg/services/alerting/notifiers/line.go
+++ b/pkg/services/alerting/notifiers/line.go
@@ -39,7 +39,7 @@ func NewLINENotifier(model *m.AlertNotification) (alerting.Notifier, error) {
 	}
 
 	return &LineNotifier{
-		NotifierBase: NewNotifierBase(model.Id, model.IsDefault, model.Name, model.Type, model.Settings),
+		NotifierBase: NewNotifierBase(model.Id, model.IsDefault, model.Name, model.Type, model.NotifyOnce, model.Frequency, model.Settings),
 		Token:        token,
 		log:          log.New("alerting.notifier.line"),
 	}, nil
diff --git a/pkg/services/alerting/notifiers/opsgenie.go b/pkg/services/alerting/notifiers/opsgenie.go
index f0f5142cf05..991afd5ce9b 100644
--- a/pkg/services/alerting/notifiers/opsgenie.go
+++ b/pkg/services/alerting/notifiers/opsgenie.go
@@ -56,7 +56,7 @@ func NewOpsGenieNotifier(model *m.AlertNotification) (alerting.Notifier, error)
 	}
 
 	return &OpsGenieNotifier{
-		NotifierBase: NewNotifierBase(model.Id, model.IsDefault, model.Name, model.Type, model.Settings),
+		NotifierBase: NewNotifierBase(model.Id, model.IsDefault, model.Name, model.Type, model.NotifyOnce, model.Frequency, model.Settings),
 		ApiKey:       apiKey,
 		ApiUrl:       apiUrl,
 		AutoClose:    autoClose,
diff --git a/pkg/services/alerting/notifiers/pagerduty.go b/pkg/services/alerting/notifiers/pagerduty.go
index 02219b2203d..afa0ba63eca 100644
--- a/pkg/services/alerting/notifiers/pagerduty.go
+++ b/pkg/services/alerting/notifiers/pagerduty.go
@@ -51,7 +51,7 @@ func NewPagerdutyNotifier(model *m.AlertNotification) (alerting.Notifier, error)
 	}
 
 	return &PagerdutyNotifier{
-		NotifierBase: NewNotifierBase(model.Id, model.IsDefault, model.Name, model.Type, model.Settings),
+		NotifierBase: NewNotifierBase(model.Id, model.IsDefault, model.Name, model.Type, model.NotifyOnce, model.Frequency, model.Settings),
 		Key:          key,
 		AutoResolve:  autoResolve,
 		log:          log.New("alerting.notifier.pagerduty"),
diff --git a/pkg/services/alerting/notifiers/pushover.go b/pkg/services/alerting/notifiers/pushover.go
index cbe9e16801a..09dfd6f0f9b 100644
--- a/pkg/services/alerting/notifiers/pushover.go
+++ b/pkg/services/alerting/notifiers/pushover.go
@@ -99,7 +99,7 @@ func NewPushoverNotifier(model *m.AlertNotification) (alerting.Notifier, error)
 		return nil, alerting.ValidationError{Reason: "API token not given"}
 	}
 	return &PushoverNotifier{
-		NotifierBase: NewNotifierBase(model.Id, model.IsDefault, model.Name, model.Type, model.Settings),
+		NotifierBase: NewNotifierBase(model.Id, model.IsDefault, model.Name, model.Type, model.NotifyOnce, model.Frequency, model.Settings),
 		UserKey:      userKey,
 		ApiToken:     apiToken,
 		Priority:     priority,
diff --git a/pkg/services/alerting/notifiers/sensu.go b/pkg/services/alerting/notifiers/sensu.go
index 9f77801d458..e6b94d3223e 100644
--- a/pkg/services/alerting/notifiers/sensu.go
+++ b/pkg/services/alerting/notifiers/sensu.go
@@ -51,7 +51,7 @@ func NewSensuNotifier(model *m.AlertNotification) (alerting.Notifier, error) {
 	}
 
 	return &SensuNotifier{
-		NotifierBase: NewNotifierBase(model.Id, model.IsDefault, model.Name, model.Type, model.Settings),
+		NotifierBase: NewNotifierBase(model.Id, model.IsDefault, model.Name, model.Type, model.NotifyOnce, model.Frequency, model.Settings),
 		Url:          url,
 		User:         model.Settings.Get("username").MustString(),
 		Source:       model.Settings.Get("source").MustString(),
diff --git a/pkg/services/alerting/notifiers/slack.go b/pkg/services/alerting/notifiers/slack.go
index a8139b62726..fbbe4b3e59d 100644
--- a/pkg/services/alerting/notifiers/slack.go
+++ b/pkg/services/alerting/notifiers/slack.go
@@ -78,7 +78,7 @@ func NewSlackNotifier(model *m.AlertNotification) (alerting.Notifier, error) {
 	uploadImage := model.Settings.Get("uploadImage").MustBool(true)
 
 	return &SlackNotifier{
-		NotifierBase: NewNotifierBase(model.Id, model.IsDefault, model.Name, model.Type, model.Settings),
+		NotifierBase: NewNotifierBase(model.Id, model.IsDefault, model.Name, model.Type, model.NotifyOnce, model.Frequency, model.Settings),
 		Url:          url,
 		Recipient:    recipient,
 		Mention:      mention,
diff --git a/pkg/services/alerting/notifiers/teams.go b/pkg/services/alerting/notifiers/teams.go
index 7f62340d0e1..362a367e1f2 100644
--- a/pkg/services/alerting/notifiers/teams.go
+++ b/pkg/services/alerting/notifiers/teams.go
@@ -33,7 +33,7 @@ func NewTeamsNotifier(model *m.AlertNotification) (alerting.Notifier, error) {
 	}
 
 	return &TeamsNotifier{
-		NotifierBase: NewNotifierBase(model.Id, model.IsDefault, model.Name, model.Type, model.Settings),
+		NotifierBase: NewNotifierBase(model.Id, model.IsDefault, model.Name, model.Type, model.NotifyOnce, model.Frequency, model.Settings),
 		Url:          url,
 		log:          log.New("alerting.notifier.teams"),
 	}, nil
diff --git a/pkg/services/alerting/notifiers/telegram.go b/pkg/services/alerting/notifiers/telegram.go
index ca24c996914..97696b2290c 100644
--- a/pkg/services/alerting/notifiers/telegram.go
+++ b/pkg/services/alerting/notifiers/telegram.go
@@ -78,7 +78,7 @@ func NewTelegramNotifier(model *m.AlertNotification) (alerting.Notifier, error)
 	}
 
 	return &TelegramNotifier{
-		NotifierBase: NewNotifierBase(model.Id, model.IsDefault, model.Name, model.Type, model.Settings),
+		NotifierBase: NewNotifierBase(model.Id, model.IsDefault, model.Name, model.Type, model.NotifyOnce, model.Frequency, model.Settings),
 		BotToken:     botToken,
 		ChatID:       chatId,
 		UploadImage:  uploadImage,
diff --git a/pkg/services/alerting/notifiers/threema.go b/pkg/services/alerting/notifiers/threema.go
index e4ffffc9108..e7fb39f27db 100644
--- a/pkg/services/alerting/notifiers/threema.go
+++ b/pkg/services/alerting/notifiers/threema.go
@@ -106,7 +106,7 @@ func NewThreemaNotifier(model *m.AlertNotification) (alerting.Notifier, error) {
 	}
 
 	return &ThreemaNotifier{
-		NotifierBase: NewNotifierBase(model.Id, model.IsDefault, model.Name, model.Type, model.Settings),
+		NotifierBase: NewNotifierBase(model.Id, model.IsDefault, model.Name, model.Type, model.NotifyOnce, model.Frequency, model.Settings),
 		GatewayID:    gatewayID,
 		RecipientID:  recipientID,
 		APISecret:    apiSecret,
diff --git a/pkg/services/alerting/notifiers/victorops.go b/pkg/services/alerting/notifiers/victorops.go
index a753ca3cbf6..c6c1cf76047 100644
--- a/pkg/services/alerting/notifiers/victorops.go
+++ b/pkg/services/alerting/notifiers/victorops.go
@@ -51,7 +51,7 @@ func NewVictoropsNotifier(model *models.AlertNotification) (alerting.Notifier, e
 	}
 
 	return &VictoropsNotifier{
-		NotifierBase: NewNotifierBase(model.Id, model.IsDefault, model.Name, model.Type, model.Settings),
+		NotifierBase: NewNotifierBase(model.Id, model.IsDefault, model.Name, model.Type, model.NotifyOnce, model.Frequency, model.Settings),
 		URL:          url,
 		AutoResolve:  autoResolve,
 		log:          log.New("alerting.notifier.victorops"),
diff --git a/pkg/services/alerting/notifiers/webhook.go b/pkg/services/alerting/notifiers/webhook.go
index 4c97ed2b75e..26989873e9e 100644
--- a/pkg/services/alerting/notifiers/webhook.go
+++ b/pkg/services/alerting/notifiers/webhook.go
@@ -47,7 +47,7 @@ func NewWebHookNotifier(model *m.AlertNotification) (alerting.Notifier, error) {
 	}
 
 	return &WebhookNotifier{
-		NotifierBase: NewNotifierBase(model.Id, model.IsDefault, model.Name, model.Type, model.Settings),
+		NotifierBase: NewNotifierBase(model.Id, model.IsDefault, model.Name, model.Type, model.NotifyOnce, model.Frequency, model.Settings),
 		Url:          url,
 		User:         model.Settings.Get("username").MustString(),
 		Password:     model.Settings.Get("password").MustString(),
diff --git a/pkg/services/alerting/result_handler.go b/pkg/services/alerting/result_handler.go
index 56d299001f0..c57b28c7c3e 100644
--- a/pkg/services/alerting/result_handler.go
+++ b/pkg/services/alerting/result_handler.go
@@ -88,7 +88,6 @@ func (handler *DefaultResultHandler) Handle(evalContext *EvalContext) error {
 		}
 	}
 
-	bus.Dispatch(&m.IncAlertEvalCommand{AlertId: evalContext.Rule.Id})
 	handler.notifier.SendIfNeeded(evalContext)
 
 	return nil
diff --git a/pkg/services/alerting/rule.go b/pkg/services/alerting/rule.go
index 0003fe791e3..018d138dbe4 100644
--- a/pkg/services/alerting/rule.go
+++ b/pkg/services/alerting/rule.go
@@ -23,9 +23,6 @@ type Rule struct {
 	State               m.AlertStateType
 	Conditions          []Condition
 	Notifications       []int64
-	NotifyOnce          bool
-	NotifyFreq          uint64
-	NotifyEval          uint64
 }
 
 type ValidationError struct {
@@ -100,9 +97,6 @@ func NewRuleFromDBAlert(ruleDef *m.Alert) (*Rule, error) {
 	model.Name = ruleDef.Name
 	model.Message = ruleDef.Message
 	model.Frequency = ruleDef.Frequency
-	model.NotifyOnce = ruleDef.NotifyOnce
-	model.NotifyFreq = ruleDef.NotifyFreq
-	model.NotifyEval = ruleDef.NotifyEval
 	model.State = ruleDef.State
 	model.NoDataState = m.NoDataOption(ruleDef.Settings.Get("noDataState").MustString("no_data"))
 	model.ExecutionErrorState = m.ExecutionErrorOption(ruleDef.Settings.Get("executionErrorState").MustString("alerting"))
diff --git a/pkg/services/sqlstore/alert.go b/pkg/services/sqlstore/alert.go
index 9ab28be84ee..58ec7e2857a 100644
--- a/pkg/services/sqlstore/alert.go
+++ b/pkg/services/sqlstore/alert.go
@@ -22,7 +22,6 @@ func init() {
 	bus.AddHandler("sql", GetAlertStatesForDashboard)
 	bus.AddHandler("sql", PauseAlert)
 	bus.AddHandler("sql", PauseAllAlerts)
-	bus.AddHandler("sql", IncAlertEval)
 }
 
 func GetAlertById(query *m.GetAlertByIdQuery) error {
@@ -189,7 +188,7 @@ func updateAlerts(existingAlerts []*m.Alert, cmd *m.SaveAlertsCommand, sess *DBS
 			if alertToUpdate.ContainsUpdates(alert) {
 				alert.Updated = timeNow()
 				alert.State = alertToUpdate.State
-				sess.MustCols("message", "notify_freq", "notify_once")
+				sess.MustCols("message")
 				_, err := sess.Id(alert.Id).Update(alert)
 				if err != nil {
 					return err
@@ -344,22 +343,3 @@ func GetAlertStatesForDashboard(query *m.GetAlertStatesForDashboardQuery) error
 
 	return err
 }
-
-func IncAlertEval(cmd *m.IncAlertEvalCommand) error {
-	return inTransaction(func(sess *DBSession) error {
-		alert := m.Alert{}
-
-		if _, err := sess.Id(cmd.AlertId).Get(&alert); err != nil {
-			return err
-		}
-
-		alert.NotifyEval = (alert.NotifyEval + 1) % alert.NotifyFreq
-
-		sess.MustCols("notify_eval")
-		if _, err := sess.Id(cmd.AlertId).Update(alert); err != nil {
-			return err
-		}
-
-		return nil
-	})
-}
diff --git a/pkg/services/sqlstore/alert_notification.go b/pkg/services/sqlstore/alert_notification.go
index 651241f7714..8bb17143042 100644
--- a/pkg/services/sqlstore/alert_notification.go
+++ b/pkg/services/sqlstore/alert_notification.go
@@ -17,6 +17,9 @@ func init() {
 	bus.AddHandler("sql", DeleteAlertNotification)
 	bus.AddHandler("sql", GetAlertNotificationsToSend)
 	bus.AddHandler("sql", GetAllAlertNotifications)
+	bus.AddHandler("sql", RecordNotificationJournal)
+	bus.AddHandler("sql", GetLatestNotification)
+	bus.AddHandler("sql", CleanNotificationJournal)
 }
 
 func DeleteAlertNotification(cmd *m.DeleteAlertNotificationCommand) error {
@@ -138,13 +141,15 @@ func CreateAlertNotificationCommand(cmd *m.CreateAlertNotificationCommand) error
 		}
 
 		alertNotification := &m.AlertNotification{
-			OrgId:     cmd.OrgId,
-			Name:      cmd.Name,
-			Type:      cmd.Type,
-			Settings:  cmd.Settings,
-			Created:   time.Now(),
-			Updated:   time.Now(),
-			IsDefault: cmd.IsDefault,
+			OrgId:      cmd.OrgId,
+			Name:       cmd.Name,
+			Type:       cmd.Type,
+			Settings:   cmd.Settings,
+			NotifyOnce: cmd.NotifyOnce,
+			Frequency:  cmd.Frequency,
+			Created:    time.Now(),
+			Updated:    time.Now(),
+			IsDefault:  cmd.IsDefault,
 		}
 
 		if _, err = sess.Insert(alertNotification); err != nil {
@@ -192,3 +197,42 @@ func UpdateAlertNotification(cmd *m.UpdateAlertNotificationCommand) error {
 		return nil
 	})
 }
+
+func RecordNotificationJournal(cmd *m.RecordNotificationJournalCommand) error {
+	return inTransaction(func(sess *DBSession) error {
+		journalEntry := &m.NotificationJournal{
+			OrgId:      cmd.OrgId,
+			AlertId:    cmd.AlertId,
+			NotifierId: cmd.NotifierId,
+			SentAt:     cmd.SentAt,
+			Success:    cmd.Success,
+		}
+
+		if _, err := sess.Insert(journalEntry); err != nil {
+			return err
+		}
+
+		return nil
+	})
+}
+
+func GetLatestNotification(cmd *m.GetLatestNotificationQuery) error {
+	return inTransaction(func(sess *DBSession) error {
+		notificationJournal := &m.NotificationJournal{}
+		_, err := sess.OrderBy("notification_journal.sent_at").Desc().Where("notification_journal.org_id = ? AND notification_journal.alert_id = ? AND notification_journal.notifier_id = ?", cmd.OrgId, cmd.AlertId, cmd.NotifierId).Get(notificationJournal)
+		if err != nil {
+			return err
+		}
+
+		cmd.Result = notificationJournal
+		return nil
+	})
+}
+
+func CleanNotificationJournal(cmd *m.CleanNotificationJournalCommand) error {
+	return inTransaction(func(sess *DBSession) error {
+		sql := "DELETE FROM notification_journal WHERE notification_journal.org_id = ? AND notification_journal.alert_id = ? AND notification_journal.notifier_id = ?"
+		_, err := sess.Exec(sql, cmd.OrgId, cmd.AlertId, cmd.NotifierId)
+		return err
+	})
+}
diff --git a/pkg/services/sqlstore/migrations/alert_mig.go b/pkg/services/sqlstore/migrations/alert_mig.go
index 3452e5710cc..d045f611fb2 100644
--- a/pkg/services/sqlstore/migrations/alert_mig.go
+++ b/pkg/services/sqlstore/migrations/alert_mig.go
@@ -29,9 +29,6 @@ func addAlertMigrations(mg *Migrator) {
 			{Name: "state_changes", Type: DB_Int, Nullable: false},
 			{Name: "created", Type: DB_DateTime, Nullable: false},
 			{Name: "updated", Type: DB_DateTime, Nullable: false},
-			{Name: "notify_once", Type: DB_Bool, Nullable: false},
-			{Name: "notify_freq", Type: DB_Int, Nullable: false},
-			{Name: "notify_eval", Type: DB_Int, Nullable: false},
 		},
 		Indices: []*Index{
 			{Cols: []string{"org_id", "id"}, Type: IndexType},
@@ -68,8 +65,32 @@ func addAlertMigrations(mg *Migrator) {
 	mg.AddMigration("Add column is_default", NewAddColumnMigration(alert_notification, &Column{
 		Name: "is_default", Type: DB_Bool, Nullable: false, Default: "0",
 	}))
+	mg.AddMigration("Add column frequency", NewAddColumnMigration(alert_notification, &Column{
+		Name: "frequency", Type: DB_BigInt, Nullable: true,
+	}))
+	mg.AddMigration("Add column notify_once", NewAddColumnMigration(alert_notification, &Column{
+		Name: "notify_once", Type: DB_Bool, Nullable: false, Default: "1",
+	}))
 	mg.AddMigration("add index alert_notification org_id & name", NewAddIndexMigration(alert_notification, alert_notification.Indices[0]))
 
+	notification_journal := Table{
+		Name: "notification_journal",
+		Columns: []*Column{
+			{Name: "id", Type: DB_BigInt, IsPrimaryKey: true, IsAutoIncrement: true},
+			{Name: "org_id", Type: DB_BigInt, Nullable: false},
+			{Name: "alert_id", Type: DB_BigInt, Nullable: false},
+			{Name: "notifier_id", Type: DB_BigInt, Nullable: false},
+			{Name: "sent_at", Type: DB_DateTime, Nullable: false},
+			{Name: "success", Type: DB_Bool, Nullable: false},
+		},
+		Indices: []*Index{
+			{Cols: []string{"org_id", "alert_id", "notifier_id"}, Type: IndexType},
+		},
+	}
+
+	mg.AddMigration("create notification_journal table v1", NewAddTableMigration(notification_journal))
+	mg.AddMigration("add index notification_journal org_id & alert_id & notifier_id", NewAddIndexMigration(notification_journal, notification_journal.Indices[0]))
+
 	mg.AddMigration("Update alert table charset", NewTableCharsetMigration("alert", []*Column{
 		{Name: "name", Type: DB_NVarchar, Length: 255, Nullable: false},
 		{Name: "message", Type: DB_Text, Nullable: false},
diff --git a/public/app/features/alerting/alert_tab_ctrl.ts b/public/app/features/alerting/alert_tab_ctrl.ts
index f0d965ae81e..79baa1e3f5a 100644
--- a/public/app/features/alerting/alert_tab_ctrl.ts
+++ b/public/app/features/alerting/alert_tab_ctrl.ts
@@ -167,9 +167,6 @@ export class AlertTabCtrl {
     alert.noDataState = alert.noDataState || 'no_data';
     alert.executionErrorState = alert.executionErrorState || 'alerting';
     alert.frequency = alert.frequency || '60s';
-    alert.notifyFrequency = alert.notifyFrequency || 10;
-    alert.notifyOnce = alert.notifyOnce == null ? true : alert.notifyOnce;
-    alert.frequency = alert.frequency || '60s';
     alert.handler = alert.handler || 1;
     alert.notifications = alert.notifications || [];
 
diff --git a/public/app/features/alerting/notification_edit_ctrl.ts b/public/app/features/alerting/notification_edit_ctrl.ts
index 18b1c4d1d55..2fd185bee29 100644
--- a/public/app/features/alerting/notification_edit_ctrl.ts
+++ b/public/app/features/alerting/notification_edit_ctrl.ts
@@ -11,6 +11,7 @@ export class AlertNotificationEditCtrl {
   model: any;
   defaults: any = {
     type: 'email',
+    notifyOnce: true,
     settings: {
       httpMethod: 'POST',
       autoResolve: true,
@@ -102,6 +103,7 @@ export class AlertNotificationEditCtrl {
     var payload = {
       name: this.model.name,
       type: this.model.type,
+      frequency: this.model.frequency,
       settings: this.model.settings,
     };
 
diff --git a/public/app/features/alerting/partials/alert_tab.html b/public/app/features/alerting/partials/alert_tab.html
index 084aeb2036a..cb101672aa4 100644
--- a/public/app/features/alerting/partials/alert_tab.html
+++ b/public/app/features/alerting/partials/alert_tab.html
@@ -31,9 +31,6 @@
 					<input type="text" class="gf-form-input width-20" ng-model="ctrl.alert.name">
 					<span class="gf-form-label">Evaluate every</span>
 					<input class="gf-form-input max-width-5" type="text" ng-model="ctrl.alert.frequency"></input>
-					<a class="gf-form-label" ng-click="ctrl.alert.notifyOnce = !ctrl.alert.notifyOnce;">{{ ctrl.alert.notifyOnce ? 'Notify on state change' : 'Notify every' }}</a>
-					<input class="gf-form-input max-width-5" type="number" ng-model="ctrl.alert.notifyFrequency" ng-hide="ctrl.alert.notifyOnce"></input>
-					<span class="gf-form-label" ng-hide="ctrl.alert.notifyOnce">evaluations</span>
 				</div>
 			</div>
 
diff --git a/public/app/features/alerting/partials/notification_edit.html b/public/app/features/alerting/partials/notification_edit.html
index d20b9031a8f..ccdb9ef1073 100644
--- a/public/app/features/alerting/partials/notification_edit.html
+++ b/public/app/features/alerting/partials/notification_edit.html
@@ -18,6 +18,11 @@
           </select>
         </div>
       </div>
+      <div class="gf-form">
+        <a class="gf-form-label width-12" ng-click="ctrl.model.notifyOnce = !ctrl.model.notifyOnce;">{{ ctrl.model.notifyOnce ? 'Notify on state change' : 'Notify at most every' }}</a>
+        <input class="gf-form-input max-width-10" type="number" required ng-model="ctrl.model.frequency" required ng-hide="ctrl.model.notifyOnce"></input>
+        <span class="gf-form-label max-width-5" ng-hide="ctrl.model.notifyOnce">seconds</span>
+      </div>
       <gf-form-switch
           class="gf-form"
           label="Send on all alerts"

From fca97535d121a0d316dfa0dc9e62d5d8839ee46a Mon Sep 17 00:00:00 2001
From: John Baublitz <john.baublitz@threatstack.com>
Date: Sun, 20 May 2018 12:12:10 -0400
Subject: [PATCH 03/39] Fix multiple bugs

---
 pkg/api/alerting.go                           | 57 ++++++++++++++++---
 pkg/api/dtos/alerting.go                      | 19 ++++---
 pkg/models/alert_notifications.go             |  6 +-
 pkg/services/alerting/eval_context.go         |  4 +-
 pkg/services/alerting/notifiers/base.go       |  1 +
 pkg/services/alerting/notifiers/base_test.go  | 13 +++--
 pkg/services/sqlstore/alert_notification.go   | 28 +++++++--
 .../alerting/partials/notification_edit.html  |  3 +-
 8 files changed, 95 insertions(+), 36 deletions(-)

diff --git a/pkg/api/alerting.go b/pkg/api/alerting.go
index 961fc11b2dc..c5b47270f4d 100644
--- a/pkg/api/alerting.go
+++ b/pkg/api/alerting.go
@@ -193,12 +193,15 @@ func GetAlertNotifications(c *m.ReqContext) Response {
 
 	for _, notification := range query.Result {
 		result = append(result, &dtos.AlertNotification{
-			Id:        notification.Id,
-			Name:      notification.Name,
-			Type:      notification.Type,
-			IsDefault: notification.IsDefault,
-			Created:   notification.Created,
-			Updated:   notification.Updated,
+			Id:         notification.Id,
+			Name:       notification.Name,
+			Type:       notification.Type,
+			IsDefault:  notification.IsDefault,
+			Created:    notification.Created,
+			Updated:    notification.Updated,
+			Frequency:  notification.Frequency.String(),
+			NotifyOnce: notification.NotifyOnce,
+			Settings:   notification.Settings,
 		})
 	}
 
@@ -215,7 +218,19 @@ func GetAlertNotificationByID(c *m.ReqContext) Response {
 		return Error(500, "Failed to get alert notifications", err)
 	}
 
-	return JSON(200, query.Result)
+	result := &dtos.AlertNotification{
+		Id:         query.Result.Id,
+		Name:       query.Result.Name,
+		Type:       query.Result.Type,
+		IsDefault:  query.Result.IsDefault,
+		Created:    query.Result.Created,
+		Updated:    query.Result.Updated,
+		Frequency:  query.Result.Frequency.String(),
+		NotifyOnce: query.Result.NotifyOnce,
+		Settings:   query.Result.Settings,
+	}
+
+	return JSON(200, result)
 }
 
 func CreateAlertNotification(c *m.ReqContext, cmd m.CreateAlertNotificationCommand) Response {
@@ -225,7 +240,19 @@ func CreateAlertNotification(c *m.ReqContext, cmd m.CreateAlertNotificationComma
 		return Error(500, "Failed to create alert notification", err)
 	}
 
-	return JSON(200, cmd.Result)
+	result := &dtos.AlertNotification{
+		Id:         cmd.Result.Id,
+		Name:       cmd.Result.Name,
+		Type:       cmd.Result.Type,
+		IsDefault:  cmd.Result.IsDefault,
+		Created:    cmd.Result.Created,
+		Updated:    cmd.Result.Updated,
+		Frequency:  cmd.Result.Frequency.String(),
+		NotifyOnce: cmd.Result.NotifyOnce,
+		Settings:   cmd.Result.Settings,
+	}
+
+	return JSON(200, result)
 }
 
 func UpdateAlertNotification(c *m.ReqContext, cmd m.UpdateAlertNotificationCommand) Response {
@@ -235,7 +262,19 @@ func UpdateAlertNotification(c *m.ReqContext, cmd m.UpdateAlertNotificationComma
 		return Error(500, "Failed to update alert notification", err)
 	}
 
-	return JSON(200, cmd.Result)
+	result := &dtos.AlertNotification{
+		Id:         cmd.Result.Id,
+		Name:       cmd.Result.Name,
+		Type:       cmd.Result.Type,
+		IsDefault:  cmd.Result.IsDefault,
+		Created:    cmd.Result.Created,
+		Updated:    cmd.Result.Updated,
+		Frequency:  cmd.Result.Frequency.String(),
+		NotifyOnce: cmd.Result.NotifyOnce,
+		Settings:   cmd.Result.Settings,
+	}
+
+	return JSON(200, result)
 }
 
 func DeleteAlertNotification(c *m.ReqContext) Response {
diff --git a/pkg/api/dtos/alerting.go b/pkg/api/dtos/alerting.go
index 5e0196c20d1..7d4201fba87 100644
--- a/pkg/api/dtos/alerting.go
+++ b/pkg/api/dtos/alerting.go
@@ -24,14 +24,15 @@ type AlertRule struct {
 }
 
 type AlertNotification struct {
-	Id         int64     `json:"id"`
-	Name       string    `json:"name"`
-	Type       string    `json:"type"`
-	IsDefault  bool      `json:"isDefault"`
-	NotifyOnce bool      `json:"notifyOnce"`
-	Frequency  bool      `json:"frequency"`
-	Created    time.Time `json:"created"`
-	Updated    time.Time `json:"updated"`
+	Id         int64            `json:"id"`
+	Name       string           `json:"name"`
+	Type       string           `json:"type"`
+	IsDefault  bool             `json:"isDefault"`
+	NotifyOnce bool             `json:"notifyOnce"`
+	Frequency  string           `json:"frequency"`
+	Created    time.Time        `json:"created"`
+	Updated    time.Time        `json:"updated"`
+	Settings   *simplejson.Json `json:"settings"`
 }
 
 type AlertTestCommand struct {
@@ -64,7 +65,7 @@ type NotificationTestCommand struct {
 	Name       string           `json:"name"`
 	Type       string           `json:"type"`
 	NotifyOnce bool             `json:"notifyOnce"`
-	Frequency  time.Duration    `json:"frequency"`
+	Frequency  string           `json:"frequency"`
 	Settings   *simplejson.Json `json:"settings"`
 }
 
diff --git a/pkg/models/alert_notifications.go b/pkg/models/alert_notifications.go
index cba62a51527..6715eb21395 100644
--- a/pkg/models/alert_notifications.go
+++ b/pkg/models/alert_notifications.go
@@ -22,8 +22,8 @@ type AlertNotification struct {
 type CreateAlertNotificationCommand struct {
 	Name       string           `json:"name"  binding:"Required"`
 	Type       string           `json:"type"  binding:"Required"`
-	NotifyOnce bool             `json:"notifyOnce"  binding:"Required"`
-	Frequency  time.Duration    `json:"frequency"`
+	NotifyOnce bool             `json:"notifyOnce"`
+	Frequency  string           `json:"frequency"`
 	IsDefault  bool             `json:"isDefault"`
 	Settings   *simplejson.Json `json:"settings"`
 
@@ -35,7 +35,7 @@ type UpdateAlertNotificationCommand struct {
 	Id         int64            `json:"id"  binding:"Required"`
 	Name       string           `json:"name"  binding:"Required"`
 	Type       string           `json:"type"  binding:"Required"`
-	NotifyOnce string           `json:"notifyOnce"  binding:"Required"`
+	NotifyOnce bool             `json:"notifyOnce"`
 	Frequency  string           `json:"frequency"`
 	IsDefault  bool             `json:"isDefault"`
 	Settings   *simplejson.Json `json:"settings"  binding:"Required"`
diff --git a/pkg/services/alerting/eval_context.go b/pkg/services/alerting/eval_context.go
index b451d188a64..3817f4b4a3c 100644
--- a/pkg/services/alerting/eval_context.go
+++ b/pkg/services/alerting/eval_context.go
@@ -151,8 +151,8 @@ func (c *EvalContext) LastNotify(notifierId int64) *time.Time {
 		NotifierId: notifierId,
 	}
 	if err := bus.Dispatch(cmd); err != nil {
-		c.log.Warn("Could not determine last time alert",
-			c.Rule.Name, "notified")
+		c.log.Warn("Could not determine last time alert notifier fired",
+			"Alert name", c.Rule.Name, "Error", err)
 		return nil
 	}
 
diff --git a/pkg/services/alerting/notifiers/base.go b/pkg/services/alerting/notifiers/base.go
index e9e32020c6d..734b5e56b28 100644
--- a/pkg/services/alerting/notifiers/base.go
+++ b/pkg/services/alerting/notifiers/base.go
@@ -41,6 +41,7 @@ func defaultShouldNotify(context *alerting.EvalContext, notifyOnce bool, frequen
 	if context.PrevAlertState == context.Rule.State && notifyOnce {
 		return false
 	}
+	// Do not notify if interval has not elapsed
 	if !notifyOnce && lastNotify != nil && lastNotify.Add(frequency).After(time.Now()) {
 		return false
 	}
diff --git a/pkg/services/alerting/notifiers/base_test.go b/pkg/services/alerting/notifiers/base_test.go
index b7142d144cc..5f2d4989063 100644
--- a/pkg/services/alerting/notifiers/base_test.go
+++ b/pkg/services/alerting/notifiers/base_test.go
@@ -3,6 +3,7 @@ package notifiers
 import (
 	"context"
 	"testing"
+	"time"
 
 	"github.com/grafana/grafana/pkg/components/simplejson"
 	m "github.com/grafana/grafana/pkg/models"
@@ -18,19 +19,19 @@ func TestBaseNotifier(t *testing.T) {
 			Convey("can parse false value", func() {
 				bJson.Set("uploadImage", false)
 
-				base := NewNotifierBase(1, false, "name", "email", bJson)
+				base := NewNotifierBase(1, false, "name", "email", true, 0, bJson)
 				So(base.UploadImage, ShouldBeFalse)
 			})
 
 			Convey("can parse true value", func() {
 				bJson.Set("uploadImage", true)
 
-				base := NewNotifierBase(1, false, "name", "email", bJson)
+				base := NewNotifierBase(1, false, "name", "email", true, 0, bJson)
 				So(base.UploadImage, ShouldBeTrue)
 			})
 
 			Convey("default value should be true for backwards compatibility", func() {
-				base := NewNotifierBase(1, false, "name", "email", bJson)
+				base := NewNotifierBase(1, false, "name", "email", true, 0, bJson)
 				So(base.UploadImage, ShouldBeTrue)
 			})
 		})
@@ -41,7 +42,8 @@ func TestBaseNotifier(t *testing.T) {
 					State: m.AlertStatePending,
 				})
 				context.Rule.State = m.AlertStateOK
-				So(defaultShouldNotify(context), ShouldBeFalse)
+				timeNow := time.Now()
+				So(defaultShouldNotify(context, true, 0, &timeNow), ShouldBeFalse)
 			})
 
 			Convey("ok -> alerting", func() {
@@ -49,7 +51,8 @@ func TestBaseNotifier(t *testing.T) {
 					State: m.AlertStateOK,
 				})
 				context.Rule.State = m.AlertStateAlerting
-				So(defaultShouldNotify(context), ShouldBeTrue)
+				timeNow := time.Now()
+				So(defaultShouldNotify(context, true, 0, &timeNow), ShouldBeTrue)
 			})
 		})
 	})
diff --git a/pkg/services/sqlstore/alert_notification.go b/pkg/services/sqlstore/alert_notification.go
index 8bb17143042..a2cfa37ce5c 100644
--- a/pkg/services/sqlstore/alert_notification.go
+++ b/pkg/services/sqlstore/alert_notification.go
@@ -56,7 +56,9 @@ func GetAlertNotificationsToSend(query *m.GetAlertNotificationsToSendQuery) erro
 										alert_notification.created,
 										alert_notification.updated,
 										alert_notification.settings,
-										alert_notification.is_default
+										alert_notification.is_default,
+										alert_notification.notify_once,
+										alert_notification.frequency
 										FROM alert_notification
 	  							`)
 
@@ -94,7 +96,9 @@ func getAlertNotificationInternal(query *m.GetAlertNotificationsQuery, sess *DBS
 										alert_notification.created,
 										alert_notification.updated,
 										alert_notification.settings,
-										alert_notification.is_default
+										alert_notification.is_default,
+										alert_notification.notify_once,
+										alert_notification.frequency
 										FROM alert_notification
 	  							`)
 
@@ -140,19 +144,24 @@ func CreateAlertNotificationCommand(cmd *m.CreateAlertNotificationCommand) error
 			return fmt.Errorf("Alert notification name %s already exists", cmd.Name)
 		}
 
+		frequency, err_convert := time.ParseDuration(cmd.Frequency)
+		if err_convert != nil {
+			return err
+		}
+
 		alertNotification := &m.AlertNotification{
 			OrgId:      cmd.OrgId,
 			Name:       cmd.Name,
 			Type:       cmd.Type,
 			Settings:   cmd.Settings,
 			NotifyOnce: cmd.NotifyOnce,
-			Frequency:  cmd.Frequency,
+			Frequency:  frequency,
 			Created:    time.Now(),
 			Updated:    time.Now(),
 			IsDefault:  cmd.IsDefault,
 		}
 
-		if _, err = sess.Insert(alertNotification); err != nil {
+		if _, err = sess.MustCols("notify_once").Insert(alertNotification); err != nil {
 			return err
 		}
 
@@ -184,8 +193,15 @@ func UpdateAlertNotification(cmd *m.UpdateAlertNotificationCommand) error {
 		current.Name = cmd.Name
 		current.Type = cmd.Type
 		current.IsDefault = cmd.IsDefault
+		current.NotifyOnce = cmd.NotifyOnce
 
-		sess.UseBool("is_default")
+		frequency, err_convert := time.ParseDuration(cmd.Frequency)
+		if err_convert != nil {
+			return err
+		}
+		current.Frequency = frequency
+
+		sess.UseBool("is_default", "notify_once")
 
 		if affected, err := sess.ID(cmd.Id).Update(current); err != nil {
 			return err
@@ -219,7 +235,7 @@ func RecordNotificationJournal(cmd *m.RecordNotificationJournalCommand) error {
 func GetLatestNotification(cmd *m.GetLatestNotificationQuery) error {
 	return inTransaction(func(sess *DBSession) error {
 		notificationJournal := &m.NotificationJournal{}
-		_, err := sess.OrderBy("notification_journal.sent_at").Desc().Where("notification_journal.org_id = ? AND notification_journal.alert_id = ? AND notification_journal.notifier_id = ?", cmd.OrgId, cmd.AlertId, cmd.NotifierId).Get(notificationJournal)
+		_, err := sess.Desc("notification_journal.sent_at").Limit(1).Where("notification_journal.org_id = ? AND notification_journal.alert_id = ? AND notification_journal.notifier_id = ?", cmd.OrgId, cmd.AlertId, cmd.NotifierId).Get(notificationJournal)
 		if err != nil {
 			return err
 		}
diff --git a/public/app/features/alerting/partials/notification_edit.html b/public/app/features/alerting/partials/notification_edit.html
index ccdb9ef1073..dd56564cb95 100644
--- a/public/app/features/alerting/partials/notification_edit.html
+++ b/public/app/features/alerting/partials/notification_edit.html
@@ -20,8 +20,7 @@
       </div>
       <div class="gf-form">
         <a class="gf-form-label width-12" ng-click="ctrl.model.notifyOnce = !ctrl.model.notifyOnce;">{{ ctrl.model.notifyOnce ? 'Notify on state change' : 'Notify at most every' }}</a>
-        <input class="gf-form-input max-width-10" type="number" required ng-model="ctrl.model.frequency" required ng-hide="ctrl.model.notifyOnce"></input>
-        <span class="gf-form-label max-width-5" ng-hide="ctrl.model.notifyOnce">seconds</span>
+        <input class="gf-form-input max-width-15" type="text" ng-model="ctrl.model.frequency" ng-if="!ctrl.model.notifyOnce"></input>
       </div>
       <gf-form-switch
           class="gf-form"

From c0a9e5cf1dd749da7ec7fc9c277e5174e8d8fcc9 Mon Sep 17 00:00:00 2001
From: John Baublitz <john.baublitz@threatstack.com>
Date: Sun, 20 May 2018 16:08:42 -0400
Subject: [PATCH 04/39] Fix tests

---
 pkg/services/sqlstore/alert_notification.go   |  8 +++++
 .../sqlstore/alert_notification_test.go       | 32 +++++++++++--------
 2 files changed, 26 insertions(+), 14 deletions(-)

diff --git a/pkg/services/sqlstore/alert_notification.go b/pkg/services/sqlstore/alert_notification.go
index a2cfa37ce5c..0ecd6a18818 100644
--- a/pkg/services/sqlstore/alert_notification.go
+++ b/pkg/services/sqlstore/alert_notification.go
@@ -144,6 +144,10 @@ func CreateAlertNotificationCommand(cmd *m.CreateAlertNotificationCommand) error
 			return fmt.Errorf("Alert notification name %s already exists", cmd.Name)
 		}
 
+		if cmd.Frequency == "" {
+			return fmt.Errorf("Alert notification frequency required")
+		}
+
 		frequency, err_convert := time.ParseDuration(cmd.Frequency)
 		if err_convert != nil {
 			return err
@@ -195,6 +199,10 @@ func UpdateAlertNotification(cmd *m.UpdateAlertNotificationCommand) error {
 		current.IsDefault = cmd.IsDefault
 		current.NotifyOnce = cmd.NotifyOnce
 
+		if cmd.Frequency == "" {
+			return fmt.Errorf("Alert notification frequency required")
+		}
+
 		frequency, err_convert := time.ParseDuration(cmd.Frequency)
 		if err_convert != nil {
 			return err
diff --git a/pkg/services/sqlstore/alert_notification_test.go b/pkg/services/sqlstore/alert_notification_test.go
index 2dbf9de5ca8..01c6c3aebd6 100644
--- a/pkg/services/sqlstore/alert_notification_test.go
+++ b/pkg/services/sqlstore/alert_notification_test.go
@@ -26,10 +26,12 @@ func TestAlertNotificationSQLAccess(t *testing.T) {
 
 		Convey("Can save Alert Notification", func() {
 			cmd := &m.CreateAlertNotificationCommand{
-				Name:     "ops",
-				Type:     "email",
-				OrgId:    1,
-				Settings: simplejson.New(),
+				Name:       "ops",
+				Type:       "email",
+				OrgId:      1,
+				NotifyOnce: true,
+				Frequency:  "10s",
+				Settings:   simplejson.New(),
 			}
 
 			err = CreateAlertNotificationCommand(cmd)
@@ -45,11 +47,13 @@ func TestAlertNotificationSQLAccess(t *testing.T) {
 
 			Convey("Can update alert notification", func() {
 				newCmd := &m.UpdateAlertNotificationCommand{
-					Name:     "NewName",
-					Type:     "webhook",
-					OrgId:    cmd.Result.OrgId,
-					Settings: simplejson.New(),
-					Id:       cmd.Result.Id,
+					Name:       "NewName",
+					Type:       "webhook",
+					OrgId:      cmd.Result.OrgId,
+					NotifyOnce: true,
+					Frequency:  "10s",
+					Settings:   simplejson.New(),
+					Id:         cmd.Result.Id,
 				}
 				err := UpdateAlertNotification(newCmd)
 				So(err, ShouldBeNil)
@@ -58,12 +62,12 @@ func TestAlertNotificationSQLAccess(t *testing.T) {
 		})
 
 		Convey("Can search using an array of ids", func() {
-			cmd1 := m.CreateAlertNotificationCommand{Name: "nagios", Type: "webhook", OrgId: 1, Settings: simplejson.New()}
-			cmd2 := m.CreateAlertNotificationCommand{Name: "slack", Type: "webhook", OrgId: 1, Settings: simplejson.New()}
-			cmd3 := m.CreateAlertNotificationCommand{Name: "ops2", Type: "email", OrgId: 1, Settings: simplejson.New()}
-			cmd4 := m.CreateAlertNotificationCommand{IsDefault: true, Name: "default", Type: "email", OrgId: 1, Settings: simplejson.New()}
+			cmd1 := m.CreateAlertNotificationCommand{Name: "nagios", Type: "webhook", OrgId: 1, NotifyOnce: true, Frequency: "10s", Settings: simplejson.New()}
+			cmd2 := m.CreateAlertNotificationCommand{Name: "slack", Type: "webhook", OrgId: 1, NotifyOnce: true, Frequency: "10s", Settings: simplejson.New()}
+			cmd3 := m.CreateAlertNotificationCommand{Name: "ops2", Type: "email", OrgId: 1, NotifyOnce: true, Frequency: "10s", Settings: simplejson.New()}
+			cmd4 := m.CreateAlertNotificationCommand{IsDefault: true, Name: "default", Type: "email", OrgId: 1, NotifyOnce: true, Frequency: "10s", Settings: simplejson.New()}
 
-			otherOrg := m.CreateAlertNotificationCommand{Name: "default", Type: "email", OrgId: 2, Settings: simplejson.New()}
+			otherOrg := m.CreateAlertNotificationCommand{Name: "default", Type: "email", OrgId: 2, NotifyOnce: true, Frequency: "10s", Settings: simplejson.New()}
 
 			So(CreateAlertNotificationCommand(&cmd1), ShouldBeNil)
 			So(CreateAlertNotificationCommand(&cmd2), ShouldBeNil)

From 5c5951bc4274f3b4ff1ea3b41507e394faaeb22f Mon Sep 17 00:00:00 2001
From: John Baublitz <john.baublitz@threatstack.com>
Date: Sun, 20 May 2018 19:01:10 -0400
Subject: [PATCH 05/39] Bug fix for repeated alerting even on OK state and add
 notification_journal cleanup when alert resolves

---
 pkg/services/alerting/engine.go         | 14 ++++++++++++++
 pkg/services/alerting/notifiers/base.go |  4 ++++
 2 files changed, 18 insertions(+)

diff --git a/pkg/services/alerting/engine.go b/pkg/services/alerting/engine.go
index 0f8e24bcef5..43f6db66771 100644
--- a/pkg/services/alerting/engine.go
+++ b/pkg/services/alerting/engine.go
@@ -10,7 +10,9 @@ import (
 	tlog "github.com/opentracing/opentracing-go/log"
 
 	"github.com/benbjohnson/clock"
+	"github.com/grafana/grafana/pkg/bus"
 	"github.com/grafana/grafana/pkg/log"
+	m "github.com/grafana/grafana/pkg/models"
 	"github.com/grafana/grafana/pkg/registry"
 	"github.com/grafana/grafana/pkg/services/rendering"
 	"github.com/grafana/grafana/pkg/setting"
@@ -205,6 +207,18 @@ func (e *AlertingService) processJob(attemptID int, attemptChan chan int, cancel
 		}
 
 		evalContext.Rule.State = evalContext.GetNewState()
+		if evalContext.Rule.State == m.AlertStateOK && evalContext.PrevAlertState != m.AlertStateOK {
+			for _, notifierId := range evalContext.Rule.Notifications {
+				cmd := &m.CleanNotificationJournalCommand{
+					AlertId:    evalContext.Rule.Id,
+					NotifierId: notifierId,
+					OrgId:      evalContext.Rule.OrgId,
+				}
+				if err := bus.Dispatch(cmd); err != nil {
+					e.log.Error("Failed to clean up old notification records", "notifier", notifierId, "alert", evalContext.Rule.Id, "Error", err)
+				}
+			}
+		}
 		e.resultHandler.Handle(evalContext)
 		span.Finish()
 		e.log.Debug("Job Execution completed", "timeMs", evalContext.GetDurationMs(), "alertId", evalContext.Rule.Id, "name", evalContext.Rule.Name, "firing", evalContext.Firing, "attemptID", attemptID)
diff --git a/pkg/services/alerting/notifiers/base.go b/pkg/services/alerting/notifiers/base.go
index 734b5e56b28..7672d397491 100644
--- a/pkg/services/alerting/notifiers/base.go
+++ b/pkg/services/alerting/notifiers/base.go
@@ -45,6 +45,10 @@ func defaultShouldNotify(context *alerting.EvalContext, notifyOnce bool, frequen
 	if !notifyOnce && lastNotify != nil && lastNotify.Add(frequency).After(time.Now()) {
 		return false
 	}
+	// Do not notify if alert state if OK or pending even on repeated notify
+	if !notifyOnce && (context.Rule.State == m.AlertStateOK || context.Rule.State == m.AlertStatePending) {
+		return false
+	}
 	// Do not notify when we become OK for the first time.
 	if (context.PrevAlertState == m.AlertStatePending) && (context.Rule.State == m.AlertStateOK) {
 		return false

From bdf433594add113b05b2bbd4f0381c1090fd2d6b Mon Sep 17 00:00:00 2001
From: John Baublitz <john.baublitz@threatstack.com>
Date: Fri, 25 May 2018 14:14:33 -0400
Subject: [PATCH 06/39] Implement code review changes

---
 pkg/models/alert_notifications.go                 |  5 +++++
 pkg/services/alerting/engine.go                   | 14 --------------
 pkg/services/alerting/result_handler.go           | 12 ++++++++++++
 pkg/services/sqlstore/alert_notification.go       |  7 ++++---
 .../features/alerting/notification_edit_ctrl.ts   |  1 +
 .../alerting/partials/notification_edit.html      | 15 +++++++++++----
 6 files changed, 33 insertions(+), 21 deletions(-)

diff --git a/pkg/models/alert_notifications.go b/pkg/models/alert_notifications.go
index 6715eb21395..ed6b8f372d1 100644
--- a/pkg/models/alert_notifications.go
+++ b/pkg/models/alert_notifications.go
@@ -1,11 +1,16 @@
 package models
 
 import (
+	"errors"
 	"time"
 
 	"github.com/grafana/grafana/pkg/components/simplejson"
 )
 
+var (
+	ErrNotificationFrequencyNotFound = errors.New("Notification frequency not specified")
+)
+
 type AlertNotification struct {
 	Id         int64            `json:"id"`
 	OrgId      int64            `json:"-"`
diff --git a/pkg/services/alerting/engine.go b/pkg/services/alerting/engine.go
index 43f6db66771..0f8e24bcef5 100644
--- a/pkg/services/alerting/engine.go
+++ b/pkg/services/alerting/engine.go
@@ -10,9 +10,7 @@ import (
 	tlog "github.com/opentracing/opentracing-go/log"
 
 	"github.com/benbjohnson/clock"
-	"github.com/grafana/grafana/pkg/bus"
 	"github.com/grafana/grafana/pkg/log"
-	m "github.com/grafana/grafana/pkg/models"
 	"github.com/grafana/grafana/pkg/registry"
 	"github.com/grafana/grafana/pkg/services/rendering"
 	"github.com/grafana/grafana/pkg/setting"
@@ -207,18 +205,6 @@ func (e *AlertingService) processJob(attemptID int, attemptChan chan int, cancel
 		}
 
 		evalContext.Rule.State = evalContext.GetNewState()
-		if evalContext.Rule.State == m.AlertStateOK && evalContext.PrevAlertState != m.AlertStateOK {
-			for _, notifierId := range evalContext.Rule.Notifications {
-				cmd := &m.CleanNotificationJournalCommand{
-					AlertId:    evalContext.Rule.Id,
-					NotifierId: notifierId,
-					OrgId:      evalContext.Rule.OrgId,
-				}
-				if err := bus.Dispatch(cmd); err != nil {
-					e.log.Error("Failed to clean up old notification records", "notifier", notifierId, "alert", evalContext.Rule.Id, "Error", err)
-				}
-			}
-		}
 		e.resultHandler.Handle(evalContext)
 		span.Finish()
 		e.log.Debug("Job Execution completed", "timeMs", evalContext.GetDurationMs(), "alertId", evalContext.Rule.Id, "name", evalContext.Rule.Name, "firing", evalContext.Firing, "attemptID", attemptID)
diff --git a/pkg/services/alerting/result_handler.go b/pkg/services/alerting/result_handler.go
index c57b28c7c3e..c4c20bd8beb 100644
--- a/pkg/services/alerting/result_handler.go
+++ b/pkg/services/alerting/result_handler.go
@@ -88,6 +88,18 @@ func (handler *DefaultResultHandler) Handle(evalContext *EvalContext) error {
 		}
 	}
 
+	if evalContext.Rule.State == m.AlertStateOK && evalContext.PrevAlertState != m.AlertStateOK {
+		for _, notifierId := range evalContext.Rule.Notifications {
+			cmd := &m.CleanNotificationJournalCommand{
+				AlertId:    evalContext.Rule.Id,
+				NotifierId: notifierId,
+				OrgId:      evalContext.Rule.OrgId,
+			}
+			if err := bus.Dispatch(cmd); err != nil {
+				handler.log.Error("Failed to clean up old notification records", "notifier", notifierId, "alert", evalContext.Rule.Id, "Error", err)
+			}
+		}
+	}
 	handler.notifier.SendIfNeeded(evalContext)
 
 	return nil
diff --git a/pkg/services/sqlstore/alert_notification.go b/pkg/services/sqlstore/alert_notification.go
index 0ecd6a18818..6913009a163 100644
--- a/pkg/services/sqlstore/alert_notification.go
+++ b/pkg/services/sqlstore/alert_notification.go
@@ -148,8 +148,9 @@ func CreateAlertNotificationCommand(cmd *m.CreateAlertNotificationCommand) error
 			return fmt.Errorf("Alert notification frequency required")
 		}
 
-		frequency, err_convert := time.ParseDuration(cmd.Frequency)
-		if err_convert != nil {
+		var frequency time.Duration
+		frequency, err = time.ParseDuration(cmd.Frequency)
+		if err != nil {
 			return err
 		}
 
@@ -200,7 +201,7 @@ func UpdateAlertNotification(cmd *m.UpdateAlertNotificationCommand) error {
 		current.NotifyOnce = cmd.NotifyOnce
 
 		if cmd.Frequency == "" {
-			return fmt.Errorf("Alert notification frequency required")
+			return m.ErrNotificationFrequencyNotFound
 		}
 
 		frequency, err_convert := time.ParseDuration(cmd.Frequency)
diff --git a/public/app/features/alerting/notification_edit_ctrl.ts b/public/app/features/alerting/notification_edit_ctrl.ts
index 2fd185bee29..9d20e871c7c 100644
--- a/public/app/features/alerting/notification_edit_ctrl.ts
+++ b/public/app/features/alerting/notification_edit_ctrl.ts
@@ -12,6 +12,7 @@ export class AlertNotificationEditCtrl {
   defaults: any = {
     type: 'email',
     notifyOnce: true,
+    frequency: '15m',
     settings: {
       httpMethod: 'POST',
       autoResolve: true,
diff --git a/public/app/features/alerting/partials/notification_edit.html b/public/app/features/alerting/partials/notification_edit.html
index dd56564cb95..48d44b74581 100644
--- a/public/app/features/alerting/partials/notification_edit.html
+++ b/public/app/features/alerting/partials/notification_edit.html
@@ -18,10 +18,6 @@
           </select>
         </div>
       </div>
-      <div class="gf-form">
-        <a class="gf-form-label width-12" ng-click="ctrl.model.notifyOnce = !ctrl.model.notifyOnce;">{{ ctrl.model.notifyOnce ? 'Notify on state change' : 'Notify at most every' }}</a>
-        <input class="gf-form-input max-width-15" type="text" ng-model="ctrl.model.frequency" ng-if="!ctrl.model.notifyOnce"></input>
-      </div>
       <gf-form-switch
           class="gf-form"
           label="Send on all alerts"
@@ -36,6 +32,17 @@
           checked="ctrl.model.settings.uploadImage"
           tooltip="Captures an image and include it in the notification">
       </gf-form-switch>
+      <gf-form-switch
+          class="gf-form"
+          label="Notify once"
+          label-class="width-12"
+          checked="ctrl.model.notifyOnce"
+          tooltip="Choose to either notify on state change or at every interval">
+      </gf-form-switch>
+      <div class="gf-form">
+        <span class="gf-form-label width-12" ng-if="!ctrl.model.notifyOnce">Notify every</span>
+        <input class="gf-form-input max-width-15" type="text" required ng-model="ctrl.model.frequency" required ng-if="!ctrl.model.notifyOnce"></input>
+      </div>
     </div>
 
     <div class="gf-form-group" ng-include src="ctrl.notifierTemplateId">

From 86e65f84f9ba86b202ff38d7f51f1b7d2e75f02f Mon Sep 17 00:00:00 2001
From: bergquist <carl.bergquist@gmail.com>
Date: Mon, 4 Jun 2018 17:30:57 +0200
Subject: [PATCH 07/39] alerting: fixes invalid error handling

---
 pkg/services/sqlstore/alert_notification.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/pkg/services/sqlstore/alert_notification.go b/pkg/services/sqlstore/alert_notification.go
index 6913009a163..4f79035063e 100644
--- a/pkg/services/sqlstore/alert_notification.go
+++ b/pkg/services/sqlstore/alert_notification.go
@@ -204,8 +204,8 @@ func UpdateAlertNotification(cmd *m.UpdateAlertNotificationCommand) error {
 			return m.ErrNotificationFrequencyNotFound
 		}
 
-		frequency, err_convert := time.ParseDuration(cmd.Frequency)
-		if err_convert != nil {
+		frequency, err := time.ParseDuration(cmd.Frequency)
+		if err != nil {
 			return err
 		}
 		current.Frequency = frequency

From 93124f38fae77627cdb0a7b4a5f71171c5088eca Mon Sep 17 00:00:00 2001
From: bergquist <carl.bergquist@gmail.com>
Date: Mon, 4 Jun 2018 22:19:27 +0200
Subject: [PATCH 08/39] alerting: only check frequency when not send once

---
 pkg/api/alerting.go                           | 54 ++----------------
 pkg/api/alerting_test.go                      | 19 +++++++
 pkg/api/dtos/alerting.go                      | 53 +++++++++++++-----
 pkg/services/sqlstore/alert_notification.go   | 35 +++++++-----
 .../sqlstore/alert_notification_test.go       | 55 ++++++++++++++++++-
 5 files changed, 135 insertions(+), 81 deletions(-)

diff --git a/pkg/api/alerting.go b/pkg/api/alerting.go
index 4e9b89fefd6..a936d696207 100644
--- a/pkg/api/alerting.go
+++ b/pkg/api/alerting.go
@@ -192,17 +192,7 @@ func GetAlertNotifications(c *m.ReqContext) Response {
 	result := make([]*dtos.AlertNotification, 0)
 
 	for _, notification := range query.Result {
-		result = append(result, &dtos.AlertNotification{
-			Id:         notification.Id,
-			Name:       notification.Name,
-			Type:       notification.Type,
-			IsDefault:  notification.IsDefault,
-			Created:    notification.Created,
-			Updated:    notification.Updated,
-			Frequency:  notification.Frequency.String(),
-			NotifyOnce: notification.NotifyOnce,
-			Settings:   notification.Settings,
-		})
+		result = append(result, dtos.NewAlertNotification(notification))
 	}
 
 	return JSON(200, result)
@@ -218,19 +208,7 @@ func GetAlertNotificationByID(c *m.ReqContext) Response {
 		return Error(500, "Failed to get alert notifications", err)
 	}
 
-	result := &dtos.AlertNotification{
-		Id:         query.Result.Id,
-		Name:       query.Result.Name,
-		Type:       query.Result.Type,
-		IsDefault:  query.Result.IsDefault,
-		Created:    query.Result.Created,
-		Updated:    query.Result.Updated,
-		Frequency:  query.Result.Frequency.String(),
-		NotifyOnce: query.Result.NotifyOnce,
-		Settings:   query.Result.Settings,
-	}
-
-	return JSON(200, result)
+	return JSON(200, dtos.NewAlertNotification(query.Result))
 }
 
 func CreateAlertNotification(c *m.ReqContext, cmd m.CreateAlertNotificationCommand) Response {
@@ -240,19 +218,7 @@ func CreateAlertNotification(c *m.ReqContext, cmd m.CreateAlertNotificationComma
 		return Error(500, "Failed to create alert notification", err)
 	}
 
-	result := &dtos.AlertNotification{
-		Id:         cmd.Result.Id,
-		Name:       cmd.Result.Name,
-		Type:       cmd.Result.Type,
-		IsDefault:  cmd.Result.IsDefault,
-		Created:    cmd.Result.Created,
-		Updated:    cmd.Result.Updated,
-		Frequency:  cmd.Result.Frequency.String(),
-		NotifyOnce: cmd.Result.NotifyOnce,
-		Settings:   cmd.Result.Settings,
-	}
-
-	return JSON(200, result)
+	return JSON(200, dtos.NewAlertNotification(cmd.Result))
 }
 
 func UpdateAlertNotification(c *m.ReqContext, cmd m.UpdateAlertNotificationCommand) Response {
@@ -262,19 +228,7 @@ func UpdateAlertNotification(c *m.ReqContext, cmd m.UpdateAlertNotificationComma
 		return Error(500, "Failed to update alert notification", err)
 	}
 
-	result := &dtos.AlertNotification{
-		Id:         cmd.Result.Id,
-		Name:       cmd.Result.Name,
-		Type:       cmd.Result.Type,
-		IsDefault:  cmd.Result.IsDefault,
-		Created:    cmd.Result.Created,
-		Updated:    cmd.Result.Updated,
-		Frequency:  cmd.Result.Frequency.String(),
-		NotifyOnce: cmd.Result.NotifyOnce,
-		Settings:   cmd.Result.Settings,
-	}
-
-	return JSON(200, result)
+	return JSON(200, dtos.NewAlertNotification(cmd.Result))
 }
 
 func DeleteAlertNotification(c *m.ReqContext) Response {
diff --git a/pkg/api/alerting_test.go b/pkg/api/alerting_test.go
index abfdfb66322..3e50487190c 100644
--- a/pkg/api/alerting_test.go
+++ b/pkg/api/alerting_test.go
@@ -2,6 +2,7 @@ package api
 
 import (
 	"testing"
+	"time"
 
 	"github.com/grafana/grafana/pkg/api/dtos"
 	"github.com/grafana/grafana/pkg/bus"
@@ -11,6 +12,24 @@ import (
 	. "github.com/smartystreets/goconvey/convey"
 )
 
+func TestRemoveZeroUnitsFromInterval(t *testing.T) {
+	tcs := []struct {
+		interval time.Duration
+		expected string
+	}{
+		{interval: time.Duration(time.Hour), expected: "1h"},
+		{interval: time.Duration(time.Hour + time.Minute), expected: "1h1m"},
+		{interval: time.Duration((time.Hour * 10) + time.Minute), expected: "10h1m"},
+	}
+
+	for _, tc := range tcs {
+		got := removeZeroesFromDuration(tc.interval)
+		if got != tc.expected {
+			t.Errorf("expected %s got %s internval: %v", tc.expected, got, tc.interval)
+		}
+	}
+}
+
 func TestAlertingApiEndpoint(t *testing.T) {
 	Convey("Given an alert in a dashboard with an acl", t, func() {
 
diff --git a/pkg/api/dtos/alerting.go b/pkg/api/dtos/alerting.go
index 7d4201fba87..786fccc10b5 100644
--- a/pkg/api/dtos/alerting.go
+++ b/pkg/api/dtos/alerting.go
@@ -1,26 +1,51 @@
 package dtos
 
 import (
+	"strings"
 	"time"
 
 	"github.com/grafana/grafana/pkg/components/null"
 	"github.com/grafana/grafana/pkg/components/simplejson"
-	m "github.com/grafana/grafana/pkg/models"
+	"github.com/grafana/grafana/pkg/models"
 )
 
 type AlertRule struct {
-	Id             int64            `json:"id"`
-	DashboardId    int64            `json:"dashboardId"`
-	PanelId        int64            `json:"panelId"`
-	Name           string           `json:"name"`
-	Message        string           `json:"message"`
-	State          m.AlertStateType `json:"state"`
-	NewStateDate   time.Time        `json:"newStateDate"`
-	EvalDate       time.Time        `json:"evalDate"`
-	EvalData       *simplejson.Json `json:"evalData"`
-	ExecutionError string           `json:"executionError"`
-	Url            string           `json:"url"`
-	CanEdit        bool             `json:"canEdit"`
+	Id             int64                 `json:"id"`
+	DashboardId    int64                 `json:"dashboardId"`
+	PanelId        int64                 `json:"panelId"`
+	Name           string                `json:"name"`
+	Message        string                `json:"message"`
+	State          models.AlertStateType `json:"state"`
+	NewStateDate   time.Time             `json:"newStateDate"`
+	EvalDate       time.Time             `json:"evalDate"`
+	EvalData       *simplejson.Json      `json:"evalData"`
+	ExecutionError string                `json:"executionError"`
+	Url            string                `json:"url"`
+	CanEdit        bool                  `json:"canEdit"`
+}
+
+func removeZeroesFromDuration(interval time.Duration) string {
+	frequency := interval.String()
+
+	frequency = strings.Replace(frequency, "0h", "", 1)
+	frequency = strings.Replace(frequency, "0m", "", 1)
+	frequency = strings.Replace(frequency, "0s", "", 1)
+
+	return frequency
+}
+
+func NewAlertNotification(notification *models.AlertNotification) *AlertNotification {
+	return &AlertNotification{
+		Id:         notification.Id,
+		Name:       notification.Name,
+		Type:       notification.Type,
+		IsDefault:  notification.IsDefault,
+		Created:    notification.Created,
+		Updated:    notification.Updated,
+		Frequency:  removeZeroesFromDuration(notification.Frequency),
+		NotifyOnce: notification.NotifyOnce,
+		Settings:   notification.Settings,
+	}
 }
 
 type AlertNotification struct {
@@ -42,7 +67,7 @@ type AlertTestCommand struct {
 
 type AlertTestResult struct {
 	Firing         bool                  `json:"firing"`
-	State          m.AlertStateType      `json:"state"`
+	State          models.AlertStateType `json:"state"`
 	ConditionEvals string                `json:"conditionEvals"`
 	TimeMs         string                `json:"timeMs"`
 	Error          string                `json:"error,omitempty"`
diff --git a/pkg/services/sqlstore/alert_notification.go b/pkg/services/sqlstore/alert_notification.go
index 4f79035063e..ff36c38b1a5 100644
--- a/pkg/services/sqlstore/alert_notification.go
+++ b/pkg/services/sqlstore/alert_notification.go
@@ -144,14 +144,16 @@ func CreateAlertNotificationCommand(cmd *m.CreateAlertNotificationCommand) error
 			return fmt.Errorf("Alert notification name %s already exists", cmd.Name)
 		}
 
-		if cmd.Frequency == "" {
-			return fmt.Errorf("Alert notification frequency required")
-		}
-
 		var frequency time.Duration
-		frequency, err = time.ParseDuration(cmd.Frequency)
-		if err != nil {
-			return err
+		if !cmd.NotifyOnce {
+			if cmd.Frequency == "" {
+				return m.ErrNotificationFrequencyNotFound
+			}
+
+			frequency, err = time.ParseDuration(cmd.Frequency)
+			if err != nil {
+				return err
+			}
 		}
 
 		alertNotification := &m.AlertNotification{
@@ -200,22 +202,25 @@ func UpdateAlertNotification(cmd *m.UpdateAlertNotificationCommand) error {
 		current.IsDefault = cmd.IsDefault
 		current.NotifyOnce = cmd.NotifyOnce
 
-		if cmd.Frequency == "" {
-			return m.ErrNotificationFrequencyNotFound
-		}
+		if !current.NotifyOnce {
+			if cmd.Frequency == "" {
+				return m.ErrNotificationFrequencyNotFound
+			}
 
-		frequency, err := time.ParseDuration(cmd.Frequency)
-		if err != nil {
-			return err
+			frequency, err := time.ParseDuration(cmd.Frequency)
+			if err != nil {
+				return err
+			}
+
+			current.Frequency = frequency
 		}
-		current.Frequency = frequency
 
 		sess.UseBool("is_default", "notify_once")
 
 		if affected, err := sess.ID(cmd.Id).Update(current); err != nil {
 			return err
 		} else if affected == 0 {
-			return fmt.Errorf("Could not find alert notification")
+			return fmt.Errorf("Could not update alert notification")
 		}
 
 		cmd.Result = &current
diff --git a/pkg/services/sqlstore/alert_notification_test.go b/pkg/services/sqlstore/alert_notification_test.go
index 01c6c3aebd6..578a53f34ad 100644
--- a/pkg/services/sqlstore/alert_notification_test.go
+++ b/pkg/services/sqlstore/alert_notification_test.go
@@ -11,7 +11,6 @@ import (
 func TestAlertNotificationSQLAccess(t *testing.T) {
 	Convey("Testing Alert notification sql access", t, func() {
 		InitTestDB(t)
-		var err error
 
 		Convey("Alert notifications should be empty", func() {
 			cmd := &m.GetAlertNotificationsQuery{
@@ -24,6 +23,58 @@ func TestAlertNotificationSQLAccess(t *testing.T) {
 			So(cmd.Result, ShouldBeNil)
 		})
 
+		Convey("Cannot save alert notifier with notitfyonce = false", func() {
+			cmd := &m.CreateAlertNotificationCommand{
+				Name:       "ops",
+				Type:       "email",
+				OrgId:      1,
+				NotifyOnce: false,
+				Settings:   simplejson.New(),
+			}
+
+			Convey("and missing frequency", func() {
+				err := CreateAlertNotificationCommand(cmd)
+				So(err, ShouldEqual, m.ErrNotificationFrequencyNotFound)
+			})
+
+			Convey("invalid frequency", func() {
+				cmd.Frequency = "invalid duration"
+
+				err := CreateAlertNotificationCommand(cmd)
+				So(err.Error(), ShouldEqual, "time: invalid duration invalid duration")
+			})
+		})
+
+		Convey("Cannot update alert notifier with notitfyonce = false", func() {
+			cmd := &m.CreateAlertNotificationCommand{
+				Name:       "ops update",
+				Type:       "email",
+				OrgId:      1,
+				NotifyOnce: true,
+				Settings:   simplejson.New(),
+			}
+
+			err := CreateAlertNotificationCommand(cmd)
+			So(err, ShouldBeNil)
+
+			updateCmd := &m.UpdateAlertNotificationCommand{
+				Id:         cmd.Result.Id,
+				NotifyOnce: false,
+			}
+
+			Convey("and missing frequency", func() {
+				err := UpdateAlertNotification(updateCmd)
+				So(err, ShouldEqual, m.ErrNotificationFrequencyNotFound)
+			})
+
+			Convey("invalid frequency", func() {
+				updateCmd.Frequency = "invalid duration"
+
+				err := UpdateAlertNotification(updateCmd)
+				So(err.Error(), ShouldEqual, "time: invalid duration invalid duration")
+			})
+		})
+
 		Convey("Can save Alert Notification", func() {
 			cmd := &m.CreateAlertNotificationCommand{
 				Name:       "ops",
@@ -34,7 +85,7 @@ func TestAlertNotificationSQLAccess(t *testing.T) {
 				Settings:   simplejson.New(),
 			}
 
-			err = CreateAlertNotificationCommand(cmd)
+			err := CreateAlertNotificationCommand(cmd)
 			So(err, ShouldBeNil)
 			So(cmd.Result.Id, ShouldNotEqual, 0)
 			So(cmd.Result.OrgId, ShouldNotEqual, 0)

From 0c6d8398a14e3a4c15b10af5ad0b4eda2965d988 Mon Sep 17 00:00:00 2001
From: bergquist <carl.bergquist@gmail.com>
Date: Tue, 5 Jun 2018 08:42:39 +0200
Subject: [PATCH 09/39] alerting: remove zero units from duration

---
 pkg/api/alerting_test.go      | 19 -------------------
 pkg/api/dtos/alerting.go      | 29 +++++++++++++++++++++--------
 pkg/api/dtos/alerting_test.go | 34 ++++++++++++++++++++++++++++++++++
 3 files changed, 55 insertions(+), 27 deletions(-)
 create mode 100644 pkg/api/dtos/alerting_test.go

diff --git a/pkg/api/alerting_test.go b/pkg/api/alerting_test.go
index 3e50487190c..abfdfb66322 100644
--- a/pkg/api/alerting_test.go
+++ b/pkg/api/alerting_test.go
@@ -2,7 +2,6 @@ package api
 
 import (
 	"testing"
-	"time"
 
 	"github.com/grafana/grafana/pkg/api/dtos"
 	"github.com/grafana/grafana/pkg/bus"
@@ -12,24 +11,6 @@ import (
 	. "github.com/smartystreets/goconvey/convey"
 )
 
-func TestRemoveZeroUnitsFromInterval(t *testing.T) {
-	tcs := []struct {
-		interval time.Duration
-		expected string
-	}{
-		{interval: time.Duration(time.Hour), expected: "1h"},
-		{interval: time.Duration(time.Hour + time.Minute), expected: "1h1m"},
-		{interval: time.Duration((time.Hour * 10) + time.Minute), expected: "10h1m"},
-	}
-
-	for _, tc := range tcs {
-		got := removeZeroesFromDuration(tc.interval)
-		if got != tc.expected {
-			t.Errorf("expected %s got %s internval: %v", tc.expected, got, tc.interval)
-		}
-	}
-}
-
 func TestAlertingApiEndpoint(t *testing.T) {
 	Convey("Given an alert in a dashboard with an acl", t, func() {
 
diff --git a/pkg/api/dtos/alerting.go b/pkg/api/dtos/alerting.go
index 786fccc10b5..f8671978148 100644
--- a/pkg/api/dtos/alerting.go
+++ b/pkg/api/dtos/alerting.go
@@ -1,7 +1,7 @@
 package dtos
 
 import (
-	"strings"
+	"fmt"
 	"time"
 
 	"github.com/grafana/grafana/pkg/components/null"
@@ -24,14 +24,27 @@ type AlertRule struct {
 	CanEdit        bool                  `json:"canEdit"`
 }
 
-func removeZeroesFromDuration(interval time.Duration) string {
-	frequency := interval.String()
+func formatShort(interval time.Duration) string {
+	var result string
 
-	frequency = strings.Replace(frequency, "0h", "", 1)
-	frequency = strings.Replace(frequency, "0m", "", 1)
-	frequency = strings.Replace(frequency, "0s", "", 1)
+	hours := interval / time.Hour
+	if hours > 0 {
+		result += fmt.Sprintf("%dh", hours)
+	}
 
-	return frequency
+	remaining := interval - (hours * time.Hour)
+	mins := remaining / time.Minute
+	if mins > 0 {
+		result += fmt.Sprintf("%dm", mins)
+	}
+
+	remaining = remaining - (mins * time.Minute)
+	seconds := remaining / time.Second
+	if seconds > 0 {
+		result += fmt.Sprintf("%ds", seconds)
+	}
+
+	return result
 }
 
 func NewAlertNotification(notification *models.AlertNotification) *AlertNotification {
@@ -42,7 +55,7 @@ func NewAlertNotification(notification *models.AlertNotification) *AlertNotifica
 		IsDefault:  notification.IsDefault,
 		Created:    notification.Created,
 		Updated:    notification.Updated,
-		Frequency:  removeZeroesFromDuration(notification.Frequency),
+		Frequency:  formatShort(notification.Frequency),
 		NotifyOnce: notification.NotifyOnce,
 		Settings:   notification.Settings,
 	}
diff --git a/pkg/api/dtos/alerting_test.go b/pkg/api/dtos/alerting_test.go
new file mode 100644
index 00000000000..ea4c97fb4cc
--- /dev/null
+++ b/pkg/api/dtos/alerting_test.go
@@ -0,0 +1,34 @@
+package dtos
+
+import (
+	"testing"
+	"time"
+)
+
+func TestFormatShort(t *testing.T) {
+	tcs := []struct {
+		interval time.Duration
+		expected string
+	}{
+		{interval: time.Duration(time.Hour), expected: "1h"},
+		{interval: time.Duration(time.Hour + time.Minute), expected: "1h1m"},
+		{interval: time.Duration((time.Hour * 10) + time.Minute), expected: "10h1m"},
+		{interval: time.Duration((time.Hour * 10) + (time.Minute * 10) + time.Second), expected: "10h10m1s"},
+	}
+
+	for _, tc := range tcs {
+		got := formatShort(tc.interval)
+		if got != tc.expected {
+			t.Errorf("expected %s got %s interval: %v", tc.expected, got, tc.interval)
+		}
+
+		parsed, err := time.ParseDuration(tc.expected)
+		if err != nil {
+			t.Fatalf("could not parse expected duration")
+		}
+
+		if parsed != tc.interval {
+			t.Errorf("expectes the parsed duration to equal the interval. Got %v expected: %v", parsed, tc.interval)
+		}
+	}
+}

From 7333d7b8d4c128092b39c1e5616977c0a6aff015 Mon Sep 17 00:00:00 2001
From: bergquist <carl.bergquist@gmail.com>
Date: Tue, 5 Jun 2018 10:27:29 +0200
Subject: [PATCH 10/39] alerting: invert sendOnce to sendReminder

---
 pkg/api/dtos/alerting.go                      | 46 +++++++-------
 pkg/api/dtos/alerting_test.go                 |  1 +
 pkg/models/alert_notifications.go             | 46 +++++++-------
 pkg/services/alerting/interfaces.go           |  2 +-
 .../alerting/notifiers/alertmanager.go        |  2 +-
 pkg/services/alerting/notifiers/base.go       | 54 ++++++++--------
 pkg/services/alerting/notifiers/base_test.go  | 13 +++-
 pkg/services/alerting/notifiers/dingding.go   |  2 +-
 pkg/services/alerting/notifiers/discord.go    |  2 +-
 pkg/services/alerting/notifiers/email.go      |  2 +-
 pkg/services/alerting/notifiers/hipchat.go    |  2 +-
 pkg/services/alerting/notifiers/kafka.go      |  2 +-
 pkg/services/alerting/notifiers/line.go       |  2 +-
 pkg/services/alerting/notifiers/opsgenie.go   |  2 +-
 pkg/services/alerting/notifiers/pagerduty.go  |  2 +-
 pkg/services/alerting/notifiers/pushover.go   |  2 +-
 pkg/services/alerting/notifiers/sensu.go      |  2 +-
 pkg/services/alerting/notifiers/slack.go      |  2 +-
 pkg/services/alerting/notifiers/teams.go      |  2 +-
 pkg/services/alerting/notifiers/telegram.go   |  2 +-
 pkg/services/alerting/notifiers/threema.go    |  2 +-
 pkg/services/alerting/notifiers/victorops.go  |  2 +-
 pkg/services/alerting/notifiers/webhook.go    |  2 +-
 pkg/services/sqlstore/alert_notification.go   | 32 +++++-----
 .../sqlstore/alert_notification_test.go       | 63 ++++++++++---------
 pkg/services/sqlstore/migrations/alert_mig.go |  5 +-
 .../alerting/notification_edit_ctrl.ts        |  2 +-
 .../alerting/partials/notification_edit.html  | 23 +++++--
 28 files changed, 173 insertions(+), 148 deletions(-)

diff --git a/pkg/api/dtos/alerting.go b/pkg/api/dtos/alerting.go
index f8671978148..697d0a35a08 100644
--- a/pkg/api/dtos/alerting.go
+++ b/pkg/api/dtos/alerting.go
@@ -49,28 +49,28 @@ func formatShort(interval time.Duration) string {
 
 func NewAlertNotification(notification *models.AlertNotification) *AlertNotification {
 	return &AlertNotification{
-		Id:         notification.Id,
-		Name:       notification.Name,
-		Type:       notification.Type,
-		IsDefault:  notification.IsDefault,
-		Created:    notification.Created,
-		Updated:    notification.Updated,
-		Frequency:  formatShort(notification.Frequency),
-		NotifyOnce: notification.NotifyOnce,
-		Settings:   notification.Settings,
+		Id:           notification.Id,
+		Name:         notification.Name,
+		Type:         notification.Type,
+		IsDefault:    notification.IsDefault,
+		Created:      notification.Created,
+		Updated:      notification.Updated,
+		Frequency:    formatShort(notification.Frequency),
+		SendReminder: notification.SendReminder,
+		Settings:     notification.Settings,
 	}
 }
 
 type AlertNotification struct {
-	Id         int64            `json:"id"`
-	Name       string           `json:"name"`
-	Type       string           `json:"type"`
-	IsDefault  bool             `json:"isDefault"`
-	NotifyOnce bool             `json:"notifyOnce"`
-	Frequency  string           `json:"frequency"`
-	Created    time.Time        `json:"created"`
-	Updated    time.Time        `json:"updated"`
-	Settings   *simplejson.Json `json:"settings"`
+	Id           int64            `json:"id"`
+	Name         string           `json:"name"`
+	Type         string           `json:"type"`
+	IsDefault    bool             `json:"isDefault"`
+	SendReminder bool             `json:"sendReminder"`
+	Frequency    string           `json:"frequency"`
+	Created      time.Time        `json:"created"`
+	Updated      time.Time        `json:"updated"`
+	Settings     *simplejson.Json `json:"settings"`
 }
 
 type AlertTestCommand struct {
@@ -100,11 +100,11 @@ type EvalMatch struct {
 }
 
 type NotificationTestCommand struct {
-	Name       string           `json:"name"`
-	Type       string           `json:"type"`
-	NotifyOnce bool             `json:"notifyOnce"`
-	Frequency  string           `json:"frequency"`
-	Settings   *simplejson.Json `json:"settings"`
+	Name         string           `json:"name"`
+	Type         string           `json:"type"`
+	SendReminder bool             `json:"sendReminder"`
+	Frequency    string           `json:"frequency"`
+	Settings     *simplejson.Json `json:"settings"`
 }
 
 type PauseAlertCommand struct {
diff --git a/pkg/api/dtos/alerting_test.go b/pkg/api/dtos/alerting_test.go
index ea4c97fb4cc..bd0d9ff8feb 100644
--- a/pkg/api/dtos/alerting_test.go
+++ b/pkg/api/dtos/alerting_test.go
@@ -14,6 +14,7 @@ func TestFormatShort(t *testing.T) {
 		{interval: time.Duration(time.Hour + time.Minute), expected: "1h1m"},
 		{interval: time.Duration((time.Hour * 10) + time.Minute), expected: "10h1m"},
 		{interval: time.Duration((time.Hour * 10) + (time.Minute * 10) + time.Second), expected: "10h10m1s"},
+		{interval: time.Duration(time.Minute * 10), expected: "10m"},
 	}
 
 	for _, tc := range tcs {
diff --git a/pkg/models/alert_notifications.go b/pkg/models/alert_notifications.go
index ed6b8f372d1..c17124dd6ef 100644
--- a/pkg/models/alert_notifications.go
+++ b/pkg/models/alert_notifications.go
@@ -12,38 +12,38 @@ var (
 )
 
 type AlertNotification struct {
-	Id         int64            `json:"id"`
-	OrgId      int64            `json:"-"`
-	Name       string           `json:"name"`
-	Type       string           `json:"type"`
-	NotifyOnce bool             `json:"notifyOnce"`
-	Frequency  time.Duration    `json:"frequency"`
-	IsDefault  bool             `json:"isDefault"`
-	Settings   *simplejson.Json `json:"settings"`
-	Created    time.Time        `json:"created"`
-	Updated    time.Time        `json:"updated"`
+	Id           int64            `json:"id"`
+	OrgId        int64            `json:"-"`
+	Name         string           `json:"name"`
+	Type         string           `json:"type"`
+	SendReminder bool             `json:"sendReminder"`
+	Frequency    time.Duration    `json:"frequency"`
+	IsDefault    bool             `json:"isDefault"`
+	Settings     *simplejson.Json `json:"settings"`
+	Created      time.Time        `json:"created"`
+	Updated      time.Time        `json:"updated"`
 }
 
 type CreateAlertNotificationCommand struct {
-	Name       string           `json:"name"  binding:"Required"`
-	Type       string           `json:"type"  binding:"Required"`
-	NotifyOnce bool             `json:"notifyOnce"`
-	Frequency  string           `json:"frequency"`
-	IsDefault  bool             `json:"isDefault"`
-	Settings   *simplejson.Json `json:"settings"`
+	Name         string           `json:"name"  binding:"Required"`
+	Type         string           `json:"type"  binding:"Required"`
+	SendReminder bool             `json:"sendReminder"`
+	Frequency    string           `json:"frequency"`
+	IsDefault    bool             `json:"isDefault"`
+	Settings     *simplejson.Json `json:"settings"`
 
 	OrgId  int64 `json:"-"`
 	Result *AlertNotification
 }
 
 type UpdateAlertNotificationCommand struct {
-	Id         int64            `json:"id"  binding:"Required"`
-	Name       string           `json:"name"  binding:"Required"`
-	Type       string           `json:"type"  binding:"Required"`
-	NotifyOnce bool             `json:"notifyOnce"`
-	Frequency  string           `json:"frequency"`
-	IsDefault  bool             `json:"isDefault"`
-	Settings   *simplejson.Json `json:"settings"  binding:"Required"`
+	Id           int64            `json:"id"  binding:"Required"`
+	Name         string           `json:"name"  binding:"Required"`
+	Type         string           `json:"type"  binding:"Required"`
+	SendReminder bool             `json:"sendReminder"`
+	Frequency    string           `json:"frequency"`
+	IsDefault    bool             `json:"isDefault"`
+	Settings     *simplejson.Json `json:"settings"  binding:"Required"`
 
 	OrgId  int64 `json:"-"`
 	Result *AlertNotification
diff --git a/pkg/services/alerting/interfaces.go b/pkg/services/alerting/interfaces.go
index 8842b35fba2..95fd4b5d04e 100644
--- a/pkg/services/alerting/interfaces.go
+++ b/pkg/services/alerting/interfaces.go
@@ -19,7 +19,7 @@ type Notifier interface {
 
 	GetNotifierId() int64
 	GetIsDefault() bool
-	GetNotifyOnce() bool
+	GetSendReminder() bool
 	GetFrequency() time.Duration
 }
 
diff --git a/pkg/services/alerting/notifiers/alertmanager.go b/pkg/services/alerting/notifiers/alertmanager.go
index 3eeb25986e0..42ffa9b2d6e 100644
--- a/pkg/services/alerting/notifiers/alertmanager.go
+++ b/pkg/services/alerting/notifiers/alertmanager.go
@@ -33,7 +33,7 @@ func NewAlertmanagerNotifier(model *m.AlertNotification) (alerting.Notifier, err
 	}
 
 	return &AlertmanagerNotifier{
-		NotifierBase: NewNotifierBase(model.Id, model.IsDefault, model.Name, model.Type, model.NotifyOnce, model.Frequency, model.Settings),
+		NotifierBase: NewNotifierBase(model),
 		Url:          url,
 		log:          log.New("alerting.notifier.prometheus-alertmanager"),
 	}, nil
diff --git a/pkg/services/alerting/notifiers/base.go b/pkg/services/alerting/notifiers/base.go
index 7672d397491..1d0d904457f 100644
--- a/pkg/services/alerting/notifiers/base.go
+++ b/pkg/services/alerting/notifiers/base.go
@@ -3,54 +3,56 @@ package notifiers
 import (
 	"time"
 
-	"github.com/grafana/grafana/pkg/components/simplejson"
-	m "github.com/grafana/grafana/pkg/models"
+	"github.com/grafana/grafana/pkg/models"
 	"github.com/grafana/grafana/pkg/services/alerting"
 )
 
 type NotifierBase struct {
-	Name        string
-	Type        string
-	Id          int64
-	IsDeault    bool
-	UploadImage bool
-	NotifyOnce  bool
-	Frequency   time.Duration
+	Name         string
+	Type         string
+	Id           int64
+	IsDeault     bool
+	UploadImage  bool
+	SendReminder bool
+	Frequency    time.Duration
 }
 
-func NewNotifierBase(id int64, isDefault bool, name, notifierType string, notifyOnce bool, frequency time.Duration, model *simplejson.Json) NotifierBase {
+func NewNotifierBase(model *models.AlertNotification) NotifierBase {
 	uploadImage := true
-	value, exist := model.CheckGet("uploadImage")
+	value, exist := model.Settings.CheckGet("uploadImage")
 	if exist {
 		uploadImage = value.MustBool()
 	}
 
 	return NotifierBase{
-		Id:          id,
-		Name:        name,
-		IsDeault:    isDefault,
-		Type:        notifierType,
-		UploadImage: uploadImage,
-		NotifyOnce:  notifyOnce,
-		Frequency:   frequency,
+		Id:           model.Id,
+		Name:         model.Name,
+		IsDeault:     model.IsDefault,
+		Type:         model.Type,
+		UploadImage:  uploadImage,
+		SendReminder: model.SendReminder,
+		Frequency:    model.Frequency,
 	}
 }
 
-func defaultShouldNotify(context *alerting.EvalContext, notifyOnce bool, frequency time.Duration, lastNotify *time.Time) bool {
+func defaultShouldNotify(context *alerting.EvalContext, sendReminder bool, frequency time.Duration, lastNotify *time.Time) bool {
 	// Only notify on state change.
-	if context.PrevAlertState == context.Rule.State && notifyOnce {
+	if context.PrevAlertState == context.Rule.State && !sendReminder {
 		return false
 	}
+
 	// Do not notify if interval has not elapsed
-	if !notifyOnce && lastNotify != nil && lastNotify.Add(frequency).After(time.Now()) {
+	if sendReminder && lastNotify != nil && lastNotify.Add(frequency).After(time.Now()) {
 		return false
 	}
+
 	// Do not notify if alert state if OK or pending even on repeated notify
-	if !notifyOnce && (context.Rule.State == m.AlertStateOK || context.Rule.State == m.AlertStatePending) {
+	if sendReminder && (context.Rule.State == models.AlertStateOK || context.Rule.State == models.AlertStatePending) {
 		return false
 	}
+
 	// Do not notify when we become OK for the first time.
-	if (context.PrevAlertState == m.AlertStatePending) && (context.Rule.State == m.AlertStateOK) {
+	if (context.PrevAlertState == models.AlertStatePending) && (context.Rule.State == models.AlertStateOK) {
 		return false
 	}
 	return true
@@ -58,7 +60,7 @@ func defaultShouldNotify(context *alerting.EvalContext, notifyOnce bool, frequen
 
 func (n *NotifierBase) ShouldNotify(context *alerting.EvalContext) bool {
 	lastNotify := context.LastNotify(n.Id)
-	return defaultShouldNotify(context, n.NotifyOnce, n.Frequency, lastNotify)
+	return defaultShouldNotify(context, n.SendReminder, n.Frequency, lastNotify)
 }
 
 func (n *NotifierBase) GetType() string {
@@ -77,8 +79,8 @@ func (n *NotifierBase) GetIsDefault() bool {
 	return n.IsDeault
 }
 
-func (n *NotifierBase) GetNotifyOnce() bool {
-	return n.NotifyOnce
+func (n *NotifierBase) GetSendReminder() bool {
+	return n.SendReminder
 }
 
 func (n *NotifierBase) GetFrequency() time.Duration {
diff --git a/pkg/services/alerting/notifiers/base_test.go b/pkg/services/alerting/notifiers/base_test.go
index 5f2d4989063..28e2ff9f024 100644
--- a/pkg/services/alerting/notifiers/base_test.go
+++ b/pkg/services/alerting/notifiers/base_test.go
@@ -16,22 +16,29 @@ func TestBaseNotifier(t *testing.T) {
 		Convey("default constructor for notifiers", func() {
 			bJson := simplejson.New()
 
+			model := &m.AlertNotification{
+				Id:       1,
+				Name:     "name",
+				Type:     "email",
+				Settings: bJson,
+			}
+
 			Convey("can parse false value", func() {
 				bJson.Set("uploadImage", false)
 
-				base := NewNotifierBase(1, false, "name", "email", true, 0, bJson)
+				base := NewNotifierBase(model)
 				So(base.UploadImage, ShouldBeFalse)
 			})
 
 			Convey("can parse true value", func() {
 				bJson.Set("uploadImage", true)
 
-				base := NewNotifierBase(1, false, "name", "email", true, 0, bJson)
+				base := NewNotifierBase(model)
 				So(base.UploadImage, ShouldBeTrue)
 			})
 
 			Convey("default value should be true for backwards compatibility", func() {
-				base := NewNotifierBase(1, false, "name", "email", true, 0, bJson)
+				base := NewNotifierBase(model)
 				So(base.UploadImage, ShouldBeTrue)
 			})
 		})
diff --git a/pkg/services/alerting/notifiers/dingding.go b/pkg/services/alerting/notifiers/dingding.go
index 78446c56f88..738e43af2d2 100644
--- a/pkg/services/alerting/notifiers/dingding.go
+++ b/pkg/services/alerting/notifiers/dingding.go
@@ -32,7 +32,7 @@ func NewDingDingNotifier(model *m.AlertNotification) (alerting.Notifier, error)
 	}
 
 	return &DingDingNotifier{
-		NotifierBase: NewNotifierBase(model.Id, model.IsDefault, model.Name, model.Type, model.NotifyOnce, model.Frequency, model.Settings),
+		NotifierBase: NewNotifierBase(model),
 		Url:          url,
 		log:          log.New("alerting.notifier.dingding"),
 	}, nil
diff --git a/pkg/services/alerting/notifiers/discord.go b/pkg/services/alerting/notifiers/discord.go
index 693ed31e206..57d9d438fa2 100644
--- a/pkg/services/alerting/notifiers/discord.go
+++ b/pkg/services/alerting/notifiers/discord.go
@@ -39,7 +39,7 @@ func NewDiscordNotifier(model *m.AlertNotification) (alerting.Notifier, error) {
 	}
 
 	return &DiscordNotifier{
-		NotifierBase: NewNotifierBase(model.Id, model.IsDefault, model.Name, model.Type, model.NotifyOnce, model.Frequency, model.Settings),
+		NotifierBase: NewNotifierBase(model),
 		WebhookURL:   url,
 		log:          log.New("alerting.notifier.discord"),
 	}, nil
diff --git a/pkg/services/alerting/notifiers/email.go b/pkg/services/alerting/notifiers/email.go
index 234a4f8e756..17b88f7d97f 100644
--- a/pkg/services/alerting/notifiers/email.go
+++ b/pkg/services/alerting/notifiers/email.go
@@ -52,7 +52,7 @@ func NewEmailNotifier(model *m.AlertNotification) (alerting.Notifier, error) {
 	})
 
 	return &EmailNotifier{
-		NotifierBase: NewNotifierBase(model.Id, model.IsDefault, model.Name, model.Type, model.NotifyOnce, model.Frequency, model.Settings),
+		NotifierBase: NewNotifierBase(model),
 		Addresses:    addresses,
 		log:          log.New("alerting.notifier.email"),
 	}, nil
diff --git a/pkg/services/alerting/notifiers/hipchat.go b/pkg/services/alerting/notifiers/hipchat.go
index 4eb5b78811e..1c284ec3d2b 100644
--- a/pkg/services/alerting/notifiers/hipchat.go
+++ b/pkg/services/alerting/notifiers/hipchat.go
@@ -59,7 +59,7 @@ func NewHipChatNotifier(model *models.AlertNotification) (alerting.Notifier, err
 	roomId := model.Settings.Get("roomid").MustString()
 
 	return &HipChatNotifier{
-		NotifierBase: NewNotifierBase(model.Id, model.IsDefault, model.Name, model.Type, model.NotifyOnce, model.Frequency, model.Settings),
+		NotifierBase: NewNotifierBase(model),
 		Url:          url,
 		ApiKey:       apikey,
 		RoomId:       roomId,
diff --git a/pkg/services/alerting/notifiers/kafka.go b/pkg/services/alerting/notifiers/kafka.go
index 0dab556d5e1..d8d19fc5dae 100644
--- a/pkg/services/alerting/notifiers/kafka.go
+++ b/pkg/services/alerting/notifiers/kafka.go
@@ -43,7 +43,7 @@ func NewKafkaNotifier(model *m.AlertNotification) (alerting.Notifier, error) {
 	}
 
 	return &KafkaNotifier{
-		NotifierBase: NewNotifierBase(model.Id, model.IsDefault, model.Name, model.Type, model.NotifyOnce, model.Frequency, model.Settings),
+		NotifierBase: NewNotifierBase(model),
 		Endpoint:     endpoint,
 		Topic:        topic,
 		log:          log.New("alerting.notifier.kafka"),
diff --git a/pkg/services/alerting/notifiers/line.go b/pkg/services/alerting/notifiers/line.go
index 0ee252e6447..9e3888b8f95 100644
--- a/pkg/services/alerting/notifiers/line.go
+++ b/pkg/services/alerting/notifiers/line.go
@@ -39,7 +39,7 @@ func NewLINENotifier(model *m.AlertNotification) (alerting.Notifier, error) {
 	}
 
 	return &LineNotifier{
-		NotifierBase: NewNotifierBase(model.Id, model.IsDefault, model.Name, model.Type, model.NotifyOnce, model.Frequency, model.Settings),
+		NotifierBase: NewNotifierBase(model),
 		Token:        token,
 		log:          log.New("alerting.notifier.line"),
 	}, nil
diff --git a/pkg/services/alerting/notifiers/opsgenie.go b/pkg/services/alerting/notifiers/opsgenie.go
index 991afd5ce9b..84148a0d99c 100644
--- a/pkg/services/alerting/notifiers/opsgenie.go
+++ b/pkg/services/alerting/notifiers/opsgenie.go
@@ -56,7 +56,7 @@ func NewOpsGenieNotifier(model *m.AlertNotification) (alerting.Notifier, error)
 	}
 
 	return &OpsGenieNotifier{
-		NotifierBase: NewNotifierBase(model.Id, model.IsDefault, model.Name, model.Type, model.NotifyOnce, model.Frequency, model.Settings),
+		NotifierBase: NewNotifierBase(model),
 		ApiKey:       apiKey,
 		ApiUrl:       apiUrl,
 		AutoClose:    autoClose,
diff --git a/pkg/services/alerting/notifiers/pagerduty.go b/pkg/services/alerting/notifiers/pagerduty.go
index afa0ba63eca..bf85466388f 100644
--- a/pkg/services/alerting/notifiers/pagerduty.go
+++ b/pkg/services/alerting/notifiers/pagerduty.go
@@ -51,7 +51,7 @@ func NewPagerdutyNotifier(model *m.AlertNotification) (alerting.Notifier, error)
 	}
 
 	return &PagerdutyNotifier{
-		NotifierBase: NewNotifierBase(model.Id, model.IsDefault, model.Name, model.Type, model.NotifyOnce, model.Frequency, model.Settings),
+		NotifierBase: NewNotifierBase(model),
 		Key:          key,
 		AutoResolve:  autoResolve,
 		log:          log.New("alerting.notifier.pagerduty"),
diff --git a/pkg/services/alerting/notifiers/pushover.go b/pkg/services/alerting/notifiers/pushover.go
index 09dfd6f0f9b..55dc02c5f4a 100644
--- a/pkg/services/alerting/notifiers/pushover.go
+++ b/pkg/services/alerting/notifiers/pushover.go
@@ -99,7 +99,7 @@ func NewPushoverNotifier(model *m.AlertNotification) (alerting.Notifier, error)
 		return nil, alerting.ValidationError{Reason: "API token not given"}
 	}
 	return &PushoverNotifier{
-		NotifierBase: NewNotifierBase(model.Id, model.IsDefault, model.Name, model.Type, model.NotifyOnce, model.Frequency, model.Settings),
+		NotifierBase: NewNotifierBase(model),
 		UserKey:      userKey,
 		ApiToken:     apiToken,
 		Priority:     priority,
diff --git a/pkg/services/alerting/notifiers/sensu.go b/pkg/services/alerting/notifiers/sensu.go
index e6b94d3223e..21d5d3d9d9e 100644
--- a/pkg/services/alerting/notifiers/sensu.go
+++ b/pkg/services/alerting/notifiers/sensu.go
@@ -51,7 +51,7 @@ func NewSensuNotifier(model *m.AlertNotification) (alerting.Notifier, error) {
 	}
 
 	return &SensuNotifier{
-		NotifierBase: NewNotifierBase(model.Id, model.IsDefault, model.Name, model.Type, model.NotifyOnce, model.Frequency, model.Settings),
+		NotifierBase: NewNotifierBase(model),
 		Url:          url,
 		User:         model.Settings.Get("username").MustString(),
 		Source:       model.Settings.Get("source").MustString(),
diff --git a/pkg/services/alerting/notifiers/slack.go b/pkg/services/alerting/notifiers/slack.go
index fbbe4b3e59d..93c9a0accc0 100644
--- a/pkg/services/alerting/notifiers/slack.go
+++ b/pkg/services/alerting/notifiers/slack.go
@@ -78,7 +78,7 @@ func NewSlackNotifier(model *m.AlertNotification) (alerting.Notifier, error) {
 	uploadImage := model.Settings.Get("uploadImage").MustBool(true)
 
 	return &SlackNotifier{
-		NotifierBase: NewNotifierBase(model.Id, model.IsDefault, model.Name, model.Type, model.NotifyOnce, model.Frequency, model.Settings),
+		NotifierBase: NewNotifierBase(model),
 		Url:          url,
 		Recipient:    recipient,
 		Mention:      mention,
diff --git a/pkg/services/alerting/notifiers/teams.go b/pkg/services/alerting/notifiers/teams.go
index 362a367e1f2..58dd4b22bb7 100644
--- a/pkg/services/alerting/notifiers/teams.go
+++ b/pkg/services/alerting/notifiers/teams.go
@@ -33,7 +33,7 @@ func NewTeamsNotifier(model *m.AlertNotification) (alerting.Notifier, error) {
 	}
 
 	return &TeamsNotifier{
-		NotifierBase: NewNotifierBase(model.Id, model.IsDefault, model.Name, model.Type, model.NotifyOnce, model.Frequency, model.Settings),
+		NotifierBase: NewNotifierBase(model),
 		Url:          url,
 		log:          log.New("alerting.notifier.teams"),
 	}, nil
diff --git a/pkg/services/alerting/notifiers/telegram.go b/pkg/services/alerting/notifiers/telegram.go
index 97696b2290c..b03f7ca38c5 100644
--- a/pkg/services/alerting/notifiers/telegram.go
+++ b/pkg/services/alerting/notifiers/telegram.go
@@ -78,7 +78,7 @@ func NewTelegramNotifier(model *m.AlertNotification) (alerting.Notifier, error)
 	}
 
 	return &TelegramNotifier{
-		NotifierBase: NewNotifierBase(model.Id, model.IsDefault, model.Name, model.Type, model.NotifyOnce, model.Frequency, model.Settings),
+		NotifierBase: NewNotifierBase(model),
 		BotToken:     botToken,
 		ChatID:       chatId,
 		UploadImage:  uploadImage,
diff --git a/pkg/services/alerting/notifiers/threema.go b/pkg/services/alerting/notifiers/threema.go
index e7fb39f27db..28a62fade17 100644
--- a/pkg/services/alerting/notifiers/threema.go
+++ b/pkg/services/alerting/notifiers/threema.go
@@ -106,7 +106,7 @@ func NewThreemaNotifier(model *m.AlertNotification) (alerting.Notifier, error) {
 	}
 
 	return &ThreemaNotifier{
-		NotifierBase: NewNotifierBase(model.Id, model.IsDefault, model.Name, model.Type, model.NotifyOnce, model.Frequency, model.Settings),
+		NotifierBase: NewNotifierBase(model),
 		GatewayID:    gatewayID,
 		RecipientID:  recipientID,
 		APISecret:    apiSecret,
diff --git a/pkg/services/alerting/notifiers/victorops.go b/pkg/services/alerting/notifiers/victorops.go
index c6c1cf76047..3093aec9957 100644
--- a/pkg/services/alerting/notifiers/victorops.go
+++ b/pkg/services/alerting/notifiers/victorops.go
@@ -51,7 +51,7 @@ func NewVictoropsNotifier(model *models.AlertNotification) (alerting.Notifier, e
 	}
 
 	return &VictoropsNotifier{
-		NotifierBase: NewNotifierBase(model.Id, model.IsDefault, model.Name, model.Type, model.NotifyOnce, model.Frequency, model.Settings),
+		NotifierBase: NewNotifierBase(model),
 		URL:          url,
 		AutoResolve:  autoResolve,
 		log:          log.New("alerting.notifier.victorops"),
diff --git a/pkg/services/alerting/notifiers/webhook.go b/pkg/services/alerting/notifiers/webhook.go
index 26989873e9e..4045e496af9 100644
--- a/pkg/services/alerting/notifiers/webhook.go
+++ b/pkg/services/alerting/notifiers/webhook.go
@@ -47,7 +47,7 @@ func NewWebHookNotifier(model *m.AlertNotification) (alerting.Notifier, error) {
 	}
 
 	return &WebhookNotifier{
-		NotifierBase: NewNotifierBase(model.Id, model.IsDefault, model.Name, model.Type, model.NotifyOnce, model.Frequency, model.Settings),
+		NotifierBase: NewNotifierBase(model),
 		Url:          url,
 		User:         model.Settings.Get("username").MustString(),
 		Password:     model.Settings.Get("password").MustString(),
diff --git a/pkg/services/sqlstore/alert_notification.go b/pkg/services/sqlstore/alert_notification.go
index ff36c38b1a5..8c136bd3887 100644
--- a/pkg/services/sqlstore/alert_notification.go
+++ b/pkg/services/sqlstore/alert_notification.go
@@ -57,7 +57,7 @@ func GetAlertNotificationsToSend(query *m.GetAlertNotificationsToSendQuery) erro
 										alert_notification.updated,
 										alert_notification.settings,
 										alert_notification.is_default,
-										alert_notification.notify_once,
+										alert_notification.send_reminder,
 										alert_notification.frequency
 										FROM alert_notification
 	  							`)
@@ -97,7 +97,7 @@ func getAlertNotificationInternal(query *m.GetAlertNotificationsQuery, sess *DBS
 										alert_notification.updated,
 										alert_notification.settings,
 										alert_notification.is_default,
-										alert_notification.notify_once,
+										alert_notification.send_reminder,
 										alert_notification.frequency
 										FROM alert_notification
 	  							`)
@@ -145,7 +145,7 @@ func CreateAlertNotificationCommand(cmd *m.CreateAlertNotificationCommand) error
 		}
 
 		var frequency time.Duration
-		if !cmd.NotifyOnce {
+		if cmd.SendReminder {
 			if cmd.Frequency == "" {
 				return m.ErrNotificationFrequencyNotFound
 			}
@@ -157,18 +157,18 @@ func CreateAlertNotificationCommand(cmd *m.CreateAlertNotificationCommand) error
 		}
 
 		alertNotification := &m.AlertNotification{
-			OrgId:      cmd.OrgId,
-			Name:       cmd.Name,
-			Type:       cmd.Type,
-			Settings:   cmd.Settings,
-			NotifyOnce: cmd.NotifyOnce,
-			Frequency:  frequency,
-			Created:    time.Now(),
-			Updated:    time.Now(),
-			IsDefault:  cmd.IsDefault,
+			OrgId:        cmd.OrgId,
+			Name:         cmd.Name,
+			Type:         cmd.Type,
+			Settings:     cmd.Settings,
+			SendReminder: cmd.SendReminder,
+			Frequency:    frequency,
+			Created:      time.Now(),
+			Updated:      time.Now(),
+			IsDefault:    cmd.IsDefault,
 		}
 
-		if _, err = sess.MustCols("notify_once").Insert(alertNotification); err != nil {
+		if _, err = sess.MustCols("send_reminder").Insert(alertNotification); err != nil {
 			return err
 		}
 
@@ -200,9 +200,9 @@ func UpdateAlertNotification(cmd *m.UpdateAlertNotificationCommand) error {
 		current.Name = cmd.Name
 		current.Type = cmd.Type
 		current.IsDefault = cmd.IsDefault
-		current.NotifyOnce = cmd.NotifyOnce
+		current.SendReminder = cmd.SendReminder
 
-		if !current.NotifyOnce {
+		if current.SendReminder {
 			if cmd.Frequency == "" {
 				return m.ErrNotificationFrequencyNotFound
 			}
@@ -215,7 +215,7 @@ func UpdateAlertNotification(cmd *m.UpdateAlertNotificationCommand) error {
 			current.Frequency = frequency
 		}
 
-		sess.UseBool("is_default", "notify_once")
+		sess.UseBool("is_default", "send_reminder")
 
 		if affected, err := sess.ID(cmd.Id).Update(current); err != nil {
 			return err
diff --git a/pkg/services/sqlstore/alert_notification_test.go b/pkg/services/sqlstore/alert_notification_test.go
index 578a53f34ad..fe7f02b22b0 100644
--- a/pkg/services/sqlstore/alert_notification_test.go
+++ b/pkg/services/sqlstore/alert_notification_test.go
@@ -23,13 +23,13 @@ func TestAlertNotificationSQLAccess(t *testing.T) {
 			So(cmd.Result, ShouldBeNil)
 		})
 
-		Convey("Cannot save alert notifier with notitfyonce = false", func() {
+		Convey("Cannot save alert notifier with send reminder = true", func() {
 			cmd := &m.CreateAlertNotificationCommand{
-				Name:       "ops",
-				Type:       "email",
-				OrgId:      1,
-				NotifyOnce: false,
-				Settings:   simplejson.New(),
+				Name:         "ops",
+				Type:         "email",
+				OrgId:        1,
+				SendReminder: true,
+				Settings:     simplejson.New(),
 			}
 
 			Convey("and missing frequency", func() {
@@ -47,19 +47,19 @@ func TestAlertNotificationSQLAccess(t *testing.T) {
 
 		Convey("Cannot update alert notifier with notitfyonce = false", func() {
 			cmd := &m.CreateAlertNotificationCommand{
-				Name:       "ops update",
-				Type:       "email",
-				OrgId:      1,
-				NotifyOnce: true,
-				Settings:   simplejson.New(),
+				Name:         "ops update",
+				Type:         "email",
+				OrgId:        1,
+				SendReminder: false,
+				Settings:     simplejson.New(),
 			}
 
 			err := CreateAlertNotificationCommand(cmd)
 			So(err, ShouldBeNil)
 
 			updateCmd := &m.UpdateAlertNotificationCommand{
-				Id:         cmd.Result.Id,
-				NotifyOnce: false,
+				Id:           cmd.Result.Id,
+				SendReminder: true,
 			}
 
 			Convey("and missing frequency", func() {
@@ -71,18 +71,19 @@ func TestAlertNotificationSQLAccess(t *testing.T) {
 				updateCmd.Frequency = "invalid duration"
 
 				err := UpdateAlertNotification(updateCmd)
+				So(err, ShouldNotBeNil)
 				So(err.Error(), ShouldEqual, "time: invalid duration invalid duration")
 			})
 		})
 
 		Convey("Can save Alert Notification", func() {
 			cmd := &m.CreateAlertNotificationCommand{
-				Name:       "ops",
-				Type:       "email",
-				OrgId:      1,
-				NotifyOnce: true,
-				Frequency:  "10s",
-				Settings:   simplejson.New(),
+				Name:         "ops",
+				Type:         "email",
+				OrgId:        1,
+				SendReminder: true,
+				Frequency:    "10s",
+				Settings:     simplejson.New(),
 			}
 
 			err := CreateAlertNotificationCommand(cmd)
@@ -98,13 +99,13 @@ func TestAlertNotificationSQLAccess(t *testing.T) {
 
 			Convey("Can update alert notification", func() {
 				newCmd := &m.UpdateAlertNotificationCommand{
-					Name:       "NewName",
-					Type:       "webhook",
-					OrgId:      cmd.Result.OrgId,
-					NotifyOnce: true,
-					Frequency:  "10s",
-					Settings:   simplejson.New(),
-					Id:         cmd.Result.Id,
+					Name:         "NewName",
+					Type:         "webhook",
+					OrgId:        cmd.Result.OrgId,
+					SendReminder: true,
+					Frequency:    "10s",
+					Settings:     simplejson.New(),
+					Id:           cmd.Result.Id,
 				}
 				err := UpdateAlertNotification(newCmd)
 				So(err, ShouldBeNil)
@@ -113,12 +114,12 @@ func TestAlertNotificationSQLAccess(t *testing.T) {
 		})
 
 		Convey("Can search using an array of ids", func() {
-			cmd1 := m.CreateAlertNotificationCommand{Name: "nagios", Type: "webhook", OrgId: 1, NotifyOnce: true, Frequency: "10s", Settings: simplejson.New()}
-			cmd2 := m.CreateAlertNotificationCommand{Name: "slack", Type: "webhook", OrgId: 1, NotifyOnce: true, Frequency: "10s", Settings: simplejson.New()}
-			cmd3 := m.CreateAlertNotificationCommand{Name: "ops2", Type: "email", OrgId: 1, NotifyOnce: true, Frequency: "10s", Settings: simplejson.New()}
-			cmd4 := m.CreateAlertNotificationCommand{IsDefault: true, Name: "default", Type: "email", OrgId: 1, NotifyOnce: true, Frequency: "10s", Settings: simplejson.New()}
+			cmd1 := m.CreateAlertNotificationCommand{Name: "nagios", Type: "webhook", OrgId: 1, SendReminder: true, Frequency: "10s", Settings: simplejson.New()}
+			cmd2 := m.CreateAlertNotificationCommand{Name: "slack", Type: "webhook", OrgId: 1, SendReminder: true, Frequency: "10s", Settings: simplejson.New()}
+			cmd3 := m.CreateAlertNotificationCommand{Name: "ops2", Type: "email", OrgId: 1, SendReminder: true, Frequency: "10s", Settings: simplejson.New()}
+			cmd4 := m.CreateAlertNotificationCommand{IsDefault: true, Name: "default", Type: "email", OrgId: 1, SendReminder: true, Frequency: "10s", Settings: simplejson.New()}
 
-			otherOrg := m.CreateAlertNotificationCommand{Name: "default", Type: "email", OrgId: 2, NotifyOnce: true, Frequency: "10s", Settings: simplejson.New()}
+			otherOrg := m.CreateAlertNotificationCommand{Name: "default", Type: "email", OrgId: 2, SendReminder: true, Frequency: "10s", Settings: simplejson.New()}
 
 			So(CreateAlertNotificationCommand(&cmd1), ShouldBeNil)
 			So(CreateAlertNotificationCommand(&cmd2), ShouldBeNil)
diff --git a/pkg/services/sqlstore/migrations/alert_mig.go b/pkg/services/sqlstore/migrations/alert_mig.go
index d045f611fb2..51509099c7d 100644
--- a/pkg/services/sqlstore/migrations/alert_mig.go
+++ b/pkg/services/sqlstore/migrations/alert_mig.go
@@ -68,9 +68,10 @@ func addAlertMigrations(mg *Migrator) {
 	mg.AddMigration("Add column frequency", NewAddColumnMigration(alert_notification, &Column{
 		Name: "frequency", Type: DB_BigInt, Nullable: true,
 	}))
-	mg.AddMigration("Add column notify_once", NewAddColumnMigration(alert_notification, &Column{
-		Name: "notify_once", Type: DB_Bool, Nullable: false, Default: "1",
+	mg.AddMigration("Add column send_reminder", NewAddColumnMigration(alert_notification, &Column{
+		Name: "send_reminder", Type: DB_Bool, Nullable: true, Default: "0",
 	}))
+
 	mg.AddMigration("add index alert_notification org_id & name", NewAddIndexMigration(alert_notification, alert_notification.Indices[0]))
 
 	notification_journal := Table{
diff --git a/public/app/features/alerting/notification_edit_ctrl.ts b/public/app/features/alerting/notification_edit_ctrl.ts
index 9d20e871c7c..e066406bc43 100644
--- a/public/app/features/alerting/notification_edit_ctrl.ts
+++ b/public/app/features/alerting/notification_edit_ctrl.ts
@@ -11,7 +11,7 @@ export class AlertNotificationEditCtrl {
   model: any;
   defaults: any = {
     type: 'email',
-    notifyOnce: true,
+    sendReminder: false,
     frequency: '15m',
     settings: {
       httpMethod: 'POST',
diff --git a/public/app/features/alerting/partials/notification_edit.html b/public/app/features/alerting/partials/notification_edit.html
index 48d44b74581..2a2fbb131b8 100644
--- a/public/app/features/alerting/partials/notification_edit.html
+++ b/public/app/features/alerting/partials/notification_edit.html
@@ -34,14 +34,27 @@
       </gf-form-switch>
       <gf-form-switch
           class="gf-form"
-          label="Notify once"
+          label="Send reminder"
           label-class="width-12"
-          checked="ctrl.model.notifyOnce"
+          checked="ctrl.model.sendReminder"
           tooltip="Choose to either notify on state change or at every interval">
       </gf-form-switch>
-      <div class="gf-form">
-        <span class="gf-form-label width-12" ng-if="!ctrl.model.notifyOnce">Notify every</span>
-        <input class="gf-form-input max-width-15" type="text" required ng-model="ctrl.model.frequency" required ng-if="!ctrl.model.notifyOnce"></input>
+      <div class="gf-form-inline">
+        <div class="gf-form">
+          <span class="gf-form-label width-12" ng-if="ctrl.model.sendReminder">Send reminder every</span>
+          <input
+            class="gf-form-input max-width-15"
+            type="text"
+            ng-model="ctrl.model.frequency"
+            ng-required="ctrl.model.sendReminder"
+            ng-if="ctrl.model.sendReminder"
+            spellcheck='false'
+            placeholder='15m'></input>
+          <info-popover mode="right-absolute">
+            Specify at what interval you want reminder's about this alerting beeing triggered.
+            Ex. 60s, 10m, 30m, 1h
+          </info-popover>
+        </div>
       </div>
     </div>
 

From bcbae7aa6240e974d47f6c55d83195e4ef2348ad Mon Sep 17 00:00:00 2001
From: bergquist <carl.bergquist@gmail.com>
Date: Tue, 5 Jun 2018 12:07:02 +0200
Subject: [PATCH 11/39] alerting: move queries from evalcontext to notifier
 base

---
 pkg/services/alerting/eval_context.go   | 15 ---------------
 pkg/services/alerting/interfaces.go     |  2 ++
 pkg/services/alerting/notifier.go       |  1 +
 pkg/services/alerting/notifiers/base.go | 23 ++++++++++++++++++++---
 4 files changed, 23 insertions(+), 18 deletions(-)

diff --git a/pkg/services/alerting/eval_context.go b/pkg/services/alerting/eval_context.go
index 3817f4b4a3c..d0441d379b7 100644
--- a/pkg/services/alerting/eval_context.go
+++ b/pkg/services/alerting/eval_context.go
@@ -143,18 +143,3 @@ func (c *EvalContext) GetNewState() m.AlertStateType {
 
 	return m.AlertStateOK
 }
-
-func (c *EvalContext) LastNotify(notifierId int64) *time.Time {
-	cmd := &m.GetLatestNotificationQuery{
-		OrgId:      c.Rule.OrgId,
-		AlertId:    c.Rule.Id,
-		NotifierId: notifierId,
-	}
-	if err := bus.Dispatch(cmd); err != nil {
-		c.log.Warn("Could not determine last time alert notifier fired",
-			"Alert name", c.Rule.Name, "Error", err)
-		return nil
-	}
-
-	return &cmd.Result.SentAt
-}
diff --git a/pkg/services/alerting/interfaces.go b/pkg/services/alerting/interfaces.go
index 95fd4b5d04e..b4376191df0 100644
--- a/pkg/services/alerting/interfaces.go
+++ b/pkg/services/alerting/interfaces.go
@@ -15,6 +15,8 @@ type Notifier interface {
 	Notify(evalContext *EvalContext) error
 	GetType() string
 	NeedsImage() bool
+
+	// ShouldNotify checks this evaluation should send an alert notification
 	ShouldNotify(evalContext *EvalContext) bool
 
 	GetNotifierId() int64
diff --git a/pkg/services/alerting/notifier.go b/pkg/services/alerting/notifier.go
index 53923a420fe..363a156c5ec 100644
--- a/pkg/services/alerting/notifier.go
+++ b/pkg/services/alerting/notifier.go
@@ -131,6 +131,7 @@ func (n *notificationService) getNeededNotifiers(orgId int64, notificationIds []
 		if err != nil {
 			return nil, err
 		}
+
 		if not.ShouldNotify(context) {
 			result = append(result, not)
 		}
diff --git a/pkg/services/alerting/notifiers/base.go b/pkg/services/alerting/notifiers/base.go
index 1d0d904457f..fc00bd5265e 100644
--- a/pkg/services/alerting/notifiers/base.go
+++ b/pkg/services/alerting/notifiers/base.go
@@ -3,6 +3,8 @@ package notifiers
 import (
 	"time"
 
+	"github.com/grafana/grafana/pkg/bus"
+	"github.com/grafana/grafana/pkg/log"
 	"github.com/grafana/grafana/pkg/models"
 	"github.com/grafana/grafana/pkg/services/alerting"
 )
@@ -15,6 +17,8 @@ type NotifierBase struct {
 	UploadImage  bool
 	SendReminder bool
 	Frequency    time.Duration
+
+	log log.Logger
 }
 
 func NewNotifierBase(model *models.AlertNotification) NotifierBase {
@@ -32,6 +36,7 @@ func NewNotifierBase(model *models.AlertNotification) NotifierBase {
 		UploadImage:  uploadImage,
 		SendReminder: model.SendReminder,
 		Frequency:    model.Frequency,
+		log:          log.New("alerting.notifier." + model.Name),
 	}
 }
 
@@ -55,12 +60,24 @@ func defaultShouldNotify(context *alerting.EvalContext, sendReminder bool, frequ
 	if (context.PrevAlertState == models.AlertStatePending) && (context.Rule.State == models.AlertStateOK) {
 		return false
 	}
+
 	return true
 }
 
-func (n *NotifierBase) ShouldNotify(context *alerting.EvalContext) bool {
-	lastNotify := context.LastNotify(n.Id)
-	return defaultShouldNotify(context, n.SendReminder, n.Frequency, lastNotify)
+// ShouldNotify checks this evaluation should send an alert notification
+func (n *NotifierBase) ShouldNotify(c *alerting.EvalContext) bool {
+	cmd := &models.GetLatestNotificationQuery{
+		OrgId:      c.Rule.OrgId,
+		AlertId:    c.Rule.Id,
+		NotifierId: n.Id,
+	}
+
+	if err := bus.Dispatch(cmd); err != nil {
+		n.log.Error("Could not determine last time alert notifier fired", "Alert name", c.Rule.Name, "Error", err)
+		return false
+	}
+
+	return defaultShouldNotify(c, n.SendReminder, n.Frequency, &cmd.Result.SentAt)
 }
 
 func (n *NotifierBase) GetType() string {

From ab70ead5e4d7ab1dabb204e02a4e27e453d6bd67 Mon Sep 17 00:00:00 2001
From: bergquist <carl.bergquist@gmail.com>
Date: Tue, 5 Jun 2018 13:10:38 +0200
Subject: [PATCH 12/39] alerting: renames journal table to
 alert_notification_journal

---
 pkg/services/sqlstore/migrations/alert_mig.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pkg/services/sqlstore/migrations/alert_mig.go b/pkg/services/sqlstore/migrations/alert_mig.go
index 51509099c7d..e58959929d1 100644
--- a/pkg/services/sqlstore/migrations/alert_mig.go
+++ b/pkg/services/sqlstore/migrations/alert_mig.go
@@ -75,7 +75,7 @@ func addAlertMigrations(mg *Migrator) {
 	mg.AddMigration("add index alert_notification org_id & name", NewAddIndexMigration(alert_notification, alert_notification.Indices[0]))
 
 	notification_journal := Table{
-		Name: "notification_journal",
+		Name: "alert_notification_journal",
 		Columns: []*Column{
 			{Name: "id", Type: DB_BigInt, IsPrimaryKey: true, IsAutoIncrement: true},
 			{Name: "org_id", Type: DB_BigInt, Nullable: false},

From 171a38df999f919d2bbd59344483759fc83c7af0 Mon Sep 17 00:00:00 2001
From: bergquist <carl.bergquist@gmail.com>
Date: Tue, 5 Jun 2018 14:29:48 +0200
Subject: [PATCH 13/39] alerting: fixes broken table rename

---
 pkg/models/alert_notifications.go           |  4 +--
 pkg/services/sqlstore/alert_notification.go | 40 ++++++++++++++++++---
 2 files changed, 38 insertions(+), 6 deletions(-)

diff --git a/pkg/models/alert_notifications.go b/pkg/models/alert_notifications.go
index c17124dd6ef..8df7a830d8b 100644
--- a/pkg/models/alert_notifications.go
+++ b/pkg/models/alert_notifications.go
@@ -75,7 +75,7 @@ type GetAllAlertNotificationsQuery struct {
 	Result []*AlertNotification
 }
 
-type NotificationJournal struct {
+type AlertNotificationJournal struct {
 	Id         int64
 	OrgId      int64
 	AlertId    int64
@@ -97,7 +97,7 @@ type GetLatestNotificationQuery struct {
 	AlertId    int64
 	NotifierId int64
 
-	Result *NotificationJournal
+	Result *AlertNotificationJournal
 }
 
 type CleanNotificationJournalCommand struct {
diff --git a/pkg/services/sqlstore/alert_notification.go b/pkg/services/sqlstore/alert_notification.go
index 8c136bd3887..0223636bd9a 100644
--- a/pkg/services/sqlstore/alert_notification.go
+++ b/pkg/services/sqlstore/alert_notification.go
@@ -2,10 +2,12 @@ package sqlstore
 
 import (
 	"bytes"
+	"context"
 	"fmt"
 	"strings"
 	"time"
 
+	"github.com/go-xorm/xorm"
 	"github.com/grafana/grafana/pkg/bus"
 	m "github.com/grafana/grafana/pkg/models"
 )
@@ -20,6 +22,8 @@ func init() {
 	bus.AddHandler("sql", RecordNotificationJournal)
 	bus.AddHandler("sql", GetLatestNotification)
 	bus.AddHandler("sql", CleanNotificationJournal)
+
+	bus.AddCtxHandler("sql", GetLastestNotification2)
 }
 
 func DeleteAlertNotification(cmd *m.DeleteAlertNotificationCommand) error {
@@ -230,7 +234,7 @@ func UpdateAlertNotification(cmd *m.UpdateAlertNotificationCommand) error {
 
 func RecordNotificationJournal(cmd *m.RecordNotificationJournalCommand) error {
 	return inTransaction(func(sess *DBSession) error {
-		journalEntry := &m.NotificationJournal{
+		journalEntry := &m.AlertNotificationJournal{
 			OrgId:      cmd.OrgId,
 			AlertId:    cmd.AlertId,
 			NotifierId: cmd.NotifierId,
@@ -246,10 +250,38 @@ func RecordNotificationJournal(cmd *m.RecordNotificationJournalCommand) error {
 	})
 }
 
+func startSession(ctx context.Context) *DBSession {
+	value := ctx.Value("db-session")
+	var sess *xorm.Session
+	sess, ok := value.(*xorm.Session)
+
+	if !ok {
+		return newSession()
+	}
+
+	old := newSession()
+	old.Session = sess
+
+	return old
+}
+
+func GetLastestNotification2(ctx context.Context, cmd *m.GetLatestNotificationQuery) error {
+	sess := startSession(ctx)
+
+	notificationJournal := &m.AlertNotificationJournal{}
+	_, err := sess.Desc("alert_notification_journal.sent_at").Limit(1).Where("alert_notification_journal.org_id = ? AND alert_notification_journal.alert_id = ? AND alert_notification_journal.notifier_id = ?", cmd.OrgId, cmd.AlertId, cmd.NotifierId).Get(notificationJournal)
+	if err != nil {
+		return err
+	}
+
+	cmd.Result = notificationJournal
+	return nil
+}
+
 func GetLatestNotification(cmd *m.GetLatestNotificationQuery) error {
 	return inTransaction(func(sess *DBSession) error {
-		notificationJournal := &m.NotificationJournal{}
-		_, err := sess.Desc("notification_journal.sent_at").Limit(1).Where("notification_journal.org_id = ? AND notification_journal.alert_id = ? AND notification_journal.notifier_id = ?", cmd.OrgId, cmd.AlertId, cmd.NotifierId).Get(notificationJournal)
+		notificationJournal := &m.AlertNotificationJournal{}
+		_, err := sess.Desc("alert_notification_journal.sent_at").Limit(1).Where("alert_notification_journal.org_id = ? AND alert_notification_journal.alert_id = ? AND alert_notification_journal.notifier_id = ?", cmd.OrgId, cmd.AlertId, cmd.NotifierId).Get(notificationJournal)
 		if err != nil {
 			return err
 		}
@@ -261,7 +293,7 @@ func GetLatestNotification(cmd *m.GetLatestNotificationQuery) error {
 
 func CleanNotificationJournal(cmd *m.CleanNotificationJournalCommand) error {
 	return inTransaction(func(sess *DBSession) error {
-		sql := "DELETE FROM notification_journal WHERE notification_journal.org_id = ? AND notification_journal.alert_id = ? AND notification_journal.notifier_id = ?"
+		sql := "DELETE FROM alert_notification_journal WHERE notification_journal.org_id = ? AND alert_notification_journal.alert_id = ? AND alert_notification_journal.notifier_id = ?"
 		_, err := sess.Exec(sql, cmd.OrgId, cmd.AlertId, cmd.NotifierId)
 		return err
 	})

From 850aa21d451618590a8530b02d4ffd68e02bbeb6 Mon Sep 17 00:00:00 2001
From: bergquist <carl.bergquist@gmail.com>
Date: Thu, 7 Jun 2018 07:06:13 +0200
Subject: [PATCH 14/39] removes unused code

---
 pkg/services/sqlstore/alert_notification.go | 15 ---------------
 1 file changed, 15 deletions(-)

diff --git a/pkg/services/sqlstore/alert_notification.go b/pkg/services/sqlstore/alert_notification.go
index 0223636bd9a..a2f3e629ae6 100644
--- a/pkg/services/sqlstore/alert_notification.go
+++ b/pkg/services/sqlstore/alert_notification.go
@@ -22,8 +22,6 @@ func init() {
 	bus.AddHandler("sql", RecordNotificationJournal)
 	bus.AddHandler("sql", GetLatestNotification)
 	bus.AddHandler("sql", CleanNotificationJournal)
-
-	bus.AddCtxHandler("sql", GetLastestNotification2)
 }
 
 func DeleteAlertNotification(cmd *m.DeleteAlertNotificationCommand) error {
@@ -265,19 +263,6 @@ func startSession(ctx context.Context) *DBSession {
 	return old
 }
 
-func GetLastestNotification2(ctx context.Context, cmd *m.GetLatestNotificationQuery) error {
-	sess := startSession(ctx)
-
-	notificationJournal := &m.AlertNotificationJournal{}
-	_, err := sess.Desc("alert_notification_journal.sent_at").Limit(1).Where("alert_notification_journal.org_id = ? AND alert_notification_journal.alert_id = ? AND alert_notification_journal.notifier_id = ?", cmd.OrgId, cmd.AlertId, cmd.NotifierId).Get(notificationJournal)
-	if err != nil {
-		return err
-	}
-
-	cmd.Result = notificationJournal
-	return nil
-}
-
 func GetLatestNotification(cmd *m.GetLatestNotificationQuery) error {
 	return inTransaction(func(sess *DBSession) error {
 		notificationJournal := &m.AlertNotificationJournal{}

From 4a8e9cf93f9a04a30be8269ac68653889394a53d Mon Sep 17 00:00:00 2001
From: bergquist <carl.bergquist@gmail.com>
Date: Thu, 7 Jun 2018 07:15:50 +0200
Subject: [PATCH 15/39] removes more unused code

---
 pkg/services/sqlstore/alert_notification.go | 17 -----------------
 1 file changed, 17 deletions(-)

diff --git a/pkg/services/sqlstore/alert_notification.go b/pkg/services/sqlstore/alert_notification.go
index a2f3e629ae6..167b9b3bd61 100644
--- a/pkg/services/sqlstore/alert_notification.go
+++ b/pkg/services/sqlstore/alert_notification.go
@@ -2,12 +2,10 @@ package sqlstore
 
 import (
 	"bytes"
-	"context"
 	"fmt"
 	"strings"
 	"time"
 
-	"github.com/go-xorm/xorm"
 	"github.com/grafana/grafana/pkg/bus"
 	m "github.com/grafana/grafana/pkg/models"
 )
@@ -248,21 +246,6 @@ func RecordNotificationJournal(cmd *m.RecordNotificationJournalCommand) error {
 	})
 }
 
-func startSession(ctx context.Context) *DBSession {
-	value := ctx.Value("db-session")
-	var sess *xorm.Session
-	sess, ok := value.(*xorm.Session)
-
-	if !ok {
-		return newSession()
-	}
-
-	old := newSession()
-	old.Session = sess
-
-	return old
-}
-
 func GetLatestNotification(cmd *m.GetLatestNotificationQuery) error {
 	return inTransaction(func(sess *DBSession) error {
 		notificationJournal := &m.AlertNotificationJournal{}

From 7632983c627b9e5f36e9b13455dc6a45031629eb Mon Sep 17 00:00:00 2001
From: bergquist <carl.bergquist@gmail.com>
Date: Fri, 8 Jun 2018 15:51:26 +0200
Subject: [PATCH 16/39] notifications: gather actions in one transaction

---
 pkg/services/alerting/notifier.go       | 47 +++++++++++++++++--------
 pkg/services/alerting/notifiers/base.go |  6 +++-
 2 files changed, 37 insertions(+), 16 deletions(-)

diff --git a/pkg/services/alerting/notifier.go b/pkg/services/alerting/notifier.go
index 363a156c5ec..a3d016211c3 100644
--- a/pkg/services/alerting/notifier.go
+++ b/pkg/services/alerting/notifier.go
@@ -1,6 +1,7 @@
 package alerting
 
 import (
+	"context"
 	"errors"
 	"fmt"
 	"time"
@@ -59,23 +60,39 @@ func (n *notificationService) SendIfNeeded(context *EvalContext) error {
 	return n.sendNotifications(context, notifiers)
 }
 
-func (n *notificationService) sendNotifications(context *EvalContext, notifiers []Notifier) error {
-	g, _ := errgroup.WithContext(context.Ctx)
+func (n *notificationService) sendNotifications(evalContext *EvalContext, notifiers []Notifier) error {
+	g, _ := errgroup.WithContext(evalContext.Ctx)
 
 	for _, notifier := range notifiers {
 		not := notifier //avoid updating scope variable in go routine
-		n.log.Debug("Sending notification", "type", not.GetType(), "id", not.GetNotifierId(), "isDefault", not.GetIsDefault())
-		metrics.M_Alerting_Notification_Sent.WithLabelValues(not.GetType()).Inc()
+
 		g.Go(func() error {
-			success := not.Notify(context) == nil
-			cmd := &m.RecordNotificationJournalCommand{
-				OrgId:      context.Rule.OrgId,
-				AlertId:    context.Rule.Id,
-				NotifierId: not.GetNotifierId(),
-				SentAt:     time.Now(),
-				Success:    success,
-			}
-			return bus.Dispatch(cmd)
+			return bus.InTransaction(evalContext.Ctx, func(ctx context.Context) error {
+				n.log.Debug("trying to send notification", "id", not.GetNotifierId())
+
+				// Verify that we can send the notification again
+				// but this time within the same transaction.
+				if !evalContext.IsTestRun && !not.ShouldNotify(evalContext) {
+					return nil
+				}
+
+				n.log.Debug("Sending notification", "type", not.GetType(), "id", not.GetNotifierId(), "isDefault", not.GetIsDefault())
+				metrics.M_Alerting_Notification_Sent.WithLabelValues(not.GetType()).Inc()
+
+				//send notification
+				success := not.Notify(evalContext) == nil
+
+				//write result to db.
+				cmd := &m.RecordNotificationJournalCommand{
+					OrgId:      evalContext.Rule.OrgId,
+					AlertId:    evalContext.Rule.Id,
+					NotifierId: not.GetNotifierId(),
+					SentAt:     time.Now(),
+					Success:    success,
+				}
+
+				return bus.DispatchCtx(evalContext.Ctx, cmd)
+			})
 		})
 	}
 
@@ -118,7 +135,7 @@ func (n *notificationService) uploadImage(context *EvalContext) (err error) {
 	return nil
 }
 
-func (n *notificationService) getNeededNotifiers(orgId int64, notificationIds []int64, context *EvalContext) (NotifierSlice, error) {
+func (n *notificationService) getNeededNotifiers(orgId int64, notificationIds []int64, evalContext *EvalContext) (NotifierSlice, error) {
 	query := &m.GetAlertNotificationsToSendQuery{OrgId: orgId, Ids: notificationIds}
 
 	if err := bus.Dispatch(query); err != nil {
@@ -132,7 +149,7 @@ func (n *notificationService) getNeededNotifiers(orgId int64, notificationIds []
 			return nil, err
 		}
 
-		if not.ShouldNotify(context) {
+		if not.ShouldNotify(evalContext) {
 			result = append(result, not)
 		}
 	}
diff --git a/pkg/services/alerting/notifiers/base.go b/pkg/services/alerting/notifiers/base.go
index d8cb740daa1..8450816f97e 100644
--- a/pkg/services/alerting/notifiers/base.go
+++ b/pkg/services/alerting/notifiers/base.go
@@ -73,11 +73,15 @@ func (n *NotifierBase) ShouldNotify(c *alerting.EvalContext) bool {
 		NotifierId: n.Id,
 	}
 
-	if err := bus.Dispatch(cmd); err != nil {
+	if err := bus.DispatchCtx(c.Ctx, cmd); err != nil {
 		n.log.Error("Could not determine last time alert notifier fired", "Alert name", c.Rule.Name, "Error", err)
 		return false
 	}
 
+	if !cmd.Result.Success {
+		return true
+	}
+
 	return defaultShouldNotify(c, n.SendReminder, n.Frequency, &cmd.Result.SentAt)
 }
 

From f4b089d5519fbd352874282f771c8dc66731dde2 Mon Sep 17 00:00:00 2001
From: bergquist <carl.bergquist@gmail.com>
Date: Fri, 15 Jun 2018 15:30:17 +0200
Subject: [PATCH 17/39] notifications: make journaling ctx aware

---
 pkg/services/alerting/notifiers/base_test.go | 94 +++++++++++---------
 pkg/services/alerting/result_handler.go      |  2 +-
 pkg/services/sqlstore/alert_notification.go  | 19 ++--
 pkg/services/sqlstore/transactions.go        |  4 +
 4 files changed, 66 insertions(+), 53 deletions(-)

diff --git a/pkg/services/alerting/notifiers/base_test.go b/pkg/services/alerting/notifiers/base_test.go
index 28e2ff9f024..b7395030e5b 100644
--- a/pkg/services/alerting/notifiers/base_test.go
+++ b/pkg/services/alerting/notifiers/base_test.go
@@ -11,56 +11,64 @@ import (
 	. "github.com/smartystreets/goconvey/convey"
 )
 
+func TestShouldSendAlertNotification(t *testing.T) {
+	tcs := []struct {
+		prevState m.AlertStateType
+		newState  m.AlertStateType
+		expected  bool
+	}{
+		{
+			newState:  m.AlertStatePending,
+			prevState: m.AlertStateOK,
+			expected:  false,
+		},
+		{
+			newState:  m.AlertStateOK,
+			prevState: m.AlertStateAlerting,
+			expected:  true,
+		},
+	}
+
+	for _, tc := range tcs {
+		context := alerting.NewEvalContext(context.TODO(), &alerting.Rule{
+			State: tc.newState,
+		})
+		context.Rule.State = tc.prevState
+		timeNow := time.Now()
+		if defaultShouldNotify(context, true, 0, &timeNow) != tc.expected {
+			t.Errorf("expected %v to return %v", tc, tc.expected)
+		}
+	}
+}
+
 func TestBaseNotifier(t *testing.T) {
-	Convey("Base notifier tests", t, func() {
-		Convey("default constructor for notifiers", func() {
-			bJson := simplejson.New()
+	Convey("default constructor for notifiers", t, func() {
+		bJson := simplejson.New()
 
-			model := &m.AlertNotification{
-				Id:       1,
-				Name:     "name",
-				Type:     "email",
-				Settings: bJson,
-			}
+		model := &m.AlertNotification{
+			Id:       1,
+			Name:     "name",
+			Type:     "email",
+			Settings: bJson,
+		}
 
-			Convey("can parse false value", func() {
-				bJson.Set("uploadImage", false)
+		Convey("can parse false value", func() {
+			bJson.Set("uploadImage", false)
 
-				base := NewNotifierBase(model)
-				So(base.UploadImage, ShouldBeFalse)
-			})
-
-			Convey("can parse true value", func() {
-				bJson.Set("uploadImage", true)
-
-				base := NewNotifierBase(model)
-				So(base.UploadImage, ShouldBeTrue)
-			})
-
-			Convey("default value should be true for backwards compatibility", func() {
-				base := NewNotifierBase(model)
-				So(base.UploadImage, ShouldBeTrue)
-			})
+			base := NewNotifierBase(model)
+			So(base.UploadImage, ShouldBeFalse)
 		})
 
-		Convey("should notify", func() {
-			Convey("pending -> ok", func() {
-				context := alerting.NewEvalContext(context.TODO(), &alerting.Rule{
-					State: m.AlertStatePending,
-				})
-				context.Rule.State = m.AlertStateOK
-				timeNow := time.Now()
-				So(defaultShouldNotify(context, true, 0, &timeNow), ShouldBeFalse)
-			})
+		Convey("can parse true value", func() {
+			bJson.Set("uploadImage", true)
 
-			Convey("ok -> alerting", func() {
-				context := alerting.NewEvalContext(context.TODO(), &alerting.Rule{
-					State: m.AlertStateOK,
-				})
-				context.Rule.State = m.AlertStateAlerting
-				timeNow := time.Now()
-				So(defaultShouldNotify(context, true, 0, &timeNow), ShouldBeTrue)
-			})
+			base := NewNotifierBase(model)
+			So(base.UploadImage, ShouldBeTrue)
+		})
+
+		Convey("default value should be true for backwards compatibility", func() {
+			base := NewNotifierBase(model)
+			So(base.UploadImage, ShouldBeTrue)
 		})
 	})
 }
diff --git a/pkg/services/alerting/result_handler.go b/pkg/services/alerting/result_handler.go
index c4c20bd8beb..363d06d1132 100644
--- a/pkg/services/alerting/result_handler.go
+++ b/pkg/services/alerting/result_handler.go
@@ -95,7 +95,7 @@ func (handler *DefaultResultHandler) Handle(evalContext *EvalContext) error {
 				NotifierId: notifierId,
 				OrgId:      evalContext.Rule.OrgId,
 			}
-			if err := bus.Dispatch(cmd); err != nil {
+			if err := bus.DispatchCtx(evalContext.Ctx, cmd); err != nil {
 				handler.log.Error("Failed to clean up old notification records", "notifier", notifierId, "alert", evalContext.Rule.Id, "Error", err)
 			}
 		}
diff --git a/pkg/services/sqlstore/alert_notification.go b/pkg/services/sqlstore/alert_notification.go
index 167b9b3bd61..26362dcb750 100644
--- a/pkg/services/sqlstore/alert_notification.go
+++ b/pkg/services/sqlstore/alert_notification.go
@@ -2,6 +2,7 @@ package sqlstore
 
 import (
 	"bytes"
+	"context"
 	"fmt"
 	"strings"
 	"time"
@@ -17,9 +18,9 @@ func init() {
 	bus.AddHandler("sql", DeleteAlertNotification)
 	bus.AddHandler("sql", GetAlertNotificationsToSend)
 	bus.AddHandler("sql", GetAllAlertNotifications)
-	bus.AddHandler("sql", RecordNotificationJournal)
-	bus.AddHandler("sql", GetLatestNotification)
-	bus.AddHandler("sql", CleanNotificationJournal)
+	bus.AddHandlerCtx("sql", RecordNotificationJournal)
+	bus.AddHandlerCtx("sql", GetLatestNotification)
+	bus.AddHandlerCtx("sql", CleanNotificationJournal)
 }
 
 func DeleteAlertNotification(cmd *m.DeleteAlertNotificationCommand) error {
@@ -228,8 +229,8 @@ func UpdateAlertNotification(cmd *m.UpdateAlertNotificationCommand) error {
 	})
 }
 
-func RecordNotificationJournal(cmd *m.RecordNotificationJournalCommand) error {
-	return inTransaction(func(sess *DBSession) error {
+func RecordNotificationJournal(ctx context.Context, cmd *m.RecordNotificationJournalCommand) error {
+	return inTransactionCtx(ctx, func(sess *DBSession) error {
 		journalEntry := &m.AlertNotificationJournal{
 			OrgId:      cmd.OrgId,
 			AlertId:    cmd.AlertId,
@@ -246,8 +247,8 @@ func RecordNotificationJournal(cmd *m.RecordNotificationJournalCommand) error {
 	})
 }
 
-func GetLatestNotification(cmd *m.GetLatestNotificationQuery) error {
-	return inTransaction(func(sess *DBSession) error {
+func GetLatestNotification(ctx context.Context, cmd *m.GetLatestNotificationQuery) error {
+	return inTransactionCtx(ctx, func(sess *DBSession) error {
 		notificationJournal := &m.AlertNotificationJournal{}
 		_, err := sess.Desc("alert_notification_journal.sent_at").Limit(1).Where("alert_notification_journal.org_id = ? AND alert_notification_journal.alert_id = ? AND alert_notification_journal.notifier_id = ?", cmd.OrgId, cmd.AlertId, cmd.NotifierId).Get(notificationJournal)
 		if err != nil {
@@ -259,8 +260,8 @@ func GetLatestNotification(cmd *m.GetLatestNotificationQuery) error {
 	})
 }
 
-func CleanNotificationJournal(cmd *m.CleanNotificationJournalCommand) error {
-	return inTransaction(func(sess *DBSession) error {
+func CleanNotificationJournal(ctx context.Context, cmd *m.CleanNotificationJournalCommand) error {
+	return inTransactionCtx(ctx, func(sess *DBSession) error {
 		sql := "DELETE FROM alert_notification_journal WHERE notification_journal.org_id = ? AND alert_notification_journal.alert_id = ? AND alert_notification_journal.notifier_id = ?"
 		_, err := sess.Exec(sql, cmd.OrgId, cmd.AlertId, cmd.NotifierId)
 		return err
diff --git a/pkg/services/sqlstore/transactions.go b/pkg/services/sqlstore/transactions.go
index f72b0bb8500..59290f83121 100644
--- a/pkg/services/sqlstore/transactions.go
+++ b/pkg/services/sqlstore/transactions.go
@@ -103,3 +103,7 @@ func inTransactionWithRetryCtx(ctx context.Context, callback dbTransactionFunc,
 func inTransaction(callback dbTransactionFunc) error {
 	return inTransactionWithRetry(callback, 0)
 }
+
+func inTransactionCtx(ctx context.Context, callback dbTransactionFunc) error {
+	return inTransactionWithRetryCtx(ctx, callback, 0)
+}

From 12bf5c225a98121a9198ffd54a80b609e8364657 Mon Sep 17 00:00:00 2001
From: bergquist <carl.bergquist@gmail.com>
Date: Fri, 15 Jun 2018 16:27:20 +0200
Subject: [PATCH 18/39] tests for defaultShouldNotify

---
 pkg/services/alerting/notifiers/base.go      |  4 ++
 pkg/services/alerting/notifiers/base_test.go | 45 +++++++++++++++++---
 pkg/services/sqlstore/alert_notification.go  |  2 +-
 3 files changed, 43 insertions(+), 8 deletions(-)

diff --git a/pkg/services/alerting/notifiers/base.go b/pkg/services/alerting/notifiers/base.go
index 8450816f97e..8178054f4d3 100644
--- a/pkg/services/alerting/notifiers/base.go
+++ b/pkg/services/alerting/notifiers/base.go
@@ -78,6 +78,10 @@ func (n *NotifierBase) ShouldNotify(c *alerting.EvalContext) bool {
 		return false
 	}
 
+	// this currently serves two purposes.
+	// 1. make sure failed notifications try again
+	// 2. make sure we send notifications if no previous exist
+	// this should be refactored //Carl Bergquist
 	if !cmd.Result.Success {
 		return true
 	}
diff --git a/pkg/services/alerting/notifiers/base_test.go b/pkg/services/alerting/notifiers/base_test.go
index b7395030e5b..96c80cf03bc 100644
--- a/pkg/services/alerting/notifiers/base_test.go
+++ b/pkg/services/alerting/notifiers/base_test.go
@@ -13,30 +13,61 @@ import (
 
 func TestShouldSendAlertNotification(t *testing.T) {
 	tcs := []struct {
-		prevState m.AlertStateType
-		newState  m.AlertStateType
-		expected  bool
+		name         string
+		prevState    m.AlertStateType
+		newState     m.AlertStateType
+		expected     bool
+		sendReminder bool
 	}{
 		{
+			name:      "pending -> ok should not trigger an notification",
 			newState:  m.AlertStatePending,
 			prevState: m.AlertStateOK,
 			expected:  false,
 		},
 		{
+			name:      "ok -> alerting should trigger an notification",
 			newState:  m.AlertStateOK,
 			prevState: m.AlertStateAlerting,
 			expected:  true,
 		},
+		{
+			name:      "ok -> pending should not trigger an notification",
+			newState:  m.AlertStateOK,
+			prevState: m.AlertStatePending,
+			expected:  false,
+		},
+		{
+			name:         "ok -> ok should not trigger an notification",
+			newState:     m.AlertStateOK,
+			prevState:    m.AlertStateOK,
+			expected:     false,
+			sendReminder: false,
+		},
+		{
+			name:         "ok -> alerting should not trigger an notification",
+			newState:     m.AlertStateOK,
+			prevState:    m.AlertStateAlerting,
+			expected:     true,
+			sendReminder: true,
+		},
+		{
+			name:         "ok -> ok with reminder should not trigger an notification",
+			newState:     m.AlertStateOK,
+			prevState:    m.AlertStateOK,
+			expected:     false,
+			sendReminder: true,
+		},
 	}
 
 	for _, tc := range tcs {
-		context := alerting.NewEvalContext(context.TODO(), &alerting.Rule{
+		evalContext := alerting.NewEvalContext(context.TODO(), &alerting.Rule{
 			State: tc.newState,
 		})
-		context.Rule.State = tc.prevState
+		evalContext.Rule.State = tc.prevState
 		timeNow := time.Now()
-		if defaultShouldNotify(context, true, 0, &timeNow) != tc.expected {
-			t.Errorf("expected %v to return %v", tc, tc.expected)
+		if defaultShouldNotify(evalContext, true, 0, &timeNow) != tc.expected {
+			t.Errorf("failed %s. expected %+v to return %v", tc.name, tc, tc.expected)
 		}
 	}
 }
diff --git a/pkg/services/sqlstore/alert_notification.go b/pkg/services/sqlstore/alert_notification.go
index 26362dcb750..5f08a70eff3 100644
--- a/pkg/services/sqlstore/alert_notification.go
+++ b/pkg/services/sqlstore/alert_notification.go
@@ -262,7 +262,7 @@ func GetLatestNotification(ctx context.Context, cmd *m.GetLatestNotificationQuer
 
 func CleanNotificationJournal(ctx context.Context, cmd *m.CleanNotificationJournalCommand) error {
 	return inTransactionCtx(ctx, func(sess *DBSession) error {
-		sql := "DELETE FROM alert_notification_journal WHERE notification_journal.org_id = ? AND alert_notification_journal.alert_id = ? AND alert_notification_journal.notifier_id = ?"
+		sql := "DELETE FROM alert_notification_journal WHERE alert_notification_journal.org_id = ? AND alert_notification_journal.alert_id = ? AND alert_notification_journal.notifier_id = ?"
 		_, err := sess.Exec(sql, cmd.OrgId, cmd.AlertId, cmd.NotifierId)
 		return err
 	})

From 72224dbe377e5c065624065724ab9ebfb6011ea3 Mon Sep 17 00:00:00 2001
From: bergquist <carl.bergquist@gmail.com>
Date: Fri, 15 Jun 2018 16:53:35 +0200
Subject: [PATCH 19/39] adds info about eval/reminder interval

---
 .../features/alerting/partials/notification_edit.html    | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/public/app/features/alerting/partials/notification_edit.html b/public/app/features/alerting/partials/notification_edit.html
index 2a2fbb131b8..7132ed41f3c 100644
--- a/public/app/features/alerting/partials/notification_edit.html
+++ b/public/app/features/alerting/partials/notification_edit.html
@@ -39,6 +39,11 @@
           checked="ctrl.model.sendReminder"
           tooltip="Choose to either notify on state change or at every interval">
       </gf-form-switch>
+      <div class="gf-form">
+        <span class="alert alert-info width-30" ng-if="ctrl.model.sendReminder">
+          Alert reminders are sent after rules are evaluated. Therefore the alert rule interval has to be lower than the reminder frequency
+        </span>
+      </div>
       <div class="gf-form-inline">
         <div class="gf-form">
           <span class="gf-form-label width-12" ng-if="ctrl.model.sendReminder">Send reminder every</span>
@@ -50,8 +55,8 @@
             ng-if="ctrl.model.sendReminder"
             spellcheck='false'
             placeholder='15m'></input>
-          <info-popover mode="right-absolute">
-            Specify at what interval you want reminder's about this alerting beeing triggered.
+          <info-popover mode="right-absolute" ng-if="ctrl.model.sendReminder">
+            Specify at what interval you want reminder's about this alerting being triggered.
             Ex. 60s, 10m, 30m, 1h
           </info-popover>
         </div>

From c21938d4c44367a35d7004f5919023f50b467f73 Mon Sep 17 00:00:00 2001
From: bergquist <carl.bergquist@gmail.com>
Date: Sat, 16 Jun 2018 00:03:13 +0200
Subject: [PATCH 20/39] use epoch to compare timestamp

---
 pkg/models/alert_notifications.go             |  4 +--
 pkg/services/alerting/notifier.go             |  2 +-
 pkg/services/alerting/notifiers/base.go       |  6 ++--
 pkg/services/alerting/notifiers/base_test.go  |  3 +-
 pkg/services/sqlstore/migrations/alert_mig.go | 36 +++++++++----------
 5 files changed, 25 insertions(+), 26 deletions(-)

diff --git a/pkg/models/alert_notifications.go b/pkg/models/alert_notifications.go
index 8df7a830d8b..6be2b02c96f 100644
--- a/pkg/models/alert_notifications.go
+++ b/pkg/models/alert_notifications.go
@@ -80,7 +80,7 @@ type AlertNotificationJournal struct {
 	OrgId      int64
 	AlertId    int64
 	NotifierId int64
-	SentAt     time.Time
+	SentAt     int64
 	Success    bool
 }
 
@@ -88,7 +88,7 @@ type RecordNotificationJournalCommand struct {
 	OrgId      int64
 	AlertId    int64
 	NotifierId int64
-	SentAt     time.Time
+	SentAt     int64
 	Success    bool
 }
 
diff --git a/pkg/services/alerting/notifier.go b/pkg/services/alerting/notifier.go
index a3d016211c3..61526ed642c 100644
--- a/pkg/services/alerting/notifier.go
+++ b/pkg/services/alerting/notifier.go
@@ -87,7 +87,7 @@ func (n *notificationService) sendNotifications(evalContext *EvalContext, notifi
 					OrgId:      evalContext.Rule.OrgId,
 					AlertId:    evalContext.Rule.Id,
 					NotifierId: not.GetNotifierId(),
-					SentAt:     time.Now(),
+					SentAt:     time.Now().Unix(),
 					Success:    success,
 				}
 
diff --git a/pkg/services/alerting/notifiers/base.go b/pkg/services/alerting/notifiers/base.go
index 8178054f4d3..6245650ab4e 100644
--- a/pkg/services/alerting/notifiers/base.go
+++ b/pkg/services/alerting/notifiers/base.go
@@ -41,14 +41,14 @@ func NewNotifierBase(model *models.AlertNotification) NotifierBase {
 	}
 }
 
-func defaultShouldNotify(context *alerting.EvalContext, sendReminder bool, frequency time.Duration, lastNotify *time.Time) bool {
+func defaultShouldNotify(context *alerting.EvalContext, sendReminder bool, frequency time.Duration, lastNotify time.Time) bool {
 	// Only notify on state change.
 	if context.PrevAlertState == context.Rule.State && !sendReminder {
 		return false
 	}
 
 	// Do not notify if interval has not elapsed
-	if sendReminder && lastNotify != nil && lastNotify.Add(frequency).After(time.Now()) {
+	if sendReminder && !lastNotify.IsZero() && lastNotify.Add(frequency).After(time.Now()) {
 		return false
 	}
 
@@ -86,7 +86,7 @@ func (n *NotifierBase) ShouldNotify(c *alerting.EvalContext) bool {
 		return true
 	}
 
-	return defaultShouldNotify(c, n.SendReminder, n.Frequency, &cmd.Result.SentAt)
+	return defaultShouldNotify(c, n.SendReminder, n.Frequency, time.Unix(cmd.Result.SentAt, 0))
 }
 
 func (n *NotifierBase) GetType() string {
diff --git a/pkg/services/alerting/notifiers/base_test.go b/pkg/services/alerting/notifiers/base_test.go
index 96c80cf03bc..5b75ea4d59b 100644
--- a/pkg/services/alerting/notifiers/base_test.go
+++ b/pkg/services/alerting/notifiers/base_test.go
@@ -65,8 +65,7 @@ func TestShouldSendAlertNotification(t *testing.T) {
 			State: tc.newState,
 		})
 		evalContext.Rule.State = tc.prevState
-		timeNow := time.Now()
-		if defaultShouldNotify(evalContext, true, 0, &timeNow) != tc.expected {
+		if defaultShouldNotify(evalContext, true, 0, time.Now()) != tc.expected {
 			t.Errorf("failed %s. expected %+v to return %v", tc.name, tc, tc.expected)
 		}
 	}
diff --git a/pkg/services/sqlstore/migrations/alert_mig.go b/pkg/services/sqlstore/migrations/alert_mig.go
index e58959929d1..e27e64c6124 100644
--- a/pkg/services/sqlstore/migrations/alert_mig.go
+++ b/pkg/services/sqlstore/migrations/alert_mig.go
@@ -74,24 +74,6 @@ func addAlertMigrations(mg *Migrator) {
 
 	mg.AddMigration("add index alert_notification org_id & name", NewAddIndexMigration(alert_notification, alert_notification.Indices[0]))
 
-	notification_journal := Table{
-		Name: "alert_notification_journal",
-		Columns: []*Column{
-			{Name: "id", Type: DB_BigInt, IsPrimaryKey: true, IsAutoIncrement: true},
-			{Name: "org_id", Type: DB_BigInt, Nullable: false},
-			{Name: "alert_id", Type: DB_BigInt, Nullable: false},
-			{Name: "notifier_id", Type: DB_BigInt, Nullable: false},
-			{Name: "sent_at", Type: DB_DateTime, Nullable: false},
-			{Name: "success", Type: DB_Bool, Nullable: false},
-		},
-		Indices: []*Index{
-			{Cols: []string{"org_id", "alert_id", "notifier_id"}, Type: IndexType},
-		},
-	}
-
-	mg.AddMigration("create notification_journal table v1", NewAddTableMigration(notification_journal))
-	mg.AddMigration("add index notification_journal org_id & alert_id & notifier_id", NewAddIndexMigration(notification_journal, notification_journal.Indices[0]))
-
 	mg.AddMigration("Update alert table charset", NewTableCharsetMigration("alert", []*Column{
 		{Name: "name", Type: DB_NVarchar, Length: 255, Nullable: false},
 		{Name: "message", Type: DB_Text, Nullable: false},
@@ -107,4 +89,22 @@ func addAlertMigrations(mg *Migrator) {
 		{Name: "type", Type: DB_NVarchar, Length: 255, Nullable: false},
 		{Name: "settings", Type: DB_Text, Nullable: false},
 	}))
+
+	notification_journal := Table{
+		Name: "alert_notification_journal",
+		Columns: []*Column{
+			{Name: "id", Type: DB_BigInt, IsPrimaryKey: true, IsAutoIncrement: true},
+			{Name: "org_id", Type: DB_BigInt, Nullable: false},
+			{Name: "alert_id", Type: DB_BigInt, Nullable: false},
+			{Name: "notifier_id", Type: DB_BigInt, Nullable: false},
+			{Name: "sent_at", Type: DB_BigInt, Nullable: false},
+			{Name: "success", Type: DB_Bool, Nullable: false},
+		},
+		Indices: []*Index{
+			{Cols: []string{"org_id", "alert_id", "notifier_id"}, Type: IndexType},
+		},
+	}
+
+	mg.AddMigration("create notification_journal table v1", NewAddTableMigration(notification_journal))
+	mg.AddMigration("add index notification_journal org_id & alert_id & notifier_id", NewAddIndexMigration(notification_journal, notification_journal.Indices[0]))
 }

From 83a12afc07ef0d6891e9d9f76736359ec067c024 Mon Sep 17 00:00:00 2001
From: bergquist <carl.bergquist@gmail.com>
Date: Sat, 16 Jun 2018 11:27:04 +0200
Subject: [PATCH 21/39] adds tests for journaling sql operations

---
 pkg/models/alert_notifications.go             |  1 +
 pkg/services/alerting/notifiers/base.go       | 11 ++---
 pkg/services/sqlstore/alert_notification.go   | 13 ++++--
 .../sqlstore/alert_notification_test.go       | 43 +++++++++++++++++++
 4 files changed, 60 insertions(+), 8 deletions(-)

diff --git a/pkg/models/alert_notifications.go b/pkg/models/alert_notifications.go
index 6be2b02c96f..42d33d5ed22 100644
--- a/pkg/models/alert_notifications.go
+++ b/pkg/models/alert_notifications.go
@@ -9,6 +9,7 @@ import (
 
 var (
 	ErrNotificationFrequencyNotFound = errors.New("Notification frequency not specified")
+	ErrJournalingNotFound            = errors.New("alert notification journaling not found")
 )
 
 type AlertNotification struct {
diff --git a/pkg/services/alerting/notifiers/base.go b/pkg/services/alerting/notifiers/base.go
index 6245650ab4e..4869c40f436 100644
--- a/pkg/services/alerting/notifiers/base.go
+++ b/pkg/services/alerting/notifiers/base.go
@@ -73,15 +73,16 @@ func (n *NotifierBase) ShouldNotify(c *alerting.EvalContext) bool {
 		NotifierId: n.Id,
 	}
 
-	if err := bus.DispatchCtx(c.Ctx, cmd); err != nil {
+	err := bus.DispatchCtx(c.Ctx, cmd)
+	if err != nil {
 		n.log.Error("Could not determine last time alert notifier fired", "Alert name", c.Rule.Name, "Error", err)
 		return false
 	}
 
-	// this currently serves two purposes.
-	// 1. make sure failed notifications try again
-	// 2. make sure we send notifications if no previous exist
-	// this should be refactored //Carl Bergquist
+	if err == models.ErrJournalingNotFound {
+		return true
+	}
+
 	if !cmd.Result.Success {
 		return true
 	}
diff --git a/pkg/services/sqlstore/alert_notification.go b/pkg/services/sqlstore/alert_notification.go
index 5f08a70eff3..3f2ca109c1a 100644
--- a/pkg/services/sqlstore/alert_notification.go
+++ b/pkg/services/sqlstore/alert_notification.go
@@ -249,13 +249,20 @@ func RecordNotificationJournal(ctx context.Context, cmd *m.RecordNotificationJou
 
 func GetLatestNotification(ctx context.Context, cmd *m.GetLatestNotificationQuery) error {
 	return inTransactionCtx(ctx, func(sess *DBSession) error {
-		notificationJournal := &m.AlertNotificationJournal{}
-		_, err := sess.Desc("alert_notification_journal.sent_at").Limit(1).Where("alert_notification_journal.org_id = ? AND alert_notification_journal.alert_id = ? AND alert_notification_journal.notifier_id = ?", cmd.OrgId, cmd.AlertId, cmd.NotifierId).Get(notificationJournal)
+		nj := &m.AlertNotificationJournal{}
+		_, err := sess.Desc("alert_notification_journal.sent_at").
+			Limit(1).
+			Where("alert_notification_journal.org_id = ? AND alert_notification_journal.alert_id = ? AND alert_notification_journal.notifier_id = ?", cmd.OrgId, cmd.AlertId, cmd.NotifierId).Get(nj)
+
 		if err != nil {
 			return err
 		}
 
-		cmd.Result = notificationJournal
+		if nj.AlertId == 0 && nj.Id == 0 && nj.NotifierId == 0 && nj.OrgId == 0 {
+			return m.ErrJournalingNotFound
+		}
+
+		cmd.Result = nj
 		return nil
 	})
 }
diff --git a/pkg/services/sqlstore/alert_notification_test.go b/pkg/services/sqlstore/alert_notification_test.go
index fe7f02b22b0..aba437f427e 100644
--- a/pkg/services/sqlstore/alert_notification_test.go
+++ b/pkg/services/sqlstore/alert_notification_test.go
@@ -1,6 +1,7 @@
 package sqlstore
 
 import (
+	"context"
 	"testing"
 
 	"github.com/grafana/grafana/pkg/components/simplejson"
@@ -12,6 +13,48 @@ func TestAlertNotificationSQLAccess(t *testing.T) {
 	Convey("Testing Alert notification sql access", t, func() {
 		InitTestDB(t)
 
+		Convey("Alert notification journal", func() {
+			var alertId int64 = 5
+			var orgId int64 = 5
+			var notifierId int64 = 5
+
+			Convey("Getting last journal should raise error if no one exists", func() {
+				query := &m.GetLatestNotificationQuery{AlertId: alertId, OrgId: orgId, NotifierId: notifierId}
+				err := GetLatestNotification(context.Background(), query)
+				So(err, ShouldEqual, m.ErrJournalingNotFound)
+
+				Convey("shoulbe be able to record two journaling events", func() {
+					createCmd := &m.RecordNotificationJournalCommand{AlertId: alertId, NotifierId: notifierId, OrgId: orgId, Success: true, SentAt: 1}
+
+					err := RecordNotificationJournal(context.Background(), createCmd)
+					So(err, ShouldBeNil)
+
+					createCmd.SentAt += 1000 //increase epoch
+
+					err = RecordNotificationJournal(context.Background(), createCmd)
+					So(err, ShouldBeNil)
+
+					Convey("get last journaling event", func() {
+						err := GetLatestNotification(context.Background(), query)
+						So(err, ShouldBeNil)
+						So(query.Result.SentAt, ShouldEqual, 1001)
+
+						Convey("be able to clear all journaling for an notifier", func() {
+							cmd := &m.CleanNotificationJournalCommand{AlertId: alertId, NotifierId: notifierId, OrgId: orgId}
+							err := CleanNotificationJournal(context.Background(), cmd)
+							So(err, ShouldBeNil)
+
+							Convey("querying for last junaling should raise error", func() {
+								query := &m.GetLatestNotificationQuery{AlertId: alertId, OrgId: orgId, NotifierId: notifierId}
+								err := GetLatestNotification(context.Background(), query)
+								So(err, ShouldEqual, m.ErrJournalingNotFound)
+							})
+						})
+					})
+				})
+			})
+		})
+
 		Convey("Alert notifications should be empty", func() {
 			cmd := &m.GetAlertNotificationsQuery{
 				OrgId: 2,

From 8ff538be074f228239e1694c617fc57c89d5d5cf Mon Sep 17 00:00:00 2001
From: bergquist <carl.bergquist@gmail.com>
Date: Tue, 26 Jun 2018 14:13:45 +0200
Subject: [PATCH 22/39] notifier: handle known error first

---
 pkg/services/alerting/notifiers/base.go      |  8 ++---
 pkg/services/alerting/notifiers/base_test.go | 38 ++++++++++++++++++++
 2 files changed, 42 insertions(+), 4 deletions(-)

diff --git a/pkg/services/alerting/notifiers/base.go b/pkg/services/alerting/notifiers/base.go
index 4869c40f436..31ec77cbf25 100644
--- a/pkg/services/alerting/notifiers/base.go
+++ b/pkg/services/alerting/notifiers/base.go
@@ -74,15 +74,15 @@ func (n *NotifierBase) ShouldNotify(c *alerting.EvalContext) bool {
 	}
 
 	err := bus.DispatchCtx(c.Ctx, cmd)
+	if err == models.ErrJournalingNotFound {
+		return true
+	}
+
 	if err != nil {
 		n.log.Error("Could not determine last time alert notifier fired", "Alert name", c.Rule.Name, "Error", err)
 		return false
 	}
 
-	if err == models.ErrJournalingNotFound {
-		return true
-	}
-
 	if !cmd.Result.Success {
 		return true
 	}
diff --git a/pkg/services/alerting/notifiers/base_test.go b/pkg/services/alerting/notifiers/base_test.go
index 5b75ea4d59b..3fd23b69c6e 100644
--- a/pkg/services/alerting/notifiers/base_test.go
+++ b/pkg/services/alerting/notifiers/base_test.go
@@ -2,9 +2,12 @@ package notifiers
 
 import (
 	"context"
+	"errors"
 	"testing"
 	"time"
 
+	"github.com/grafana/grafana/pkg/bus"
+
 	"github.com/grafana/grafana/pkg/components/simplejson"
 	m "github.com/grafana/grafana/pkg/models"
 	"github.com/grafana/grafana/pkg/services/alerting"
@@ -64,6 +67,7 @@ func TestShouldSendAlertNotification(t *testing.T) {
 		evalContext := alerting.NewEvalContext(context.TODO(), &alerting.Rule{
 			State: tc.newState,
 		})
+
 		evalContext.Rule.State = tc.prevState
 		if defaultShouldNotify(evalContext, true, 0, time.Now()) != tc.expected {
 			t.Errorf("failed %s. expected %+v to return %v", tc.name, tc, tc.expected)
@@ -71,6 +75,40 @@ func TestShouldSendAlertNotification(t *testing.T) {
 	}
 }
 
+func TestShouldNotifyWhenNoJournalingIsFound(t *testing.T) {
+	Convey("base notifier", t, func() {
+		bus.ClearBusHandlers()
+
+		notifier := NewNotifierBase(&m.AlertNotification{
+			Id:       1,
+			Name:     "name",
+			Type:     "email",
+			Settings: simplejson.New(),
+		})
+		evalContext := alerting.NewEvalContext(context.TODO(), &alerting.Rule{})
+
+		Convey("should notify if no journaling is found", func() {
+			bus.AddHandlerCtx("", func(ctx context.Context, q *m.GetLatestNotificationQuery) error {
+				return m.ErrJournalingNotFound
+			})
+
+			if !notifier.ShouldNotify(evalContext) {
+				t.Errorf("should send notifications when ErrJournalingNotFound is returned")
+			}
+		})
+
+		Convey("should not notify query returns error", func() {
+			bus.AddHandlerCtx("", func(ctx context.Context, q *m.GetLatestNotificationQuery) error {
+				return errors.New("some kind of error unknown error")
+			})
+
+			if notifier.ShouldNotify(evalContext) {
+				t.Errorf("should not send notifications when query returns error")
+			}
+		})
+	})
+}
+
 func TestBaseNotifier(t *testing.T) {
 	Convey("default constructor for notifiers", t, func() {
 		bJson := simplejson.New()

From 396f8e6464a38e6ac925ecb8465a71e5118b79c5 Mon Sep 17 00:00:00 2001
From: bergquist <carl.bergquist@gmail.com>
Date: Fri, 29 Jun 2018 15:15:31 +0200
Subject: [PATCH 23/39] notifications: read without tran, write with tran

---
 pkg/services/alerting/interfaces.go             | 7 +++++--
 pkg/services/alerting/notifier.go               | 6 +++---
 pkg/services/alerting/notifiers/alertmanager.go | 3 ++-
 pkg/services/alerting/notifiers/base.go         | 5 +++--
 pkg/services/alerting/notifiers/base_test.go    | 4 ++--
 pkg/services/sqlstore/alert_notification.go     | 1 +
 6 files changed, 16 insertions(+), 10 deletions(-)

diff --git a/pkg/services/alerting/interfaces.go b/pkg/services/alerting/interfaces.go
index b4376191df0..46f8b3c769c 100644
--- a/pkg/services/alerting/interfaces.go
+++ b/pkg/services/alerting/interfaces.go
@@ -1,6 +1,9 @@
 package alerting
 
-import "time"
+import (
+	"context"
+	"time"
+)
 
 type EvalHandler interface {
 	Eval(evalContext *EvalContext)
@@ -17,7 +20,7 @@ type Notifier interface {
 	NeedsImage() bool
 
 	// ShouldNotify checks this evaluation should send an alert notification
-	ShouldNotify(evalContext *EvalContext) bool
+	ShouldNotify(ctx context.Context, evalContext *EvalContext) bool
 
 	GetNotifierId() int64
 	GetIsDefault() bool
diff --git a/pkg/services/alerting/notifier.go b/pkg/services/alerting/notifier.go
index 61526ed642c..a19e44a8f99 100644
--- a/pkg/services/alerting/notifier.go
+++ b/pkg/services/alerting/notifier.go
@@ -72,7 +72,7 @@ func (n *notificationService) sendNotifications(evalContext *EvalContext, notifi
 
 				// Verify that we can send the notification again
 				// but this time within the same transaction.
-				if !evalContext.IsTestRun && !not.ShouldNotify(evalContext) {
+				if !evalContext.IsTestRun && !not.ShouldNotify(context.Background(), evalContext) {
 					return nil
 				}
 
@@ -91,7 +91,7 @@ func (n *notificationService) sendNotifications(evalContext *EvalContext, notifi
 					Success:    success,
 				}
 
-				return bus.DispatchCtx(evalContext.Ctx, cmd)
+				return bus.DispatchCtx(ctx, cmd)
 			})
 		})
 	}
@@ -149,7 +149,7 @@ func (n *notificationService) getNeededNotifiers(orgId int64, notificationIds []
 			return nil, err
 		}
 
-		if not.ShouldNotify(evalContext) {
+		if not.ShouldNotify(evalContext.Ctx, evalContext) {
 			result = append(result, not)
 		}
 	}
diff --git a/pkg/services/alerting/notifiers/alertmanager.go b/pkg/services/alerting/notifiers/alertmanager.go
index 42ffa9b2d6e..9826dd1dffb 100644
--- a/pkg/services/alerting/notifiers/alertmanager.go
+++ b/pkg/services/alerting/notifiers/alertmanager.go
@@ -1,6 +1,7 @@
 package notifiers
 
 import (
+	"context"
 	"time"
 
 	"github.com/grafana/grafana/pkg/bus"
@@ -45,7 +46,7 @@ type AlertmanagerNotifier struct {
 	log log.Logger
 }
 
-func (this *AlertmanagerNotifier) ShouldNotify(evalContext *alerting.EvalContext) bool {
+func (this *AlertmanagerNotifier) ShouldNotify(ctx context.Context, evalContext *alerting.EvalContext) bool {
 	this.log.Debug("Should notify", "ruleId", evalContext.Rule.Id, "state", evalContext.Rule.State, "previousState", evalContext.PrevAlertState)
 
 	// Do not notify when we become OK for the first time.
diff --git a/pkg/services/alerting/notifiers/base.go b/pkg/services/alerting/notifiers/base.go
index 31ec77cbf25..ca011356247 100644
--- a/pkg/services/alerting/notifiers/base.go
+++ b/pkg/services/alerting/notifiers/base.go
@@ -1,6 +1,7 @@
 package notifiers
 
 import (
+	"context"
 	"time"
 
 	"github.com/grafana/grafana/pkg/bus"
@@ -66,14 +67,14 @@ func defaultShouldNotify(context *alerting.EvalContext, sendReminder bool, frequ
 }
 
 // ShouldNotify checks this evaluation should send an alert notification
-func (n *NotifierBase) ShouldNotify(c *alerting.EvalContext) bool {
+func (n *NotifierBase) ShouldNotify(ctx context.Context, c *alerting.EvalContext) bool {
 	cmd := &models.GetLatestNotificationQuery{
 		OrgId:      c.Rule.OrgId,
 		AlertId:    c.Rule.Id,
 		NotifierId: n.Id,
 	}
 
-	err := bus.DispatchCtx(c.Ctx, cmd)
+	err := bus.DispatchCtx(ctx, cmd)
 	if err == models.ErrJournalingNotFound {
 		return true
 	}
diff --git a/pkg/services/alerting/notifiers/base_test.go b/pkg/services/alerting/notifiers/base_test.go
index 3fd23b69c6e..57b82f32466 100644
--- a/pkg/services/alerting/notifiers/base_test.go
+++ b/pkg/services/alerting/notifiers/base_test.go
@@ -92,7 +92,7 @@ func TestShouldNotifyWhenNoJournalingIsFound(t *testing.T) {
 				return m.ErrJournalingNotFound
 			})
 
-			if !notifier.ShouldNotify(evalContext) {
+			if !notifier.ShouldNotify(context.Background(), evalContext) {
 				t.Errorf("should send notifications when ErrJournalingNotFound is returned")
 			}
 		})
@@ -102,7 +102,7 @@ func TestShouldNotifyWhenNoJournalingIsFound(t *testing.T) {
 				return errors.New("some kind of error unknown error")
 			})
 
-			if notifier.ShouldNotify(evalContext) {
+			if notifier.ShouldNotify(context.Background(), evalContext) {
 				t.Errorf("should not send notifications when query returns error")
 			}
 		})
diff --git a/pkg/services/sqlstore/alert_notification.go b/pkg/services/sqlstore/alert_notification.go
index 3f2ca109c1a..8fb1e2212a9 100644
--- a/pkg/services/sqlstore/alert_notification.go
+++ b/pkg/services/sqlstore/alert_notification.go
@@ -250,6 +250,7 @@ func RecordNotificationJournal(ctx context.Context, cmd *m.RecordNotificationJou
 func GetLatestNotification(ctx context.Context, cmd *m.GetLatestNotificationQuery) error {
 	return inTransactionCtx(ctx, func(sess *DBSession) error {
 		nj := &m.AlertNotificationJournal{}
+
 		_, err := sess.Desc("alert_notification_journal.sent_at").
 			Limit(1).
 			Where("alert_notification_journal.org_id = ? AND alert_notification_journal.alert_id = ? AND alert_notification_journal.notifier_id = ?", cmd.OrgId, cmd.AlertId, cmd.NotifierId).Get(nj)

From e91e3ea771228af267178f6fd927b4afcf3b6e75 Mon Sep 17 00:00:00 2001
From: bergquist <carl.bergquist@gmail.com>
Date: Fri, 29 Jun 2018 16:16:09 +0200
Subject: [PATCH 24/39] notifications: send notifications synchronous

---
 pkg/services/alerting/notifier.go | 54 +++++++++++++++----------------
 1 file changed, 26 insertions(+), 28 deletions(-)

diff --git a/pkg/services/alerting/notifier.go b/pkg/services/alerting/notifier.go
index 7fe97596494..fb2933f6e26 100644
--- a/pkg/services/alerting/notifier.go
+++ b/pkg/services/alerting/notifier.go
@@ -6,8 +6,6 @@ import (
 	"fmt"
 	"time"
 
-	"golang.org/x/sync/errgroup"
-
 	"github.com/grafana/grafana/pkg/bus"
 	"github.com/grafana/grafana/pkg/components/imguploader"
 	"github.com/grafana/grafana/pkg/log"
@@ -61,42 +59,42 @@ func (n *notificationService) SendIfNeeded(context *EvalContext) error {
 }
 
 func (n *notificationService) sendNotifications(evalContext *EvalContext, notifiers []Notifier) error {
-	g, _ := errgroup.WithContext(evalContext.Ctx)
-
 	for _, notifier := range notifiers {
-		not := notifier //avoid updating scope variable in go routine
+		not := notifier
 
-		g.Go(func() error {
-			return bus.InTransaction(evalContext.Ctx, func(ctx context.Context) error {
-				n.log.Debug("trying to send notification", "id", not.GetNotifierId())
+		err := bus.InTransaction(evalContext.Ctx, func(ctx context.Context) error {
+			n.log.Debug("trying to send notification", "id", not.GetNotifierId())
 
-				// Verify that we can send the notification again
-				// but this time within the same transaction.
-				if !evalContext.IsTestRun && !not.ShouldNotify(context.Background(), evalContext) {
-					return nil
-				}
+			// Verify that we can send the notification again
+			// but this time within the same transaction.
+			if !evalContext.IsTestRun && !not.ShouldNotify(context.Background(), evalContext) {
+				return nil
+			}
 
-				n.log.Debug("Sending notification", "type", not.GetType(), "id", not.GetNotifierId(), "isDefault", not.GetIsDefault())
-				metrics.M_Alerting_Notification_Sent.WithLabelValues(not.GetType()).Inc()
+			n.log.Debug("Sending notification", "type", not.GetType(), "id", not.GetNotifierId(), "isDefault", not.GetIsDefault())
+			metrics.M_Alerting_Notification_Sent.WithLabelValues(not.GetType()).Inc()
 
-				//send notification
-				success := not.Notify(evalContext) == nil
+			//send notification
+			success := not.Notify(evalContext) == nil
 
-				//write result to db.
-				cmd := &m.RecordNotificationJournalCommand{
-					OrgId:      evalContext.Rule.OrgId,
-					AlertId:    evalContext.Rule.Id,
-					NotifierId: not.GetNotifierId(),
-					SentAt:     time.Now().Unix(),
-					Success:    success,
-				}
+			//write result to db.
+			cmd := &m.RecordNotificationJournalCommand{
+				OrgId:      evalContext.Rule.OrgId,
+				AlertId:    evalContext.Rule.Id,
+				NotifierId: not.GetNotifierId(),
+				SentAt:     time.Now().Unix(),
+				Success:    success,
+			}
 
-				return bus.DispatchCtx(ctx, cmd)
-			})
+			return bus.DispatchCtx(ctx, cmd)
 		})
+
+		if err != nil {
+			return err
+		}
 	}
 
-	return g.Wait()
+	return nil
 }
 
 func (n *notificationService) uploadImage(context *EvalContext) (err error) {

From 66c56e7bbebd188a232ae6ac7a444c6463a7e9ef Mon Sep 17 00:00:00 2001
From: bergquist <carl.bergquist@gmail.com>
Date: Sat, 30 Jun 2018 23:14:18 +0200
Subject: [PATCH 25/39] notifications: dont return error if one notifer failed

---
 pkg/services/alerting/notifier.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pkg/services/alerting/notifier.go b/pkg/services/alerting/notifier.go
index fb2933f6e26..41c4e79445c 100644
--- a/pkg/services/alerting/notifier.go
+++ b/pkg/services/alerting/notifier.go
@@ -90,7 +90,7 @@ func (n *notificationService) sendNotifications(evalContext *EvalContext, notifi
 		})
 
 		if err != nil {
-			return err
+			n.log.Error("failed to send notification", "id", not.GetNotifierId())
 		}
 	}
 

From 3350a3477de5cef3112306ed2a14b2fae6ce2354 Mon Sep 17 00:00:00 2001
From: Marcus Efraimsson <marcus.efraimsson@gmail.com>
Date: Mon, 20 Aug 2018 13:31:06 +0200
Subject: [PATCH 26/39] fix after merge with master

---
 pkg/services/alerting/notifier.go | 1 +
 1 file changed, 1 insertion(+)

diff --git a/pkg/services/alerting/notifier.go b/pkg/services/alerting/notifier.go
index b458656404a..39f91c1caea 100644
--- a/pkg/services/alerting/notifier.go
+++ b/pkg/services/alerting/notifier.go
@@ -4,6 +4,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"time"
 
 	"github.com/grafana/grafana/pkg/bus"
 	"github.com/grafana/grafana/pkg/components/imguploader"

From 86bc462bc18e4cfb489236e1295b66fb655a6799 Mon Sep 17 00:00:00 2001
From: Marcus Efraimsson <marcus.efraimsson@gmail.com>
Date: Mon, 20 Aug 2018 13:35:36 +0200
Subject: [PATCH 27/39] remove unnecessary conversion (metalinter)

---
 pkg/api/dtos/alerting_test.go | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/pkg/api/dtos/alerting_test.go b/pkg/api/dtos/alerting_test.go
index bd0d9ff8feb..c38f281be9c 100644
--- a/pkg/api/dtos/alerting_test.go
+++ b/pkg/api/dtos/alerting_test.go
@@ -10,11 +10,11 @@ func TestFormatShort(t *testing.T) {
 		interval time.Duration
 		expected string
 	}{
-		{interval: time.Duration(time.Hour), expected: "1h"},
-		{interval: time.Duration(time.Hour + time.Minute), expected: "1h1m"},
-		{interval: time.Duration((time.Hour * 10) + time.Minute), expected: "10h1m"},
-		{interval: time.Duration((time.Hour * 10) + (time.Minute * 10) + time.Second), expected: "10h10m1s"},
-		{interval: time.Duration(time.Minute * 10), expected: "10m"},
+		{interval: time.Hour, expected: "1h"},
+		{interval: time.Hour + time.Minute, expected: "1h1m"},
+		{interval: (time.Hour * 10) + time.Minute, expected: "10h1m"},
+		{interval: (time.Hour * 10) + (time.Minute * 10) + time.Second, expected: "10h10m1s"},
+		{interval: time.Minute * 10, expected: "10m"},
 	}
 
 	for _, tc := range tcs {

From dfa5d176704139119aa649a19c773603e4627d7b Mon Sep 17 00:00:00 2001
From: Marcus Efraimsson <marcus.efraimsson@gmail.com>
Date: Mon, 20 Aug 2018 16:27:13 +0200
Subject: [PATCH 28/39] don't write to notification journal when testing
 notifier/rule

---
 pkg/services/alerting/notifier.go | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/pkg/services/alerting/notifier.go b/pkg/services/alerting/notifier.go
index 39f91c1caea..7fbd956f4f9 100644
--- a/pkg/services/alerting/notifier.go
+++ b/pkg/services/alerting/notifier.go
@@ -77,6 +77,10 @@ func (n *notificationService) sendNotifications(evalContext *EvalContext, notifi
 			//send notification
 			success := not.Notify(evalContext) == nil
 
+			if evalContext.IsTestRun {
+				return nil
+			}
+
 			//write result to db.
 			cmd := &m.RecordNotificationJournalCommand{
 				OrgId:      evalContext.Rule.OrgId,

From 470e7cc6dbaa8ce20220d5560f1d1fb4838c17b7 Mon Sep 17 00:00:00 2001
From: Marcus Efraimsson <marcus.efraimsson@gmail.com>
Date: Mon, 20 Aug 2018 18:23:48 +0200
Subject: [PATCH 29/39] add suggestions for reminder frequency and change copy

---
 .../alerting/notification_edit_ctrl.ts        |  5 ++++
 .../alerting/partials/notification_edit.html  | 25 +++++++------------
 2 files changed, 14 insertions(+), 16 deletions(-)

diff --git a/public/app/features/alerting/notification_edit_ctrl.ts b/public/app/features/alerting/notification_edit_ctrl.ts
index e066406bc43..92781d42a23 100644
--- a/public/app/features/alerting/notification_edit_ctrl.ts
+++ b/public/app/features/alerting/notification_edit_ctrl.ts
@@ -20,12 +20,17 @@ export class AlertNotificationEditCtrl {
     },
     isDefault: false,
   };
+  getFrequencySuggestion: any;
 
   /** @ngInject */
   constructor(private $routeParams, private backendSrv, private $location, private $templateCache, navModelSrv) {
     this.navModel = navModelSrv.getNav('alerting', 'channels', 0);
     this.isNew = !this.$routeParams.id;
 
+    this.getFrequencySuggestion = () => {
+      return ['1m', '5m', '10m', '15m', '30m', '1h'];
+    };
+
     this.backendSrv
       .get(`/api/alert-notifiers`)
       .then(notifiers => {
diff --git a/public/app/features/alerting/partials/notification_edit.html b/public/app/features/alerting/partials/notification_edit.html
index 7132ed41f3c..7e9997452f9 100644
--- a/public/app/features/alerting/partials/notification_edit.html
+++ b/public/app/features/alerting/partials/notification_edit.html
@@ -37,28 +37,21 @@
           label="Send reminder"
           label-class="width-12"
           checked="ctrl.model.sendReminder"
-          tooltip="Choose to either notify on state change or at every interval">
+          tooltip="Choose to either notify on state change (default) or at every interval">
       </gf-form-switch>
       <div class="gf-form">
         <span class="alert alert-info width-30" ng-if="ctrl.model.sendReminder">
-          Alert reminders are sent after rules are evaluated. Therefore the alert rule interval has to be lower than the reminder frequency
+          Alert reminders are sent after rules are evaluated. Therefore the highest alert rule interval (of alert rules using this notification channel) must be lower than the reminder frequency.
         </span>
       </div>
       <div class="gf-form-inline">
-        <div class="gf-form">
-          <span class="gf-form-label width-12" ng-if="ctrl.model.sendReminder">Send reminder every</span>
-          <input
-            class="gf-form-input max-width-15"
-            type="text"
-            ng-model="ctrl.model.frequency"
-            ng-required="ctrl.model.sendReminder"
-            ng-if="ctrl.model.sendReminder"
-            spellcheck='false'
-            placeholder='15m'></input>
-          <info-popover mode="right-absolute" ng-if="ctrl.model.sendReminder">
-            Specify at what interval you want reminder's about this alerting being triggered.
-            Ex. 60s, 10m, 30m, 1h
-          </info-popover>
+        <div class="gf-form" ng-if="ctrl.model.sendReminder">
+            <span class="gf-form-label width-12">Reminder frequency</span>
+            <input type="text" placeholder="Select or specify custom" class="gf-form-input width-15" ng-model="ctrl.model.frequency"
+              bs-typeahead="ctrl.getFrequencySuggestion" data-min-length=0 ng-required="ctrl.model.sendReminder">
+            <info-popover mode="right-absolute">
+              Select at what interval you want reminder's to be sent after alerts being triggered, e.g. 30s, 1m, 10m, 30m or 1h etc.
+            </info-popover>
         </div>
       </div>
     </div>

From 15d950ce35ffeeae8e9398a92b8739e4d1e7e997 Mon Sep 17 00:00:00 2001
From: Marcus Efraimsson <marcus.efraimsson@gmail.com>
Date: Wed, 22 Aug 2018 18:06:05 +0200
Subject: [PATCH 30/39] update copy/ux for configuring alerting notification
 reminders

---
 .../alerting/partials/notification_edit.html  | 25 ++++++++++---------
 1 file changed, 13 insertions(+), 12 deletions(-)

diff --git a/public/app/features/alerting/partials/notification_edit.html b/public/app/features/alerting/partials/notification_edit.html
index 7e9997452f9..faa168d0acd 100644
--- a/public/app/features/alerting/partials/notification_edit.html
+++ b/public/app/features/alerting/partials/notification_edit.html
@@ -34,26 +34,27 @@
       </gf-form-switch>
       <gf-form-switch
           class="gf-form"
-          label="Send reminder"
+          label="Send reminders"
           label-class="width-12"
           checked="ctrl.model.sendReminder"
-          tooltip="Choose to either notify on state change (default) or at every interval">
+          tooltip="Send additional notifications for triggered alerts">
       </gf-form-switch>
-      <div class="gf-form">
-        <span class="alert alert-info width-30" ng-if="ctrl.model.sendReminder">
-          Alert reminders are sent after rules are evaluated. Therefore the highest alert rule interval (of alert rules using this notification channel) must be lower than the reminder frequency.
-        </span>
-      </div>
       <div class="gf-form-inline">
         <div class="gf-form" ng-if="ctrl.model.sendReminder">
-            <span class="gf-form-label width-12">Reminder frequency</span>
-            <input type="text" placeholder="Select or specify custom" class="gf-form-input width-15" ng-model="ctrl.model.frequency"
-              bs-typeahead="ctrl.getFrequencySuggestion" data-min-length=0 ng-required="ctrl.model.sendReminder">
-            <info-popover mode="right-absolute">
-              Select at what interval you want reminder's to be sent after alerts being triggered, e.g. 30s, 1m, 10m, 30m or 1h etc.
+          <span class="gf-form-label width-12">Send reminder every
+            <info-popover mode="right-normal" position="top center">
+              Specify how often reminders should be sent, e.g. every 30s, 1m, 10m, 30m or 1h etc.
             </info-popover>
+          </span>
+          <input type="text" placeholder="Select or specify custom" class="gf-form-input width-15" ng-model="ctrl.model.frequency"
+            bs-typeahead="ctrl.getFrequencySuggestion" data-min-length=0 ng-required="ctrl.model.sendReminder">
         </div>
       </div>
+      <div class="gf-form">
+          <span class="alert alert-info width-30" ng-if="ctrl.model.sendReminder">
+            Alert reminders are sent after rules are evaluated. Therefore a reminder can never be sent earlier than a configured alert rule evaluation interval.
+          </span>
+        </div>
     </div>
 
     <div class="gf-form-group" ng-include src="ctrl.notifierTemplateId">

From 2e1c4b3d76b13300f23e7799370acb8deeb754d1 Mon Sep 17 00:00:00 2001
From: Marcus Efraimsson <marcus.efraimsson@gmail.com>
Date: Wed, 22 Aug 2018 18:06:51 +0200
Subject: [PATCH 31/39] docs: alerting notification reminders

---
 docs/sources/alerting/notifications.md | 63 ++++++++++++++++++--------
 docs/sources/alerting/rules.md         |  7 +++
 2 files changed, 50 insertions(+), 20 deletions(-)

diff --git a/docs/sources/alerting/notifications.md b/docs/sources/alerting/notifications.md
index b3b4305a748..64742a6c7f1 100644
--- a/docs/sources/alerting/notifications.md
+++ b/docs/sources/alerting/notifications.md
@@ -16,12 +16,11 @@ weight = 2
 
 When an alert changes state, it sends out notifications. Each alert rule can have
 multiple notifications. In order to add a notification to an alert rule you first need
-to add and configure a `notification` channel (can be email, PagerDuty or other integration). This is done from the Notification Channels page.
+to add and configure a `notification` channel (can be email, PagerDuty or other integration).
+This is done from the Notification Channels page.
 
 ## Notification Channel Setup
 
-{{< imgbox max-width="30%" img="/img/docs/v50/alerts_notifications_menu.png" caption="Alerting Notification Channels" >}}
-
 On the Notification Channels page hit the `New Channel` button to go the page where you
 can configure and setup a new Notification Channel.
 
@@ -30,7 +29,31 @@ sure it's setup correctly.
 
 ### Send on all alerts
 
-When checked, this option will nofity for all alert rules - existing and new.
+When checked, this option will notify for all alert rules - existing and new.
+
+### Send reminders
+
+> Only available in Grafana v5.3 and above.
+
+{{< docs-imagebox max-width="600px" img="/img/docs/v53/alerting_notification_reminders.png" class="docs-image--right" caption="Alerting notification reminders setup" >}}
+
+When this option is checked additional notifications (reminders) will be sent for triggered alerts. You can specify how often reminders
+should be sent using number of seconds (s), minutes (m) or hours (h), for example `30s`, `3m`, `5m` or `1h` etc.
+
+**Important:** Alert reminders are sent after rules are evaluated. Therefore a reminder can never be sent earlier than a configured [alert rule evaluation interval](/alerting/rules/#name-evaluation-interval).
+
+These examples shows how often and when reminders are sent for a triggered alert.
+
+Alert rule evaluation interval | Send reminders every | Reminder sent every (after last alerting notification)
+---------- | ----------- | -----------
+`30s` | `15s` | ~30 seconds
+`1m` | `5m` | ~5 minutes
+`5m` | `15m` | ~15 minutes
+`6m` | `20m` | ~24 minutes
+`1h` | `15m` | ~1 hour
+`1h` | `2h` | ~2 hours
+
+<div class="clearfix"></div>
 
 ## Supported Notification Types
 
@@ -132,22 +155,22 @@ Once these two properties are set, you can send the alerts to Kafka for further
 
 ### All supported notifier
 
-Name | Type |Support images
------|------------ | ------
-Slack | `slack` | yes
-Pagerduty | `pagerduty` | yes
-Email | `email` | yes
-Webhook | `webhook` | link
-Kafka | `kafka` | no
-Hipchat | `hipchat` | yes
-VictorOps | `victorops` | yes
-Sensu | `sensu` | yes
-OpsGenie | `opsgenie` | yes
-Threema | `threema` | yes
-Pushover | `pushover` | no
-Telegram | `telegram` | no
-Line | `line` | no
-Prometheus Alertmanager | `prometheus-alertmanager` | no
+Name | Type |Support images | Support reminders
+-----|------------ | ------ | ------ |
+Slack | `slack` | yes | yes
+Pagerduty | `pagerduty` | yes | yes
+Email | `email` | yes | yes
+Webhook | `webhook` | link | yes
+Kafka | `kafka` | no | yes
+Hipchat | `hipchat` | yes | yes
+VictorOps | `victorops` | yes | yes
+Sensu | `sensu` | yes | yes
+OpsGenie | `opsgenie` | yes | yes
+Threema | `threema` | yes | yes
+Pushover | `pushover` | no | yes
+Telegram | `telegram` | no | yes
+Line | `line` | no | yes
+Prometheus Alertmanager | `prometheus-alertmanager` | no | no
 
 
 
diff --git a/docs/sources/alerting/rules.md b/docs/sources/alerting/rules.md
index fa7332e7145..844e91b8768 100644
--- a/docs/sources/alerting/rules.md
+++ b/docs/sources/alerting/rules.md
@@ -88,6 +88,13 @@ So as you can see from the above scenario Grafana will not send out notification
 to fire if the rule already is in state `Alerting`. To improve support for queries that return multiple series
 we plan to track state **per series** in a future release.
 
+> Starting with Grafana v5.3 you can configure reminders to be sent for triggered alerts. This will send additional notifications
+> when an alert continues to fire. If other series cause the alert they'll be included in the reminder notification. Depending on
+> what notification channel you're using you may be able to take advantage of this feature for identifying new/existing series
+> causing alert to fire. [Read more about notification reminders here](/alerting/notifications/#send-reminders).
+>
+> Please note that the track state **per series** feature still is needed for proper handling of notifications for multiple series.
+
 ### No Data / Null values
 
 Below your conditions you can configure how the rule evaluation engine should handle queries that return no data or only null values.

From 51069c9ccba11554a5bcea191d64c4c31b0040dc Mon Sep 17 00:00:00 2001
From: Marcus Efraimsson <marcus.efraimsson@gmail.com>
Date: Thu, 23 Aug 2018 18:56:37 +0200
Subject: [PATCH 32/39] docs: reminder notifications update

---
 docs/sources/alerting/notifications.md |  4 +--
 docs/sources/alerting/rules.md         |  8 ++---
 docs/sources/http_api/alerting.md      | 48 +++++++++++++++++++-------
 3 files changed, 41 insertions(+), 19 deletions(-)

diff --git a/docs/sources/alerting/notifications.md b/docs/sources/alerting/notifications.md
index 361f086105c..5262cc2bc48 100644
--- a/docs/sources/alerting/notifications.md
+++ b/docs/sources/alerting/notifications.md
@@ -42,9 +42,9 @@ should be sent using number of seconds (s), minutes (m) or hours (h), for exampl
 
 **Important:** Alert reminders are sent after rules are evaluated. Therefore a reminder can never be sent earlier than a configured [alert rule evaluation interval](/alerting/rules/#name-evaluation-interval).
 
-These examples shows how often and when reminders are sent for a triggered alert.
+These examples show how often and when reminders are sent for a triggered alert.
 
-Alert rule evaluation interval | Send reminders every | Reminder sent every (after last alerting notification)
+Alert rule evaluation interval | Send reminders every | Reminder sent every (after last alert notification)
 ---------- | ----------- | -----------
 `30s` | `15s` | ~30 seconds
 `1m` | `5m` | ~5 minutes
diff --git a/docs/sources/alerting/rules.md b/docs/sources/alerting/rules.md
index 844e91b8768..488619055e2 100644
--- a/docs/sources/alerting/rules.md
+++ b/docs/sources/alerting/rules.md
@@ -89,11 +89,9 @@ to fire if the rule already is in state `Alerting`. To improve support for queri
 we plan to track state **per series** in a future release.
 
 > Starting with Grafana v5.3 you can configure reminders to be sent for triggered alerts. This will send additional notifications
-> when an alert continues to fire. If other series cause the alert they'll be included in the reminder notification. Depending on
-> what notification channel you're using you may be able to take advantage of this feature for identifying new/existing series
-> causing alert to fire. [Read more about notification reminders here](/alerting/notifications/#send-reminders).
->
-> Please note that the track state **per series** feature still is needed for proper handling of notifications for multiple series.
+> when an alert continues to fire. If other series (like server2 in the example above) also cause the alert rule to fire they will
+> be included in the reminder notification. Depending on what notification channel you're using you may be able to take advantage
+> of this feature for identifying new/existing series causing alert to fire. [Read more about notification reminders here](/alerting/notifications/#send-reminders).
 
 ### No Data / Null values
 
diff --git a/docs/sources/http_api/alerting.md b/docs/sources/http_api/alerting.md
index 80b6e283be3..032fd508dd0 100644
--- a/docs/sources/http_api/alerting.md
+++ b/docs/sources/http_api/alerting.md
@@ -50,6 +50,7 @@ Authorization: Bearer eyJrIjoiT0tTcG1pUlY2RnVKZTFVaDFsNFZXdE9ZWmNrMkZYbk
 ```http
 HTTP/1.1 200
 Content-Type: application/json
+
 [
   {
     "id": 1,
@@ -86,6 +87,7 @@ Authorization: Bearer eyJrIjoiT0tTcG1pUlY2RnVKZTFVaDFsNFZXdE9ZWmNrMkZYbk
 ```http
 HTTP/1.1 200
 Content-Type: application/json
+
 {
   "id": 1,
   "dashboardId": 1,
@@ -146,6 +148,7 @@ JSON Body Schema:
 ```http
 HTTP/1.1 200
 Content-Type: application/json
+
 {
   "alertId": 1,
   "state":   "Paused",
@@ -177,6 +180,7 @@ JSON Body Schema:
 ```http
 HTTP/1.1 200
 Content-Type: application/json
+
 {
   "state":   "Paused",
   "message": "alert paused",
@@ -204,14 +208,21 @@ Authorization: Bearer eyJrIjoiT0tTcG1pUlY2RnVKZTFVaDFsNFZXdE9ZWmNrMkZYbk
 HTTP/1.1 200
 Content-Type: application/json
 
-{
-  "id": 1,
-  "name": "Team A",
-  "type": "email",
-  "isDefault": true,
-  "created": "2017-01-01 12:45",
-  "updated": "2017-01-01 12:45"
-}
+[
+  {
+    "id": 1,
+    "name": "Team A",
+    "type": "email",
+    "isDefault": false,
+    "sendReminder": false,
+    "settings": {
+      "addresses": "carl@grafana.com;dev@grafana.com"
+    },
+    "created": "2018-04-23T14:44:09+02:00",
+    "updated": "2018-08-20T15:47:49+02:00"
+  }
+]
+
 ```
 
 ## Create alert notification
@@ -232,6 +243,7 @@ Authorization: Bearer eyJrIjoiT0tTcG1pUlY2RnVKZTFVaDFsNFZXdE9ZWmNrMkZYbk
   "name": "new alert notification",  //Required
   "type":  "email", //Required
   "isDefault": false,
+  "sendReminder": false,
   "settings": {
     "addresses": "carl@grafana.com;dev@grafana.com"
   }
@@ -243,14 +255,18 @@ Authorization: Bearer eyJrIjoiT0tTcG1pUlY2RnVKZTFVaDFsNFZXdE9ZWmNrMkZYbk
 ```http
 HTTP/1.1 200
 Content-Type: application/json
+
 {
   "id": 1,
   "name": "new alert notification",
   "type": "email",
   "isDefault": false,
-  "settings": { addresses: "carl@grafana.com;dev@grafana.com"} }
-  "created": "2017-01-01 12:34",
-  "updated": "2017-01-01 12:34"
+  "sendReminder": false,
+  "settings": {
+    "addresses": "carl@grafana.com;dev@grafana.com"
+  },
+  "created": "2018-04-23T14:44:09+02:00",
+  "updated": "2018-08-20T15:47:49+02:00"
 }
 ```
 
@@ -271,6 +287,8 @@ Authorization: Bearer eyJrIjoiT0tTcG1pUlY2RnVKZTFVaDFsNFZXdE9ZWmNrMkZYbk
   "name": "new alert notification",  //Required
   "type":  "email", //Required
   "isDefault": false,
+  "sendReminder": true,
+  "frequency": "15m",
   "settings": {
     "addresses: "carl@grafana.com;dev@grafana.com"
   }
@@ -282,12 +300,17 @@ Authorization: Bearer eyJrIjoiT0tTcG1pUlY2RnVKZTFVaDFsNFZXdE9ZWmNrMkZYbk
 ```http
 HTTP/1.1 200
 Content-Type: application/json
+
 {
   "id": 1,
   "name": "new alert notification",
   "type": "email",
   "isDefault": false,
-  "settings": { addresses: "carl@grafana.com;dev@grafana.com"} }
+  "sendReminder": true,
+  "frequency": "15m",
+  "settings": {
+    "addresses": "carl@grafana.com;dev@grafana.com"
+  },
   "created": "2017-01-01 12:34",
   "updated": "2017-01-01 12:34"
 }
@@ -311,6 +334,7 @@ Authorization: Bearer eyJrIjoiT0tTcG1pUlY2RnVKZTFVaDFsNFZXdE9ZWmNrMkZYbk
 ```http
 HTTP/1.1 200
 Content-Type: application/json
+
 {
   "message": "Notification deleted"
 }

From eba147c1a3f45f0b76399b87a288a612f240bfbf Mon Sep 17 00:00:00 2001
From: Marcus Efraimsson <marcus.efraimsson@gmail.com>
Date: Fri, 24 Aug 2018 19:09:19 +0200
Subject: [PATCH 33/39] change/add tests for alerting notification reminders

---
 .../sqlstore/alert_notification_test.go       | 21 +++++++++++++++++--
 1 file changed, 19 insertions(+), 2 deletions(-)

diff --git a/pkg/services/sqlstore/alert_notification_test.go b/pkg/services/sqlstore/alert_notification_test.go
index aba437f427e..83fb42db9bb 100644
--- a/pkg/services/sqlstore/alert_notification_test.go
+++ b/pkg/services/sqlstore/alert_notification_test.go
@@ -3,6 +3,7 @@ package sqlstore
 import (
 	"context"
 	"testing"
+	"time"
 
 	"github.com/grafana/grafana/pkg/components/simplejson"
 	m "github.com/grafana/grafana/pkg/models"
@@ -88,7 +89,7 @@ func TestAlertNotificationSQLAccess(t *testing.T) {
 			})
 		})
 
-		Convey("Cannot update alert notifier with notitfyonce = false", func() {
+		Convey("Cannot update alert notifier with send reminder = false", func() {
 			cmd := &m.CreateAlertNotificationCommand{
 				Name:         "ops update",
 				Type:         "email",
@@ -134,6 +135,7 @@ func TestAlertNotificationSQLAccess(t *testing.T) {
 			So(cmd.Result.Id, ShouldNotEqual, 0)
 			So(cmd.Result.OrgId, ShouldNotEqual, 0)
 			So(cmd.Result.Type, ShouldEqual, "email")
+			So(cmd.Result.Frequency, ShouldEqual, 10*time.Second)
 
 			Convey("Cannot save Alert Notification with the same name", func() {
 				err = CreateAlertNotificationCommand(cmd)
@@ -146,13 +148,28 @@ func TestAlertNotificationSQLAccess(t *testing.T) {
 					Type:         "webhook",
 					OrgId:        cmd.Result.OrgId,
 					SendReminder: true,
-					Frequency:    "10s",
+					Frequency:    "60s",
 					Settings:     simplejson.New(),
 					Id:           cmd.Result.Id,
 				}
 				err := UpdateAlertNotification(newCmd)
 				So(err, ShouldBeNil)
 				So(newCmd.Result.Name, ShouldEqual, "NewName")
+				So(newCmd.Result.Frequency, ShouldEqual, 60*time.Second)
+			})
+
+			Convey("Can update alert notification to disable sending of reminders", func() {
+				newCmd := &m.UpdateAlertNotificationCommand{
+					Name:         "NewName",
+					Type:         "webhook",
+					OrgId:        cmd.Result.OrgId,
+					SendReminder: false,
+					Settings:     simplejson.New(),
+					Id:           cmd.Result.Id,
+				}
+				err := UpdateAlertNotification(newCmd)
+				So(err, ShouldBeNil)
+				So(newCmd.Result.SendReminder, ShouldBeFalse)
 			})
 		})
 

From 6995242b8b2db85fd934c2cdedb15a9c0797de85 Mon Sep 17 00:00:00 2001
From: Marcus Efraimsson <marcus.efraimsson@gmail.com>
Date: Fri, 24 Aug 2018 19:13:52 +0200
Subject: [PATCH 34/39] copy and docs update for alert notification reminders

---
 docs/sources/alerting/notifications.md                       | 2 +-
 public/app/features/alerting/partials/notification_edit.html | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/sources/alerting/notifications.md b/docs/sources/alerting/notifications.md
index 5262cc2bc48..a5b7f4264e0 100644
--- a/docs/sources/alerting/notifications.md
+++ b/docs/sources/alerting/notifications.md
@@ -40,7 +40,7 @@ When checked, this option will notify for all alert rules - existing and new.
 When this option is checked additional notifications (reminders) will be sent for triggered alerts. You can specify how often reminders
 should be sent using number of seconds (s), minutes (m) or hours (h), for example `30s`, `3m`, `5m` or `1h` etc.
 
-**Important:** Alert reminders are sent after rules are evaluated. Therefore a reminder can never be sent earlier than a configured [alert rule evaluation interval](/alerting/rules/#name-evaluation-interval).
+**Important:** Alert reminders are sent after rules are evaluated. Therefore a reminder can never be sent more frequently than a configured [alert rule evaluation interval](/alerting/rules/#name-evaluation-interval).
 
 These examples show how often and when reminders are sent for a triggered alert.
 
diff --git a/public/app/features/alerting/partials/notification_edit.html b/public/app/features/alerting/partials/notification_edit.html
index faa168d0acd..7b198736b83 100644
--- a/public/app/features/alerting/partials/notification_edit.html
+++ b/public/app/features/alerting/partials/notification_edit.html
@@ -52,7 +52,7 @@
       </div>
       <div class="gf-form">
           <span class="alert alert-info width-30" ng-if="ctrl.model.sendReminder">
-            Alert reminders are sent after rules are evaluated. Therefore a reminder can never be sent earlier than a configured alert rule evaluation interval.
+            Alert reminders are sent after rules are evaluated. Therefore a reminder can never be sent more frequently than a configured alert rule evaluation interval.
           </span>
         </div>
     </div>

From c1efa13018bd3d4b872e936f04f659a534e641c2 Mon Sep 17 00:00:00 2001
From: Marcus Efraimsson <marcus.efraimsson@gmail.com>
Date: Mon, 3 Sep 2018 14:56:50 +0200
Subject: [PATCH 35/39] changelog: add notes about closing #7330

[skip ci]
---
 CHANGELOG.md | 17 +++++++++++------
 1 file changed, 11 insertions(+), 6 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 41b07e4ad7a..a6a0f6d7f8c 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,15 +1,21 @@
 # 5.3.0 (unreleased)
 
+### New Major Features
+
+* **Alerting**: Notification reminders [#7330](https://github.com/grafana/grafana/issues/7330), thx [@jbaublitz](https://github.com/jbaublitz)
 * **Dashboard**: TV & Kiosk mode changes, new cycle view mode button in dashboard toolbar [#13025](https://github.com/grafana/grafana/pull/13025)
 * **OAuth**: Gitlab OAuth with support for filter by groups [#5623](https://github.com/grafana/grafana/issues/5623), thx [@BenoitKnecht](https://github.com/BenoitKnecht)
-* **Dataproxy**: Pass configured/auth headers to a Datasource [#10971](https://github.com/grafana/grafana/issues/10971), thx [@mrsiano](https://github.com/mrsiano)
-* **Cleanup**: Make temp file time to live configurable [#11607](https://github.com/grafana/grafana/issues/11607), thx [@xapon](https://github.com/xapon)
+
+### New Features
+
 * **LDAP**: Define Grafana Admin permission in ldap group mappings [#2469](https://github.com/grafana/grafana/issues/2496), PR [#12622](https://github.com/grafana/grafana/issues/12622)
-* **Cloudwatch**: CloudWatch GetMetricData support [#11487](https://github.com/grafana/grafana/issues/11487), thx [@mtanda](https://github.com/mtanda)
-* **Configuration**: Allow auto-assigning users to specific organization (other than Main. Org) [#1823](https://github.com/grafana/grafana/issues/1823) [#12801](https://github.com/grafana/grafana/issues/12801), thx [@gzzo](https://github.com/gzzo) and [@ofosos](https://github.com/ofosos)
-* **Profile**: List teams that the user is member of in current/active organization [#12476](https://github.com/grafana/grafana/issues/12476)
 * **LDAP**: Client certificates support [#12805](https://github.com/grafana/grafana/issues/12805), thx [@nyxi](https://github.com/nyxi)
+* **Profile**: List teams that the user is member of in current/active organization [#12476](https://github.com/grafana/grafana/issues/12476)
+* **Configuration**: Allow auto-assigning users to specific organization (other than Main. Org) [#1823](https://github.com/grafana/grafana/issues/1823) [#12801](https://github.com/grafana/grafana/issues/12801), thx [@gzzo](https://github.com/gzzo) and [@ofosos](https://github.com/ofosos)
+* **Dataproxy**: Pass configured/auth headers to a Datasource [#10971](https://github.com/grafana/grafana/issues/10971), thx [@mrsiano](https://github.com/mrsiano)
+* **Cloudwatch**: CloudWatch GetMetricData support [#11487](https://github.com/grafana/grafana/issues/11487), thx [@mtanda](https://github.com/mtanda)
 * **Postgres**: TimescaleDB support, e.g. use `time_bucket` for grouping by time when option enabled [#12680](https://github.com/grafana/grafana/pull/12680), thx [svenklemm](https://github.com/svenklemm)
+* **Cleanup**: Make temp file time to live configurable [#11607](https://github.com/grafana/grafana/issues/11607), thx [@xapon](https://github.com/xapon)
 
 ### Minor
 
@@ -42,7 +48,6 @@
 * **Cloudwatch**: Add new Redshift metrics and dimensions [#12063](https://github.com/grafana/grafana/pulls/12063), thx [@A21z](https://github.com/A21z)
 * **Table**: Adjust header contrast for the light theme [#12668](https://github.com/grafana/grafana/issues/12668)
 * **Table**: Fix link color when using light theme and thresholds in use [#12766](https://github.com/grafana/grafana/issues/12766)
-om/grafana/grafana/issues/12668)
 * **Table**: Fix for useless horizontal scrollbar for table panel [#9964](https://github.com/grafana/grafana/issues/9964)
 * **Table**: Make table sorting stable when null values exist [#12362](https://github.com/grafana/grafana/pull/12362), thx [@bz2](https://github.com/bz2)
 * **Elasticsearch**: For alerting/backend, support having index name to the right of pattern in index pattern [#12731](https://github.com/grafana/grafana/issues/12731)

From db2264c554688ab569f40a8655d688f8930c27dd Mon Sep 17 00:00:00 2001
From: Marcus Efraimsson <marcus.efraimsson@gmail.com>
Date: Mon, 3 Sep 2018 15:12:17 +0200
Subject: [PATCH 36/39] docs: v5.2 upgrade notice, ref #13084

---
 docs/sources/installation/upgrading.md | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/docs/sources/installation/upgrading.md b/docs/sources/installation/upgrading.md
index c72bb4c0921..a476a38c3c5 100644
--- a/docs/sources/installation/upgrading.md
+++ b/docs/sources/installation/upgrading.md
@@ -109,3 +109,11 @@ positioning system when you load them in v5. Dashboards saved in v5 will not wor
 external panel plugins might need to be updated to work properly.
 
 For more details on the new panel positioning system, [click here]({{< relref "reference/dashboard.md#panel-size-position" >}})
+
+## Upgrading to v5.2
+
+One of the database migrations included in this release will update all annotation timestamps from second to millisecond precision. If you have a large amount of annotations the database migration may take a long time to complete which may cause problems if you use systemd to run Grafana.
+
+We've got one report where using systemd, PostgreSQL and a large amount of annotations (table size 1645mb) took 8-20 minutes for the database migration to complete. However, the grafana-server process was killed after 90 seconds by systemd. Any database migration queries in progress when systemd kills the grafana-server process continues to execute in database until finished.
+
+If you're using systemd and have a large amount of annotations consider temporary adjusting the systemd `TimeoutStartSec` setting to something high like `30m` before upgrading.

From ee1083d9b4c3bef1626909f3b86ac455256aa0a8 Mon Sep 17 00:00:00 2001
From: Carl Bergquist <carl@grafana.com>
Date: Mon, 3 Sep 2018 20:33:21 +0200
Subject: [PATCH 37/39] cli: avoid rely on response.ContentLength (#13120)

response.ContentLength might be invalid if the http response
is chunked.

fixes #13079
---
 pkg/cmd/grafana-cli/commands/install_command.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pkg/cmd/grafana-cli/commands/install_command.go b/pkg/cmd/grafana-cli/commands/install_command.go
index 9bdb73a5858..5d4969e06af 100644
--- a/pkg/cmd/grafana-cli/commands/install_command.go
+++ b/pkg/cmd/grafana-cli/commands/install_command.go
@@ -152,7 +152,7 @@ func downloadFile(pluginName, filePath, url string) (err error) {
 		return err
 	}
 
-	r, err := zip.NewReader(bytes.NewReader(body), resp.ContentLength)
+	r, err := zip.NewReader(bytes.NewReader(body), int64(len(body)))
 	if err != nil {
 		return err
 	}

From b0134d30ae41a7373ce6bbc44a33e930837a1746 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Torkel=20=C3=96degaard?= <torkel@grafana.com>
Date: Mon, 3 Sep 2018 20:35:28 +0200
Subject: [PATCH 38/39] Update CHANGELOG.md

---
 CHANGELOG.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index a6a0f6d7f8c..30fa0cd0b94 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -19,6 +19,7 @@
 
 ### Minor
 
+* **GrafanaCli**: Fixed issue with grafana-cli install plugin resulting in corrupt http response from source error. Fixes [#13079](https://github.com/grafana/grafana/issues/13079)
 * **Api**: Delete nonexistent datasource should return 404 [#12313](https://github.com/grafana/grafana/issues/12313), thx [@AustinWinstanley](https://github.com/AustinWinstanley)
 * **Dashboard**: Fix selecting current dashboard from search should not reload dashboard [#12248](https://github.com/grafana/grafana/issues/12248)
 * **Dashboard**: Use uid when linking to dashboards internally in a dashboard [#10705](https://github.com/grafana/grafana/issues/10705)

From dac2c6254593b3283f740ad09adfcf4b76ff137d Mon Sep 17 00:00:00 2001
From: Patrick O'Carroll <ocarroll.patrick@gmail.com>
Date: Mon, 3 Sep 2018 20:36:01 +0200
Subject: [PATCH 39/39] added new-parens rule (#13119)

---
 tslint.json | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tslint.json b/tslint.json
index f53eb541643..0d525bea3f0 100644
--- a/tslint.json
+++ b/tslint.json
@@ -29,6 +29,7 @@
     "label-position": true,
     "max-line-length": [true, 150],
     "member-access": [true, "no-public"],
+    "new-parens": true,
     "no-angle-bracket-type-assertion": true,
     "no-arg": true,
     "no-bitwise": false,