mirror of
https://github.com/grafana/grafana.git
synced 2025-07-30 21:53:00 +08:00
feat(alerting): working on state management
This commit is contained in:
@ -49,6 +49,7 @@ func GetAlerts(c *middleware.Context) Response {
|
||||
Name: alert.Name,
|
||||
Description: alert.Description,
|
||||
State: alert.State,
|
||||
Severity: alert.Severity,
|
||||
})
|
||||
}
|
||||
|
||||
@ -92,7 +93,7 @@ func AlertTest(c *middleware.Context, dto dtos.AlertTestCommand) Response {
|
||||
res := backendCmd.Result
|
||||
|
||||
dtoRes := &dtos.AlertTestResult{
|
||||
Triggered: res.Triggered,
|
||||
Firing: res.Firing,
|
||||
}
|
||||
|
||||
if res.Error != nil {
|
||||
@ -138,41 +139,41 @@ func DelAlert(c *middleware.Context) Response {
|
||||
return Json(200, resp)
|
||||
}
|
||||
|
||||
// GET /api/alerts/events/:id
|
||||
func GetAlertStates(c *middleware.Context) Response {
|
||||
alertId := c.ParamsInt64(":alertId")
|
||||
|
||||
query := models.GetAlertsStateQuery{
|
||||
AlertId: alertId,
|
||||
}
|
||||
|
||||
if err := bus.Dispatch(&query); err != nil {
|
||||
return ApiError(500, "Failed get alert state log", err)
|
||||
}
|
||||
|
||||
return Json(200, query.Result)
|
||||
}
|
||||
|
||||
// PUT /api/alerts/events/:id
|
||||
func PutAlertState(c *middleware.Context, cmd models.UpdateAlertStateCommand) Response {
|
||||
cmd.AlertId = c.ParamsInt64(":alertId")
|
||||
cmd.OrgId = c.OrgId
|
||||
|
||||
query := models.GetAlertByIdQuery{Id: cmd.AlertId}
|
||||
if err := bus.Dispatch(&query); err != nil {
|
||||
return ApiError(500, "Failed to get alertstate", err)
|
||||
}
|
||||
|
||||
if query.Result.OrgId != 0 && query.Result.OrgId != c.OrgId {
|
||||
return ApiError(500, "Alert not found", nil)
|
||||
}
|
||||
|
||||
if err := bus.Dispatch(&cmd); err != nil {
|
||||
return ApiError(500, "Failed to set new state", err)
|
||||
}
|
||||
|
||||
return Json(200, cmd.Result)
|
||||
}
|
||||
// // GET /api/alerts/events/:id
|
||||
// func GetAlertStates(c *middleware.Context) Response {
|
||||
// alertId := c.ParamsInt64(":alertId")
|
||||
//
|
||||
// query := models.GetAlertsStateQuery{
|
||||
// AlertId: alertId,
|
||||
// }
|
||||
//
|
||||
// if err := bus.Dispatch(&query); err != nil {
|
||||
// return ApiError(500, "Failed get alert state log", err)
|
||||
// }
|
||||
//
|
||||
// return Json(200, query.Result)
|
||||
// }
|
||||
//
|
||||
// // PUT /api/alerts/events/:id
|
||||
// func PutAlertState(c *middleware.Context, cmd models.UpdateAlertStateCommand) Response {
|
||||
// cmd.AlertId = c.ParamsInt64(":alertId")
|
||||
// cmd.OrgId = c.OrgId
|
||||
//
|
||||
// query := models.GetAlertByIdQuery{Id: cmd.AlertId}
|
||||
// if err := bus.Dispatch(&query); err != nil {
|
||||
// return ApiError(500, "Failed to get alertstate", err)
|
||||
// }
|
||||
//
|
||||
// if query.Result.OrgId != 0 && query.Result.OrgId != c.OrgId {
|
||||
// return ApiError(500, "Alert not found", nil)
|
||||
// }
|
||||
//
|
||||
// if err := bus.Dispatch(&cmd); err != nil {
|
||||
// return ApiError(500, "Failed to set new state", err)
|
||||
// }
|
||||
//
|
||||
// return Json(200, cmd.Result)
|
||||
// }
|
||||
|
||||
func GetAlertNotifications(c *middleware.Context) Response {
|
||||
query := &models.GetAlertNotificationQuery{
|
||||
|
@ -247,7 +247,7 @@ func Register(r *macaron.Macaron) {
|
||||
|
||||
r.Group("/alerts", func() {
|
||||
r.Post("/test", bind(dtos.AlertTestCommand{}), wrap(AlertTest))
|
||||
r.Get("/:alertId/states", wrap(GetAlertStates))
|
||||
//r.Get("/:alertId/states", wrap(GetAlertStates))
|
||||
//r.Put("/:alertId/state", bind(m.UpdateAlertStateCommand{}), wrap(PutAlertState))
|
||||
r.Get("/:alertId", ValidateOrgAlert, wrap(GetAlert))
|
||||
//r.Delete("/:alertId", ValidateOrgAlert, wrap(DelAlert)) disabled until we know how to handle it dashboard updates
|
||||
|
@ -4,24 +4,17 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/grafana/grafana/pkg/components/simplejson"
|
||||
m "github.com/grafana/grafana/pkg/models"
|
||||
)
|
||||
|
||||
type AlertRuleDTO struct {
|
||||
Id int64 `json:"id"`
|
||||
DashboardId int64 `json:"dashboardId"`
|
||||
PanelId int64 `json:"panelId"`
|
||||
Query string `json:"query"`
|
||||
QueryRefId string `json:"queryRefId"`
|
||||
WarnLevel float64 `json:"warnLevel"`
|
||||
CritLevel float64 `json:"critLevel"`
|
||||
WarnOperator string `json:"warnOperator"`
|
||||
CritOperator string `json:"critOperator"`
|
||||
Frequency int64 `json:"frequency"`
|
||||
Name string `json:"name"`
|
||||
Description string `json:"description"`
|
||||
QueryRange int `json:"queryRange"`
|
||||
Aggregator string `json:"aggregator"`
|
||||
State string `json:"state"`
|
||||
Id int64 `json:"id"`
|
||||
DashboardId int64 `json:"dashboardId"`
|
||||
PanelId int64 `json:"panelId"`
|
||||
Name string `json:"name"`
|
||||
Description string `json:"description"`
|
||||
State m.AlertStateType `json:"state"`
|
||||
Severity m.AlertSeverityType `json:"severity"`
|
||||
|
||||
DashbboardUri string `json:"dashboardUri"`
|
||||
}
|
||||
@ -40,10 +33,10 @@ type AlertTestCommand struct {
|
||||
}
|
||||
|
||||
type AlertTestResult struct {
|
||||
Triggered bool `json:"triggerd"`
|
||||
Timing string `json:"timing"`
|
||||
Error string `json:"error,omitempty"`
|
||||
Logs []*AlertTestResultLog `json:"logs,omitempty"`
|
||||
Firing bool `json:"firing"`
|
||||
Timing string `json:"timing"`
|
||||
Error string `json:"error,omitempty"`
|
||||
Logs []*AlertTestResultLog `json:"logs,omitempty"`
|
||||
}
|
||||
|
||||
type AlertTestResultLog struct {
|
||||
|
@ -6,6 +6,29 @@ import (
|
||||
"github.com/grafana/grafana/pkg/components/simplejson"
|
||||
)
|
||||
|
||||
type AlertStateType string
|
||||
type AlertSeverityType string
|
||||
|
||||
const (
|
||||
AlertStatePending AlertStateType = "pending"
|
||||
AlertStateFiring AlertStateType = "firing"
|
||||
AlertStateOK AlertStateType = "ok"
|
||||
)
|
||||
|
||||
func (s AlertStateType) IsValid() bool {
|
||||
return s == AlertStatePending || s == AlertStateFiring || s == AlertStateOK
|
||||
}
|
||||
|
||||
const (
|
||||
AlertSeverityCritical AlertSeverityType = "critical"
|
||||
AlertSeverityWarning AlertSeverityType = "warning"
|
||||
AlertSeverityInfo AlertSeverityType = "info"
|
||||
)
|
||||
|
||||
func (s AlertSeverityType) IsValid() bool {
|
||||
return s == AlertSeverityCritical || s == AlertSeverityInfo || s == AlertSeverityWarning
|
||||
}
|
||||
|
||||
type Alert struct {
|
||||
Id int64
|
||||
OrgId int64
|
||||
@ -13,8 +36,8 @@ type Alert struct {
|
||||
PanelId int64
|
||||
Name string
|
||||
Description string
|
||||
Severity string
|
||||
State string
|
||||
Severity AlertSeverityType
|
||||
State AlertStateType
|
||||
Handler int64
|
||||
Enabled bool
|
||||
Frequency int64
|
||||
@ -32,7 +55,7 @@ func (alert *Alert) ValidToSave() bool {
|
||||
return alert.DashboardId != 0 && alert.OrgId != 0 && alert.PanelId != 0
|
||||
}
|
||||
|
||||
func (alert *Alert) ShouldUpdateState(newState string) bool {
|
||||
func (alert *Alert) ShouldUpdateState(newState AlertStateType) bool {
|
||||
return alert.State != newState
|
||||
}
|
||||
|
||||
@ -74,25 +97,6 @@ type HeartBeatCommand struct {
|
||||
Result AlertingClusterInfo
|
||||
}
|
||||
|
||||
type AlertChange struct {
|
||||
Id int64 `json:"id"`
|
||||
OrgId int64 `json:"-"`
|
||||
AlertId int64 `json:"alertId"`
|
||||
UpdatedBy int64 `json:"updatedBy"`
|
||||
NewAlertSettings *simplejson.Json `json:"newAlertSettings"`
|
||||
Type string `json:"type"`
|
||||
Created time.Time `json:"created"`
|
||||
}
|
||||
|
||||
// Commands
|
||||
type CreateAlertChangeCommand struct {
|
||||
OrgId int64
|
||||
AlertId int64
|
||||
UpdatedBy int64
|
||||
NewAlertSettings *simplejson.Json
|
||||
Type string
|
||||
}
|
||||
|
||||
type SaveAlertsCommand struct {
|
||||
DashboardId int64
|
||||
UserId int64
|
||||
@ -101,6 +105,13 @@ type SaveAlertsCommand struct {
|
||||
Alerts []*Alert
|
||||
}
|
||||
|
||||
type SetAlertStateCommand struct {
|
||||
AlertId int64
|
||||
OrgId int64
|
||||
State AlertStateType
|
||||
Timestamp time.Time
|
||||
}
|
||||
|
||||
type DeleteAlertCommand struct {
|
||||
AlertId int64
|
||||
}
|
||||
@ -124,11 +135,3 @@ type GetAlertByIdQuery struct {
|
||||
|
||||
Result *Alert
|
||||
}
|
||||
|
||||
type GetAlertChangesQuery struct {
|
||||
OrgId int64
|
||||
Limit int64
|
||||
SinceId int64
|
||||
|
||||
Result []*AlertChange
|
||||
}
|
||||
|
@ -1,54 +1,47 @@
|
||||
package models
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"github.com/grafana/grafana/pkg/components/simplejson"
|
||||
"github.com/grafana/grafana/pkg/services/alerting/alertstates"
|
||||
)
|
||||
|
||||
type AlertState struct {
|
||||
Id int64 `json:"-"`
|
||||
OrgId int64 `json:"-"`
|
||||
AlertId int64 `json:"alertId"`
|
||||
State string `json:"state"`
|
||||
Created time.Time `json:"created"`
|
||||
Info string `json:"info"`
|
||||
TriggeredAlerts *simplejson.Json `json:"triggeredAlerts"`
|
||||
}
|
||||
|
||||
func (this *UpdateAlertStateCommand) IsValidState() bool {
|
||||
for _, v := range alertstates.ValidStates {
|
||||
if this.State == v {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// Commands
|
||||
|
||||
type UpdateAlertStateCommand struct {
|
||||
AlertId int64 `json:"alertId" binding:"Required"`
|
||||
OrgId int64 `json:"orgId" binding:"Required"`
|
||||
State string `json:"state" binding:"Required"`
|
||||
Info string `json:"info"`
|
||||
|
||||
Result *Alert
|
||||
}
|
||||
|
||||
// Queries
|
||||
|
||||
type GetAlertsStateQuery struct {
|
||||
OrgId int64 `json:"orgId" binding:"Required"`
|
||||
AlertId int64 `json:"alertId" binding:"Required"`
|
||||
|
||||
Result *[]AlertState
|
||||
}
|
||||
|
||||
type GetLastAlertStateQuery struct {
|
||||
AlertId int64
|
||||
OrgId int64
|
||||
|
||||
Result *AlertState
|
||||
}
|
||||
// type AlertState struct {
|
||||
// Id int64 `json:"-"`
|
||||
// OrgId int64 `json:"-"`
|
||||
// AlertId int64 `json:"alertId"`
|
||||
// State string `json:"state"`
|
||||
// Created time.Time `json:"created"`
|
||||
// Info string `json:"info"`
|
||||
// TriggeredAlerts *simplejson.Json `json:"triggeredAlerts"`
|
||||
// }
|
||||
//
|
||||
// func (this *UpdateAlertStateCommand) IsValidState() bool {
|
||||
// for _, v := range alertstates.ValidStates {
|
||||
// if this.State == v {
|
||||
// return true
|
||||
// }
|
||||
// }
|
||||
// return false
|
||||
// }
|
||||
//
|
||||
// // Commands
|
||||
//
|
||||
// type UpdateAlertStateCommand struct {
|
||||
// AlertId int64 `json:"alertId" binding:"Required"`
|
||||
// OrgId int64 `json:"orgId" binding:"Required"`
|
||||
// State string `json:"state" binding:"Required"`
|
||||
// Info string `json:"info"`
|
||||
//
|
||||
// Result *Alert
|
||||
// }
|
||||
//
|
||||
// // Queries
|
||||
//
|
||||
// type GetAlertsStateQuery struct {
|
||||
// OrgId int64 `json:"orgId" binding:"Required"`
|
||||
// AlertId int64 `json:"alertId" binding:"Required"`
|
||||
//
|
||||
// Result *[]AlertState
|
||||
// }
|
||||
//
|
||||
// type GetLastAlertStateQuery struct {
|
||||
// AlertId int64
|
||||
// OrgId int64
|
||||
//
|
||||
// Result *AlertState
|
||||
// }
|
||||
|
@ -18,7 +18,8 @@ type AlertRule struct {
|
||||
Frequency int64
|
||||
Name string
|
||||
Description string
|
||||
Severity string
|
||||
State m.AlertStateType
|
||||
Severity m.AlertSeverityType
|
||||
Conditions []AlertCondition
|
||||
Notifications []int64
|
||||
}
|
||||
@ -63,6 +64,7 @@ func NewAlertRuleFromDBModel(ruleDef *m.Alert) (*AlertRule, error) {
|
||||
model.Description = ruleDef.Description
|
||||
model.Frequency = ruleDef.Frequency
|
||||
model.Severity = ruleDef.Severity
|
||||
model.State = ruleDef.State
|
||||
|
||||
for _, v := range ruleDef.Settings.Get("notifications").MustArray() {
|
||||
if id, ok := v.(int64); ok {
|
||||
|
@ -1,16 +0,0 @@
|
||||
package alertstates
|
||||
|
||||
var (
|
||||
ValidStates = []string{
|
||||
Ok,
|
||||
Warn,
|
||||
Critical,
|
||||
Unknown,
|
||||
}
|
||||
|
||||
Ok = "OK"
|
||||
Warn = "WARN"
|
||||
Critical = "CRITICAL"
|
||||
Pending = "PENDING"
|
||||
Unknown = "UNKNOWN"
|
||||
)
|
@ -40,7 +40,7 @@ func (c *QueryCondition) Eval(context *AlertResultContext) {
|
||||
Metric: series.Name,
|
||||
Value: reducedValue,
|
||||
})
|
||||
context.Triggered = true
|
||||
context.Firing = true
|
||||
break
|
||||
}
|
||||
}
|
||||
|
@ -19,20 +19,20 @@ func TestQueryCondition(t *testing.T) {
|
||||
ctx.reducer = `{"type": "avg"}`
|
||||
ctx.evaluator = `{"type": ">", "params": [100]}`
|
||||
|
||||
Convey("should trigger when avg is above 100", func() {
|
||||
Convey("should fire when avg is above 100", func() {
|
||||
ctx.series = tsdb.TimeSeriesSlice{tsdb.NewTimeSeries("test1", [][2]float64{{120, 0}})}
|
||||
ctx.exec()
|
||||
|
||||
So(ctx.result.Error, ShouldBeNil)
|
||||
So(ctx.result.Triggered, ShouldBeTrue)
|
||||
So(ctx.result.Firing, ShouldBeTrue)
|
||||
})
|
||||
|
||||
Convey("Should not trigger when avg is below 100", func() {
|
||||
Convey("Should not fire when avg is below 100", func() {
|
||||
ctx.series = tsdb.TimeSeriesSlice{tsdb.NewTimeSeries("test1", [][2]float64{{90, 0}})}
|
||||
ctx.exec()
|
||||
|
||||
So(ctx.result.Error, ShouldBeNil)
|
||||
So(ctx.result.Triggered, ShouldBeFalse)
|
||||
So(ctx.result.Firing, ShouldBeFalse)
|
||||
})
|
||||
})
|
||||
})
|
||||
|
@ -100,7 +100,7 @@ func (e *Engine) resultHandler() {
|
||||
}()
|
||||
|
||||
for result := range e.resultQueue {
|
||||
e.log.Debug("Alert Rule Result", "ruleId", result.Rule.Id, "triggered", result.Triggered)
|
||||
e.log.Debug("Alert Rule Result", "ruleId", result.Rule.Id, "firing", result.Firing)
|
||||
|
||||
if result.Error != nil {
|
||||
e.log.Error("Alert Rule Result Error", "ruleId", result.Rule.Id, "error", result.Error, "retry")
|
||||
|
@ -2,7 +2,6 @@ package alerting
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
|
||||
"github.com/grafana/grafana/pkg/bus"
|
||||
"github.com/grafana/grafana/pkg/components/simplejson"
|
||||
@ -90,10 +89,14 @@ func (e *DashAlertExtractor) GetAlerts() ([]*m.Alert, error) {
|
||||
Handler: jsonAlert.Get("handler").MustInt64(),
|
||||
Enabled: jsonAlert.Get("enabled").MustBool(),
|
||||
Description: jsonAlert.Get("description").MustString(),
|
||||
Severity: jsonAlert.Get("severity").MustString(),
|
||||
Severity: m.AlertSeverityType(jsonAlert.Get("severity").MustString()),
|
||||
Frequency: getTimeDurationStringToSeconds(jsonAlert.Get("frequency").MustString()),
|
||||
}
|
||||
|
||||
if !alert.Severity.IsValid() {
|
||||
return nil, AlertValidationError{Reason: "Invalid alert Severity"}
|
||||
}
|
||||
|
||||
for _, condition := range jsonAlert.Get("conditions").MustArray() {
|
||||
jsonCondition := simplejson.NewFromAny(condition)
|
||||
|
||||
@ -102,7 +105,7 @@ func (e *DashAlertExtractor) GetAlerts() ([]*m.Alert, error) {
|
||||
panelQuery := findPanelQueryByRefId(panel, queryRefId)
|
||||
|
||||
if panelQuery == nil {
|
||||
return nil, fmt.Errorf("Alert referes to query %s, that could not be found", queryRefId)
|
||||
return nil, AlertValidationError{Reason: "Alert refes to query that cannot be found"}
|
||||
}
|
||||
|
||||
dsName := ""
|
||||
|
@ -33,7 +33,7 @@ func (e *HandlerImpl) Execute(context *AlertResultContext) {
|
||||
context.EndTime = time.Now()
|
||||
e.log.Debug("Job Execution timeout", "alertId", context.Rule.Id)
|
||||
case <-context.DoneChan:
|
||||
e.log.Debug("Job Execution done", "timing", context.GetDurationSeconds(), "alertId", context.Rule.Id, "triggered", context.Triggered)
|
||||
e.log.Debug("Job Execution done", "timing", context.GetDurationSeconds(), "alertId", context.Rule.Id, "firing", context.Firing)
|
||||
}
|
||||
|
||||
}
|
||||
@ -49,7 +49,7 @@ func (e *HandlerImpl) eval(context *AlertResultContext) {
|
||||
}
|
||||
|
||||
// break if result has not triggered yet
|
||||
if context.Triggered == false {
|
||||
if context.Firing == false {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
@ -7,11 +7,11 @@ import (
|
||||
)
|
||||
|
||||
type conditionStub struct {
|
||||
triggered bool
|
||||
firing bool
|
||||
}
|
||||
|
||||
func (c *conditionStub) Eval(context *AlertResultContext) {
|
||||
context.Triggered = c.triggered
|
||||
context.Firing = c.firing
|
||||
}
|
||||
|
||||
func TestAlertingExecutor(t *testing.T) {
|
||||
@ -21,24 +21,24 @@ func TestAlertingExecutor(t *testing.T) {
|
||||
Convey("Show return triggered with single passing condition", func() {
|
||||
context := NewAlertResultContext(&AlertRule{
|
||||
Conditions: []AlertCondition{&conditionStub{
|
||||
triggered: true,
|
||||
firing: true,
|
||||
}},
|
||||
})
|
||||
|
||||
handler.eval(context)
|
||||
So(context.Triggered, ShouldEqual, true)
|
||||
So(context.Firing, ShouldEqual, true)
|
||||
})
|
||||
|
||||
Convey("Show return false with not passing condition", func() {
|
||||
context := NewAlertResultContext(&AlertRule{
|
||||
Conditions: []AlertCondition{
|
||||
&conditionStub{triggered: true},
|
||||
&conditionStub{triggered: false},
|
||||
&conditionStub{firing: true},
|
||||
&conditionStub{firing: false},
|
||||
},
|
||||
})
|
||||
|
||||
handler.eval(context)
|
||||
So(context.Triggered, ShouldEqual, false)
|
||||
So(context.Firing, ShouldEqual, false)
|
||||
})
|
||||
|
||||
// Convey("Show return critical since below 2", func() {
|
||||
|
@ -28,7 +28,7 @@ func (aj *AlertJob) IncRetry() {
|
||||
}
|
||||
|
||||
type AlertResultContext struct {
|
||||
Triggered bool
|
||||
Firing bool
|
||||
IsTestRun bool
|
||||
Events []*AlertEvent
|
||||
Logs []*AlertResultLogEntry
|
||||
|
@ -1,12 +1,9 @@
|
||||
package alerting
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"github.com/grafana/grafana/pkg/bus"
|
||||
"github.com/grafana/grafana/pkg/log"
|
||||
m "github.com/grafana/grafana/pkg/models"
|
||||
"github.com/grafana/grafana/pkg/services/alerting/alertstates"
|
||||
)
|
||||
|
||||
type ResultHandler interface {
|
||||
@ -20,24 +17,27 @@ type ResultHandlerImpl struct {
|
||||
|
||||
func NewResultHandler() *ResultHandlerImpl {
|
||||
return &ResultHandlerImpl{
|
||||
log: log.New("alerting.responseHandler"),
|
||||
//notifier: NewNotifier(),
|
||||
log: log.New("alerting.resultHandler"),
|
||||
}
|
||||
}
|
||||
|
||||
func (handler *ResultHandlerImpl) Handle(result *AlertResultContext) {
|
||||
newState := alertstates.Ok
|
||||
if result.Triggered {
|
||||
newState = result.Rule.Severity
|
||||
var newState m.AlertStateType
|
||||
|
||||
if result.Error != nil {
|
||||
handler.log.Error("Alert Rule Result Error", "ruleId", result.Rule.Id, "error", result.Error)
|
||||
newState = m.AlertStatePending
|
||||
} else if result.Firing {
|
||||
newState = m.AlertStateFiring
|
||||
} else {
|
||||
newState = m.AlertStateOK
|
||||
}
|
||||
|
||||
handler.log.Info("Handle result", "newState", newState)
|
||||
handler.log.Info("Handle result", "triggered", result.Triggered)
|
||||
if result.Rule.State != newState {
|
||||
handler.log.Info("New state change", "alertId", result.Rule.Id, "newState", newState, "oldState", result.Rule.State)
|
||||
|
||||
if handler.shouldUpdateState(result, newState) {
|
||||
cmd := &m.UpdateAlertStateCommand{
|
||||
cmd := &m.SetAlertStateCommand{
|
||||
AlertId: result.Rule.Id,
|
||||
Info: result.Description,
|
||||
OrgId: result.Rule.OrgId,
|
||||
State: newState,
|
||||
}
|
||||
@ -46,30 +46,8 @@ func (handler *ResultHandlerImpl) Handle(result *AlertResultContext) {
|
||||
handler.log.Error("Failed to save state", "error", err)
|
||||
}
|
||||
|
||||
result.Rule.State = newState
|
||||
//handler.log.Debug("will notify about new state", "new state", result.State)
|
||||
//handler.notifier.Notify(result)
|
||||
}
|
||||
}
|
||||
|
||||
func (handler *ResultHandlerImpl) shouldUpdateState(result *AlertResultContext, newState string) bool {
|
||||
query := &m.GetLastAlertStateQuery{
|
||||
AlertId: result.Rule.Id,
|
||||
OrgId: result.Rule.OrgId,
|
||||
}
|
||||
|
||||
if err := bus.Dispatch(query); err != nil {
|
||||
log.Error2("Failed to read last alert state", "error", err)
|
||||
return false
|
||||
}
|
||||
|
||||
if query.Result == nil {
|
||||
return true
|
||||
}
|
||||
|
||||
lastExecution := query.Result.Created
|
||||
asdf := result.StartTime.Add(time.Minute * -15)
|
||||
olderThen15Min := lastExecution.Before(asdf)
|
||||
changedState := query.Result.State != newState
|
||||
|
||||
return changedState || olderThen15Min
|
||||
}
|
||||
|
@ -17,52 +17,9 @@ func init() {
|
||||
bus.AddHandler("sql", GetAlertById)
|
||||
bus.AddHandler("sql", DeleteAlertById)
|
||||
bus.AddHandler("sql", GetAllAlertQueryHandler)
|
||||
//bus.AddHandler("sql", HeartBeat)
|
||||
bus.AddHandler("sql", SetAlertState)
|
||||
}
|
||||
|
||||
/*
|
||||
func HeartBeat(query *m.HeartBeatCommand) error {
|
||||
return inTransaction(func(sess *xorm.Session) error {
|
||||
now := time.Now().Sub(0, 0, 0, 5)
|
||||
activeTime := time.Now().Sub(0, 0, 0, 5)
|
||||
ownHeartbeats := make([]m.HeartBeat, 0)
|
||||
err := x.Where("server_id = ?", query.ServerId).Find(&ownHeartbeats)
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if (len(ownHeartbeats)) > 0 && ownHeartbeats[0].Updated > activeTime {
|
||||
//update
|
||||
x.Insert(&m.HeartBeat{ServerId: query.ServerId, Created: now, Updated: now})
|
||||
} else {
|
||||
thisServer := ownHeartbeats[0]
|
||||
thisServer.Updated = now
|
||||
x.Id(thisServer.Id).Update(&thisServer)
|
||||
}
|
||||
|
||||
activeServers := make([]m.HeartBeat, 0)
|
||||
err = x.Where("server_id = ? and updated > ", query.ServerId, now.String()).OrderBy("id").Find(&activeServers)
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for i, pos := range activeServers {
|
||||
if pos.ServerId == query.ServerId {
|
||||
query.Result = &m.AlertingClusterInfo{
|
||||
ClusterSize: len(activeServers),
|
||||
UptimePosition: i,
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
}
|
||||
*/
|
||||
|
||||
func GetAlertById(query *m.GetAlertByIdQuery) error {
|
||||
alert := m.Alert{}
|
||||
has, err := x.Id(query.Id).Get(&alert)
|
||||
@ -203,7 +160,7 @@ func upsertAlerts(existingAlerts []*m.Alert, cmd *m.SaveAlertsCommand, sess *xor
|
||||
} else {
|
||||
alert.Updated = time.Now()
|
||||
alert.Created = time.Now()
|
||||
alert.State = "UNKNOWN"
|
||||
alert.State = m.AlertStatePending
|
||||
alert.CreatedBy = cmd.UserId
|
||||
alert.UpdatedBy = cmd.UserId
|
||||
|
||||
@ -253,3 +210,20 @@ func GetAlertsByDashboardId2(dashboardId int64, sess *xorm.Session) ([]*m.Alert,
|
||||
|
||||
return alerts, nil
|
||||
}
|
||||
|
||||
func SetAlertState(cmd *m.SetAlertStateCommand) error {
|
||||
return inTransaction(func(sess *xorm.Session) error {
|
||||
alert := m.Alert{}
|
||||
|
||||
if has, err := sess.Id(cmd.AlertId).Get(&alert); err != nil {
|
||||
return err
|
||||
} else if !has {
|
||||
return fmt.Errorf("Could not find alert")
|
||||
}
|
||||
|
||||
alert.State = cmd.State
|
||||
sess.Id(alert.Id).Update(&alert)
|
||||
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
@ -1,77 +1,77 @@
|
||||
package sqlstore
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/go-xorm/xorm"
|
||||
"github.com/grafana/grafana/pkg/bus"
|
||||
m "github.com/grafana/grafana/pkg/models"
|
||||
)
|
||||
|
||||
func init() {
|
||||
bus.AddHandler("sql", SetNewAlertState)
|
||||
bus.AddHandler("sql", GetAlertStateLogByAlertId)
|
||||
bus.AddHandler("sql", GetLastAlertStateQuery)
|
||||
}
|
||||
|
||||
func GetLastAlertStateQuery(cmd *m.GetLastAlertStateQuery) error {
|
||||
states := make([]m.AlertState, 0)
|
||||
|
||||
if err := x.Where("alert_id = ? and org_id = ? ", cmd.AlertId, cmd.OrgId).Desc("created").Find(&states); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if len(states) == 0 {
|
||||
cmd.Result = nil
|
||||
return nil
|
||||
}
|
||||
|
||||
cmd.Result = &states[0]
|
||||
return nil
|
||||
}
|
||||
|
||||
func SetNewAlertState(cmd *m.UpdateAlertStateCommand) error {
|
||||
return inTransaction(func(sess *xorm.Session) error {
|
||||
if !cmd.IsValidState() {
|
||||
return fmt.Errorf("new state is invalid")
|
||||
}
|
||||
|
||||
alert := m.Alert{}
|
||||
has, err := sess.Id(cmd.AlertId).Get(&alert)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if !has {
|
||||
return fmt.Errorf("Could not find alert")
|
||||
}
|
||||
|
||||
alert.State = cmd.State
|
||||
sess.Id(alert.Id).Update(&alert)
|
||||
|
||||
alertState := m.AlertState{
|
||||
AlertId: cmd.AlertId,
|
||||
OrgId: cmd.OrgId,
|
||||
State: cmd.State,
|
||||
Info: cmd.Info,
|
||||
Created: time.Now(),
|
||||
}
|
||||
|
||||
sess.Insert(&alertState)
|
||||
|
||||
cmd.Result = &alert
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
func GetAlertStateLogByAlertId(cmd *m.GetAlertsStateQuery) error {
|
||||
states := make([]m.AlertState, 0)
|
||||
|
||||
if err := x.Where("alert_id = ?", cmd.AlertId).Desc("created").Find(&states); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
cmd.Result = &states
|
||||
return nil
|
||||
}
|
||||
// import (
|
||||
// "fmt"
|
||||
// "time"
|
||||
//
|
||||
// "github.com/go-xorm/xorm"
|
||||
// "github.com/grafana/grafana/pkg/bus"
|
||||
// m "github.com/grafana/grafana/pkg/models"
|
||||
// )
|
||||
//
|
||||
// func init() {
|
||||
// bus.AddHandler("sql", SetNewAlertState)
|
||||
// bus.AddHandler("sql", GetAlertStateLogByAlertId)
|
||||
// bus.AddHandler("sql", GetLastAlertStateQuery)
|
||||
// }
|
||||
//
|
||||
// func GetLastAlertStateQuery(cmd *m.GetLastAlertStateQuery) error {
|
||||
// states := make([]m.AlertState, 0)
|
||||
//
|
||||
// if err := x.Where("alert_id = ? and org_id = ? ", cmd.AlertId, cmd.OrgId).Desc("created").Find(&states); err != nil {
|
||||
// return err
|
||||
// }
|
||||
//
|
||||
// if len(states) == 0 {
|
||||
// cmd.Result = nil
|
||||
// return nil
|
||||
// }
|
||||
//
|
||||
// cmd.Result = &states[0]
|
||||
// return nil
|
||||
// }
|
||||
//
|
||||
// func SetNewAlertState(cmd *m.UpdateAlertStateCommand) error {
|
||||
// return inTransaction(func(sess *xorm.Session) error {
|
||||
// if !cmd.IsValidState() {
|
||||
// return fmt.Errorf("new state is invalid")
|
||||
// }
|
||||
//
|
||||
// alert := m.Alert{}
|
||||
// has, err := sess.Id(cmd.AlertId).Get(&alert)
|
||||
// if err != nil {
|
||||
// return err
|
||||
// }
|
||||
//
|
||||
// if !has {
|
||||
// return fmt.Errorf("Could not find alert")
|
||||
// }
|
||||
//
|
||||
// alert.State = cmd.State
|
||||
// sess.Id(alert.Id).Update(&alert)
|
||||
//
|
||||
// alertState := m.AlertState{
|
||||
// AlertId: cmd.AlertId,
|
||||
// OrgId: cmd.OrgId,
|
||||
// State: cmd.State,
|
||||
// Info: cmd.Info,
|
||||
// Created: time.Now(),
|
||||
// }
|
||||
//
|
||||
// sess.Insert(&alertState)
|
||||
//
|
||||
// cmd.Result = &alert
|
||||
// return nil
|
||||
// })
|
||||
// }
|
||||
//
|
||||
// func GetAlertStateLogByAlertId(cmd *m.GetAlertsStateQuery) error {
|
||||
// states := make([]m.AlertState, 0)
|
||||
//
|
||||
// if err := x.Where("alert_id = ?", cmd.AlertId).Desc("created").Find(&states); err != nil {
|
||||
// return err
|
||||
// }
|
||||
//
|
||||
// cmd.Result = &states
|
||||
// return nil
|
||||
// }
|
||||
|
@ -1,100 +1,100 @@
|
||||
package sqlstore
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
m "github.com/grafana/grafana/pkg/models"
|
||||
. "github.com/smartystreets/goconvey/convey"
|
||||
)
|
||||
|
||||
func TestAlertingStateAccess(t *testing.T) {
|
||||
Convey("Test alerting state changes", t, func() {
|
||||
InitTestDB(t)
|
||||
|
||||
testDash := insertTestDashboard("dashboard with alerts", 1, "alert")
|
||||
|
||||
items := []*m.Alert{
|
||||
{
|
||||
PanelId: 1,
|
||||
DashboardId: testDash.Id,
|
||||
OrgId: testDash.OrgId,
|
||||
Name: "Alerting title",
|
||||
Description: "Alerting description",
|
||||
},
|
||||
}
|
||||
|
||||
cmd := m.SaveAlertsCommand{
|
||||
Alerts: items,
|
||||
DashboardId: testDash.Id,
|
||||
OrgId: 1,
|
||||
UserId: 1,
|
||||
}
|
||||
|
||||
err := SaveAlerts(&cmd)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
Convey("Cannot insert invalid states", func() {
|
||||
err = SetNewAlertState(&m.UpdateAlertStateCommand{
|
||||
AlertId: 1,
|
||||
NewState: "maybe ok",
|
||||
Info: "Shit just hit the fan",
|
||||
})
|
||||
|
||||
So(err, ShouldNotBeNil)
|
||||
})
|
||||
|
||||
Convey("Changes state to alert", func() {
|
||||
|
||||
err = SetNewAlertState(&m.UpdateAlertStateCommand{
|
||||
AlertId: 1,
|
||||
NewState: "CRITICAL",
|
||||
Info: "Shit just hit the fan",
|
||||
})
|
||||
|
||||
Convey("can get new state for alert", func() {
|
||||
query := &m.GetAlertByIdQuery{Id: 1}
|
||||
err := GetAlertById(query)
|
||||
So(err, ShouldBeNil)
|
||||
So(query.Result.State, ShouldEqual, "CRITICAL")
|
||||
})
|
||||
|
||||
Convey("Changes state to ok", func() {
|
||||
err = SetNewAlertState(&m.UpdateAlertStateCommand{
|
||||
AlertId: 1,
|
||||
NewState: "OK",
|
||||
Info: "Shit just hit the fan",
|
||||
})
|
||||
|
||||
Convey("get ok state for alert", func() {
|
||||
query := &m.GetAlertByIdQuery{Id: 1}
|
||||
err := GetAlertById(query)
|
||||
So(err, ShouldBeNil)
|
||||
So(query.Result.State, ShouldEqual, "OK")
|
||||
})
|
||||
|
||||
Convey("should have two event state logs", func() {
|
||||
query := &m.GetAlertsStateQuery{
|
||||
AlertId: 1,
|
||||
OrgId: 1,
|
||||
}
|
||||
|
||||
err := GetAlertStateLogByAlertId(query)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
So(len(*query.Result), ShouldEqual, 2)
|
||||
})
|
||||
|
||||
Convey("should not get any alerts with critical state", func() {
|
||||
query := &m.GetAlertsQuery{
|
||||
OrgId: 1,
|
||||
State: []string{"Critical", "Warn"},
|
||||
}
|
||||
|
||||
err := HandleAlertsQuery(query)
|
||||
So(err, ShouldBeNil)
|
||||
So(len(query.Result), ShouldEqual, 0)
|
||||
})
|
||||
})
|
||||
})
|
||||
})
|
||||
}
|
||||
// import (
|
||||
// "testing"
|
||||
//
|
||||
// m "github.com/grafana/grafana/pkg/models"
|
||||
// . "github.com/smartystreets/goconvey/convey"
|
||||
// )
|
||||
//
|
||||
// func TestAlertingStateAccess(t *testing.T) {
|
||||
// Convey("Test alerting state changes", t, func() {
|
||||
// InitTestDB(t)
|
||||
//
|
||||
// testDash := insertTestDashboard("dashboard with alerts", 1, "alert")
|
||||
//
|
||||
// items := []*m.Alert{
|
||||
// {
|
||||
// PanelId: 1,
|
||||
// DashboardId: testDash.Id,
|
||||
// OrgId: testDash.OrgId,
|
||||
// Name: "Alerting title",
|
||||
// Description: "Alerting description",
|
||||
// },
|
||||
// }
|
||||
//
|
||||
// cmd := m.SaveAlertsCommand{
|
||||
// Alerts: items,
|
||||
// DashboardId: testDash.Id,
|
||||
// OrgId: 1,
|
||||
// UserId: 1,
|
||||
// }
|
||||
//
|
||||
// err := SaveAlerts(&cmd)
|
||||
// So(err, ShouldBeNil)
|
||||
//
|
||||
// Convey("Cannot insert invalid states", func() {
|
||||
// err = SetNewAlertState(&m.UpdateAlertStateCommand{
|
||||
// AlertId: 1,
|
||||
// NewState: "maybe ok",
|
||||
// Info: "Shit just hit the fan",
|
||||
// })
|
||||
//
|
||||
// So(err, ShouldNotBeNil)
|
||||
// })
|
||||
//
|
||||
// Convey("Changes state to alert", func() {
|
||||
//
|
||||
// err = SetNewAlertState(&m.UpdateAlertStateCommand{
|
||||
// AlertId: 1,
|
||||
// NewState: "CRITICAL",
|
||||
// Info: "Shit just hit the fan",
|
||||
// })
|
||||
//
|
||||
// Convey("can get new state for alert", func() {
|
||||
// query := &m.GetAlertByIdQuery{Id: 1}
|
||||
// err := GetAlertById(query)
|
||||
// So(err, ShouldBeNil)
|
||||
// So(query.Result.State, ShouldEqual, "CRITICAL")
|
||||
// })
|
||||
//
|
||||
// Convey("Changes state to ok", func() {
|
||||
// err = SetNewAlertState(&m.UpdateAlertStateCommand{
|
||||
// AlertId: 1,
|
||||
// NewState: "OK",
|
||||
// Info: "Shit just hit the fan",
|
||||
// })
|
||||
//
|
||||
// Convey("get ok state for alert", func() {
|
||||
// query := &m.GetAlertByIdQuery{Id: 1}
|
||||
// err := GetAlertById(query)
|
||||
// So(err, ShouldBeNil)
|
||||
// So(query.Result.State, ShouldEqual, "OK")
|
||||
// })
|
||||
//
|
||||
// Convey("should have two event state logs", func() {
|
||||
// query := &m.GetAlertsStateQuery{
|
||||
// AlertId: 1,
|
||||
// OrgId: 1,
|
||||
// }
|
||||
//
|
||||
// err := GetAlertStateLogByAlertId(query)
|
||||
// So(err, ShouldBeNil)
|
||||
//
|
||||
// So(len(*query.Result), ShouldEqual, 2)
|
||||
// })
|
||||
//
|
||||
// Convey("should not get any alerts with critical state", func() {
|
||||
// query := &m.GetAlertsQuery{
|
||||
// OrgId: 1,
|
||||
// State: []string{"Critical", "Warn"},
|
||||
// }
|
||||
//
|
||||
// err := HandleAlertsQuery(query)
|
||||
// So(err, ShouldBeNil)
|
||||
// So(len(query.Result), ShouldEqual, 0)
|
||||
// })
|
||||
// })
|
||||
// })
|
||||
// })
|
||||
// }
|
||||
|
@ -1,16 +1,15 @@
|
||||
///<reference path="../../headers/common.d.ts" />
|
||||
|
||||
var alertStateToCssMap = {
|
||||
"OK": "icon-gf-online alert-icon-online",
|
||||
"WARN": "icon-gf-warn alert-icon-warn",
|
||||
"CRITICAL": "icon-gf-critical alert-icon-critical",
|
||||
"ACKNOWLEDGED": "icon-gf-alert-disabled"
|
||||
var alertSeverityIconMap = {
|
||||
"ok": "icon-gf-online alert-icon-online",
|
||||
"warning": "icon-gf-warn alert-icon-warn",
|
||||
"critical": "icon-gf-critical alert-icon-critical",
|
||||
};
|
||||
|
||||
function getCssForState(alertState) {
|
||||
return alertStateToCssMap[alertState];
|
||||
function getSeverityIconClass(alertState) {
|
||||
return alertSeverityIconMap[alertState];
|
||||
}
|
||||
|
||||
export default {
|
||||
getCssForState
|
||||
getSeverityIconClass,
|
||||
};
|
||||
|
@ -28,10 +28,9 @@ export class AlertListCtrl {
|
||||
updateFilter() {
|
||||
var stats = [];
|
||||
|
||||
this.filter.ok && stats.push('Ok');
|
||||
this.filter.ok && stats.push('OK');
|
||||
this.filter.warn && stats.push('Warn');
|
||||
this.filter.critical && stats.push('critical');
|
||||
this.filter.acknowleged && stats.push('acknowleged');
|
||||
|
||||
this.$route.current.params.state = stats;
|
||||
this.$route.updateParams();
|
||||
@ -40,10 +39,9 @@ export class AlertListCtrl {
|
||||
loadAlerts() {
|
||||
var stats = [];
|
||||
|
||||
this.filter.ok && stats.push('Ok');
|
||||
this.filter.ok && stats.push('OK');
|
||||
this.filter.warn && stats.push('Warn');
|
||||
this.filter.critical && stats.push('critical');
|
||||
this.filter.acknowleged && stats.push('acknowleged');
|
||||
|
||||
var params = {
|
||||
state: stats
|
||||
@ -51,7 +49,8 @@ export class AlertListCtrl {
|
||||
|
||||
this.backendSrv.get('/api/alerts', params).then(result => {
|
||||
this.alerts = _.map(result, alert => {
|
||||
alert.iconCss = alertDef.getCssForState(alert.state);
|
||||
alert.severityClass = alertDef.getSeverityClass(alert.severity);
|
||||
alert.stateClass = alertDef.getStateClass(alert.state);
|
||||
return alert;
|
||||
});
|
||||
});
|
||||
|
@ -7,28 +7,29 @@
|
||||
</div>
|
||||
|
||||
<div class="gf-form-inline">
|
||||
<gf-form-switch class="gf-form" label="Ok" label-class="width-5" checked="ctrl.filter.ok" on-change="ctrl.updateFilter()"></gf-form-switch>
|
||||
<gf-form-switch class="gf-form" label="OK" label-class="width-5" checked="ctrl.filter.ok" on-change="ctrl.updateFilter()"></gf-form-switch>
|
||||
<gf-form-switch class="gf-form" label="Warn" label-class="width-5" checked="ctrl.filter.warn" on-change="ctrl.updateFilter()"></gf-form-switch>
|
||||
<gf-form-switch class="gf-form" label="Critical" label-class="width-5" checked="ctrl.filter.critical" on-change="ctrl.updateFilter()"></gf-form-switch>
|
||||
<gf-form-switch class="gf-form" label="Acknowleged" label-class="width-7" checked="ctrl.filter.acknowleged" on-change="ctrl.updateFilter()"></gf-form-switch>
|
||||
</div>
|
||||
|
||||
<table class="grafana-options-table">
|
||||
<thead>
|
||||
<th style="min-width: 200px"><strong>Name</strong></th>
|
||||
<th style="width: 1%">State</th>
|
||||
<th style="width: 1%">Severity</th>
|
||||
<th style="width: 1%"></th>
|
||||
</thead>
|
||||
<tr ng-repeat="alert in ctrl.alerts">
|
||||
<td>
|
||||
<a href="alerting/{{alert.id}}/states">
|
||||
<a href="dashboard/{{alert.dashboardUri}}?panelId={{alert.panelId}}&fullscreen&edit&editorTab=Alerting">
|
||||
{{alert.name}}
|
||||
</a>
|
||||
</td>
|
||||
<td class="text-center">
|
||||
<a href="alerting/{{alert.id}}/states">
|
||||
<i class="icon-gf {{alert.iconCss}}"></i>
|
||||
</a>
|
||||
{{alert.state}}
|
||||
</td>
|
||||
<td class="text-center">
|
||||
{{alert.severity}}
|
||||
</td>
|
||||
<td class="text-center">
|
||||
<a href="dashboard/{{alert.dashboardUri}}?panelId={{alert.panelId}}&fullscreen&edit&editorTab=Alerting" class="btn btn-inverse btn-small">
|
||||
|
@ -46,8 +46,8 @@ export class AlertTabCtrl {
|
||||
{text: '<', value: '<'},
|
||||
];
|
||||
severityLevels = [
|
||||
{text: 'Critical', value: 'CRITICAL'},
|
||||
{text: 'Warning', value: 'WARN'},
|
||||
{text: 'Critical', value: 'critical'},
|
||||
{text: 'Warning', value: 'warning'},
|
||||
];
|
||||
|
||||
/** @ngInject */
|
||||
|
@ -34,7 +34,7 @@
|
||||
}
|
||||
|
||||
.label-tag:hover {
|
||||
opacity: 0.85;
|
||||
opacity: 0.85;
|
||||
background-color: darken($purple, 10%);
|
||||
}
|
||||
|
||||
|
Reference in New Issue
Block a user