opensource/grafana
1
0
Fork 0
mirror of https://github.com/grafana/grafana.git synced 2025-08-02 18:32:23 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(alerting): more work on handling result and saving state

Browse Source
This commit is contained in:
Torkel Ödegaard
2016-07-21 17:31:46 +02:00
parent f0fc336e88
commit b073fe0eba
6 changed files with 88 additions and 58 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
pkg/services/alerting/engine.go
View File

@ -71,23 +71,25 @@ func (e *Engine) alertingTicker() {
}
func (e *Engine) execDispatch() {
defer func() {
if err := recover(); err != nil {
e.log.Error("Scheduler Panic: stopping executor", "error", err, "stack", log.Stack(1))
}
}()
for job := range e.execQueue {
log.Trace("Alerting: engine:execDispatch() starting job %s", job.Rule.Name)
e.executeJob(job)
e.log.Debug("Starting executing alert rule %s", job.Rule.Name)
go e.executeJob(job)
}
}
func (e *Engine) executeJob(job *AlertJob) {
defer func() {
if err := recover(); err != nil {
e.log.Error("Execute Alert Panic", "error", err, "stack", log.Stack(1))
}
}()
job.Running = true
context := NewAlertResultContext(job.Rule)
e.handler.Execute(context)
job.Running = false
e.resultQueue <- context
}
func (e *Engine) resultHandler() {

1
pkg/services/alerting/handler.go
View File

@ -55,6 +55,7 @@ func (e *HandlerImpl) eval(context *AlertResultContext) {
}
context.EndTime = time.Now()
context.DoneChan <- true
}
// func (e *HandlerImpl) executeQuery(job *AlertJob) (tsdb.TimeSeriesSlice, error) {

3
pkg/services/alerting/models.go
View File

@ -4,6 +4,7 @@ import (
"time"
"github.com/grafana/grafana/pkg/components/simplejson"
"github.com/grafana/grafana/pkg/log"
)
type AlertJob struct {
@ -38,6 +39,7 @@ type AlertResultContext struct {
Rule *AlertRule
DoneChan chan bool
CancelChan chan bool
log log.Logger
}
func (a *AlertResultContext) GetDurationSeconds() float64 {
@ -51,6 +53,7 @@ func NewAlertResultContext(rule *AlertRule) *AlertResultContext {
Logs: make([]*AlertResultLogEntry, 0),
DoneChan: make(chan bool, 1),
CancelChan: make(chan bool, 1),
log: log.New("alerting.engine"),
}
}

93
pkg/services/alerting/result_handler.go
View File

@ -1,6 +1,14 @@
package alerting
import "github.com/grafana/grafana/pkg/log"
import (
"time"
"github.com/grafana/grafana/pkg/bus"
"github.com/grafana/grafana/pkg/components/simplejson"
"github.com/grafana/grafana/pkg/log"
m "github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/services/alerting/alertstates"
)
type ResultHandler interface {
Handle(result *AlertResultContext)
@ -19,44 +27,51 @@ func NewResultHandler() *ResultHandlerImpl {
}
func (handler *ResultHandlerImpl) Handle(result *AlertResultContext) {
// if handler.shouldUpdateState(result) {
// cmd := &m.UpdateAlertStateCommand{
// AlertId: result.Rule.Id,
// State: result.Rule.Severity,
// Info: result.Description,
// OrgId: result.Rule.OrgId,
// TriggeredAlerts: simplejson.NewFromAny(result.Details),
// }
//
// if err := bus.Dispatch(cmd); err != nil {
// handler.log.Error("Failed to save state", "error", err)
// }
//
// handler.log.Debug("will notify about new state", "new state", result.State)
// handler.notifier.Notify(result)
// }
newState := alertstates.Ok
if result.Triggered {
newState = result.Rule.Severity
}
handler.log.Info("Handle result", "newState", newState)
handler.log.Info("Handle result", "triggered", result.Triggered)
if handler.shouldUpdateState(result, newState) {
cmd := &m.UpdateAlertStateCommand{
AlertId: result.Rule.Id,
Info: result.Description,
OrgId: result.Rule.OrgId,
State: newState,
TriggeredAlerts: simplejson.NewFromAny(result.Details),
}
if err := bus.Dispatch(cmd); err != nil {
handler.log.Error("Failed to save state", "error", err)
}
//handler.log.Debug("will notify about new state", "new state", result.State)
//handler.notifier.Notify(result)
}
}
func (handler *ResultHandlerImpl) shouldUpdateState(result *AlertResultContext) bool {
// query := &m.GetLastAlertStateQuery{
// AlertId: result.AlertJob.Rule.Id,
// OrgId: result.AlertJob.Rule.OrgId,
// }
//
// if err := bus.Dispatch(query); err != nil {
// log.Error2("Failed to read last alert state", "error", err)
// return false
// }
//
// if query.Result == nil {
// return true
// }
//
// lastExecution := query.Result.Created
// asdf := result.StartTime.Add(time.Minute * -15)
// olderThen15Min := lastExecution.Before(asdf)
// changedState := query.Result.State != result.State
//
// return changedState || olderThen15Min
return false
func (handler *ResultHandlerImpl) shouldUpdateState(result *AlertResultContext, newState string) bool {
query := &m.GetLastAlertStateQuery{
AlertId: result.Rule.Id,
OrgId: result.Rule.OrgId,
}
if err := bus.Dispatch(query); err != nil {
log.Error2("Failed to read last alert state", "error", err)
return false
}
if query.Result == nil {
return true
}
lastExecution := query.Result.Created
asdf := result.StartTime.Add(time.Minute * -15)
olderThen15Min := lastExecution.Before(asdf)
changedState := query.Result.State != newState
return changedState || olderThen15Min
}

27
pkg/services/alerting/result_handler_test.go
View File

@ -1,15 +1,24 @@
package alerting
// import (
// "testing"
// "time"
//
// "github.com/grafana/grafana/pkg/bus"
// m "github.com/grafana/grafana/pkg/models"
// "github.com/grafana/grafana/pkg/services/alerting/alertstates"
//
// . "github.com/smartystreets/goconvey/convey"
// )
//
// func TestAlertResultHandler(t *testing.T) {
// Convey("Test result Handler", t, func() {
// resultHandler := ResultHandlerImpl{}
// mockResult := &AlertResult{
// State: alertstates.Ok,
// AlertJob: &AlertJob{
// Rule: &AlertRule{
// Id: 1,
// OrgId: 1,
// },
// mockResult := &AlertResultContext{
// Triggered: false,
// Rule: &AlertRule{
// Id: 1,
// OrgId 1,
// },
// }
// mockAlertState := &m.AlertState{}
@ -30,7 +39,7 @@ package alerting
// mockAlertState = &m.AlertState{
// State: alertstates.Critical,
// }
// mockResult.State = alertstates.Ok
// mockResult.Triggered = false
// So(resultHandler.shouldUpdateState(mockResult), ShouldBeTrue)
// })
//
@ -40,7 +49,7 @@ package alerting
// State: alertstates.Critical,
// Created: now.Add(time.Minute * -30),
// }
// mockResult.State = alertstates.Critical
// mockResult.Triggered = true
// mockResult.StartTime = time.Now()
// So(resultHandler.shouldUpdateState(mockResult), ShouldBeTrue)
// })

4
public/app/plugins/panel/graph/alert_tab_ctrl.ts
View File

@ -46,8 +46,8 @@ export class AlertTabCtrl {
{text: '<', value: '<'},
];
severityLevels = [
{text: 'Critical', value: 'critical'},
{text: 'Warning', value: 'warning'},
{text: 'Critical', value: 'CRITICAL'},
{text: 'Warning', value: 'WARN'},
];
/** @ngInject */