mirror of
https://github.com/grafana/grafana.git
synced 2025-09-28 11:53:59 +08:00
15e4556c2f
* require legacy Editor for post, put, delete endpoints * require user to be signed in on group level because handler that checks that user has role Editor does not check it is signed in
509 lines
17 KiB
Go
509 lines
17 KiB
Go
package api
|
|
|
|
import (
|
|
"context"
|
|
"encoding/base64"
|
|
"errors"
|
|
"fmt"
|
|
"net/http"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/grafana/grafana/pkg/api/response"
|
|
"github.com/grafana/grafana/pkg/infra/log"
|
|
"github.com/grafana/grafana/pkg/models"
|
|
apimodels "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
|
|
ngmodels "github.com/grafana/grafana/pkg/services/ngalert/models"
|
|
"github.com/grafana/grafana/pkg/services/ngalert/notifier"
|
|
"github.com/grafana/grafana/pkg/services/ngalert/store"
|
|
"github.com/grafana/grafana/pkg/services/secrets"
|
|
"github.com/grafana/grafana/pkg/util"
|
|
"github.com/grafana/grafana/pkg/web"
|
|
)
|
|
|
|
const (
|
|
defaultTestReceiversTimeout = 15 * time.Second
|
|
maxTestReceiversTimeout = 30 * time.Second
|
|
)
|
|
|
|
type AlertmanagerSrv struct {
|
|
mam *notifier.MultiOrgAlertmanager
|
|
secrets secrets.Service
|
|
store AlertingStore
|
|
log log.Logger
|
|
}
|
|
|
|
type UnknownReceiverError struct {
|
|
UID string
|
|
}
|
|
|
|
func (e UnknownReceiverError) Error() string {
|
|
return fmt.Sprintf("unknown receiver: %s", e.UID)
|
|
}
|
|
|
|
func (srv AlertmanagerSrv) loadSecureSettings(ctx context.Context, orgId int64, receivers []*apimodels.PostableApiReceiver) error {
|
|
// Get the last known working configuration
|
|
query := ngmodels.GetLatestAlertmanagerConfigurationQuery{OrgID: orgId}
|
|
if err := srv.store.GetLatestAlertmanagerConfiguration(ctx, &query); err != nil {
|
|
// If we don't have a configuration there's nothing for us to know and we should just continue saving the new one
|
|
if !errors.Is(err, store.ErrNoAlertmanagerConfiguration) {
|
|
return fmt.Errorf("failed to get latest configuration: %w", err)
|
|
}
|
|
}
|
|
|
|
currentReceiverMap := make(map[string]*apimodels.PostableGrafanaReceiver)
|
|
if query.Result != nil {
|
|
currentConfig, err := notifier.Load([]byte(query.Result.AlertmanagerConfiguration))
|
|
if err != nil {
|
|
return fmt.Errorf("failed to load latest configuration: %w", err)
|
|
}
|
|
currentReceiverMap = currentConfig.GetGrafanaReceiverMap()
|
|
}
|
|
|
|
// Copy the previously known secure settings
|
|
for i, r := range receivers {
|
|
for j, gr := range r.PostableGrafanaReceivers.GrafanaManagedReceivers {
|
|
if gr.UID == "" { // new receiver
|
|
continue
|
|
}
|
|
|
|
cgmr, ok := currentReceiverMap[gr.UID]
|
|
if !ok {
|
|
// it tries to update a receiver that didn't previously exist
|
|
return UnknownReceiverError{UID: gr.UID}
|
|
}
|
|
|
|
// frontend sends only the secure settings that have to be updated
|
|
// therefore we have to copy from the last configuration only those secure settings not included in the request
|
|
for key := range cgmr.SecureSettings {
|
|
_, ok := gr.SecureSettings[key]
|
|
if !ok {
|
|
decryptedValue, err := srv.getDecryptedSecret(cgmr, key)
|
|
if err != nil {
|
|
return fmt.Errorf("failed to decrypt stored secure setting: %s: %w", key, err)
|
|
}
|
|
|
|
if receivers[i].PostableGrafanaReceivers.GrafanaManagedReceivers[j].SecureSettings == nil {
|
|
receivers[i].PostableGrafanaReceivers.GrafanaManagedReceivers[j].SecureSettings = make(map[string]string, len(cgmr.SecureSettings))
|
|
}
|
|
|
|
receivers[i].PostableGrafanaReceivers.GrafanaManagedReceivers[j].SecureSettings[key] = decryptedValue
|
|
}
|
|
}
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (srv AlertmanagerSrv) getDecryptedSecret(r *apimodels.PostableGrafanaReceiver, key string) (string, error) {
|
|
storedValue, ok := r.SecureSettings[key]
|
|
if !ok {
|
|
return "", nil
|
|
}
|
|
|
|
decodeValue, err := base64.StdEncoding.DecodeString(storedValue)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
|
|
decryptedValue, err := srv.secrets.Decrypt(context.Background(), decodeValue)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
|
|
return string(decryptedValue), nil
|
|
}
|
|
|
|
func (srv AlertmanagerSrv) RouteGetAMStatus(c *models.ReqContext) response.Response {
|
|
am, errResp := srv.AlertmanagerFor(c.OrgId)
|
|
if errResp != nil {
|
|
return errResp
|
|
}
|
|
|
|
return response.JSON(http.StatusOK, am.GetStatus())
|
|
}
|
|
|
|
func (srv AlertmanagerSrv) RouteCreateSilence(c *models.ReqContext, postableSilence apimodels.PostableSilence) response.Response {
|
|
am, errResp := srv.AlertmanagerFor(c.OrgId)
|
|
if errResp != nil {
|
|
return errResp
|
|
}
|
|
|
|
silenceID, err := am.CreateSilence(&postableSilence)
|
|
if err != nil {
|
|
if errors.Is(err, notifier.ErrSilenceNotFound) {
|
|
return ErrResp(http.StatusNotFound, err, "")
|
|
}
|
|
|
|
if errors.Is(err, notifier.ErrCreateSilenceBadPayload) {
|
|
return ErrResp(http.StatusBadRequest, err, "")
|
|
}
|
|
|
|
return ErrResp(http.StatusInternalServerError, err, "failed to create silence")
|
|
}
|
|
return response.JSON(http.StatusAccepted, util.DynMap{"message": "silence created", "id": silenceID})
|
|
}
|
|
|
|
func (srv AlertmanagerSrv) RouteDeleteAlertingConfig(c *models.ReqContext) response.Response {
|
|
am, errResp := srv.AlertmanagerFor(c.OrgId)
|
|
if errResp != nil {
|
|
return errResp
|
|
}
|
|
|
|
if err := am.SaveAndApplyDefaultConfig(c.Req.Context()); err != nil {
|
|
srv.log.Error("unable to save and apply default alertmanager configuration", "err", err)
|
|
return ErrResp(http.StatusInternalServerError, err, "failed to save and apply default Alertmanager configuration")
|
|
}
|
|
|
|
return response.JSON(http.StatusAccepted, util.DynMap{"message": "configuration deleted; the default is applied"})
|
|
}
|
|
|
|
func (srv AlertmanagerSrv) RouteDeleteSilence(c *models.ReqContext) response.Response {
|
|
am, errResp := srv.AlertmanagerFor(c.OrgId)
|
|
if errResp != nil {
|
|
return errResp
|
|
}
|
|
|
|
silenceID := web.Params(c.Req)[":SilenceId"]
|
|
if err := am.DeleteSilence(silenceID); err != nil {
|
|
if errors.Is(err, notifier.ErrSilenceNotFound) {
|
|
return ErrResp(http.StatusNotFound, err, "")
|
|
}
|
|
return ErrResp(http.StatusInternalServerError, err, "")
|
|
}
|
|
return response.JSON(http.StatusOK, util.DynMap{"message": "silence deleted"})
|
|
}
|
|
|
|
func (srv AlertmanagerSrv) RouteGetAlertingConfig(c *models.ReqContext) response.Response {
|
|
query := ngmodels.GetLatestAlertmanagerConfigurationQuery{OrgID: c.OrgId}
|
|
if err := srv.store.GetLatestAlertmanagerConfiguration(c.Req.Context(), &query); err != nil {
|
|
if errors.Is(err, store.ErrNoAlertmanagerConfiguration) {
|
|
return ErrResp(http.StatusNotFound, err, "")
|
|
}
|
|
return ErrResp(http.StatusInternalServerError, err, "failed to get latest configuration")
|
|
}
|
|
|
|
cfg, err := notifier.Load([]byte(query.Result.AlertmanagerConfiguration))
|
|
if err != nil {
|
|
return ErrResp(http.StatusInternalServerError, err, "failed to unmarshal alertmanager configuration")
|
|
}
|
|
|
|
result := apimodels.GettableUserConfig{
|
|
TemplateFiles: cfg.TemplateFiles,
|
|
AlertmanagerConfig: apimodels.GettableApiAlertingConfig{
|
|
Config: cfg.AlertmanagerConfig.Config,
|
|
},
|
|
}
|
|
for _, recv := range cfg.AlertmanagerConfig.Receivers {
|
|
receivers := make([]*apimodels.GettableGrafanaReceiver, 0, len(recv.PostableGrafanaReceivers.GrafanaManagedReceivers))
|
|
for _, pr := range recv.PostableGrafanaReceivers.GrafanaManagedReceivers {
|
|
secureFields := make(map[string]bool, len(pr.SecureSettings))
|
|
for k := range pr.SecureSettings {
|
|
decryptedValue, err := srv.getDecryptedSecret(pr, k)
|
|
if err != nil {
|
|
return ErrResp(http.StatusInternalServerError, err, "failed to decrypt stored secure setting: %s", k)
|
|
}
|
|
if decryptedValue == "" {
|
|
continue
|
|
}
|
|
secureFields[k] = true
|
|
}
|
|
gr := apimodels.GettableGrafanaReceiver{
|
|
UID: pr.UID,
|
|
Name: pr.Name,
|
|
Type: pr.Type,
|
|
DisableResolveMessage: pr.DisableResolveMessage,
|
|
Settings: pr.Settings,
|
|
SecureFields: secureFields,
|
|
}
|
|
receivers = append(receivers, &gr)
|
|
}
|
|
gettableApiReceiver := apimodels.GettableApiReceiver{
|
|
GettableGrafanaReceivers: apimodels.GettableGrafanaReceivers{
|
|
GrafanaManagedReceivers: receivers,
|
|
},
|
|
}
|
|
gettableApiReceiver.Name = recv.Name
|
|
result.AlertmanagerConfig.Receivers = append(result.AlertmanagerConfig.Receivers, &gettableApiReceiver)
|
|
}
|
|
|
|
return response.JSON(http.StatusOK, result)
|
|
}
|
|
|
|
func (srv AlertmanagerSrv) RouteGetAMAlertGroups(c *models.ReqContext) response.Response {
|
|
am, errResp := srv.AlertmanagerFor(c.OrgId)
|
|
if errResp != nil {
|
|
return errResp
|
|
}
|
|
|
|
groups, err := am.GetAlertGroups(
|
|
c.QueryBoolWithDefault("active", true),
|
|
c.QueryBoolWithDefault("silenced", true),
|
|
c.QueryBoolWithDefault("inhibited", true),
|
|
c.QueryStrings("filter"),
|
|
c.Query("receiver"),
|
|
)
|
|
if err != nil {
|
|
if errors.Is(err, notifier.ErrGetAlertGroupsBadPayload) {
|
|
return ErrResp(http.StatusBadRequest, err, "")
|
|
}
|
|
// any other error here should be an unexpected failure and thus an internal error
|
|
return ErrResp(http.StatusInternalServerError, err, "")
|
|
}
|
|
|
|
return response.JSON(http.StatusOK, groups)
|
|
}
|
|
|
|
func (srv AlertmanagerSrv) RouteGetAMAlerts(c *models.ReqContext) response.Response {
|
|
am, errResp := srv.AlertmanagerFor(c.OrgId)
|
|
if errResp != nil {
|
|
return errResp
|
|
}
|
|
|
|
alerts, err := am.GetAlerts(
|
|
c.QueryBoolWithDefault("active", true),
|
|
c.QueryBoolWithDefault("silenced", true),
|
|
c.QueryBoolWithDefault("inhibited", true),
|
|
c.QueryStrings("filter"),
|
|
c.Query("receiver"),
|
|
)
|
|
if err != nil {
|
|
if errors.Is(err, notifier.ErrGetAlertsBadPayload) {
|
|
return ErrResp(http.StatusBadRequest, err, "")
|
|
}
|
|
if errors.Is(err, notifier.ErrGetAlertsUnavailable) {
|
|
return ErrResp(http.StatusServiceUnavailable, err, "")
|
|
}
|
|
// any other error here should be an unexpected failure and thus an internal error
|
|
return ErrResp(http.StatusInternalServerError, err, "")
|
|
}
|
|
|
|
return response.JSON(http.StatusOK, alerts)
|
|
}
|
|
|
|
func (srv AlertmanagerSrv) RouteGetSilence(c *models.ReqContext) response.Response {
|
|
am, errResp := srv.AlertmanagerFor(c.OrgId)
|
|
if errResp != nil {
|
|
return errResp
|
|
}
|
|
|
|
silenceID := web.Params(c.Req)[":SilenceId"]
|
|
gettableSilence, err := am.GetSilence(silenceID)
|
|
if err != nil {
|
|
if errors.Is(err, notifier.ErrSilenceNotFound) {
|
|
return ErrResp(http.StatusNotFound, err, "")
|
|
}
|
|
// any other error here should be an unexpected failure and thus an internal error
|
|
return ErrResp(http.StatusInternalServerError, err, "")
|
|
}
|
|
return response.JSON(http.StatusOK, gettableSilence)
|
|
}
|
|
|
|
func (srv AlertmanagerSrv) RouteGetSilences(c *models.ReqContext) response.Response {
|
|
am, errResp := srv.AlertmanagerFor(c.OrgId)
|
|
if errResp != nil {
|
|
return errResp
|
|
}
|
|
|
|
gettableSilences, err := am.ListSilences(c.QueryStrings("filter"))
|
|
if err != nil {
|
|
if errors.Is(err, notifier.ErrListSilencesBadPayload) {
|
|
return ErrResp(http.StatusBadRequest, err, "")
|
|
}
|
|
// any other error here should be an unexpected failure and thus an internal error
|
|
return ErrResp(http.StatusInternalServerError, err, "")
|
|
}
|
|
return response.JSON(http.StatusOK, gettableSilences)
|
|
}
|
|
|
|
func (srv AlertmanagerSrv) RoutePostAlertingConfig(c *models.ReqContext, body apimodels.PostableUserConfig) response.Response {
|
|
// Get the last known working configuration
|
|
query := ngmodels.GetLatestAlertmanagerConfigurationQuery{OrgID: c.OrgId}
|
|
if err := srv.store.GetLatestAlertmanagerConfiguration(c.Req.Context(), &query); err != nil {
|
|
// If we don't have a configuration there's nothing for us to know and we should just continue saving the new one
|
|
if !errors.Is(err, store.ErrNoAlertmanagerConfiguration) {
|
|
return ErrResp(http.StatusInternalServerError, err, "failed to get latest configuration")
|
|
}
|
|
}
|
|
|
|
if err := srv.loadSecureSettings(c.Req.Context(), c.OrgId, body.AlertmanagerConfig.Receivers); err != nil {
|
|
var unknownReceiverError UnknownReceiverError
|
|
if errors.As(err, &unknownReceiverError) {
|
|
return ErrResp(http.StatusBadRequest, err, "")
|
|
}
|
|
return ErrResp(http.StatusInternalServerError, err, "")
|
|
}
|
|
|
|
if err := body.ProcessConfig(srv.secrets.Encrypt); err != nil {
|
|
return ErrResp(http.StatusInternalServerError, err, "failed to post process Alertmanager configuration")
|
|
}
|
|
|
|
am, errResp := srv.AlertmanagerFor(c.OrgId)
|
|
if errResp != nil {
|
|
// It's okay if the alertmanager isn't ready yet, we're changing its config anyway.
|
|
if !errors.Is(errResp.Err(), notifier.ErrAlertmanagerNotReady) {
|
|
return errResp
|
|
}
|
|
}
|
|
|
|
if err := am.SaveAndApplyConfig(c.Req.Context(), &body); err != nil {
|
|
srv.log.Error("unable to save and apply alertmanager configuration", "err", err)
|
|
return ErrResp(http.StatusBadRequest, err, "failed to save and apply Alertmanager configuration")
|
|
}
|
|
|
|
return response.JSON(http.StatusAccepted, util.DynMap{"message": "configuration created"})
|
|
}
|
|
|
|
func (srv AlertmanagerSrv) RoutePostAMAlerts(_ *models.ReqContext, _ apimodels.PostableAlerts) response.Response {
|
|
return NotImplementedResp
|
|
}
|
|
|
|
func (srv AlertmanagerSrv) RoutePostTestReceivers(c *models.ReqContext, body apimodels.TestReceiversConfigBodyParams) response.Response {
|
|
if err := srv.loadSecureSettings(c.Req.Context(), c.OrgId, body.Receivers); err != nil {
|
|
var unknownReceiverError UnknownReceiverError
|
|
if errors.As(err, &unknownReceiverError) {
|
|
return ErrResp(http.StatusBadRequest, err, "")
|
|
}
|
|
return ErrResp(http.StatusInternalServerError, err, "")
|
|
}
|
|
|
|
if err := body.ProcessConfig(srv.secrets.Encrypt); err != nil {
|
|
return ErrResp(http.StatusInternalServerError, err, "failed to post process Alertmanager configuration")
|
|
}
|
|
|
|
ctx, cancelFunc, err := contextWithTimeoutFromRequest(
|
|
c.Req.Context(),
|
|
c.Req,
|
|
defaultTestReceiversTimeout,
|
|
maxTestReceiversTimeout)
|
|
if err != nil {
|
|
return ErrResp(http.StatusBadRequest, err, "")
|
|
}
|
|
defer cancelFunc()
|
|
|
|
am, errResp := srv.AlertmanagerFor(c.OrgId)
|
|
if errResp != nil {
|
|
return errResp
|
|
}
|
|
|
|
result, err := am.TestReceivers(ctx, body)
|
|
if err != nil {
|
|
if errors.Is(err, notifier.ErrNoReceivers) {
|
|
return response.Error(http.StatusBadRequest, "", err)
|
|
}
|
|
return response.Error(http.StatusInternalServerError, "", err)
|
|
}
|
|
|
|
return response.JSON(statusForTestReceivers(result.Receivers), newTestReceiversResult(result))
|
|
}
|
|
|
|
// contextWithTimeoutFromRequest returns a context with a deadline set from the
|
|
// Request-Timeout header in the HTTP request. If the header is absent then the
|
|
// context will use the default timeout. The timeout in the Request-Timeout
|
|
// header cannot exceed the maximum timeout.
|
|
func contextWithTimeoutFromRequest(ctx context.Context, r *http.Request, defaultTimeout, maxTimeout time.Duration) (context.Context, context.CancelFunc, error) {
|
|
timeout := defaultTimeout
|
|
if s := strings.TrimSpace(r.Header.Get("Request-Timeout")); s != "" {
|
|
// the timeout is measured in seconds
|
|
v, err := strconv.ParseInt(s, 10, 16)
|
|
if err != nil {
|
|
return nil, nil, err
|
|
}
|
|
if d := time.Duration(v) * time.Second; d < maxTimeout {
|
|
timeout = d
|
|
} else {
|
|
return nil, nil, fmt.Errorf("exceeded maximum timeout of %d seconds", maxTimeout)
|
|
}
|
|
}
|
|
ctx, cancelFunc := context.WithTimeout(ctx, timeout)
|
|
return ctx, cancelFunc, nil
|
|
}
|
|
|
|
func newTestReceiversResult(r *notifier.TestReceiversResult) apimodels.TestReceiversResult {
|
|
v := apimodels.TestReceiversResult{
|
|
Alert: apimodels.TestReceiversConfigAlertParams{
|
|
Annotations: r.Alert.Annotations,
|
|
Labels: r.Alert.Labels,
|
|
},
|
|
Receivers: make([]apimodels.TestReceiverResult, len(r.Receivers)),
|
|
NotifiedAt: r.NotifedAt,
|
|
}
|
|
for ix, next := range r.Receivers {
|
|
configs := make([]apimodels.TestReceiverConfigResult, len(next.Configs))
|
|
for jx, config := range next.Configs {
|
|
configs[jx].Name = config.Name
|
|
configs[jx].UID = config.UID
|
|
configs[jx].Status = config.Status
|
|
if config.Error != nil {
|
|
configs[jx].Error = config.Error.Error()
|
|
}
|
|
}
|
|
v.Receivers[ix].Configs = configs
|
|
v.Receivers[ix].Name = next.Name
|
|
}
|
|
return v
|
|
}
|
|
|
|
// statusForTestReceivers returns the appropriate status code for the response
|
|
// for the results.
|
|
//
|
|
// It returns an HTTP 200 OK status code if notifications were sent to all receivers,
|
|
// an HTTP 400 Bad Request status code if all receivers contain invalid configuration,
|
|
// an HTTP 408 Request Timeout status code if all receivers timed out when sending
|
|
// a test notification or an HTTP 207 Multi Status.
|
|
func statusForTestReceivers(v []notifier.TestReceiverResult) int {
|
|
var (
|
|
numBadRequests int
|
|
numTimeouts int
|
|
numUnknownErrors int
|
|
)
|
|
for _, receiver := range v {
|
|
for _, next := range receiver.Configs {
|
|
if next.Error != nil {
|
|
var (
|
|
invalidReceiverErr notifier.InvalidReceiverError
|
|
receiverTimeoutErr notifier.ReceiverTimeoutError
|
|
)
|
|
if errors.As(next.Error, &invalidReceiverErr) {
|
|
numBadRequests += 1
|
|
} else if errors.As(next.Error, &receiverTimeoutErr) {
|
|
numTimeouts += 1
|
|
} else {
|
|
numUnknownErrors += 1
|
|
}
|
|
}
|
|
}
|
|
}
|
|
if numBadRequests == len(v) {
|
|
// if all receivers contain invalid configuration
|
|
return http.StatusBadRequest
|
|
} else if numTimeouts == len(v) {
|
|
// if all receivers contain valid configuration but timed out
|
|
return http.StatusRequestTimeout
|
|
} else if numBadRequests+numTimeouts+numUnknownErrors > 0 {
|
|
return http.StatusMultiStatus
|
|
} else {
|
|
// all receivers were sent a notification without error
|
|
return http.StatusOK
|
|
}
|
|
}
|
|
|
|
func (srv AlertmanagerSrv) AlertmanagerFor(orgID int64) (Alertmanager, *response.NormalResponse) {
|
|
am, err := srv.mam.AlertmanagerFor(orgID)
|
|
if err == nil {
|
|
return am, nil
|
|
}
|
|
|
|
if errors.Is(err, notifier.ErrNoAlertmanagerForOrg) {
|
|
return nil, response.Error(http.StatusNotFound, err.Error(), err)
|
|
}
|
|
|
|
if errors.Is(err, notifier.ErrAlertmanagerNotReady) {
|
|
return am, response.Error(http.StatusConflict, err.Error(), err)
|
|
}
|
|
|
|
srv.log.Error("unable to obtain the org's Alertmanager", "err", err)
|
|
return nil, response.Error(http.StatusInternalServerError, "unable to obtain org's Alertmanager", err)
|
|
}
|