opensource/grafana
1
0
Fork 0
mirror of https://github.com/grafana/grafana.git synced 2025-08-03 06:12:20 +08:00
Code Issues Packages Projects Releases Wiki Activity

Alerting: send alerts to external, internal, or both alertmanagers (#40341)

Browse Source 
* (WIP) send alerts to external, internal, or both alertmanagers

* Modify admin configuration endpoint, update swagger docs

* Integration test for admin config updated

* Code review changes

* Fix alertmanagers choice not changing bug, add unit test

* Add AlertmanagersChoice as enum in swagger, code review changes

* Fix API and tests errors

* Change enum from int to string, use 'SendAlertsTo' instead of 'AlertmanagerChoice' where necessary

* Fix tests to reflect last changes

* Keep senders running when alerts are handled just internally

* Check if any external AM has been discovered before sending alerts, update tests

* remove duplicate data from logs

* update comment

* represent alertmanagers choice as an int instead of a string

* default alertmanagers choice to all alertmanagers, test cases

* update definitions and generate spec
This commit is contained in:
Santiago
2022-02-01 20:36:55 -03:00
committed by GitHub
parent 21796c61ba
commit 04d93751b8
10 changed files with 388 additions and 84 deletions
Download Patch File Download Diff File Expand all files Collapse all files

46
pkg/tests/api/alerting/api_admin_configuration_test.go
View File

@ -70,10 +70,47 @@ func TestAdminConfiguration_SendingToExternalAlertmanagers(t *testing.T) {
require.JSONEq(t, string(b), "{\"message\": \"no admin configuration available\",\"error\": \"no admin configuration available\"}")
}
// Now, lets re-set external Alertmanagers for main organisation.
// An invalid alertmanager choice should return an error.
{
ac := apimodels.PostableNGalertConfig{
Alertmanagers: []string{fakeAM1.URL(), fakeAM2.URL()},
AlertmanagersChoice: apimodels.AlertmanagersChoice("invalid"),
}
buf := bytes.Buffer{}
enc := json.NewEncoder(&buf)
err := enc.Encode(&ac)
require.NoError(t, err)
alertsURL := fmt.Sprintf("http://grafana:password@%s/api/v1/ngalert/admin_config", grafanaListedAddr)
resp := postRequest(t, alertsURL, buf.String(), http.StatusBadRequest) // nolint
b, err := ioutil.ReadAll(resp.Body)
require.NoError(t, err)
require.JSONEq(t, string(b), "{\"message\": \"Invalid alertmanager choice specified\"}")
}
// Let's try to send all the alerts to an external Alertmanager
// but never specify any. This should return an error.
{
ac := apimodels.PostableNGalertConfig{
AlertmanagersChoice: apimodels.AlertmanagersChoice(ngmodels.ExternalAlertmanagers.String()),
}
buf := bytes.Buffer{}
enc := json.NewEncoder(&buf)
err := enc.Encode(&ac)
require.NoError(t, err)
alertsURL := fmt.Sprintf("http://grafana:password@%s/api/v1/ngalert/admin_config", grafanaListedAddr)
resp := postRequest(t, alertsURL, buf.String(), http.StatusBadRequest) // nolint
b, err := ioutil.ReadAll(resp.Body)
require.NoError(t, err)
require.JSONEq(t, string(b), "{\"message\": \"At least one Alertmanager must be provided to choose this option\"}")
}
// Now, lets re-set external Alertmanagers for main organisation
// and make it so that only the external Alertmanagers handle the alerts.
{
ac := apimodels.PostableNGalertConfig{
Alertmanagers: []string{fakeAM1.URL(), fakeAM2.URL()},
AlertmanagersChoice: apimodels.AlertmanagersChoice(ngmodels.ExternalAlertmanagers.String()),
}
buf := bytes.Buffer{}
enc := json.NewEncoder(&buf)
@ -93,7 +130,7 @@ func TestAdminConfiguration_SendingToExternalAlertmanagers(t *testing.T) {
resp := getRequest(t, alertsURL, http.StatusOK) // nolint
b, err := ioutil.ReadAll(resp.Body)
require.NoError(t, err)
require.JSONEq(t, string(b), fmt.Sprintf("{\"alertmanagers\":[\"%s\",\"%s\"]}\n", fakeAM1.URL(), fakeAM2.URL()))
require.JSONEq(t, string(b), fmt.Sprintf("{\"alertmanagers\":[\"%s\",\"%s\"], \"alertmanagersChoice\": %q}\n", fakeAM1.URL(), fakeAM2.URL(), ngmodels.ExternalAlertmanagers))
}
// With the configuration set, we should eventually discover those Alertmanagers.
@ -170,6 +207,7 @@ func TestAdminConfiguration_SendingToExternalAlertmanagers(t *testing.T) {
}
// Now, lets re-set external Alertmanagers for the other organisation.
// Sending an empty value for AlertmanagersChoice should default to AllAlertmanagers.
{
ac := apimodels.PostableNGalertConfig{
Alertmanagers: []string{fakeAM3.URL()},
@ -192,7 +230,7 @@ func TestAdminConfiguration_SendingToExternalAlertmanagers(t *testing.T) {
resp := getRequest(t, alertsURL, http.StatusOK) // nolint
b, err := ioutil.ReadAll(resp.Body)
require.NoError(t, err)
require.JSONEq(t, string(b), fmt.Sprintf("{\"alertmanagers\":[\"%s\"]}\n", fakeAM3.URL()))
require.JSONEq(t, string(b), fmt.Sprintf("{\"alertmanagers\":[\"%s\"], \"alertmanagersChoice\": %q}\n", fakeAM3.URL(), ngmodels.AllAlertmanagers))
}
// With the configuration set, we should eventually not discover Alertmanagers.