Alerting: Fetch receivers from remote Alertmanager (#76841)

* Alerting: fetch receivers from remote Alertmanager

* make linter happy

* change require.Eventually() timeout and tick

pkg/services/ngalert/api/api_alertmanager.go

@@ -298,7 +298,10 @@ func (srv AlertmanagerSrv) RouteGetReceivers(c *contextmodel.ReqContext) respons
 		return errResp
 	}
 
-	rcvs := am.GetReceivers(c.Req.Context())
+	rcvs, err := am.GetReceivers(c.Req.Context())
+	if err != nil {
+		return ErrResp(http.StatusInternalServerError, err, "failed to retrieve receivers")
+	}
 	return response.JSON(http.StatusOK, rcvs)
 }
 

pkg/services/ngalert/notifier/external_alertmanager.go

@@ -3,8 +3,10 @@ package notifier
 import (
 	"context"
 	"fmt"
+	"io"
 	"net/http"
 	"net/url"
+	"strings"
 
 	httptransport "github.com/go-openapi/runtime/client"
 	"github.com/go-openapi/strfmt"
@@ -15,6 +17,7 @@ import (
 	amclient "github.com/prometheus/alertmanager/api/v2/client"
 	amalert "github.com/prometheus/alertmanager/api/v2/client/alert"
 	amalertgroup "github.com/prometheus/alertmanager/api/v2/client/alertgroup"
+	amreceiver "github.com/prometheus/alertmanager/api/v2/client/receiver"
 	amsilence "github.com/prometheus/alertmanager/api/v2/client/silence"
 )
 
@@ -220,8 +223,18 @@ func (am *externalAlertmanager) GetStatus() apimodels.GettableStatus {
 	return apimodels.GettableStatus{}
 }
 
-func (am *externalAlertmanager) GetReceivers(ctx context.Context) []apimodels.Receiver {
-	return []apimodels.Receiver{}
+func (am *externalAlertmanager) GetReceivers(ctx context.Context) ([]apimodels.Receiver, error) {
+	params := amreceiver.NewGetReceiversParamsWithContext(ctx)
+	res, err := am.amClient.Receiver.GetReceivers(params)
+	if err != nil {
+		return []apimodels.Receiver{}, err
+	}
+
+	var rcvs []apimodels.Receiver
+	for _, rcv := range res.Payload {
+		rcvs = append(rcvs, *rcv)
+	}
+	return rcvs, nil
 }
 
 func (am *externalAlertmanager) TestReceivers(ctx context.Context, c apimodels.TestReceiversConfigBodyParams) (*TestReceiversResult, error) {
@@ -267,3 +280,37 @@ func (r *roundTripper) RoundTrip(req *http.Request) (*http.Response, error) {
 
 	return r.next.RoundTrip(req)
 }
+
+// TODO: change implementation, this is only useful for testing other methods.
+func (am *externalAlertmanager) postConfig(ctx context.Context, rawConfig string) error {
+	url := strings.TrimSuffix(am.url, "/alertmanager") + "/api/v1/alerts"
+	req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, strings.NewReader(rawConfig))
+	if err != nil {
+		return fmt.Errorf("error creating request: %v", err)
+	}
+
+	res, err := am.httpClient.Do(req)
+	if err != nil {
+		return err
+	}
+
+	if res.StatusCode == http.StatusNotFound {
+		return fmt.Errorf("config not found")
+	}
+
+	defer func() {
+		if err := res.Body.Close(); err != nil {
+			am.log.Warn("Error while closing body", "err", err)
+		}
+	}()
+
+	_, err = io.ReadAll(res.Body)
+	if err != nil {
+		return fmt.Errorf("error reading request response: %w", err)
+	}
+
+	if res.StatusCode != http.StatusCreated {
+		return fmt.Errorf("setting config failed with status code %d", res.StatusCode)
+	}
+	return nil
+}

pkg/services/ngalert/notifier/external_alertmanager_test.go

@@ -14,7 +14,12 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
-const validConfig = `{"template_files":{},"alertmanager_config":{"route":{"receiver":"grafana-default-email","group_by":["grafana_folder","alertname"]},"templates":null,"receivers":[{"name":"grafana-default-email","grafana_managed_receiver_configs":[{"uid":"","name":"some other name","type":"email","disableResolveMessage":false,"settings":{"addresses":"\u003cexample@email.com\u003e"},"secureSettings":null}]}]}}`
+const (
+	validConfig = `{"template_files":{},"alertmanager_config":{"route":{"receiver":"grafana-default-email","group_by":["grafana_folder","alertname"]},"templates":null,"receivers":[{"name":"grafana-default-email","grafana_managed_receiver_configs":[{"uid":"","name":"some other name","type":"email","disableResolveMessage":false,"settings":{"addresses":"\u003cexample@email.com\u003e"},"secureSettings":null}]}]}}`
+
+	// Valid config for Cloud AM, no `grafana_managed_receievers` field.
+	upstreamConfig = `{"template_files": {}, "alertmanager_config": "{\"global\": {\"smtp_from\": \"test@test.com\"}, \"route\": {\"receiver\": \"discord\"}, \"receivers\": [{\"name\": \"discord\", \"discord_configs\": [{\"webhook_url\": \"http://localhost:1234\"}]}]}"}`
+)
 
 func TestNewExternalAlertmanager(t *testing.T) {
 	tests := []struct {
@@ -223,6 +228,43 @@ func TestIntegrationRemoteAlertmanagerAlerts(t *testing.T) {
 	require.Equal(t, 1, len(alerts))
 }
 
+func TestIntegrationRemoteAlertmanagerReceivers(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test")
+	}
+
+	amURL, ok := os.LookupEnv("AM_URL")
+	if !ok {
+		t.Skip("No Alertmanager URL provided")
+	}
+
+	tenantID := os.Getenv("AM_TENANT_ID")
+	password := os.Getenv("AM_PASSWORD")
+
+	cfg := externalAlertmanagerConfig{
+		URL:               amURL + "/alertmanager",
+		TenantID:          tenantID,
+		BasicAuthPassword: password,
+		DefaultConfig:     validConfig,
+	}
+
+	am, err := newExternalAlertmanager(cfg, 1)
+	require.NoError(t, err)
+
+	// We should start with the default config.
+	rcvs, err := am.GetReceivers(context.Background())
+	require.NoError(t, err)
+	require.Equal(t, "empty-receiver", *rcvs[0].Name)
+
+	// After changing the configuration, we should have a new `discord` receiver.
+	require.NoError(t, am.postConfig(context.Background(), upstreamConfig))
+	require.Eventually(t, func() bool {
+		rcvs, err = am.GetReceivers(context.Background())
+		require.NoError(t, err)
+		return *rcvs[0].Name == "discord"
+	}, 16*time.Second, 1*time.Second)
+}
+
 func genSilence(createdBy string) apimodels.PostableSilence {
 	starts := strfmt.DateTime(time.Now().Add(time.Duration(rand.Int63n(9)+1) * time.Second))
 	ends := strfmt.DateTime(time.Now().Add(time.Duration(rand.Int63n(9)+10) * time.Second))

pkg/services/ngalert/notifier/multiorg_alertmanager.go

@@ -49,7 +49,7 @@ type Alertmanager interface {
 	PutAlerts(context.Context, apimodels.PostableAlerts) error
 
 	// Receivers
-	GetReceivers(ctx context.Context) []apimodels.Receiver
+	GetReceivers(ctx context.Context) ([]apimodels.Receiver, error)
 	TestReceivers(ctx context.Context, c apimodels.TestReceiversConfigBodyParams) (*TestReceiversResult, error)
 	TestTemplate(ctx context.Context, c apimodels.TestTemplatesConfigBodyParams) (*TestTemplatesResults, error)
 

pkg/services/ngalert/notifier/receivers.go

@@ -91,7 +91,7 @@ func (am *alertmanager) TestReceivers(ctx context.Context, c apimodels.TestRecei
 	}, err
 }
 
-func (am *alertmanager) GetReceivers(_ context.Context) []apimodels.Receiver {
+func (am *alertmanager) GetReceivers(_ context.Context) ([]apimodels.Receiver, error) {
 	apiReceivers := make([]apimodels.Receiver, 0, len(am.Base.GetReceivers()))
 	for _, rcv := range am.Base.GetReceivers() {
 		// Build integrations slice for each receiver.
@@ -123,5 +123,5 @@ func (am *alertmanager) GetReceivers(_ context.Context) []apimodels.Receiver {
 		})
 	}
 
-	return apiReceivers
+	return apiReceivers, nil
 }