Alerting: take datasources as external alertmanagers into consideration (#52534)

pkg/services/datasources/fakes/fake_datasource_service.go

@@ -49,6 +49,9 @@ func (s *FakeDataSourceService) GetAllDataSources(ctx context.Context, query *da
 
 func (s *FakeDataSourceService) GetDataSourcesByType(ctx context.Context, query *datasources.GetDataSourcesByTypeQuery) error {
 	for _, datasource := range s.DataSources {
+		if query.OrgId > 0 && datasource.OrgId != query.OrgId {
+			continue
+		}
 		typeMatch := query.Type != "" && query.Type == datasource.Type
 		if typeMatch {
 			query.Result = append(query.Result, datasource)

pkg/services/datasources/models.go

@@ -165,6 +165,7 @@ type GetAllDataSourcesQuery struct {
 }
 
 type GetDataSourcesByTypeQuery struct {
+	OrgId  int64 // optional: filter by org_id
 	Type   string
 	Result []*DataSource
 }

pkg/services/ngalert/api/api.go

@@ -62,6 +62,7 @@ type AlertingStore interface {
 type API struct {
 	Cfg                  *setting.Cfg
 	DatasourceCache      datasources.CacheService
+	DatasourceService    datasources.DataSourceService
 	RouteRegister        routing.RouteRegister
 	ExpressionService    *expr.Service
 	QuotaService         quota.Service
@@ -130,6 +131,7 @@ func (api *API) RegisterAPIEndpoints(m *metrics.API) {
 		}), m)
 	api.RegisterConfigurationApiEndpoints(NewConfiguration(
 		&ConfigSrv{
+			datasourceService:    api.DatasourceService,
 			store:                api.AdminConfigStore,
 			log:                  logger,
 			alertmanagerProvider: api.AlertsRouter,

pkg/services/ngalert/api/api_configuration.go

@@ -1,12 +1,15 @@
 package api
 
 import (
+	"context"
 	"errors"
+	"fmt"
 	"net/http"
 
 	"github.com/grafana/grafana/pkg/api/response"
 	"github.com/grafana/grafana/pkg/infra/log"
 	"github.com/grafana/grafana/pkg/models"
+	"github.com/grafana/grafana/pkg/services/datasources"
 	apimodels "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
 	ngmodels "github.com/grafana/grafana/pkg/services/ngalert/models"
 	"github.com/grafana/grafana/pkg/services/ngalert/store"
@@ -16,6 +19,7 @@ import (
 )
 
 type ConfigSrv struct {
+	datasourceService    datasources.DataSourceService
 	alertmanagerProvider ExternalAlertmanagerProvider
 	store                store.AdminConfigurationStore
 	log                  log.Logger
@@ -68,11 +72,17 @@ func (srv ConfigSrv) RoutePostNGalertConfig(c *models.ReqContext, body apimodels
 
 	sendAlertsTo, err := ngmodels.StringToAlertmanagersChoice(string(body.AlertmanagersChoice))
 	if err != nil {
-		return response.Error(400, "Invalid alertmanager choice specified", nil)
+		return response.Error(400, "Invalid alertmanager choice specified", err)
 	}
 
-	if sendAlertsTo == ngmodels.ExternalAlertmanagers && len(body.Alertmanagers) == 0 {
+	externalAlertmanagers, err := srv.externalAlertmanagers(c.Req.Context(), c.OrgId)
-		return response.Error(400, "At least one Alertmanager must be provided to choose this option", nil)
+	if err != nil {
+		return response.Error(500, "Couldn't fetch the external Alertmanagers from datasources", err)
+	}
+
+	if sendAlertsTo == ngmodels.ExternalAlertmanagers &&
+		len(body.Alertmanagers)+len(externalAlertmanagers) < 1 {
+		return response.Error(400, "At least one Alertmanager must be provided or configured as a datasource that handles alerts to choose this option", nil)
 	}
 
 	cfg := &ngmodels.AdminConfiguration{
@@ -110,3 +120,25 @@ func (srv ConfigSrv) RouteDeleteNGalertConfig(c *models.ReqContext) response.Res
 
 	return response.JSON(http.StatusOK, util.DynMap{"message": "admin configuration deleted"})
 }
+
+// externalAlertmanagers returns the URL of any external alertmanager that is
+// configured as datasource. The URL does not contain any auth.
+func (srv ConfigSrv) externalAlertmanagers(ctx context.Context, orgID int64) ([]string, error) {
+	var alertmanagers []string
+	query := &datasources.GetDataSourcesByTypeQuery{
+		OrgId: orgID,
+		Type:  datasources.DS_ALERTMANAGER,
+	}
+	err := srv.datasourceService.GetDataSourcesByType(ctx, query)
+	if err != nil {
+		return nil, fmt.Errorf("failed to fetch datasources for org: %w", err)
+	}
+	for _, ds := range query.Result {
+		if ds.JsonData.Get(apimodels.HandleGrafanaManagedAlerts).MustBool(false) {
+			// we don't need to build the exact URL as we only need
+			// to know if any is set
+			alertmanagers = append(alertmanagers, ds.Uid)
+		}
+	}
+	return alertmanagers, nil
+}

pkg/services/ngalert/api/api_configuration_test.go

@@ -0,0 +1,117 @@
+package api
+
+import (
+	"encoding/json"
+	"net/http"
+	"testing"
+
+	"github.com/grafana/grafana/pkg/components/simplejson"
+	"github.com/grafana/grafana/pkg/models"
+	"github.com/grafana/grafana/pkg/services/datasources"
+	fakeDatasources "github.com/grafana/grafana/pkg/services/datasources/fakes"
+	"github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
+	"github.com/grafana/grafana/pkg/services/ngalert/store"
+	"github.com/stretchr/testify/require"
+)
+
+func TestExternalAlertmanagerChoice(t *testing.T) {
+	tests := []struct {
+		name               string
+		alertmanagerChoice definitions.AlertmanagersChoice
+		alertmanagers      []string
+		datasources        []*datasources.DataSource
+		statusCode         int
+		message            string
+	}{
+		{
+			name:               "setting the choice to external by passing a plain url should succeed",
+			alertmanagerChoice: definitions.ExternalAlertmanagers,
+			alertmanagers:      []string{"http://localhost:9000"},
+			datasources:        []*datasources.DataSource{},
+			statusCode:         http.StatusCreated,
+			message:            "admin configuration updated",
+		},
+		{
+			name:               "setting the choice to external by having a enabled external am datasource should succeed",
+			alertmanagerChoice: definitions.ExternalAlertmanagers,
+			alertmanagers:      []string{},
+			datasources: []*datasources.DataSource{
+				{
+					OrgId: 1,
+					Type:  datasources.DS_ALERTMANAGER,
+					Url:   "http://localhost:9000",
+					JsonData: simplejson.NewFromAny(map[string]interface{}{
+						definitions.HandleGrafanaManagedAlerts: true,
+					}),
+				},
+			},
+			statusCode: http.StatusCreated,
+			message:    "admin configuration updated",
+		},
+		{
+			name:               "setting the choice to external by having a disabled external am datasource should fail",
+			alertmanagerChoice: definitions.ExternalAlertmanagers,
+			alertmanagers:      []string{},
+			datasources: []*datasources.DataSource{
+				{
+					OrgId:    1,
+					Type:     datasources.DS_ALERTMANAGER,
+					Url:      "http://localhost:9000",
+					JsonData: simplejson.NewFromAny(map[string]interface{}{}),
+				},
+			},
+			statusCode: http.StatusBadRequest,
+			message:    "At least one Alertmanager must be provided or configured as a datasource that handles alerts to choose this option",
+		},
+		{
+			name:               "setting the choice to external and having no am configured should fail",
+			alertmanagerChoice: definitions.ExternalAlertmanagers,
+			alertmanagers:      []string{},
+			datasources:        []*datasources.DataSource{},
+			statusCode:         http.StatusBadRequest,
+			message:            "At least one Alertmanager must be provided or configured as a datasource that handles alerts to choose this option",
+		},
+		{
+			name:               "setting the choice to all and having no external am configured should succeed",
+			alertmanagerChoice: definitions.AllAlertmanagers,
+			alertmanagers:      []string{},
+			datasources:        []*datasources.DataSource{},
+			statusCode:         http.StatusCreated,
+			message:            "admin configuration updated",
+		},
+		{
+			name:               "setting the choice to internal should always succeed",
+			alertmanagerChoice: definitions.InternalAlertmanager,
+			alertmanagers:      []string{},
+			datasources:        []*datasources.DataSource{},
+			statusCode:         http.StatusCreated,
+			message:            "admin configuration updated",
+		},
+	}
+	ctx := createRequestCtxInOrg(1)
+	ctx.OrgRole = models.ROLE_ADMIN
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			sut := createAPIAdminSut(t, test.datasources)
+			resp := sut.RoutePostNGalertConfig(ctx, definitions.PostableNGalertConfig{
+				Alertmanagers:       test.alertmanagers,
+				AlertmanagersChoice: test.alertmanagerChoice,
+			})
+			var res map[string]interface{}
+			err := json.Unmarshal(resp.Body(), &res)
+			require.NoError(t, err)
+			require.Equal(t, test.message, res["message"])
+			require.Equal(t, test.statusCode, resp.Status())
+		})
+	}
+}
+
+func createAPIAdminSut(t *testing.T,
+	datasources []*datasources.DataSource) ConfigSrv {
+	return ConfigSrv{
+		datasourceService: &fakeDatasources.FakeDataSourceService{
+			DataSources: datasources,
+		},
+		store: store.NewFakeAdminConfigStore(t),
+	}
+}

pkg/services/ngalert/api/tooling/definitions/admin.go

@@ -58,9 +58,10 @@ type NGalertConfig struct {
 type AlertmanagersChoice string
 
 const (
-	AllAlertmanagers      AlertmanagersChoice = "all"
+	AllAlertmanagers           AlertmanagersChoice = "all"
-	InternalAlertmanager  AlertmanagersChoice = "internal"
+	InternalAlertmanager       AlertmanagersChoice = "internal"
-	ExternalAlertmanagers AlertmanagersChoice = "external"
+	ExternalAlertmanagers      AlertmanagersChoice = "external"
+	HandleGrafanaManagedAlerts                     = "handleGrafanaManagedAlerts"
 )
 
 // swagger:model

pkg/services/ngalert/ngalert.go

@@ -35,7 +35,7 @@ import (
 	"github.com/grafana/grafana/pkg/setting"
 )
 
-func ProvideService(cfg *setting.Cfg, dataSourceCache datasources.CacheService, routeRegister routing.RouteRegister,
+func ProvideService(cfg *setting.Cfg, dataSourceCache datasources.CacheService, dataSourceService datasources.DataSourceService, routeRegister routing.RouteRegister,
 	sqlStore *sqlstore.SQLStore, kvStore kvstore.KVStore, expressionService *expr.Service, dataProxy *datasourceproxy.DataSourceProxyService,
 	quotaService quota.Service, secretsService secrets.Service, notificationService notifications.Service, m *metrics.NGAlert,
 	folderService dashboards.FolderService, ac accesscontrol.AccessControl, dashboardService dashboards.DashboardService, renderService rendering.Service,
@@ -43,6 +43,7 @@ func ProvideService(cfg *setting.Cfg, dataSourceCache datasources.CacheService,
 	ng := &AlertNG{
 		Cfg:                 cfg,
 		DataSourceCache:     dataSourceCache,
+		DataSourceService:   dataSourceService,
 		RouteRegister:       routeRegister,
 		SQLStore:            sqlStore,
 		KVStore:             kvStore,
@@ -75,6 +76,7 @@ func ProvideService(cfg *setting.Cfg, dataSourceCache datasources.CacheService,
 type AlertNG struct {
 	Cfg                 *setting.Cfg
 	DataSourceCache     datasources.CacheService
+	DataSourceService   datasources.DataSourceService
 	RouteRegister       routing.RouteRegister
 	SQLStore            *sqlstore.SQLStore
 	KVStore             kvstore.KVStore
@@ -138,7 +140,8 @@ func (ng *AlertNG) init() error {
 
 	clk := clock.New()
 
-	alertsRouter := sender.NewAlertsRouter(ng.MultiOrgAlertmanager, store, clk, appUrl, ng.Cfg.UnifiedAlerting.DisabledOrgs, ng.Cfg.UnifiedAlerting.AdminConfigPollInterval)
+	alertsRouter := sender.NewAlertsRouter(ng.MultiOrgAlertmanager, store, clk, appUrl, ng.Cfg.UnifiedAlerting.DisabledOrgs,
+		ng.Cfg.UnifiedAlerting.AdminConfigPollInterval, ng.DataSourceService, ng.SecretsService)
 
 	// Make sure we sync at least once as Grafana starts to get the router up and running before we start sending any alerts.
 	if err := alertsRouter.SyncAndApplyConfigFromDatabase(); err != nil {
@@ -176,6 +179,7 @@ func (ng *AlertNG) init() error {
 	api := api.API{
 		Cfg:                  ng.Cfg,
 		DatasourceCache:      ng.DataSourceCache,
+		DatasourceService:    ng.DataSourceService,
 		RouteRegister:        ng.RouteRegister,
 		ExpressionService:    ng.ExpressionService,
 		Schedule:             ng.schedule,

pkg/services/ngalert/sender/router.go

@@ -3,6 +3,7 @@ package sender
 import (
 	"context"
 	"errors"
+	"fmt"
 	"net/url"
 	"sync"
 	"time"
@@ -10,10 +11,12 @@ import (
 	"github.com/benbjohnson/clock"
 
 	"github.com/grafana/grafana/pkg/infra/log"
+	"github.com/grafana/grafana/pkg/services/datasources"
 	"github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
 	"github.com/grafana/grafana/pkg/services/ngalert/models"
 	"github.com/grafana/grafana/pkg/services/ngalert/notifier"
 	"github.com/grafana/grafana/pkg/services/ngalert/store"
+	"github.com/grafana/grafana/pkg/services/secrets"
 )
 
 // AlertsRouter handles alerts generated during alert rule evaluation.
@@ -38,9 +41,14 @@ type AlertsRouter struct {
 	appURL                  *url.URL
 	disabledOrgs            map[int64]struct{}
 	adminConfigPollInterval time.Duration
+
+	datasourceService datasources.DataSourceService
+	secretService     secrets.Service
 }
 
-func NewAlertsRouter(multiOrgNotifier *notifier.MultiOrgAlertmanager, store store.AdminConfigurationStore, clk clock.Clock, appURL *url.URL, disabledOrgs map[int64]struct{}, configPollInterval time.Duration) *AlertsRouter {
+func NewAlertsRouter(multiOrgNotifier *notifier.MultiOrgAlertmanager, store store.AdminConfigurationStore,
+	clk clock.Clock, appURL *url.URL, disabledOrgs map[int64]struct{}, configPollInterval time.Duration,
+	datasourceService datasources.DataSourceService, secretService secrets.Service) *AlertsRouter {
 	d := &AlertsRouter{
 		logger:           log.New("alerts-router"),
 		clock:            clk,
@@ -56,6 +64,9 @@ func NewAlertsRouter(multiOrgNotifier *notifier.MultiOrgAlertmanager, store stor
 		appURL:                  appURL,
 		disabledOrgs:            disabledOrgs,
 		adminConfigPollInterval: configPollInterval,
+
+		datasourceService: datasourceService,
+		secretService:     secretService,
 	}
 	return d
 }
@@ -87,17 +98,27 @@ func (d *AlertsRouter) SyncAndApplyConfigFromDatabase() error {
 
 		existing, ok := d.externalAlertmanagers[cfg.OrgID]
 
-		// We have no running sender and no Alertmanager(s) configured, no-op.
-		if !ok && len(cfg.Alertmanagers) == 0 {
-			d.logger.Debug("no external alertmanagers configured", "org", cfg.OrgID)
-			continue
-		}
 		//  We have no running sender and alerts are handled internally, no-op.
 		if !ok && cfg.SendAlertsTo == models.InternalAlertmanager {
 			d.logger.Debug("alerts are handled internally", "org", cfg.OrgID)
 			continue
 		}
 
+		externalAlertmanagers, err := d.alertmanagersFromDatasources(cfg.OrgID)
+		if err != nil {
+			d.logger.Error("failed to get alertmanagers from datasources",
+				"org", cfg.OrgID,
+				"err", err)
+			continue
+		}
+		cfg.Alertmanagers = append(cfg.Alertmanagers, externalAlertmanagers...)
+
+		// We have no running sender and no Alertmanager(s) configured, no-op.
+		if !ok && len(cfg.Alertmanagers) == 0 {
+			d.logger.Debug("no external alertmanagers configured", "org", cfg.OrgID)
+			continue
+		}
+
 		// We have a running sender but no Alertmanager(s) configured, shut it down.
 		if ok && len(cfg.Alertmanagers) == 0 {
 			d.logger.Debug("no external alertmanager(s) configured, sender will be stopped", "org", cfg.OrgID)
@@ -165,6 +186,56 @@ func (d *AlertsRouter) SyncAndApplyConfigFromDatabase() error {
 	return nil
 }
 
+func (d *AlertsRouter) alertmanagersFromDatasources(orgID int64) ([]string, error) {
+	var alertmanagers []string
+	// We might have alertmanager datasources that are acting as external
+	// alertmanager, let's fetch them.
+	query := &datasources.GetDataSourcesByTypeQuery{
+		OrgId: orgID,
+		Type:  datasources.DS_ALERTMANAGER,
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+	defer cancel()
+	err := d.datasourceService.GetDataSourcesByType(ctx, query)
+	if err != nil {
+		return nil, fmt.Errorf("failed to fetch datasources for org: %w", err)
+	}
+	for _, ds := range query.Result {
+		if !ds.JsonData.Get(definitions.HandleGrafanaManagedAlerts).MustBool(false) {
+			continue
+		}
+		amURL, err := d.buildExternalURL(ds)
+		if err != nil {
+			d.logger.Error("failed to build external alertmanager URL",
+				"org", ds.OrgId,
+				"uid", ds.Uid,
+				"err", err)
+			continue
+		}
+		alertmanagers = append(alertmanagers, amURL)
+	}
+	return alertmanagers, nil
+}
+
+func (d *AlertsRouter) buildExternalURL(ds *datasources.DataSource) (string, error) {
+	amURL := ds.Url
+	// if basic auth is enabled we need to build the url with basic auth baked in
+	if !ds.BasicAuth {
+		return amURL, nil
+	}
+
+	parsed, err := url.Parse(ds.Url)
+	if err != nil {
+		return "", fmt.Errorf("failed to parse alertmanager datasource url: %w", err)
+	}
+	password := d.secretService.GetDecryptedValue(context.Background(), ds.SecureJsonData, "basicAuthPassword", "")
+	if password == "" {
+		return "", fmt.Errorf("basic auth enabled but no password set")
+	}
+	return fmt.Sprintf("%s://%s:%s@%s%s%s", parsed.Scheme, ds.BasicAuthUser,
+		password, parsed.Host, parsed.Path, parsed.RawQuery), nil
+}
+
 func (d *AlertsRouter) Send(key models.AlertRuleKey, alerts definitions.PostableAlerts) {
 	logger := d.logger.New("rule_uid", key.UID, "org", key.OrgID)
 	if len(alerts.PostableAlerts) == 0 {

pkg/services/ngalert/sender/router_test.go

@@ -15,13 +15,15 @@ import (
 	"github.com/stretchr/testify/require"
 
 	"github.com/grafana/grafana/pkg/infra/log"
+	"github.com/grafana/grafana/pkg/services/datasources"
+	fake_ds "github.com/grafana/grafana/pkg/services/datasources/fakes"
 	"github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
 	"github.com/grafana/grafana/pkg/services/ngalert/metrics"
 	"github.com/grafana/grafana/pkg/services/ngalert/models"
 	"github.com/grafana/grafana/pkg/services/ngalert/notifier"
 	"github.com/grafana/grafana/pkg/services/ngalert/provisioning"
 	"github.com/grafana/grafana/pkg/services/ngalert/store"
-	"github.com/grafana/grafana/pkg/services/secrets/fakes"
+	fake_secrets "github.com/grafana/grafana/pkg/services/secrets/fakes"
 	secretsManager "github.com/grafana/grafana/pkg/services/secrets/manager"
 	"github.com/grafana/grafana/pkg/setting"
 	"github.com/grafana/grafana/pkg/util"
@@ -45,7 +47,8 @@ func TestSendingToExternalAlertmanager(t *testing.T) {
 		Host:   "localhost",
 	}
 
-	alertsRouter := NewAlertsRouter(moa, fakeAdminConfigStore, mockedClock, appUrl, map[int64]struct{}{}, 10*time.Minute)
+	alertsRouter := NewAlertsRouter(moa, fakeAdminConfigStore, mockedClock, appUrl, map[int64]struct{}{}, 10*time.Minute,
+		&fake_ds.FakeDataSourceService{}, fake_secrets.NewFakeSecretsService())
 
 	mockedGetAdminConfigurations.Return([]*models.AdminConfiguration{
 		{OrgID: ruleKey.OrgID, Alertmanagers: []string{fakeAM.Server.URL}, SendAlertsTo: models.AllAlertmanagers},
@@ -102,7 +105,8 @@ func TestSendingToExternalAlertmanager_WithMultipleOrgs(t *testing.T) {
 		Host:   "localhost",
 	}
 
-	alertsRouter := NewAlertsRouter(moa, fakeAdminConfigStore, mockedClock, appUrl, map[int64]struct{}{}, 10*time.Minute)
+	alertsRouter := NewAlertsRouter(moa, fakeAdminConfigStore, mockedClock, appUrl, map[int64]struct{}{}, 10*time.Minute,
+		&fake_ds.FakeDataSourceService{}, fake_secrets.NewFakeSecretsService())
 
 	mockedGetAdminConfigurations.Return([]*models.AdminConfiguration{
 		{OrgID: ruleKey1.OrgID, Alertmanagers: []string{fakeAM.Server.URL}, SendAlertsTo: models.AllAlertmanagers},
@@ -228,7 +232,8 @@ func TestChangingAlertmanagersChoice(t *testing.T) {
 		Host:   "localhost",
 	}
 
-	alertsRouter := NewAlertsRouter(moa, fakeAdminConfigStore, mockedClock, appUrl, map[int64]struct{}{}, 10*time.Minute)
+	alertsRouter := NewAlertsRouter(moa, fakeAdminConfigStore, mockedClock, appUrl, map[int64]struct{}{},
+		10*time.Minute, &fake_ds.FakeDataSourceService{}, fake_secrets.NewFakeSecretsService())
 
 	mockedGetAdminConfigurations.Return([]*models.AdminConfiguration{
 		{OrgID: ruleKey.OrgID, Alertmanagers: []string{fakeAM.Server.URL}, SendAlertsTo: models.AllAlertmanagers},
@@ -344,7 +349,7 @@ func createMultiOrgAlertmanager(t *testing.T, orgs []int64) *notifier.MultiOrgAl
 	kvStore := notifier.NewFakeKVStore(t)
 	registry := prometheus.NewPedanticRegistry()
 	m := metrics.NewNGAlert(registry)
-	secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore())
+	secretsService := secretsManager.SetupTestService(t, fake_secrets.NewFakeSecretsStore())
 	decryptFn := secretsService.GetDecryptedValue
 	moa, err := notifier.NewMultiOrgAlertmanager(cfg, &cfgStore, &orgStore, kvStore, provisioning.NewFakeProvisioningStore(), decryptFn, m.GetMultiOrgAlertmanagerMetrics(), nil, log.New("testlogger"), secretsService)
 	require.NoError(t, err)
@@ -360,3 +365,60 @@ func createMultiOrgAlertmanager(t *testing.T, orgs []int64) *notifier.MultiOrgAl
 	}, 10*time.Second, 100*time.Millisecond)
 	return moa
 }
+
+func TestBuildExternalURL(t *testing.T) {
+	sch := AlertsRouter{
+		secretService: fake_secrets.NewFakeSecretsService(),
+	}
+	tests := []struct {
+		name        string
+		ds          *datasources.DataSource
+		expectedURL string
+	}{
+		{
+			name: "datasource without auth",
+			ds: &datasources.DataSource{
+				Url: "https://localhost:9000",
+			},
+			expectedURL: "https://localhost:9000",
+		},
+		{
+			name: "datasource without auth and with path",
+			ds: &datasources.DataSource{
+				Url: "https://localhost:9000/path/to/am",
+			},
+			expectedURL: "https://localhost:9000/path/to/am",
+		},
+		{
+			name: "datasource with auth",
+			ds: &datasources.DataSource{
+				Url:           "https://localhost:9000",
+				BasicAuth:     true,
+				BasicAuthUser: "johndoe",
+				SecureJsonData: map[string][]byte{
+					"basicAuthPassword": []byte("123"),
+				},
+			},
+			expectedURL: "https://johndoe:123@localhost:9000",
+		},
+		{
+			name: "datasource with auth and path",
+			ds: &datasources.DataSource{
+				Url:           "https://localhost:9000/path/to/am",
+				BasicAuth:     true,
+				BasicAuthUser: "johndoe",
+				SecureJsonData: map[string][]byte{
+					"basicAuthPassword": []byte("123"),
+				},
+			},
+			expectedURL: "https://johndoe:123@localhost:9000/path/to/am",
+		},
+	}
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			url, err := sch.buildExternalURL(test.ds)
+			require.NoError(t, err)
+			require.Equal(t, test.expectedURL, url)
+		})
+	}
+}

pkg/services/ngalert/tests/util.go

@@ -63,7 +63,7 @@ func SetupTestEnv(t *testing.T, baseInterval time.Duration) (*ngalert.AlertNG, *
 	)
 
 	ng, err := ngalert.ProvideService(
-		cfg, nil, routing.NewRouteRegister(), sqlStore, nil, nil, nil, nil,
+		cfg, nil, nil, routing.NewRouteRegister(), sqlStore, nil, nil, nil, nil,
 		secretsService, nil, m, folderService, ac, &dashboards.FakeDashboardService{}, nil, bus,
 	)
 	require.NoError(t, err)

pkg/services/sqlstore/datasource.go

@@ -76,6 +76,9 @@ func (ss *SQLStore) GetDataSourcesByType(ctx context.Context, query *datasources
 
 	query.Result = make([]*datasources.DataSource, 0)
 	return ss.WithDbSession(ctx, func(sess *DBSession) error {
+		if query.OrgId > 0 {
+			return sess.Where("type=? AND org_id=?", query.Type, query.OrgId).Asc("id").Find(&query.Result)
+		}
 		return sess.Where("type=?", query.Type).Asc("id").Find(&query.Result)
 	})
 }

pkg/tests/api/alerting/api_admin_configuration_test.go

@@ -84,7 +84,10 @@ func TestAdminConfiguration_SendingToExternalAlertmanagers(t *testing.T) {
 		resp := postRequest(t, alertsURL, buf.String(), http.StatusBadRequest) // nolint
 		b, err := ioutil.ReadAll(resp.Body)
 		require.NoError(t, err)
-		require.JSONEq(t, `{"message": "Invalid alertmanager choice specified"}`, string(b))
+		var res map[string]interface{}
+		err = json.Unmarshal(b, &res)
+		require.NoError(t, err)
+		require.Equal(t, "Invalid alertmanager choice specified", res["message"])
 	}
 
 	// Let's try to send all the alerts to an external Alertmanager
@@ -102,7 +105,10 @@ func TestAdminConfiguration_SendingToExternalAlertmanagers(t *testing.T) {
 		resp := postRequest(t, alertsURL, buf.String(), http.StatusBadRequest) // nolint
 		b, err := ioutil.ReadAll(resp.Body)
 		require.NoError(t, err)
-		require.JSONEq(t, `{"message": "At least one Alertmanager must be provided to choose this option"}`, string(b))
+		var res map[string]interface{}
+		err = json.Unmarshal(b, &res)
+		require.NoError(t, err)
+		require.Equal(t, "At least one Alertmanager must be provided or configured as a datasource that handles alerts to choose this option", res["message"])
 	}
 
 	// Now, lets re-set external Alertmanagers for main organisation
@@ -121,7 +127,10 @@ func TestAdminConfiguration_SendingToExternalAlertmanagers(t *testing.T) {
 		resp := postRequest(t, alertsURL, buf.String(), http.StatusCreated) // nolint
 		b, err := ioutil.ReadAll(resp.Body)
 		require.NoError(t, err)
-		require.JSONEq(t, `{"message": "admin configuration updated"}`, string(b))
+		var res map[string]interface{}
+		err = json.Unmarshal(b, &res)
+		require.NoError(t, err)
+		require.Equal(t, "admin configuration updated", res["message"])
 	}
 
 	// If we get the configuration again, it shows us what we've set.
@@ -220,7 +229,10 @@ func TestAdminConfiguration_SendingToExternalAlertmanagers(t *testing.T) {
 		resp := postRequest(t, alertsURL, buf.String(), http.StatusCreated) // nolint
 		b, err := ioutil.ReadAll(resp.Body)
 		require.NoError(t, err)
-		require.JSONEq(t, "{\"message\": \"admin configuration updated\"}", string(b))
+		var res map[string]interface{}
+		err = json.Unmarshal(b, &res)
+		require.NoError(t, err)
+		require.Equal(t, "admin configuration updated", res["message"])
 	}
 
 	// If we get the configuration again, it shows us what we've set.