diff --git a/pkg/api/datasources.go b/pkg/api/datasources.go index 61a25517107..f1d1e7c1870 100644 --- a/pkg/api/datasources.go +++ b/pkg/api/datasources.go @@ -97,7 +97,7 @@ func (hs *HTTPServer) GetDataSourceById(c *models.ReqContext) response.Response return response.Error(404, "Data source not found", err) } - dto := convertModelToDtos(filtered[0]) + dto := hs.convertModelToDtos(c.Req.Context(), filtered[0]) // Add accesscontrol metadata dto.AccessControl = hs.getAccessControlMetadata(c, c.OrgId, datasources.ScopePrefix, dto.UID) @@ -128,7 +128,7 @@ func (hs *HTTPServer) DeleteDataSourceById(c *models.ReqContext) response.Respon return response.Error(403, "Cannot delete read-only data source", nil) } - cmd := &models.DeleteDataSourceCommand{ID: id, OrgID: c.OrgId} + cmd := &models.DeleteDataSourceCommand{ID: id, OrgID: c.OrgId, Name: ds.Name} err = hs.DataSourcesService.DeleteDataSource(c.Req.Context(), cmd) if err != nil { @@ -156,7 +156,7 @@ func (hs *HTTPServer) GetDataSourceByUID(c *models.ReqContext) response.Response return response.Error(404, "Data source not found", err) } - dto := convertModelToDtos(filtered[0]) + dto := hs.convertModelToDtos(c.Req.Context(), filtered[0]) // Add accesscontrol metadata dto.AccessControl = hs.getAccessControlMetadata(c, c.OrgId, datasources.ScopePrefix, dto.UID) @@ -184,7 +184,7 @@ func (hs *HTTPServer) DeleteDataSourceByUID(c *models.ReqContext) response.Respo return response.Error(403, "Cannot delete read-only data source", nil) } - cmd := &models.DeleteDataSourceCommand{UID: uid, OrgID: c.OrgId} + cmd := &models.DeleteDataSourceCommand{UID: uid, OrgID: c.OrgId, Name: ds.Name} err = hs.DataSourcesService.DeleteDataSource(c.Req.Context(), cmd) if err != nil { @@ -265,7 +265,7 @@ func (hs *HTTPServer) AddDataSource(c *models.ReqContext) response.Response { return response.Error(500, "Failed to add datasource", err) } - ds := convertModelToDtos(cmd.Result) + ds := hs.convertModelToDtos(c.Req.Context(), cmd.Result) return response.JSON(http.StatusOK, util.DynMap{ "message": "Datasource added", "id": cmd.Result.Id, @@ -327,7 +327,7 @@ func (hs *HTTPServer) UpdateDataSource(c *models.ReqContext) response.Response { return response.Error(500, "Failed to query datasource", err) } - datasourceDTO := convertModelToDtos(query.Result) + datasourceDTO := hs.convertModelToDtos(c.Req.Context(), query.Result) hs.Live.HandleDatasourceUpdate(c.OrgId, datasourceDTO.UID) @@ -408,7 +408,7 @@ func (hs *HTTPServer) GetDataSourceByName(c *models.ReqContext) response.Respons return response.Error(404, "Data source not found", err) } - dto := convertModelToDtos(filtered[0]) + dto := hs.convertModelToDtos(c.Req.Context(), filtered[0]) return response.JSON(http.StatusOK, &dto) } @@ -457,7 +457,7 @@ func (hs *HTTPServer) CallDatasourceResource(c *models.ReqContext) { hs.callPluginResource(c, plugin.ID, ds.Uid) } -func convertModelToDtos(ds *models.DataSource) dtos.DataSource { +func (hs *HTTPServer) convertModelToDtos(ctx context.Context, ds *models.DataSource) dtos.DataSource { dto := dtos.DataSource{ Id: ds.Id, UID: ds.Uid, @@ -480,9 +480,12 @@ func convertModelToDtos(ds *models.DataSource) dtos.DataSource { ReadOnly: ds.ReadOnly, } - for k, v := range ds.SecureJsonData { - if len(v) > 0 { - dto.SecureJsonFields[k] = true + secrets, err := hs.DataSourcesService.DecryptedValues(ctx, ds) + if err == nil { + for k, v := range secrets { + if len(v) > 0 { + dto.SecureJsonFields[k] = true + } } } @@ -510,7 +513,7 @@ func (hs *HTTPServer) CheckDatasourceHealth(c *models.ReqContext) response.Respo return response.Error(http.StatusInternalServerError, "Unable to find datasource plugin", err) } - dsInstanceSettings, err := adapters.ModelToInstanceSettings(ds, hs.decryptSecureJsonDataFn()) + dsInstanceSettings, err := adapters.ModelToInstanceSettings(ds, hs.decryptSecureJsonDataFn(c.Req.Context())) if err != nil { return response.Error(http.StatusInternalServerError, "Unable to get datasource model", err) } @@ -561,9 +564,9 @@ func (hs *HTTPServer) CheckDatasourceHealth(c *models.ReqContext) response.Respo return response.JSON(http.StatusOK, payload) } -func (hs *HTTPServer) decryptSecureJsonDataFn() func(map[string][]byte) map[string]string { - return func(m map[string][]byte) map[string]string { - decryptedJsonData, err := hs.SecretsService.DecryptJsonData(context.Background(), m) +func (hs *HTTPServer) decryptSecureJsonDataFn(ctx context.Context) func(ds *models.DataSource) map[string]string { + return func(ds *models.DataSource) map[string]string { + decryptedJsonData, err := hs.DataSourcesService.DecryptedValues(ctx, ds) if err != nil { hs.log.Error("Failed to decrypt secure json data", "error", err) } diff --git a/pkg/api/datasources_test.go b/pkg/api/datasources_test.go index c490e2a0142..e4640009d67 100644 --- a/pkg/api/datasources_test.go +++ b/pkg/api/datasources_test.go @@ -583,3 +583,8 @@ func (m *dataSourcesServiceMock) UpdateDataSource(ctx context.Context, cmd *mode cmd.Result = m.expectedDatasource return m.expectedError } + +func (m *dataSourcesServiceMock) DecryptedValues(ctx context.Context, ds *models.DataSource) (map[string]string, error) { + decryptedValues := make(map[string]string) + return decryptedValues, m.expectedError +} diff --git a/pkg/api/frontendsettings.go b/pkg/api/frontendsettings.go index 2c341d53979..3eff68dea68 100644 --- a/pkg/api/frontendsettings.go +++ b/pkg/api/frontendsettings.go @@ -248,9 +248,14 @@ func (hs *HTTPServer) getFSDataSources(c *models.ReqContext, enabledPlugins Enab if ds.Access == models.DS_ACCESS_DIRECT { if ds.BasicAuth { + password, err := hs.DataSourcesService.DecryptedBasicAuthPassword(c.Req.Context(), ds) + if err != nil { + return nil, err + } + dsDTO.BasicAuth = util.GetBasicAuthHeader( ds.BasicAuthUser, - hs.DataSourcesService.DecryptedBasicAuthPassword(ds), + password, ) } if ds.WithCredentials { @@ -258,14 +263,24 @@ func (hs *HTTPServer) getFSDataSources(c *models.ReqContext, enabledPlugins Enab } if ds.Type == models.DS_INFLUXDB_08 { + password, err := hs.DataSourcesService.DecryptedPassword(c.Req.Context(), ds) + if err != nil { + return nil, err + } + dsDTO.Username = ds.User - dsDTO.Password = hs.DataSourcesService.DecryptedPassword(ds) + dsDTO.Password = password dsDTO.URL = url + "/db/" + ds.Database } if ds.Type == models.DS_INFLUXDB { + password, err := hs.DataSourcesService.DecryptedPassword(c.Req.Context(), ds) + if err != nil { + return nil, err + } + dsDTO.Username = ds.User - dsDTO.Password = hs.DataSourcesService.DecryptedPassword(ds) + dsDTO.Password = password dsDTO.URL = url } } diff --git a/pkg/api/metrics_test.go b/pkg/api/metrics_test.go index 5a32d5cfbba..0b4472b0d52 100644 --- a/pkg/api/metrics_test.go +++ b/pkg/api/metrics_test.go @@ -9,6 +9,7 @@ import ( "strings" "testing" + acmock "github.com/grafana/grafana/pkg/services/accesscontrol/mock" "github.com/grafana/grafana/pkg/services/featuremgmt" "github.com/grafana/grafana/pkg/services/sqlstore/mockstore" @@ -18,8 +19,11 @@ import ( "github.com/grafana/grafana/pkg/components/simplejson" "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/plugins" + datasources "github.com/grafana/grafana/pkg/services/datasources/service" "github.com/grafana/grafana/pkg/services/query" "github.com/grafana/grafana/pkg/services/secrets/fakes" + "github.com/grafana/grafana/pkg/services/secrets/kvstore" + secretsManager "github.com/grafana/grafana/pkg/services/secrets/manager" "github.com/stretchr/testify/assert" ) @@ -193,13 +197,17 @@ type dashboardFakePluginClient struct { func TestAPIEndpoint_Metrics_QueryMetricsFromDashboard(t *testing.T) { sc := setupHTTPServerWithMockDb(t, false, false) + secretsStore := kvstore.SetupTestService(t) + secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) + ds := datasources.ProvideService(nil, secretsService, secretsStore, nil, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) + setInitCtxSignedInViewer(sc.initCtx) sc.hs.queryDataService = query.ProvideService( nil, nil, nil, &fakePluginRequestValidator{}, - fakes.NewFakeSecretsService(), + ds, &dashboardFakePluginClient{}, &fakeOAuthTokenService{}, ) diff --git a/pkg/api/pluginproxy/ds_proxy.go b/pkg/api/pluginproxy/ds_proxy.go index 239249cf620..75e93fd513b 100644 --- a/pkg/api/pluginproxy/ds_proxy.go +++ b/pkg/api/pluginproxy/ds_proxy.go @@ -19,7 +19,6 @@ import ( "github.com/grafana/grafana/pkg/plugins" "github.com/grafana/grafana/pkg/services/datasources" "github.com/grafana/grafana/pkg/services/oauthtoken" - "github.com/grafana/grafana/pkg/services/secrets" "github.com/grafana/grafana/pkg/setting" "github.com/grafana/grafana/pkg/util" "github.com/grafana/grafana/pkg/util/proxyutil" @@ -43,7 +42,6 @@ type DataSourceProxy struct { oAuthTokenService oauthtoken.OAuthTokenService dataSourcesService datasources.DataSourceService tracer tracing.Tracer - secretsService secrets.Service } type httpClient interface { @@ -54,7 +52,7 @@ type httpClient interface { func NewDataSourceProxy(ds *models.DataSource, pluginRoutes []*plugins.Route, ctx *models.ReqContext, proxyPath string, cfg *setting.Cfg, clientProvider httpclient.Provider, oAuthTokenService oauthtoken.OAuthTokenService, dsService datasources.DataSourceService, - tracer tracing.Tracer, secretsService secrets.Service) (*DataSourceProxy, error) { + tracer tracing.Tracer) (*DataSourceProxy, error) { targetURL, err := datasource.ValidateURL(ds.Type, ds.Url) if err != nil { return nil, err @@ -71,7 +69,6 @@ func NewDataSourceProxy(ds *models.DataSource, pluginRoutes []*plugins.Route, ct oAuthTokenService: oAuthTokenService, dataSourcesService: dsService, tracer: tracer, - secretsService: secretsService, }, nil } @@ -97,7 +94,7 @@ func (proxy *DataSourceProxy) HandleRequest() { "referer", proxy.ctx.Req.Referer(), ) - transport, err := proxy.dataSourcesService.GetHTTPTransport(proxy.ds, proxy.clientProvider) + transport, err := proxy.dataSourcesService.GetHTTPTransport(proxy.ctx.Req.Context(), proxy.ds, proxy.clientProvider) if err != nil { proxy.ctx.JsonApiErr(400, "Unable to load TLS certificate", err) return @@ -169,17 +166,28 @@ func (proxy *DataSourceProxy) director(req *http.Request) { switch proxy.ds.Type { case models.DS_INFLUXDB_08: + password, err := proxy.dataSourcesService.DecryptedPassword(req.Context(), proxy.ds) + if err != nil { + logger.Error("Error interpolating proxy url", "error", err) + return + } + req.URL.RawPath = util.JoinURLFragments(proxy.targetUrl.Path, "db/"+proxy.ds.Database+"/"+proxy.proxyPath) reqQueryVals.Add("u", proxy.ds.User) - reqQueryVals.Add("p", proxy.dataSourcesService.DecryptedPassword(proxy.ds)) + reqQueryVals.Add("p", password) req.URL.RawQuery = reqQueryVals.Encode() case models.DS_INFLUXDB: + password, err := proxy.dataSourcesService.DecryptedPassword(req.Context(), proxy.ds) + if err != nil { + logger.Error("Error interpolating proxy url", "error", err) + return + } req.URL.RawPath = util.JoinURLFragments(proxy.targetUrl.Path, proxy.proxyPath) req.URL.RawQuery = reqQueryVals.Encode() if !proxy.ds.BasicAuth { req.Header.Set( "Authorization", - util.GetBasicAuthHeader(proxy.ds.User, proxy.dataSourcesService.DecryptedPassword(proxy.ds)), + util.GetBasicAuthHeader(proxy.ds.User, password), ) } default: @@ -195,8 +203,13 @@ func (proxy *DataSourceProxy) director(req *http.Request) { req.URL.Path = unescapedPath if proxy.ds.BasicAuth { + password, err := proxy.dataSourcesService.DecryptedBasicAuthPassword(req.Context(), proxy.ds) + if err != nil { + logger.Error("Error interpolating proxy url", "error", err) + return + } req.Header.Set("Authorization", util.GetBasicAuthHeader(proxy.ds.BasicAuthUser, - proxy.dataSourcesService.DecryptedBasicAuthPassword(proxy.ds))) + password)) } dsAuth := req.Header.Get("X-DS-Authorization") @@ -226,23 +239,23 @@ func (proxy *DataSourceProxy) director(req *http.Request) { } } - secureJsonData, err := proxy.secretsService.DecryptJsonData(req.Context(), proxy.ds.SecureJsonData) - if err != nil { - logger.Error("Error interpolating proxy url", "error", err) - return - } - if proxy.matchedRoute != nil { - ApplyRoute(proxy.ctx.Req.Context(), req, proxy.proxyPath, proxy.matchedRoute, DSInfo{ + decryptedValues, err := proxy.dataSourcesService.DecryptedValues(req.Context(), proxy.ds) + if err != nil { + logger.Error("Error interpolating proxy url", "error", err) + return + } + + ApplyRoute(req.Context(), req, proxy.proxyPath, proxy.matchedRoute, DSInfo{ ID: proxy.ds.Id, Updated: proxy.ds.Updated, JSONData: jsonData, - DecryptedSecureJSONData: secureJsonData, + DecryptedSecureJSONData: decryptedValues, }, proxy.cfg) } if proxy.oAuthTokenService.IsOAuthPassThruEnabled(proxy.ds) { - if token := proxy.oAuthTokenService.GetCurrentOAuthToken(proxy.ctx.Req.Context(), proxy.ctx.SignedInUser); token != nil { + if token := proxy.oAuthTokenService.GetCurrentOAuthToken(req.Context(), proxy.ctx.SignedInUser); token != nil { req.Header.Set("Authorization", fmt.Sprintf("%s %s", token.Type(), token.AccessToken)) idToken, ok := token.Extra("id_token").(string) diff --git a/pkg/api/pluginproxy/ds_proxy_test.go b/pkg/api/pluginproxy/ds_proxy_test.go index d0784e839eb..544b0997e74 100644 --- a/pkg/api/pluginproxy/ds_proxy_test.go +++ b/pkg/api/pluginproxy/ds_proxy_test.go @@ -3,6 +3,7 @@ package pluginproxy import ( "bytes" "context" + "encoding/json" "fmt" "io/ioutil" "net/http" @@ -24,6 +25,7 @@ import ( "github.com/grafana/grafana/pkg/services/oauthtoken" "github.com/grafana/grafana/pkg/services/secrets" "github.com/grafana/grafana/pkg/services/secrets/fakes" + "github.com/grafana/grafana/pkg/services/secrets/kvstore" secretsManager "github.com/grafana/grafana/pkg/services/secrets/manager" "github.com/grafana/grafana/pkg/setting" "github.com/grafana/grafana/pkg/web" @@ -90,6 +92,7 @@ func TestDataSourceProxy_routeRule(t *testing.T) { }) setting.SecretKey = "password" //nolint:goconst + secretsStore := kvstore.SetupTestService(t) secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) key, err := secretsService.Encrypt(context.Background(), []byte("123"), secrets.WithoutScope()) require.NoError(t, err) @@ -128,9 +131,9 @@ func TestDataSourceProxy_routeRule(t *testing.T) { t.Run("When matching route path", func(t *testing.T) { ctx, req := setUp() - dsService := datasourceservice.ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) + dsService := datasourceservice.ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) proxy, err := NewDataSourceProxy(ds, routes, ctx, "api/v4/some/method", cfg, httpClientProvider, - &oauthtoken.Service{}, dsService, tracer, secretsService) + &oauthtoken.Service{}, dsService, tracer) require.NoError(t, err) proxy.matchedRoute = routes[0] ApplyRoute(proxy.ctx.Req.Context(), req, proxy.proxyPath, proxy.matchedRoute, dsInfo, cfg) @@ -141,8 +144,8 @@ func TestDataSourceProxy_routeRule(t *testing.T) { t.Run("When matching route path and has dynamic url", func(t *testing.T) { ctx, req := setUp() - dsService := datasourceservice.ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) - proxy, err := NewDataSourceProxy(ds, routes, ctx, "api/common/some/method", cfg, httpClientProvider, &oauthtoken.Service{}, dsService, tracer, secretsService) + dsService := datasourceservice.ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) + proxy, err := NewDataSourceProxy(ds, routes, ctx, "api/common/some/method", cfg, httpClientProvider, &oauthtoken.Service{}, dsService, tracer) require.NoError(t, err) proxy.matchedRoute = routes[3] ApplyRoute(proxy.ctx.Req.Context(), req, proxy.proxyPath, proxy.matchedRoute, dsInfo, cfg) @@ -153,8 +156,8 @@ func TestDataSourceProxy_routeRule(t *testing.T) { t.Run("When matching route path with no url", func(t *testing.T) { ctx, req := setUp() - dsService := datasourceservice.ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) - proxy, err := NewDataSourceProxy(ds, routes, ctx, "", cfg, httpClientProvider, &oauthtoken.Service{}, dsService, tracer, secretsService) + dsService := datasourceservice.ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) + proxy, err := NewDataSourceProxy(ds, routes, ctx, "", cfg, httpClientProvider, &oauthtoken.Service{}, dsService, tracer) require.NoError(t, err) proxy.matchedRoute = routes[4] ApplyRoute(proxy.ctx.Req.Context(), req, proxy.proxyPath, proxy.matchedRoute, dsInfo, cfg) @@ -164,8 +167,8 @@ func TestDataSourceProxy_routeRule(t *testing.T) { t.Run("When matching route path and has dynamic body", func(t *testing.T) { ctx, req := setUp() - dsService := datasourceservice.ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) - proxy, err := NewDataSourceProxy(ds, routes, ctx, "api/body", cfg, httpClientProvider, &oauthtoken.Service{}, dsService, tracer, secretsService) + dsService := datasourceservice.ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) + proxy, err := NewDataSourceProxy(ds, routes, ctx, "api/body", cfg, httpClientProvider, &oauthtoken.Service{}, dsService, tracer) require.NoError(t, err) proxy.matchedRoute = routes[5] ApplyRoute(proxy.ctx.Req.Context(), req, proxy.proxyPath, proxy.matchedRoute, dsInfo, cfg) @@ -178,8 +181,8 @@ func TestDataSourceProxy_routeRule(t *testing.T) { t.Run("Validating request", func(t *testing.T) { t.Run("plugin route with valid role", func(t *testing.T) { ctx, _ := setUp() - dsService := datasourceservice.ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) - proxy, err := NewDataSourceProxy(ds, routes, ctx, "api/v4/some/method", cfg, httpClientProvider, &oauthtoken.Service{}, dsService, tracer, secretsService) + dsService := datasourceservice.ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) + proxy, err := NewDataSourceProxy(ds, routes, ctx, "api/v4/some/method", cfg, httpClientProvider, &oauthtoken.Service{}, dsService, tracer) require.NoError(t, err) err = proxy.validateRequest() require.NoError(t, err) @@ -187,8 +190,8 @@ func TestDataSourceProxy_routeRule(t *testing.T) { t.Run("plugin route with admin role and user is editor", func(t *testing.T) { ctx, _ := setUp() - dsService := datasourceservice.ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) - proxy, err := NewDataSourceProxy(ds, routes, ctx, "api/admin", cfg, httpClientProvider, &oauthtoken.Service{}, dsService, tracer, secretsService) + dsService := datasourceservice.ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) + proxy, err := NewDataSourceProxy(ds, routes, ctx, "api/admin", cfg, httpClientProvider, &oauthtoken.Service{}, dsService, tracer) require.NoError(t, err) err = proxy.validateRequest() require.Error(t, err) @@ -197,8 +200,8 @@ func TestDataSourceProxy_routeRule(t *testing.T) { t.Run("plugin route with admin role and user is admin", func(t *testing.T) { ctx, _ := setUp() ctx.SignedInUser.OrgRole = models.ROLE_ADMIN - dsService := datasourceservice.ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) - proxy, err := NewDataSourceProxy(ds, routes, ctx, "api/admin", cfg, httpClientProvider, &oauthtoken.Service{}, dsService, tracer, secretsService) + dsService := datasourceservice.ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) + proxy, err := NewDataSourceProxy(ds, routes, ctx, "api/admin", cfg, httpClientProvider, &oauthtoken.Service{}, dsService, tracer) require.NoError(t, err) err = proxy.validateRequest() require.NoError(t, err) @@ -242,6 +245,7 @@ func TestDataSourceProxy_routeRule(t *testing.T) { }) setting.SecretKey = "password" + secretsStore := kvstore.SetupTestService(t) secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) key, err := secretsService.Encrypt(context.Background(), []byte("123"), secrets.WithoutScope()) require.NoError(t, err) @@ -286,8 +290,8 @@ func TestDataSourceProxy_routeRule(t *testing.T) { }, } - dsService := datasourceservice.ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) - proxy, err := NewDataSourceProxy(ds, routes, ctx, "pathwithtoken1", cfg, httpClientProvider, &oauthtoken.Service{}, dsService, tracer, secretsService) + dsService := datasourceservice.ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) + proxy, err := NewDataSourceProxy(ds, routes, ctx, "pathwithtoken1", cfg, httpClientProvider, &oauthtoken.Service{}, dsService, tracer) require.NoError(t, err) ApplyRoute(proxy.ctx.Req.Context(), req, proxy.proxyPath, routes[0], dsInfo, cfg) @@ -302,8 +306,8 @@ func TestDataSourceProxy_routeRule(t *testing.T) { req, err := http.NewRequest("GET", "http://localhost/asd", nil) require.NoError(t, err) client = newFakeHTTPClient(t, json2) - dsService := datasourceservice.ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) - proxy, err := NewDataSourceProxy(ds, routes, ctx, "pathwithtoken2", cfg, httpClientProvider, &oauthtoken.Service{}, dsService, tracer, secretsService) + dsService := datasourceservice.ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) + proxy, err := NewDataSourceProxy(ds, routes, ctx, "pathwithtoken2", cfg, httpClientProvider, &oauthtoken.Service{}, dsService, tracer) require.NoError(t, err) ApplyRoute(proxy.ctx.Req.Context(), req, proxy.proxyPath, routes[1], dsInfo, cfg) @@ -319,8 +323,8 @@ func TestDataSourceProxy_routeRule(t *testing.T) { require.NoError(t, err) client = newFakeHTTPClient(t, []byte{}) - dsService := datasourceservice.ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) - proxy, err := NewDataSourceProxy(ds, routes, ctx, "pathwithtoken1", cfg, httpClientProvider, &oauthtoken.Service{}, dsService, tracer, secretsService) + dsService := datasourceservice.ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) + proxy, err := NewDataSourceProxy(ds, routes, ctx, "pathwithtoken1", cfg, httpClientProvider, &oauthtoken.Service{}, dsService, tracer) require.NoError(t, err) ApplyRoute(proxy.ctx.Req.Context(), req, proxy.proxyPath, routes[0], dsInfo, cfg) @@ -340,9 +344,10 @@ func TestDataSourceProxy_routeRule(t *testing.T) { ds := &models.DataSource{Url: "htttp://graphite:8080", Type: models.DS_GRAPHITE} ctx := &models.ReqContext{} + secretsStore := kvstore.SetupTestService(t) secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) - dsService := datasourceservice.ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) - proxy, err := NewDataSourceProxy(ds, routes, ctx, "/render", &setting.Cfg{BuildVersion: "5.3.0"}, httpClientProvider, &oauthtoken.Service{}, dsService, tracer, secretsService) + dsService := datasourceservice.ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) + proxy, err := NewDataSourceProxy(ds, routes, ctx, "/render", &setting.Cfg{BuildVersion: "5.3.0"}, httpClientProvider, &oauthtoken.Service{}, dsService, tracer) require.NoError(t, err) req, err := http.NewRequest(http.MethodGet, "http://grafana.com/sub", nil) require.NoError(t, err) @@ -366,9 +371,10 @@ func TestDataSourceProxy_routeRule(t *testing.T) { ctx := &models.ReqContext{} var routes []*plugins.Route + secretsStore := kvstore.SetupTestService(t) secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) - dsService := datasourceservice.ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) - proxy, err := NewDataSourceProxy(ds, routes, ctx, "", &setting.Cfg{}, httpClientProvider, &oauthtoken.Service{}, dsService, tracer, secretsService) + dsService := datasourceservice.ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) + proxy, err := NewDataSourceProxy(ds, routes, ctx, "", &setting.Cfg{}, httpClientProvider, &oauthtoken.Service{}, dsService, tracer) require.NoError(t, err) req, err := http.NewRequest(http.MethodGet, "http://grafana.com/sub", nil) @@ -390,9 +396,10 @@ func TestDataSourceProxy_routeRule(t *testing.T) { ctx := &models.ReqContext{} var routes []*plugins.Route + secretsStore := kvstore.SetupTestService(t) secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) - dsService := datasourceservice.ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) - proxy, err := NewDataSourceProxy(ds, routes, ctx, "", &setting.Cfg{}, httpClientProvider, &oauthtoken.Service{}, dsService, tracer, secretsService) + dsService := datasourceservice.ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) + proxy, err := NewDataSourceProxy(ds, routes, ctx, "", &setting.Cfg{}, httpClientProvider, &oauthtoken.Service{}, dsService, tracer) require.NoError(t, err) requestURL, err := url.Parse("http://grafana.com/sub") @@ -418,9 +425,10 @@ func TestDataSourceProxy_routeRule(t *testing.T) { ctx := &models.ReqContext{} var pluginRoutes []*plugins.Route + secretsStore := kvstore.SetupTestService(t) secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) - dsService := datasourceservice.ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) - proxy, err := NewDataSourceProxy(ds, pluginRoutes, ctx, "", &setting.Cfg{}, httpClientProvider, &oauthtoken.Service{}, dsService, tracer, secretsService) + dsService := datasourceservice.ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) + proxy, err := NewDataSourceProxy(ds, pluginRoutes, ctx, "", &setting.Cfg{}, httpClientProvider, &oauthtoken.Service{}, dsService, tracer) require.NoError(t, err) requestURL, err := url.Parse("http://grafana.com/sub") @@ -441,9 +449,10 @@ func TestDataSourceProxy_routeRule(t *testing.T) { } ctx := &models.ReqContext{} var routes []*plugins.Route + secretsStore := kvstore.SetupTestService(t) secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) - dsService := datasourceservice.ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) - proxy, err := NewDataSourceProxy(ds, routes, ctx, "/path/to/folder/", &setting.Cfg{}, httpClientProvider, &oauthtoken.Service{}, dsService, tracer, secretsService) + dsService := datasourceservice.ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) + proxy, err := NewDataSourceProxy(ds, routes, ctx, "/path/to/folder/", &setting.Cfg{}, httpClientProvider, &oauthtoken.Service{}, dsService, tracer) require.NoError(t, err) req, err := http.NewRequest(http.MethodGet, "http://grafana.com/sub", nil) req.Header.Set("Origin", "grafana.com") @@ -490,9 +499,10 @@ func TestDataSourceProxy_routeRule(t *testing.T) { } var routes []*plugins.Route + secretsStore := kvstore.SetupTestService(t) secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) - dsService := datasourceservice.ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) - proxy, err := NewDataSourceProxy(ds, routes, ctx, "/path/to/folder/", &setting.Cfg{}, httpClientProvider, &mockAuthToken, dsService, tracer, secretsService) + dsService := datasourceservice.ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) + proxy, err := NewDataSourceProxy(ds, routes, ctx, "/path/to/folder/", &setting.Cfg{}, httpClientProvider, &mockAuthToken, dsService, tracer) require.NoError(t, err) req, err = http.NewRequest(http.MethodGet, "http://grafana.com/sub", nil) require.NoError(t, err) @@ -543,24 +553,25 @@ func TestDataSourceProxy_routeRule(t *testing.T) { }) t.Run("When proxying data source proxy should handle authentication", func(t *testing.T) { + secretsStore := kvstore.SetupTestService(t) secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) tests := []*testCase{ - createAuthTest(t, secretsService, models.DS_INFLUXDB_08, "http://localhost:9090", authTypePassword, authCheckQuery, false), - createAuthTest(t, secretsService, models.DS_INFLUXDB_08, "http://localhost:9090", authTypePassword, authCheckQuery, true), - createAuthTest(t, secretsService, models.DS_INFLUXDB, "http://localhost:9090", authTypePassword, authCheckHeader, true), - createAuthTest(t, secretsService, models.DS_INFLUXDB, "http://localhost:9090", authTypePassword, authCheckHeader, false), - createAuthTest(t, secretsService, models.DS_INFLUXDB, "http://localhost:9090", authTypeBasic, authCheckHeader, true), - createAuthTest(t, secretsService, models.DS_INFLUXDB, "http://localhost:9090", authTypeBasic, authCheckHeader, false), + createAuthTest(t, secretsStore, models.DS_INFLUXDB_08, "http://localhost:9090", authTypePassword, authCheckQuery, false), + createAuthTest(t, secretsStore, models.DS_INFLUXDB_08, "http://localhost:9090", authTypePassword, authCheckQuery, true), + createAuthTest(t, secretsStore, models.DS_INFLUXDB, "http://localhost:9090", authTypePassword, authCheckHeader, true), + createAuthTest(t, secretsStore, models.DS_INFLUXDB, "http://localhost:9090", authTypePassword, authCheckHeader, false), + createAuthTest(t, secretsStore, models.DS_INFLUXDB, "http://localhost:9090", authTypeBasic, authCheckHeader, true), + createAuthTest(t, secretsStore, models.DS_INFLUXDB, "http://localhost:9090", authTypeBasic, authCheckHeader, false), // These two should be enough for any other datasource at the moment. Proxy has special handling // only for Influx, others have the same path and only BasicAuth. Non BasicAuth datasources // do not go through proxy but through TSDB API which is not tested here. - createAuthTest(t, secretsService, models.DS_ES, "http://localhost:9200", authTypeBasic, authCheckHeader, false), - createAuthTest(t, secretsService, models.DS_ES, "http://localhost:9200", authTypeBasic, authCheckHeader, true), + createAuthTest(t, secretsStore, models.DS_ES, "http://localhost:9200", authTypeBasic, authCheckHeader, false), + createAuthTest(t, secretsStore, models.DS_ES, "http://localhost:9200", authTypeBasic, authCheckHeader, true), } for _, test := range tests { - runDatasourceAuthTest(t, secretsService, cfg, test) + runDatasourceAuthTest(t, secretsService, secretsStore, cfg, test) } }) } @@ -624,9 +635,10 @@ func TestDataSourceProxy_requestHandling(t *testing.T) { t.Run("When response header Set-Cookie is not set should remove proxied Set-Cookie header", func(t *testing.T) { ctx, ds := setUp(t) var routes []*plugins.Route + secretsStore := kvstore.SetupTestService(t) secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) - dsService := datasourceservice.ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) - proxy, err := NewDataSourceProxy(ds, routes, ctx, "/render", &setting.Cfg{}, httpClientProvider, &oauthtoken.Service{}, dsService, tracer, secretsService) + dsService := datasourceservice.ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) + proxy, err := NewDataSourceProxy(ds, routes, ctx, "/render", &setting.Cfg{}, httpClientProvider, &oauthtoken.Service{}, dsService, tracer) require.NoError(t, err) proxy.HandleRequest() @@ -642,9 +654,10 @@ func TestDataSourceProxy_requestHandling(t *testing.T) { }, }) var routes []*plugins.Route + secretsStore := kvstore.SetupTestService(t) secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) - dsService := datasourceservice.ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) - proxy, err := NewDataSourceProxy(ds, routes, ctx, "/render", &setting.Cfg{}, httpClientProvider, &oauthtoken.Service{}, dsService, tracer, secretsService) + dsService := datasourceservice.ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) + proxy, err := NewDataSourceProxy(ds, routes, ctx, "/render", &setting.Cfg{}, httpClientProvider, &oauthtoken.Service{}, dsService, tracer) require.NoError(t, err) proxy.HandleRequest() @@ -656,9 +669,10 @@ func TestDataSourceProxy_requestHandling(t *testing.T) { t.Run("When response should set Content-Security-Policy header", func(t *testing.T) { ctx, ds := setUp(t) var routes []*plugins.Route + secretsStore := kvstore.SetupTestService(t) secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) - dsService := datasourceservice.ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) - proxy, err := NewDataSourceProxy(ds, routes, ctx, "/render", &setting.Cfg{}, httpClientProvider, &oauthtoken.Service{}, dsService, tracer, secretsService) + dsService := datasourceservice.ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) + proxy, err := NewDataSourceProxy(ds, routes, ctx, "/render", &setting.Cfg{}, httpClientProvider, &oauthtoken.Service{}, dsService, tracer) require.NoError(t, err) proxy.HandleRequest() @@ -678,9 +692,10 @@ func TestDataSourceProxy_requestHandling(t *testing.T) { }, }) var routes []*plugins.Route + secretsStore := kvstore.SetupTestService(t) secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) - dsService := datasourceservice.ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) - proxy, err := NewDataSourceProxy(ds, routes, ctx, "/render", &setting.Cfg{}, httpClientProvider, &oauthtoken.Service{}, dsService, tracer, secretsService) + dsService := datasourceservice.ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) + proxy, err := NewDataSourceProxy(ds, routes, ctx, "/render", &setting.Cfg{}, httpClientProvider, &oauthtoken.Service{}, dsService, tracer) require.NoError(t, err) proxy.HandleRequest() @@ -703,9 +718,10 @@ func TestDataSourceProxy_requestHandling(t *testing.T) { ctx.Req = httptest.NewRequest("GET", "/api/datasources/proxy/1/path/%2Ftest%2Ftest%2F?query=%2Ftest%2Ftest%2F", nil) var routes []*plugins.Route + secretsStore := kvstore.SetupTestService(t) secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) - dsService := datasourceservice.ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) - proxy, err := NewDataSourceProxy(ds, routes, ctx, "/path/%2Ftest%2Ftest%2F", &setting.Cfg{}, httpClientProvider, &oauthtoken.Service{}, dsService, tracer, secretsService) + dsService := datasourceservice.ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) + proxy, err := NewDataSourceProxy(ds, routes, ctx, "/path/%2Ftest%2Ftest%2F", &setting.Cfg{}, httpClientProvider, &oauthtoken.Service{}, dsService, tracer) require.NoError(t, err) proxy.HandleRequest() @@ -727,9 +743,10 @@ func TestDataSourceProxy_requestHandling(t *testing.T) { ctx.Req = httptest.NewRequest("GET", "/api/datasources/proxy/1/path/%2Ftest%2Ftest%2F?query=%2Ftest%2Ftest%2F", nil) var routes []*plugins.Route + secretsStore := kvstore.SetupTestService(t) secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) - dsService := datasourceservice.ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) - proxy, err := NewDataSourceProxy(ds, routes, ctx, "/path/%2Ftest%2Ftest%2F", &setting.Cfg{}, httpClientProvider, &oauthtoken.Service{}, dsService, tracer, secretsService) + dsService := datasourceservice.ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) + proxy, err := NewDataSourceProxy(ds, routes, ctx, "/path/%2Ftest%2Ftest%2F", &setting.Cfg{}, httpClientProvider, &oauthtoken.Service{}, dsService, tracer) require.NoError(t, err) proxy.HandleRequest() @@ -752,9 +769,10 @@ func TestNewDataSourceProxy_InvalidURL(t *testing.T) { tracer, err := tracing.InitializeTracerForTest() require.NoError(t, err) var routes []*plugins.Route + secretsStore := kvstore.SetupTestService(t) secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) - dsService := datasourceservice.ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) - _, err = NewDataSourceProxy(&ds, routes, &ctx, "api/method", cfg, httpclient.NewProvider(), &oauthtoken.Service{}, dsService, tracer, secretsService) + dsService := datasourceservice.ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) + _, err = NewDataSourceProxy(&ds, routes, &ctx, "api/method", cfg, httpclient.NewProvider(), &oauthtoken.Service{}, dsService, tracer) require.Error(t, err) assert.True(t, strings.HasPrefix(err.Error(), `validation of data source URL "://host/root" failed`)) } @@ -773,9 +791,10 @@ func TestNewDataSourceProxy_ProtocolLessURL(t *testing.T) { require.NoError(t, err) var routes []*plugins.Route + secretsStore := kvstore.SetupTestService(t) secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) - dsService := datasourceservice.ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) - _, err = NewDataSourceProxy(&ds, routes, &ctx, "api/method", cfg, httpclient.NewProvider(), &oauthtoken.Service{}, dsService, tracer, secretsService) + dsService := datasourceservice.ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) + _, err = NewDataSourceProxy(&ds, routes, &ctx, "api/method", cfg, httpclient.NewProvider(), &oauthtoken.Service{}, dsService, tracer) require.NoError(t, err) } @@ -816,9 +835,10 @@ func TestNewDataSourceProxy_MSSQL(t *testing.T) { } var routes []*plugins.Route + secretsStore := kvstore.SetupTestService(t) secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) - dsService := datasourceservice.ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) - p, err := NewDataSourceProxy(&ds, routes, &ctx, "api/method", cfg, httpclient.NewProvider(), &oauthtoken.Service{}, dsService, tracer, secretsService) + dsService := datasourceservice.ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) + p, err := NewDataSourceProxy(&ds, routes, &ctx, "api/method", cfg, httpclient.NewProvider(), &oauthtoken.Service{}, dsService, tracer) if tc.err == nil { require.NoError(t, err) assert.Equal(t, &url.URL{ @@ -843,9 +863,10 @@ func getDatasourceProxiedRequest(t *testing.T, ctx *models.ReqContext, cfg *sett require.NoError(t, err) var routes []*plugins.Route + secretsStore := kvstore.SetupTestService(t) secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) - dsService := datasourceservice.ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) - proxy, err := NewDataSourceProxy(ds, routes, ctx, "", cfg, httpclient.NewProvider(), &oauthtoken.Service{}, dsService, tracer, secretsService) + dsService := datasourceservice.ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) + proxy, err := NewDataSourceProxy(ds, routes, ctx, "", cfg, httpclient.NewProvider(), &oauthtoken.Service{}, dsService, tracer) require.NoError(t, err) req, err := http.NewRequest(http.MethodGet, "http://grafana.com/sub", nil) require.NoError(t, err) @@ -897,15 +918,15 @@ const ( authCheckHeader = "header" ) -func createAuthTest(t *testing.T, secretsService secrets.Service, dsType string, url string, authType string, authCheck string, useSecureJsonData bool) *testCase { - ctx := context.Background() - +func createAuthTest(t *testing.T, secretsStore kvstore.SecretsKVStore, dsType string, url string, authType string, authCheck string, useSecureJsonData bool) *testCase { // Basic user:password base64AuthHeader := "Basic dXNlcjpwYXNzd29yZA==" test := &testCase{ datasource: &models.DataSource{ Id: 1, + OrgId: 1, + Name: fmt.Sprintf("%s,%s,%s,%s,%t", dsType, url, authType, authCheck, useSecureJsonData), Type: dsType, JsonData: simplejson.New(), Url: url, @@ -917,11 +938,13 @@ func createAuthTest(t *testing.T, secretsService secrets.Service, dsType string, message = fmt.Sprintf("%v should add username and password", dsType) test.datasource.User = "user" if useSecureJsonData { - test.datasource.SecureJsonData, err = secretsService.EncryptJsonData( - ctx, - map[string]string{ - "password": "password", - }, secrets.WithoutScope()) + secureJsonData, err := json.Marshal(map[string]string{ + "password": "password", + }) + require.NoError(t, err) + + err = secretsStore.Set(context.Background(), test.datasource.OrgId, test.datasource.Name, "datasource", string(secureJsonData)) + require.NoError(t, err) } else { test.datasource.Password = "password" } @@ -930,11 +953,13 @@ func createAuthTest(t *testing.T, secretsService secrets.Service, dsType string, test.datasource.BasicAuth = true test.datasource.BasicAuthUser = "user" if useSecureJsonData { - test.datasource.SecureJsonData, err = secretsService.EncryptJsonData( - ctx, - map[string]string{ - "basicAuthPassword": "password", - }, secrets.WithoutScope()) + secureJsonData, err := json.Marshal(map[string]string{ + "basicAuthPassword": "password", + }) + require.NoError(t, err) + + err = secretsStore.Set(context.Background(), test.datasource.OrgId, test.datasource.Name, "datasource", string(secureJsonData)) + require.NoError(t, err) } else { test.datasource.BasicAuthPassword = "password" } @@ -962,14 +987,14 @@ func createAuthTest(t *testing.T, secretsService secrets.Service, dsType string, return test } -func runDatasourceAuthTest(t *testing.T, secretsService secrets.Service, cfg *setting.Cfg, test *testCase) { +func runDatasourceAuthTest(t *testing.T, secretsService secrets.Service, secretsStore kvstore.SecretsKVStore, cfg *setting.Cfg, test *testCase) { ctx := &models.ReqContext{} tracer, err := tracing.InitializeTracerForTest() require.NoError(t, err) var routes []*plugins.Route - dsService := datasourceservice.ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) - proxy, err := NewDataSourceProxy(test.datasource, routes, ctx, "", &setting.Cfg{}, httpclient.NewProvider(), &oauthtoken.Service{}, dsService, tracer, secretsService) + dsService := datasourceservice.ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) + proxy, err := NewDataSourceProxy(test.datasource, routes, ctx, "", &setting.Cfg{}, httpclient.NewProvider(), &oauthtoken.Service{}, dsService, tracer) require.NoError(t, err) req, err := http.NewRequest(http.MethodGet, "http://grafana.com/sub", nil) @@ -1010,9 +1035,10 @@ func Test_PathCheck(t *testing.T) { return ctx, req } ctx, _ := setUp() + secretsStore := kvstore.SetupTestService(t) secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) - dsService := datasourceservice.ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) - proxy, err := NewDataSourceProxy(&models.DataSource{}, routes, ctx, "b", &setting.Cfg{}, httpclient.NewProvider(), &oauthtoken.Service{}, dsService, tracer, secretsService) + dsService := datasourceservice.ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) + proxy, err := NewDataSourceProxy(&models.DataSource{}, routes, ctx, "b", &setting.Cfg{}, httpclient.NewProvider(), &oauthtoken.Service{}, dsService, tracer) require.NoError(t, err) require.Nil(t, proxy.validateRequest()) diff --git a/pkg/expr/service.go b/pkg/expr/service.go index 4e276683004..d01cc386e9f 100644 --- a/pkg/expr/service.go +++ b/pkg/expr/service.go @@ -7,7 +7,7 @@ import ( "github.com/grafana/grafana/pkg/components/simplejson" "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/plugins" - "github.com/grafana/grafana/pkg/services/secrets" + "github.com/grafana/grafana/pkg/services/datasources" "github.com/grafana/grafana/pkg/setting" ) @@ -36,16 +36,16 @@ func IsDataSource(uid string) bool { // Service is service representation for expression handling. type Service struct { - cfg *setting.Cfg - dataService backend.QueryDataHandler - secretsService secrets.Service + cfg *setting.Cfg + dataService backend.QueryDataHandler + dataSourceService datasources.DataSourceService } -func ProvideService(cfg *setting.Cfg, pluginClient plugins.Client, secretsService secrets.Service) *Service { +func ProvideService(cfg *setting.Cfg, pluginClient plugins.Client, dataSourceService datasources.DataSourceService) *Service { return &Service{ - cfg: cfg, - dataService: pluginClient, - secretsService: secretsService, + cfg: cfg, + dataService: pluginClient, + dataSourceService: dataSourceService, } } diff --git a/pkg/expr/service_test.go b/pkg/expr/service_test.go index 6741174baad..c47944c283a 100644 --- a/pkg/expr/service_test.go +++ b/pkg/expr/service_test.go @@ -11,8 +11,7 @@ import ( "github.com/grafana/grafana-plugin-sdk-go/backend" "github.com/grafana/grafana-plugin-sdk-go/data" "github.com/grafana/grafana/pkg/models" - "github.com/grafana/grafana/pkg/services/secrets/fakes" - secretsManager "github.com/grafana/grafana/pkg/services/secrets/manager" + datasources "github.com/grafana/grafana/pkg/services/datasources/fakes" "github.com/grafana/grafana/pkg/setting" "github.com/stretchr/testify/require" ) @@ -28,12 +27,10 @@ func TestService(t *testing.T) { cfg := setting.NewCfg() - secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) - s := Service{ - cfg: cfg, - dataService: me, - secretsService: secretsService, + cfg: cfg, + dataService: me, + dataSourceService: &datasources.FakeDataSourceService{}, } queries := []Query{ diff --git a/pkg/expr/transform.go b/pkg/expr/transform.go index fb327613fe9..5e309d84c30 100644 --- a/pkg/expr/transform.go +++ b/pkg/expr/transform.go @@ -126,9 +126,9 @@ func hiddenRefIDs(queries []Query) (map[string]struct{}, error) { return hidden, nil } -func (s *Service) decryptSecureJsonDataFn(ctx context.Context) func(map[string][]byte) map[string]string { - return func(m map[string][]byte) map[string]string { - decryptedJsonData, err := s.secretsService.DecryptJsonData(ctx, m) +func (s *Service) decryptSecureJsonDataFn(ctx context.Context) func(ds *models.DataSource) map[string]string { + return func(ds *models.DataSource) map[string]string { + decryptedJsonData, err := s.dataSourceService.DecryptedValues(ctx, ds) if err != nil { logger.Error("Failed to decrypt secure json data", "error", err) } diff --git a/pkg/infra/usagestats/statscollector/prometheus_flavor.go b/pkg/infra/usagestats/statscollector/prometheus_flavor.go index 5d79e9f2a2f..652b312ea6a 100644 --- a/pkg/infra/usagestats/statscollector/prometheus_flavor.go +++ b/pkg/infra/usagestats/statscollector/prometheus_flavor.go @@ -60,7 +60,7 @@ func (s *Service) detectPrometheusVariant(ctx context.Context, ds *models.DataSo } `json:"data"` } - c, err := s.datasources.GetHTTPTransport(ds, s.httpClientProvider) + c, err := s.datasources.GetHTTPTransport(ctx, ds, s.httpClientProvider) if err != nil { s.log.Error("Failed to get HTTP client for Prometheus data source", "error", err) return "", err diff --git a/pkg/infra/usagestats/statscollector/service_test.go b/pkg/infra/usagestats/statscollector/service_test.go index 91419ac5cbc..86320488e03 100644 --- a/pkg/infra/usagestats/statscollector/service_test.go +++ b/pkg/infra/usagestats/statscollector/service_test.go @@ -395,6 +395,6 @@ func (s mockDatasourceService) GetDataSourcesByType(ctx context.Context, query * return nil } -func (s mockDatasourceService) GetHTTPTransport(ds *models.DataSource, provider httpclient.Provider, customMiddlewares ...sdkhttpclient.Middleware) (http.RoundTripper, error) { +func (s mockDatasourceService) GetHTTPTransport(ctx context.Context, ds *models.DataSource, provider httpclient.Provider, customMiddlewares ...sdkhttpclient.Middleware) (http.RoundTripper, error) { return provider.GetTransport() } diff --git a/pkg/plugins/adapters/adapters.go b/pkg/plugins/adapters/adapters.go index 00c7933e6b3..96827e4f439 100644 --- a/pkg/plugins/adapters/adapters.go +++ b/pkg/plugins/adapters/adapters.go @@ -9,7 +9,7 @@ import ( ) // ModelToInstanceSettings converts a models.DataSource to a backend.DataSourceInstanceSettings. -func ModelToInstanceSettings(ds *models.DataSource, decryptFn func(map[string][]byte) map[string]string, +func ModelToInstanceSettings(ds *models.DataSource, decryptFn func(ds *models.DataSource) map[string]string, ) (*backend.DataSourceInstanceSettings, error) { var jsonDataBytes json.RawMessage if ds.JsonData != nil { @@ -30,7 +30,7 @@ func ModelToInstanceSettings(ds *models.DataSource, decryptFn func(map[string][] BasicAuthEnabled: ds.BasicAuth, BasicAuthUser: ds.BasicAuthUser, JSONData: jsonDataBytes, - DecryptedSecureJSONData: decryptFn(ds.SecureJsonData), + DecryptedSecureJSONData: decryptFn(ds), Updated: ds.Updated, }, nil } diff --git a/pkg/plugins/plugincontext/plugincontext.go b/pkg/plugins/plugincontext/plugincontext.go index 652417534b7..e06a7821d4a 100644 --- a/pkg/plugins/plugincontext/plugincontext.go +++ b/pkg/plugins/plugincontext/plugincontext.go @@ -15,18 +15,17 @@ import ( "github.com/grafana/grafana/pkg/plugins/adapters" "github.com/grafana/grafana/pkg/services/datasources" "github.com/grafana/grafana/pkg/services/pluginsettings" - "github.com/grafana/grafana/pkg/services/secrets" "github.com/grafana/grafana/pkg/util/errutil" ) func ProvideService(cacheService *localcache.CacheService, pluginStore plugins.Store, - dataSourceCache datasources.CacheService, secretsService secrets.Service, + dataSourceCache datasources.CacheService, dataSourceService datasources.DataSourceService, pluginSettingsService pluginsettings.Service) *Provider { return &Provider{ cacheService: cacheService, pluginStore: pluginStore, dataSourceCache: dataSourceCache, - secretsService: secretsService, + dataSourceService: dataSourceService, pluginSettingsService: pluginSettingsService, logger: log.New("plugincontext"), } @@ -36,7 +35,7 @@ type Provider struct { cacheService *localcache.CacheService pluginStore plugins.Store dataSourceCache datasources.CacheService - secretsService secrets.Service + dataSourceService datasources.DataSourceService pluginSettingsService pluginsettings.Service logger log.Logger } @@ -87,7 +86,7 @@ func (p *Provider) Get(ctx context.Context, pluginID string, datasourceUID strin if err != nil { return pc, false, errutil.Wrap("Failed to get datasource", err) } - datasourceSettings, err := adapters.ModelToInstanceSettings(ds, p.decryptSecureJsonDataFn()) + datasourceSettings, err := adapters.ModelToInstanceSettings(ds, p.decryptSecureJsonDataFn(ctx)) if err != nil { return pc, false, errutil.Wrap("Failed to convert datasource", err) } @@ -122,9 +121,9 @@ func (p *Provider) getCachedPluginSettings(ctx context.Context, pluginID string, return ps, nil } -func (p *Provider) decryptSecureJsonDataFn() func(map[string][]byte) map[string]string { - return func(m map[string][]byte) map[string]string { - decryptedJsonData, err := p.secretsService.DecryptJsonData(context.Background(), m) +func (p *Provider) decryptSecureJsonDataFn(ctx context.Context) func(ds *models.DataSource) map[string]string { + return func(ds *models.DataSource) map[string]string { + decryptedJsonData, err := p.dataSourceService.DecryptedValues(ctx, ds) if err != nil { p.logger.Error("Failed to decrypt secure json data", "error", err) } diff --git a/pkg/server/wire.go b/pkg/server/wire.go index ca33eb3eac2..47e2d1eee5d 100644 --- a/pkg/server/wire.go +++ b/pkg/server/wire.go @@ -79,6 +79,7 @@ import ( "github.com/grafana/grafana/pkg/services/searchV2" "github.com/grafana/grafana/pkg/services/secrets" secretsDatabase "github.com/grafana/grafana/pkg/services/secrets/database" + secretsStore "github.com/grafana/grafana/pkg/services/secrets/kvstore" secretsManager "github.com/grafana/grafana/pkg/services/secrets/manager" "github.com/grafana/grafana/pkg/services/serviceaccounts" serviceaccountsmanager "github.com/grafana/grafana/pkg/services/serviceaccounts/manager" @@ -239,6 +240,7 @@ var wireBasicSet = wire.NewSet( wire.Bind(new(alerting.DashAlertExtractor), new(*alerting.DashAlertExtractorService)), comments.ProvideService, guardian.ProvideService, + secretsStore.ProvideService, avatar.ProvideAvatarCacheServer, authproxy.ProvideAuthProxy, statscollector.ProvideService, diff --git a/pkg/services/datasourceproxy/datasourceproxy.go b/pkg/services/datasourceproxy/datasourceproxy.go index 75d5f506ea1..c40c4360e06 100644 --- a/pkg/services/datasourceproxy/datasourceproxy.go +++ b/pkg/services/datasourceproxy/datasourceproxy.go @@ -115,7 +115,7 @@ func (p *DataSourceProxyService) proxyDatasourceRequest(c *models.ReqContext, ds proxyPath := getProxyPath(c) proxy, err := pluginproxy.NewDataSourceProxy(ds, plugin.Routes, c, proxyPath, p.Cfg, p.HTTPClientProvider, - p.OAuthTokenService, p.DataSourcesService, p.tracer, p.secretsService) + p.OAuthTokenService, p.DataSourcesService, p.tracer) if err != nil { if errors.Is(err, datasource.URLValidationError{}) { c.JsonApiErr(http.StatusBadRequest, fmt.Sprintf("Invalid data source URL: %q", ds.Url), err) diff --git a/pkg/services/datasources/datasources.go b/pkg/services/datasources/datasources.go index cfbc2095548..b6212794328 100644 --- a/pkg/services/datasources/datasources.go +++ b/pkg/services/datasources/datasources.go @@ -33,23 +33,23 @@ type DataSourceService interface { GetDefaultDataSource(ctx context.Context, query *models.GetDefaultDataSourceQuery) error // GetHTTPTransport gets a datasource specific HTTP transport. - GetHTTPTransport(ds *models.DataSource, provider httpclient.Provider, customMiddlewares ...sdkhttpclient.Middleware) (http.RoundTripper, error) + GetHTTPTransport(ctx context.Context, ds *models.DataSource, provider httpclient.Provider, customMiddlewares ...sdkhttpclient.Middleware) (http.RoundTripper, error) // DecryptedValues decrypts the encrypted secureJSONData of the provided datasource and // returns the decrypted values. - DecryptedValues(ds *models.DataSource) map[string]string + DecryptedValues(ctx context.Context, ds *models.DataSource) (map[string]string, error) // DecryptedValue decrypts the encrypted datasource secureJSONData identified by key // and returns the decryped value. - DecryptedValue(ds *models.DataSource, key string) (string, bool) + DecryptedValue(ctx context.Context, ds *models.DataSource, key string) (string, bool, error) // DecryptedBasicAuthPassword decrypts the encrypted datasource basic authentication // password and returns the decryped value. - DecryptedBasicAuthPassword(ds *models.DataSource) string + DecryptedBasicAuthPassword(ctx context.Context, ds *models.DataSource) (string, error) // DecryptedPassword decrypts the encrypted datasource password and returns the // decryped value. - DecryptedPassword(ds *models.DataSource) string + DecryptedPassword(ctx context.Context, ds *models.DataSource) (string, error) } // CacheService interface for retrieving a cached datasource. diff --git a/pkg/services/datasources/fake_cache_service.go b/pkg/services/datasources/fakes/fake_cache_service.go similarity index 87% rename from pkg/services/datasources/fake_cache_service.go rename to pkg/services/datasources/fakes/fake_cache_service.go index e4c2ed9193b..7d7fe4c9a97 100644 --- a/pkg/services/datasources/fake_cache_service.go +++ b/pkg/services/datasources/fakes/fake_cache_service.go @@ -4,13 +4,14 @@ import ( "context" "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/services/datasources" ) type FakeCacheService struct { DataSources []*models.DataSource } -var _ CacheService = &FakeCacheService{} +var _ datasources.CacheService = &FakeCacheService{} func (c *FakeCacheService) GetDatasource(ctx context.Context, datasourceID int64, user *models.SignedInUser, skipCache bool) (*models.DataSource, error) { for _, datasource := range c.DataSources { diff --git a/pkg/services/datasources/fakes/fake_datasource_service.go b/pkg/services/datasources/fakes/fake_datasource_service.go new file mode 100644 index 00000000000..a75b40163c8 --- /dev/null +++ b/pkg/services/datasources/fakes/fake_datasource_service.go @@ -0,0 +1,124 @@ +package datasources + +import ( + "context" + "net/http" + + sdkhttpclient "github.com/grafana/grafana-plugin-sdk-go/backend/httpclient" + "github.com/grafana/grafana/pkg/infra/httpclient" + "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/services/datasources" +) + +type FakeDataSourceService struct { + lastId int64 + DataSources []*models.DataSource +} + +var _ datasources.DataSourceService = &FakeDataSourceService{} + +func (s *FakeDataSourceService) GetDataSource(ctx context.Context, query *models.GetDataSourceQuery) error { + for _, datasource := range s.DataSources { + idMatch := query.Id != 0 && query.Id == datasource.Id + uidMatch := query.Uid != "" && query.Uid == datasource.Uid + nameMatch := query.Name != "" && query.Name == datasource.Name + if idMatch || nameMatch || uidMatch { + query.Result = datasource + + return nil + } + } + return models.ErrDataSourceNotFound +} + +func (s *FakeDataSourceService) GetDataSources(ctx context.Context, query *models.GetDataSourcesQuery) error { + for _, datasource := range s.DataSources { + orgMatch := query.OrgId != 0 && query.OrgId == datasource.OrgId + if orgMatch { + query.Result = append(query.Result, datasource) + } + } + return nil +} + +func (s *FakeDataSourceService) GetDataSourcesByType(ctx context.Context, query *models.GetDataSourcesByTypeQuery) error { + for _, datasource := range s.DataSources { + typeMatch := query.Type != "" && query.Type == datasource.Type + if typeMatch { + query.Result = append(query.Result, datasource) + } + } + return nil +} + +func (s *FakeDataSourceService) AddDataSource(ctx context.Context, cmd *models.AddDataSourceCommand) error { + if s.lastId == 0 { + s.lastId = int64(len(s.DataSources) - 1) + } + cmd.Result = &models.DataSource{ + Id: s.lastId + 1, + Name: cmd.Name, + Type: cmd.Type, + Uid: cmd.Uid, + OrgId: cmd.OrgId, + } + s.DataSources = append(s.DataSources, cmd.Result) + return nil +} + +func (s *FakeDataSourceService) DeleteDataSource(ctx context.Context, cmd *models.DeleteDataSourceCommand) error { + for i, datasource := range s.DataSources { + idMatch := cmd.ID != 0 && cmd.ID == datasource.Id + uidMatch := cmd.UID != "" && cmd.UID == datasource.Uid + nameMatch := cmd.Name != "" && cmd.Name == datasource.Name + if idMatch || nameMatch || uidMatch { + s.DataSources = append(s.DataSources[:i], s.DataSources[i+1:]...) + return nil + } + } + return models.ErrDataSourceNotFound +} + +func (s *FakeDataSourceService) UpdateDataSource(ctx context.Context, cmd *models.UpdateDataSourceCommand) error { + for _, datasource := range s.DataSources { + idMatch := cmd.Id != 0 && cmd.Id == datasource.Id + uidMatch := cmd.Uid != "" && cmd.Uid == datasource.Uid + nameMatch := cmd.Name != "" && cmd.Name == datasource.Name + if idMatch || nameMatch || uidMatch { + if cmd.Name != "" { + datasource.Name = cmd.Name + } + return nil + } + } + return models.ErrDataSourceNotFound +} + +func (s *FakeDataSourceService) GetDefaultDataSource(ctx context.Context, query *models.GetDefaultDataSourceQuery) error { + return nil +} + +func (s *FakeDataSourceService) GetHTTPTransport(ctx context.Context, ds *models.DataSource, provider httpclient.Provider, customMiddlewares ...sdkhttpclient.Middleware) (http.RoundTripper, error) { + rt, err := provider.GetTransport(sdkhttpclient.Options{}) + if err != nil { + return nil, err + } + return rt, nil +} + +func (s *FakeDataSourceService) DecryptedValues(ctx context.Context, ds *models.DataSource) (map[string]string, error) { + values := make(map[string]string) + return values, nil +} + +func (s *FakeDataSourceService) DecryptedValue(ctx context.Context, ds *models.DataSource, key string) (string, bool, error) { + return "", false, nil +} + +func (s *FakeDataSourceService) DecryptedBasicAuthPassword(ctx context.Context, ds *models.DataSource) (string, error) { + return "", nil +} + +func (s *FakeDataSourceService) DecryptedPassword(ctx context.Context, ds *models.DataSource) (string, error) { + return "", nil +} diff --git a/pkg/services/datasources/service/datasource_service.go b/pkg/services/datasources/service/datasource_service.go index dbc22e6e261..7f74a8bafa0 100644 --- a/pkg/services/datasources/service/datasource_service.go +++ b/pkg/services/datasources/service/datasource_service.go @@ -3,6 +3,7 @@ package service import ( "context" "crypto/tls" + "encoding/json" "fmt" "net/http" "net/url" @@ -23,20 +24,21 @@ import ( "github.com/grafana/grafana/pkg/services/datasources" "github.com/grafana/grafana/pkg/services/featuremgmt" "github.com/grafana/grafana/pkg/services/secrets" + "github.com/grafana/grafana/pkg/services/secrets/kvstore" "github.com/grafana/grafana/pkg/services/sqlstore" "github.com/grafana/grafana/pkg/setting" ) type Service struct { SQLStore *sqlstore.SQLStore + SecretsStore kvstore.SecretsKVStore SecretsService secrets.Service cfg *setting.Cfg features featuremgmt.FeatureToggles permissionsService accesscontrol.PermissionsService ac accesscontrol.AccessControl - ptc proxyTransportCache - dsDecryptionCache secureJSONDecryptionCache + ptc proxyTransportCache } type proxyTransportCache struct { @@ -49,29 +51,17 @@ type cachedRoundTripper struct { roundTripper http.RoundTripper } -type secureJSONDecryptionCache struct { - cache map[int64]cachedDecryptedJSON - sync.Mutex -} - -type cachedDecryptedJSON struct { - updated time.Time - json map[string]string -} - func ProvideService( - store *sqlstore.SQLStore, secretsService secrets.Service, cfg *setting.Cfg, features featuremgmt.FeatureToggles, - ac accesscontrol.AccessControl, permissionsServices accesscontrol.PermissionsServices, + store *sqlstore.SQLStore, secretsService secrets.Service, secretsStore kvstore.SecretsKVStore, cfg *setting.Cfg, + features featuremgmt.FeatureToggles, ac accesscontrol.AccessControl, permissionsServices accesscontrol.PermissionsServices, ) *Service { s := &Service{ SQLStore: store, + SecretsStore: secretsStore, SecretsService: secretsService, ptc: proxyTransportCache{ cache: make(map[int64]cachedRoundTripper), }, - dsDecryptionCache: secureJSONDecryptionCache{ - cache: make(map[int64]cachedDecryptedJSON), - }, cfg: cfg, features: features, permissionsService: permissionsServices.GetDataSourceService(), @@ -90,6 +80,8 @@ type DataSourceRetriever interface { GetDataSource(ctx context.Context, query *models.GetDataSourceQuery) error } +const secretType = "datasource" + // NewNameScopeResolver provides an AttributeScopeResolver able to // translate a scope prefixed with "datasources:name:" into an uid based scope. func NewNameScopeResolver(db DataSourceRetriever) (string, accesscontrol.AttributeScopeResolveFunc) { @@ -155,12 +147,17 @@ func (s *Service) GetDataSourcesByType(ctx context.Context, query *models.GetDat func (s *Service) AddDataSource(ctx context.Context, cmd *models.AddDataSourceCommand) error { var err error - cmd.EncryptedSecureJsonData, err = s.SecretsService.EncryptJsonData(ctx, cmd.SecureJsonData, secrets.WithoutScope()) + if err := s.SQLStore.AddDataSource(ctx, cmd); err != nil { + return err + } + + secret, err := json.Marshal(cmd.SecureJsonData) if err != nil { return err } - if err := s.SQLStore.AddDataSource(ctx, cmd); err != nil { + err = s.SecretsStore.Set(ctx, cmd.OrgId, cmd.Name, secretType, string(secret)) + if err != nil { return err } @@ -186,25 +183,50 @@ func (s *Service) AddDataSource(ctx context.Context, cmd *models.AddDataSourceCo } func (s *Service) DeleteDataSource(ctx context.Context, cmd *models.DeleteDataSourceCommand) error { - return s.SQLStore.DeleteDataSource(ctx, cmd) + err := s.SQLStore.DeleteDataSource(ctx, cmd) + if err != nil { + return err + } + return s.SecretsStore.Del(ctx, cmd.OrgID, cmd.Name, secretType) } func (s *Service) UpdateDataSource(ctx context.Context, cmd *models.UpdateDataSourceCommand) error { var err error - cmd.EncryptedSecureJsonData, err = s.SecretsService.EncryptJsonData(ctx, cmd.SecureJsonData, secrets.WithoutScope()) + secret, err := json.Marshal(cmd.SecureJsonData) if err != nil { return err } - return s.SQLStore.UpdateDataSource(ctx, cmd) + query := &models.GetDataSourceQuery{ + Id: cmd.Id, + OrgId: cmd.OrgId, + } + err = s.SQLStore.GetDataSource(ctx, query) + if err != nil { + return err + } + + err = s.SQLStore.UpdateDataSource(ctx, cmd) + if err != nil { + return err + } + + if query.Result.Name != cmd.Name { + err = s.SecretsStore.Rename(ctx, cmd.OrgId, query.Result.Name, secretType, cmd.Name) + if err != nil { + return err + } + } + + return s.SecretsStore.Set(ctx, cmd.OrgId, cmd.Name, secretType, string(secret)) } func (s *Service) GetDefaultDataSource(ctx context.Context, query *models.GetDefaultDataSourceQuery) error { return s.SQLStore.GetDefaultDataSource(ctx, query) } -func (s *Service) GetHTTPClient(ds *models.DataSource, provider httpclient.Provider) (*http.Client, error) { - transport, err := s.GetHTTPTransport(ds, provider) +func (s *Service) GetHTTPClient(ctx context.Context, ds *models.DataSource, provider httpclient.Provider) (*http.Client, error) { + transport, err := s.GetHTTPTransport(ctx, ds, provider) if err != nil { return nil, err } @@ -215,7 +237,7 @@ func (s *Service) GetHTTPClient(ds *models.DataSource, provider httpclient.Provi }, nil } -func (s *Service) GetHTTPTransport(ds *models.DataSource, provider httpclient.Provider, +func (s *Service) GetHTTPTransport(ctx context.Context, ds *models.DataSource, provider httpclient.Provider, customMiddlewares ...sdkhttpclient.Middleware) (http.RoundTripper, error) { s.ptc.Lock() defer s.ptc.Unlock() @@ -224,7 +246,7 @@ func (s *Service) GetHTTPTransport(ds *models.DataSource, provider httpclient.Pr return t.roundTripper, nil } - opts, err := s.httpClientOptions(ds) + opts, err := s.httpClientOptions(ctx, ds) if err != nil { return nil, err } @@ -244,58 +266,84 @@ func (s *Service) GetHTTPTransport(ds *models.DataSource, provider httpclient.Pr return rt, nil } -func (s *Service) GetTLSConfig(ds *models.DataSource, httpClientProvider httpclient.Provider) (*tls.Config, error) { - opts, err := s.httpClientOptions(ds) +func (s *Service) GetTLSConfig(ctx context.Context, ds *models.DataSource, httpClientProvider httpclient.Provider) (*tls.Config, error) { + opts, err := s.httpClientOptions(ctx, ds) if err != nil { return nil, err } return httpClientProvider.GetTLSConfig(*opts) } -func (s *Service) DecryptedValues(ds *models.DataSource) map[string]string { - s.dsDecryptionCache.Lock() - defer s.dsDecryptionCache.Unlock() - - if item, present := s.dsDecryptionCache.cache[ds.Id]; present && ds.Updated.Equal(item.updated) { - return item.json - } - - json, err := s.SecretsService.DecryptJsonData(context.Background(), ds.SecureJsonData) +func (s *Service) DecryptedValues(ctx context.Context, ds *models.DataSource) (map[string]string, error) { + decryptedValues := make(map[string]string) + secret, exist, err := s.SecretsStore.Get(ctx, ds.OrgId, ds.Name, secretType) if err != nil { - return map[string]string{} + return nil, err } - s.dsDecryptionCache.cache[ds.Id] = cachedDecryptedJSON{ - updated: ds.Updated, - json: json, + if exist { + err := json.Unmarshal([]byte(secret), &decryptedValues) + if err != nil { + return nil, err + } + } else if len(ds.SecureJsonData) > 0 { + decryptedValues, err = s.MigrateSecrets(ctx, ds) + if err != nil { + return nil, err + } } - return json + return decryptedValues, nil } -func (s *Service) DecryptedValue(ds *models.DataSource, key string) (string, bool) { - value, exists := s.DecryptedValues(ds)[key] - return value, exists -} - -func (s *Service) DecryptedBasicAuthPassword(ds *models.DataSource) string { - if value, ok := s.DecryptedValue(ds, "basicAuthPassword"); ok { - return value +func (s *Service) MigrateSecrets(ctx context.Context, ds *models.DataSource) (map[string]string, error) { + secureJsonData, err := s.SecretsService.DecryptJsonData(ctx, ds.SecureJsonData) + if err != nil { + return nil, err } - return ds.BasicAuthPassword -} - -func (s *Service) DecryptedPassword(ds *models.DataSource) string { - if value, ok := s.DecryptedValue(ds, "password"); ok { - return value + jsonData, err := json.Marshal(secureJsonData) + if err != nil { + return nil, err } - return ds.Password + err = s.SecretsStore.Set(ctx, ds.OrgId, ds.Name, secretType, string(jsonData)) + return secureJsonData, err } -func (s *Service) httpClientOptions(ds *models.DataSource) (*sdkhttpclient.Options, error) { - tlsOptions := s.dsTLSOptions(ds) +func (s *Service) DecryptedValue(ctx context.Context, ds *models.DataSource, key string) (string, bool, error) { + values, err := s.DecryptedValues(ctx, ds) + if err != nil { + return "", false, err + } + value, exists := values[key] + return value, exists, nil +} + +func (s *Service) DecryptedBasicAuthPassword(ctx context.Context, ds *models.DataSource) (string, error) { + value, ok, err := s.DecryptedValue(ctx, ds, "basicAuthPassword") + if ok { + return value, nil + } + + return ds.BasicAuthPassword, err +} + +func (s *Service) DecryptedPassword(ctx context.Context, ds *models.DataSource) (string, error) { + value, ok, err := s.DecryptedValue(ctx, ds, "password") + if ok { + return value, nil + } + + return ds.Password, err +} + +func (s *Service) httpClientOptions(ctx context.Context, ds *models.DataSource) (*sdkhttpclient.Options, error) { + tlsOptions, err := s.dsTLSOptions(ctx, ds) + if err != nil { + return nil, err + } + timeouts := &sdkhttpclient.TimeoutOptions{ Timeout: s.getTimeout(ds), DialTimeout: sdkhttpclient.DefaultTimeoutOptions.DialTimeout, @@ -307,9 +355,15 @@ func (s *Service) httpClientOptions(ds *models.DataSource) (*sdkhttpclient.Optio MaxIdleConnsPerHost: sdkhttpclient.DefaultTimeoutOptions.MaxIdleConnsPerHost, IdleConnTimeout: sdkhttpclient.DefaultTimeoutOptions.IdleConnTimeout, } + + decryptedValues, err := s.DecryptedValues(ctx, ds) + if err != nil { + return nil, err + } + opts := &sdkhttpclient.Options{ Timeouts: timeouts, - Headers: s.getCustomHeaders(ds.JsonData, s.DecryptedValues(ds)), + Headers: s.getCustomHeaders(ds.JsonData, decryptedValues), Labels: map[string]string{ "datasource_name": ds.Name, "datasource_uid": ds.Uid, @@ -320,22 +374,30 @@ func (s *Service) httpClientOptions(ds *models.DataSource) (*sdkhttpclient.Optio if ds.JsonData != nil { opts.CustomOptions = ds.JsonData.MustMap() } - if ds.BasicAuth { + password, err := s.DecryptedBasicAuthPassword(ctx, ds) + if err != nil { + return opts, err + } + opts.BasicAuth = &sdkhttpclient.BasicAuthOptions{ User: ds.BasicAuthUser, - Password: s.DecryptedBasicAuthPassword(ds), + Password: password, } } else if ds.User != "" { + password, err := s.DecryptedPassword(ctx, ds) + if err != nil { + return opts, err + } + opts.BasicAuth = &sdkhttpclient.BasicAuthOptions{ User: ds.User, - Password: s.DecryptedPassword(ds), + Password: password, } } - // Azure authentication if ds.JsonData != nil && s.features.IsEnabled(featuremgmt.FlagHttpclientproviderAzureAuth) { - credentials, err := azcredentials.FromDatasourceData(ds.JsonData.MustMap(), s.DecryptedValues(ds)) + credentials, err := azcredentials.FromDatasourceData(ds.JsonData.MustMap(), decryptedValues) if err != nil { err = fmt.Errorf("invalid Azure credentials: %s", err) return nil, err @@ -371,19 +433,27 @@ func (s *Service) httpClientOptions(ds *models.DataSource) (*sdkhttpclient.Optio Profile: ds.JsonData.Get("sigV4Profile").MustString(), } - if val, exists := s.DecryptedValue(ds, "sigV4AccessKey"); exists { - opts.SigV4.AccessKey = val + if val, exists, err := s.DecryptedValue(ctx, ds, "sigV4AccessKey"); err == nil { + if exists { + opts.SigV4.AccessKey = val + } + } else { + return opts, err } - if val, exists := s.DecryptedValue(ds, "sigV4SecretKey"); exists { - opts.SigV4.SecretKey = val + if val, exists, err := s.DecryptedValue(ctx, ds, "sigV4SecretKey"); err == nil { + if exists { + opts.SigV4.SecretKey = val + } + } else { + return opts, err } } return opts, nil } -func (s *Service) dsTLSOptions(ds *models.DataSource) sdkhttpclient.TLSOptions { +func (s *Service) dsTLSOptions(ctx context.Context, ds *models.DataSource) (sdkhttpclient.TLSOptions, error) { var tlsSkipVerify, tlsClientAuth, tlsAuthWithCACert bool var serverName string @@ -401,22 +471,35 @@ func (s *Service) dsTLSOptions(ds *models.DataSource) sdkhttpclient.TLSOptions { if tlsClientAuth || tlsAuthWithCACert { if tlsAuthWithCACert { - if val, exists := s.DecryptedValue(ds, "tlsCACert"); exists && len(val) > 0 { - opts.CACertificate = val + if val, exists, err := s.DecryptedValue(ctx, ds, "tlsCACert"); err == nil { + if exists && len(val) > 0 { + opts.CACertificate = val + } + } else { + return opts, err } } if tlsClientAuth { - if val, exists := s.DecryptedValue(ds, "tlsClientCert"); exists && len(val) > 0 { - opts.ClientCertificate = val + if val, exists, err := s.DecryptedValue(ctx, ds, "tlsClientCert"); err == nil { + fmt.Print("\n\n\n\n", val, exists, err, "\n\n\n\n") + if exists && len(val) > 0 { + opts.ClientCertificate = val + } + } else { + return opts, err } - if val, exists := s.DecryptedValue(ds, "tlsClientKey"); exists && len(val) > 0 { - opts.ClientKey = val + if val, exists, err := s.DecryptedValue(ctx, ds, "tlsClientKey"); err == nil { + if exists && len(val) > 0 { + opts.ClientKey = val + } + } else { + return opts, err } } } - return opts + return opts, nil } func (s *Service) getTimeout(ds *models.DataSource) time.Duration { diff --git a/pkg/services/datasources/service/datasource_service_test.go b/pkg/services/datasources/service/datasource_service_test.go index 2df5cb86f50..747a34be797 100644 --- a/pkg/services/datasources/service/datasource_service_test.go +++ b/pkg/services/datasources/service/datasource_service_test.go @@ -2,6 +2,7 @@ package service import ( "context" + encJson "encoding/json" "io/ioutil" "net/http" "net/http/httptest" @@ -10,6 +11,7 @@ import ( "github.com/grafana/grafana-azure-sdk-go/azsettings" sdkhttpclient "github.com/grafana/grafana-plugin-sdk-go/backend/httpclient" + secretsManager "github.com/grafana/grafana/pkg/services/secrets/manager" "github.com/grafana/grafana/pkg/components/simplejson" "github.com/grafana/grafana/pkg/infra/httpclient" @@ -17,59 +19,13 @@ import ( "github.com/grafana/grafana/pkg/services/accesscontrol" acmock "github.com/grafana/grafana/pkg/services/accesscontrol/mock" "github.com/grafana/grafana/pkg/services/featuremgmt" - "github.com/grafana/grafana/pkg/services/secrets" - "github.com/grafana/grafana/pkg/services/secrets/database" "github.com/grafana/grafana/pkg/services/secrets/fakes" - secretsManager "github.com/grafana/grafana/pkg/services/secrets/manager" - "github.com/grafana/grafana/pkg/services/sqlstore" + "github.com/grafana/grafana/pkg/services/secrets/kvstore" "github.com/grafana/grafana/pkg/setting" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) -func TestService(t *testing.T) { - cfg := &setting.Cfg{} - sqlStore := sqlstore.InitTestDB(t) - - origSecret := setting.SecretKey - setting.SecretKey = "datasources_service_test" - t.Cleanup(func() { - setting.SecretKey = origSecret - }) - - secretsService := secretsManager.SetupTestService(t, database.ProvideSecretsStore(sqlStore)) - s := ProvideService(sqlStore, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New().WithDisabled(), acmock.NewPermissionsServicesMock()) - - var ds *models.DataSource - - t.Run("create datasource should encrypt the secure json data", func(t *testing.T) { - ctx := context.Background() - - sjd := map[string]string{"password": "12345"} - cmd := models.AddDataSourceCommand{SecureJsonData: sjd} - - err := s.AddDataSource(ctx, &cmd) - require.NoError(t, err) - - ds = cmd.Result - decrypted, err := s.SecretsService.DecryptJsonData(ctx, ds.SecureJsonData) - require.NoError(t, err) - require.Equal(t, sjd, decrypted) - }) - - t.Run("update datasource should encrypt the secure json data", func(t *testing.T) { - ctx := context.Background() - sjd := map[string]string{"password": "678910"} - cmd := models.UpdateDataSourceCommand{Id: ds.Id, OrgId: ds.OrgId, SecureJsonData: sjd} - err := s.UpdateDataSource(ctx, &cmd) - require.NoError(t, err) - - decrypted, err := s.SecretsService.DecryptJsonData(ctx, cmd.Result.SecureJsonData) - require.NoError(t, err) - require.Equal(t, sjd, decrypted) - }) -} - type dataSourceMockRetriever struct { res []*models.DataSource } @@ -237,15 +193,16 @@ func TestService_GetHttpTransport(t *testing.T) { Type: "Kubernetes", } + secretsStore := kvstore.SetupTestService(t) secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) - dsService := ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) + dsService := ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) - rt1, err := dsService.GetHTTPTransport(&ds, provider) + rt1, err := dsService.GetHTTPTransport(context.Background(), &ds, provider) require.NoError(t, err) require.NotNil(t, rt1) tr1 := configuredTransport - rt2, err := dsService.GetHTTPTransport(&ds, provider) + rt2, err := dsService.GetHTTPTransport(context.Background(), &ds, provider) require.NoError(t, err) require.NotNil(t, rt2) tr2 := configuredTransport @@ -270,21 +227,19 @@ func TestService_GetHttpTransport(t *testing.T) { json := simplejson.New() json.Set("tlsAuthWithCACert", true) + secretsStore := kvstore.SetupTestService(t) secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) - dsService := ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) - - tlsCaCert, err := secretsService.Encrypt(context.Background(), []byte(caCert), secrets.WithoutScope()) - require.NoError(t, err) + dsService := ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) ds := models.DataSource{ Id: 1, Url: "http://k8s:8001", Type: "Kubernetes", - SecureJsonData: map[string][]byte{"tlsCACert": tlsCaCert}, + SecureJsonData: map[string][]byte{"tlsCACert": []byte(caCert)}, Updated: time.Now().Add(-2 * time.Minute), } - rt1, err := dsService.GetHTTPTransport(&ds, provider) + rt1, err := dsService.GetHTTPTransport(context.Background(), &ds, provider) require.NotNil(t, rt1) require.NoError(t, err) @@ -298,7 +253,7 @@ func TestService_GetHttpTransport(t *testing.T) { ds.SecureJsonData = map[string][]byte{} ds.Updated = time.Now() - rt2, err := dsService.GetHTTPTransport(&ds, provider) + rt2, err := dsService.GetHTTPTransport(context.Background(), &ds, provider) require.NoError(t, err) require.NotNil(t, rt2) tr2 := configuredTransport @@ -320,27 +275,29 @@ func TestService_GetHttpTransport(t *testing.T) { json := simplejson.New() json.Set("tlsAuth", true) + secretsStore := kvstore.SetupTestService(t) secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) - dsService := ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) - - tlsClientCert, err := secretsService.Encrypt(context.Background(), []byte(clientCert), secrets.WithoutScope()) - require.NoError(t, err) - - tlsClientKey, err := secretsService.Encrypt(context.Background(), []byte(clientKey), secrets.WithoutScope()) - require.NoError(t, err) + dsService := ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) ds := models.DataSource{ Id: 1, + OrgId: 1, + Name: "kubernetes", Url: "http://k8s:8001", Type: "Kubernetes", JsonData: json, - SecureJsonData: map[string][]byte{ - "tlsClientCert": tlsClientCert, - "tlsClientKey": tlsClientKey, - }, } - rt, err := dsService.GetHTTPTransport(&ds, provider) + secureJsonData, err := encJson.Marshal(map[string]string{ + "tlsClientCert": clientCert, + "tlsClientKey": clientKey, + }) + require.NoError(t, err) + + err = secretsStore.Set(context.Background(), ds.OrgId, ds.Name, secretType, string(secureJsonData)) + require.NoError(t, err) + + rt, err := dsService.GetHTTPTransport(context.Background(), &ds, provider) require.NoError(t, err) require.NotNil(t, rt) tr := configuredTransport @@ -363,23 +320,28 @@ func TestService_GetHttpTransport(t *testing.T) { json.Set("tlsAuthWithCACert", true) json.Set("serverName", "server-name") + secretsStore := kvstore.SetupTestService(t) secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) - dsService := ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) - - tlsCaCert, err := secretsService.Encrypt(context.Background(), []byte(caCert), secrets.WithoutScope()) - require.NoError(t, err) + dsService := ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) ds := models.DataSource{ Id: 1, + OrgId: 1, + Name: "kubernetes", Url: "http://k8s:8001", Type: "Kubernetes", JsonData: json, - SecureJsonData: map[string][]byte{ - "tlsCACert": tlsCaCert, - }, } - rt, err := dsService.GetHTTPTransport(&ds, provider) + secureJsonData, err := encJson.Marshal(map[string]string{ + "tlsCACert": caCert, + }) + require.NoError(t, err) + + err = secretsStore.Set(context.Background(), ds.OrgId, ds.Name, secretType, string(secureJsonData)) + require.NoError(t, err) + + rt, err := dsService.GetHTTPTransport(context.Background(), &ds, provider) require.NoError(t, err) require.NotNil(t, rt) tr := configuredTransport @@ -400,8 +362,9 @@ func TestService_GetHttpTransport(t *testing.T) { json := simplejson.New() json.Set("tlsSkipVerify", true) + secretsStore := kvstore.SetupTestService(t) secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) - dsService := ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) + dsService := ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) ds := models.DataSource{ Id: 1, @@ -410,12 +373,12 @@ func TestService_GetHttpTransport(t *testing.T) { JsonData: json, } - rt1, err := dsService.GetHTTPTransport(&ds, provider) + rt1, err := dsService.GetHTTPTransport(context.Background(), &ds, provider) require.NoError(t, err) require.NotNil(t, rt1) tr1 := configuredTransport - rt2, err := dsService.GetHTTPTransport(&ds, provider) + rt2, err := dsService.GetHTTPTransport(context.Background(), &ds, provider) require.NoError(t, err) require.NotNil(t, rt2) tr2 := configuredTransport @@ -431,20 +394,27 @@ func TestService_GetHttpTransport(t *testing.T) { "httpHeaderName1": "Authorization", }) + secretsStore := kvstore.SetupTestService(t) secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) - dsService := ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) - - encryptedData, err := secretsService.Encrypt(context.Background(), []byte(`Bearer xf5yhfkpsnmgo`), secrets.WithoutScope()) - require.NoError(t, err) + dsService := ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) ds := models.DataSource{ - Id: 1, - Url: "http://k8s:8001", - Type: "Kubernetes", - JsonData: json, - SecureJsonData: map[string][]byte{"httpHeaderValue1": encryptedData}, + Id: 1, + OrgId: 1, + Name: "kubernetes", + Url: "http://k8s:8001", + Type: "Kubernetes", + JsonData: json, } + secureJsonData, err := encJson.Marshal(map[string]string{ + "httpHeaderValue1": "Bearer xf5yhfkpsnmgo", + }) + require.NoError(t, err) + + err = secretsStore.Set(context.Background(), ds.OrgId, ds.Name, secretType, string(secureJsonData)) + require.NoError(t, err) + headers := dsService.getCustomHeaders(json, map[string]string{"httpHeaderValue1": "Bearer xf5yhfkpsnmgo"}) require.Equal(t, "Bearer xf5yhfkpsnmgo", headers["Authorization"]) @@ -465,7 +435,7 @@ func TestService_GetHttpTransport(t *testing.T) { // 2. Get HTTP transport from datasource which uses the test server as backend ds.Url = backend.URL - rt, err := dsService.GetHTTPTransport(&ds, provider) + rt, err := dsService.GetHTTPTransport(context.Background(), &ds, provider) require.NoError(t, err) require.NotNil(t, rt) @@ -490,8 +460,9 @@ func TestService_GetHttpTransport(t *testing.T) { "timeout": 19, }) + secretsStore := kvstore.SetupTestService(t) secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) - dsService := ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) + dsService := ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) ds := models.DataSource{ Id: 1, @@ -500,7 +471,7 @@ func TestService_GetHttpTransport(t *testing.T) { JsonData: json, } - client, err := dsService.GetHTTPClient(&ds, provider) + client, err := dsService.GetHTTPClient(context.Background(), &ds, provider) require.NoError(t, err) require.NotNil(t, client) require.Equal(t, 19*time.Second, client.Timeout) @@ -523,15 +494,16 @@ func TestService_GetHttpTransport(t *testing.T) { json, err := simplejson.NewJson([]byte(`{ "sigV4Auth": true }`)) require.NoError(t, err) + secretsStore := kvstore.SetupTestService(t) secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) - dsService := ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) + dsService := ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) ds := models.DataSource{ Type: models.DS_ES, JsonData: json, } - _, err = dsService.GetHTTPTransport(&ds, provider) + _, err = dsService.GetHTTPTransport(context.Background(), &ds, provider) require.NoError(t, err) require.NotNil(t, configuredOpts) require.NotNil(t, configuredOpts.SigV4) @@ -558,8 +530,9 @@ func TestService_getTimeout(t *testing.T) { {jsonData: simplejson.NewFromAny(map[string]interface{}{"timeout": "2"}), expectedTimeout: 2 * time.Second}, } + secretsStore := kvstore.SetupTestService(t) secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) - dsService := ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) + dsService := ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) for _, tc := range testCases { ds := &models.DataSource{ @@ -569,86 +542,6 @@ func TestService_getTimeout(t *testing.T) { } } -func TestService_DecryptedValue(t *testing.T) { - cfg := &setting.Cfg{} - - t.Run("When datasource hasn't been updated, encrypted JSON should be fetched from cache", func(t *testing.T) { - secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) - dsService := ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) - - encryptedJsonData, err := secretsService.EncryptJsonData( - context.Background(), - map[string]string{ - "password": "password", - }, secrets.WithoutScope()) - require.NoError(t, err) - - ds := models.DataSource{ - Id: 1, - Type: models.DS_INFLUXDB_08, - JsonData: simplejson.New(), - User: "user", - SecureJsonData: encryptedJsonData, - } - - // Populate cache - password, ok := dsService.DecryptedValue(&ds, "password") - require.True(t, ok) - require.Equal(t, "password", password) - - encryptedJsonData, err = secretsService.EncryptJsonData( - context.Background(), - map[string]string{ - "password": "", - }, secrets.WithoutScope()) - require.NoError(t, err) - - ds.SecureJsonData = encryptedJsonData - - password, ok = dsService.DecryptedValue(&ds, "password") - require.True(t, ok) - require.Equal(t, "password", password) - }) - - t.Run("When datasource is updated, encrypted JSON should not be fetched from cache", func(t *testing.T) { - secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) - - encryptedJsonData, err := secretsService.EncryptJsonData( - context.Background(), - map[string]string{ - "password": "password", - }, secrets.WithoutScope()) - require.NoError(t, err) - - ds := models.DataSource{ - Id: 1, - Type: models.DS_INFLUXDB_08, - JsonData: simplejson.New(), - User: "user", - SecureJsonData: encryptedJsonData, - } - - dsService := ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) - - // Populate cache - password, ok := dsService.DecryptedValue(&ds, "password") - require.True(t, ok) - require.Equal(t, "password", password) - - ds.SecureJsonData, err = secretsService.EncryptJsonData( - context.Background(), - map[string]string{ - "password": "", - }, secrets.WithoutScope()) - ds.Updated = time.Now() - require.NoError(t, err) - - password, ok = dsService.DecryptedValue(&ds, "password") - require.True(t, ok) - require.Empty(t, password) - }) -} - func TestService_HTTPClientOptions(t *testing.T) { cfg := &setting.Cfg{ Azure: &azsettings.AzureSettings{}, @@ -678,10 +571,11 @@ func TestService_HTTPClientOptions(t *testing.T) { "azureEndpointResourceId": "https://api.example.com/abd5c4ce-ca73-41e9-9cb2-bed39aa2adb5", }) + secretsStore := kvstore.SetupTestService(t) secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) - dsService := ProvideService(nil, secretsService, cfg, features, acmock.New(), acmock.NewPermissionsServicesMock()) + dsService := ProvideService(nil, secretsService, secretsStore, cfg, features, acmock.New(), acmock.NewPermissionsServicesMock()) - opts, err := dsService.httpClientOptions(&ds) + opts, err := dsService.httpClientOptions(context.Background(), &ds) require.NoError(t, err) require.NotNil(t, opts.Middlewares) @@ -695,10 +589,11 @@ func TestService_HTTPClientOptions(t *testing.T) { "httpMethod": "POST", }) + secretsStore := kvstore.SetupTestService(t) secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) - dsService := ProvideService(nil, secretsService, cfg, features, acmock.New(), acmock.NewPermissionsServicesMock()) + dsService := ProvideService(nil, secretsService, secretsStore, cfg, features, acmock.New(), acmock.NewPermissionsServicesMock()) - opts, err := dsService.httpClientOptions(&ds) + opts, err := dsService.httpClientOptions(context.Background(), &ds) require.NoError(t, err) if opts.Middlewares != nil { @@ -714,10 +609,11 @@ func TestService_HTTPClientOptions(t *testing.T) { "azureCredentials": "invalid", }) + secretsStore := kvstore.SetupTestService(t) secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) - dsService := ProvideService(nil, secretsService, cfg, features, acmock.New(), acmock.NewPermissionsServicesMock()) + dsService := ProvideService(nil, secretsService, secretsStore, cfg, features, acmock.New(), acmock.NewPermissionsServicesMock()) - _, err := dsService.httpClientOptions(&ds) + _, err := dsService.httpClientOptions(context.Background(), &ds) assert.Error(t, err) }) @@ -732,10 +628,11 @@ func TestService_HTTPClientOptions(t *testing.T) { "azureEndpointResourceId": "https://api.example.com/abd5c4ce-ca73-41e9-9cb2-bed39aa2adb5", }) + secretsStore := kvstore.SetupTestService(t) secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) - dsService := ProvideService(nil, secretsService, cfg, features, acmock.New(), acmock.NewPermissionsServicesMock()) + dsService := ProvideService(nil, secretsService, secretsStore, cfg, features, acmock.New(), acmock.NewPermissionsServicesMock()) - opts, err := dsService.httpClientOptions(&ds) + opts, err := dsService.httpClientOptions(context.Background(), &ds) require.NoError(t, err) require.NotNil(t, opts.Middlewares) @@ -750,10 +647,11 @@ func TestService_HTTPClientOptions(t *testing.T) { "azureEndpointResourceId": "https://api.example.com/abd5c4ce-ca73-41e9-9cb2-bed39aa2adb5", }) + secretsStore := kvstore.SetupTestService(t) secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) - dsService := ProvideService(nil, secretsService, cfg, features, acmock.New(), acmock.NewPermissionsServicesMock()) + dsService := ProvideService(nil, secretsService, secretsStore, cfg, features, acmock.New(), acmock.NewPermissionsServicesMock()) - opts, err := dsService.httpClientOptions(&ds) + opts, err := dsService.httpClientOptions(context.Background(), &ds) require.NoError(t, err) if opts.Middlewares != nil { @@ -772,10 +670,11 @@ func TestService_HTTPClientOptions(t *testing.T) { "azureEndpointResourceId": "invalid", }) + secretsStore := kvstore.SetupTestService(t) secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) - dsService := ProvideService(nil, secretsService, cfg, features, acmock.New(), acmock.NewPermissionsServicesMock()) + dsService := ProvideService(nil, secretsService, secretsStore, cfg, features, acmock.New(), acmock.NewPermissionsServicesMock()) - _, err := dsService.httpClientOptions(&ds) + _, err := dsService.httpClientOptions(context.Background(), &ds) assert.Error(t, err) }) }) @@ -792,10 +691,11 @@ func TestService_HTTPClientOptions(t *testing.T) { "azureEndpointResourceId": "https://api.example.com/abd5c4ce-ca73-41e9-9cb2-bed39aa2adb5", }) + secretsStore := kvstore.SetupTestService(t) secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) - dsService := ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) + dsService := ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) - opts, err := dsService.httpClientOptions(&ds) + opts, err := dsService.httpClientOptions(context.Background(), &ds) require.NoError(t, err) if opts.Middlewares != nil { diff --git a/pkg/services/ngalert/api/api_testing_test.go b/pkg/services/ngalert/api/api_testing_test.go index 91b976b7364..384aaf45ea5 100644 --- a/pkg/services/ngalert/api/api_testing_test.go +++ b/pkg/services/ngalert/api/api_testing_test.go @@ -13,6 +13,7 @@ import ( "github.com/grafana/grafana/pkg/services/accesscontrol" acMock "github.com/grafana/grafana/pkg/services/accesscontrol/mock" "github.com/grafana/grafana/pkg/services/datasources" + fakes "github.com/grafana/grafana/pkg/services/datasources/fakes" "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions" "github.com/grafana/grafana/pkg/services/ngalert/eval" "github.com/grafana/grafana/pkg/services/ngalert/models" @@ -61,7 +62,7 @@ func TestRouteTestGrafanaRuleConfig(t *testing.T) { {Action: datasources.ActionQuery, Scope: datasources.ScopeProvider.GetResourceScopeUID(data2.DatasourceUID)}, }) - ds := &datasources.FakeCacheService{DataSources: []*models2.DataSource{ + ds := &fakes.FakeCacheService{DataSources: []*models2.DataSource{ {Uid: data1.DatasourceUID}, {Uid: data2.DatasourceUID}, }} @@ -102,7 +103,7 @@ func TestRouteTestGrafanaRuleConfig(t *testing.T) { t.Run("should require user to be signed in", func(t *testing.T) { data1 := models.GenerateAlertQuery() - ds := &datasources.FakeCacheService{DataSources: []*models2.DataSource{ + ds := &fakes.FakeCacheService{DataSources: []*models2.DataSource{ {Uid: data1.DatasourceUID}, }} @@ -182,7 +183,7 @@ func TestRouteEvalQueries(t *testing.T) { {Action: datasources.ActionQuery, Scope: datasources.ScopeProvider.GetResourceScopeUID(data2.DatasourceUID)}, }) - ds := &datasources.FakeCacheService{DataSources: []*models2.DataSource{ + ds := &fakes.FakeCacheService{DataSources: []*models2.DataSource{ {Uid: data1.DatasourceUID}, {Uid: data2.DatasourceUID}, }} @@ -226,7 +227,7 @@ func TestRouteEvalQueries(t *testing.T) { t.Run("should require user to be signed in", func(t *testing.T) { data1 := models.GenerateAlertQuery() - ds := &datasources.FakeCacheService{DataSources: []*models2.DataSource{ + ds := &fakes.FakeCacheService{DataSources: []*models2.DataSource{ {Uid: data1.DatasourceUID}, }} @@ -265,7 +266,7 @@ func TestRouteEvalQueries(t *testing.T) { }) } -func createTestingApiSrv(ds *datasources.FakeCacheService, ac *acMock.Mock, evaluator *eval.FakeEvaluator) *TestingApiSrv { +func createTestingApiSrv(ds *fakes.FakeCacheService, ac *acMock.Mock, evaluator *eval.FakeEvaluator) *TestingApiSrv { if ac == nil { ac = acMock.New().WithDisabled() } diff --git a/pkg/services/query/query.go b/pkg/services/query/query.go index 95d3e7437c3..b449648ca22 100644 --- a/pkg/services/query/query.go +++ b/pkg/services/query/query.go @@ -15,7 +15,6 @@ import ( "github.com/grafana/grafana/pkg/plugins/adapters" "github.com/grafana/grafana/pkg/services/datasources" "github.com/grafana/grafana/pkg/services/oauthtoken" - "github.com/grafana/grafana/pkg/services/secrets" "github.com/grafana/grafana/pkg/setting" "github.com/grafana/grafana/pkg/tsdb/grafanads" "github.com/grafana/grafana/pkg/tsdb/legacydata" @@ -33,7 +32,7 @@ func ProvideService( dataSourceCache datasources.CacheService, expressionService *expr.Service, pluginRequestValidator models.PluginRequestValidator, - SecretsService secrets.Service, + dataSourceService datasources.DataSourceService, pluginClient plugins.Client, oAuthTokenService oauthtoken.OAuthTokenService, ) *Service { @@ -42,7 +41,7 @@ func ProvideService( dataSourceCache: dataSourceCache, expressionService: expressionService, pluginRequestValidator: pluginRequestValidator, - secretsService: SecretsService, + dataSourceService: dataSourceService, pluginClient: pluginClient, oAuthTokenService: oAuthTokenService, log: log.New("query_data"), @@ -56,7 +55,7 @@ type Service struct { dataSourceCache datasources.CacheService expressionService *expr.Service pluginRequestValidator models.PluginRequestValidator - secretsService secrets.Service + dataSourceService datasources.DataSourceService pluginClient plugins.Client oAuthTokenService oauthtoken.OAuthTokenService log log.Logger @@ -291,9 +290,9 @@ func (s *Service) getDataSourceFromQuery(ctx context.Context, user *models.Signe return nil, NewErrBadQuery("missing data source ID/UID") } -func (s *Service) decryptSecureJsonDataFn(ctx context.Context) func(map[string][]byte) map[string]string { - return func(m map[string][]byte) map[string]string { - decryptedJsonData, err := s.secretsService.DecryptJsonData(ctx, m) +func (s *Service) decryptSecureJsonDataFn(ctx context.Context) func(ds *models.DataSource) map[string]string { + return func(ds *models.DataSource) map[string]string { + decryptedJsonData, err := s.dataSourceService.DecryptedValues(ctx, ds) if err != nil { s.log.Error("Failed to decrypt secure json data", "error", err) } diff --git a/pkg/services/query/query_test.go b/pkg/services/query/query_test.go index 3b0193302f6..b5e05ed5e35 100644 --- a/pkg/services/query/query_test.go +++ b/pkg/services/query/query_test.go @@ -2,6 +2,7 @@ package query_test import ( "context" + "encoding/json" "net/http" "testing" @@ -12,18 +13,29 @@ import ( "github.com/grafana/grafana/pkg/components/simplejson" "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/plugins" + acmock "github.com/grafana/grafana/pkg/services/accesscontrol/mock" + datasources "github.com/grafana/grafana/pkg/services/datasources/service" + "github.com/grafana/grafana/pkg/services/featuremgmt" "github.com/grafana/grafana/pkg/services/query" - "github.com/grafana/grafana/pkg/services/secrets" + "github.com/grafana/grafana/pkg/services/secrets/fakes" + "github.com/grafana/grafana/pkg/services/secrets/kvstore" + secretsManager "github.com/grafana/grafana/pkg/services/secrets/manager" + "github.com/stretchr/testify/require" ) func TestQueryData(t *testing.T) { t.Run("it attaches custom headers to the request", func(t *testing.T) { - tc := setup() + tc := setup(t) tc.dataSourceCache.ds.JsonData = simplejson.NewFromAny(map[string]interface{}{"httpHeaderName1": "foo", "httpHeaderName2": "bar"}) - tc.secretService.decryptedJson = map[string]string{"httpHeaderValue1": "test-header", "httpHeaderValue2": "test-header2"} - _, err := tc.queryService.QueryData(context.Background(), nil, true, metricRequest(), false) + secureJsonData, err := json.Marshal(map[string]string{"httpHeaderValue1": "test-header", "httpHeaderValue2": "test-header2"}) + require.NoError(t, err) + + err = tc.secretStore.Set(context.Background(), tc.dataSourceCache.ds.OrgId, tc.dataSourceCache.ds.Name, "datasource", string(secureJsonData)) + require.NoError(t, err) + + _, err = tc.queryService.QueryData(context.Background(), nil, true, metricRequest(), false) require.Nil(t, err) require.Equal(t, map[string]string{"foo": "test-header", "bar": "test-header2"}, tc.pluginContext.req.Headers) @@ -36,7 +48,7 @@ func TestQueryData(t *testing.T) { } token = token.WithExtra(map[string]interface{}{"id_token": "id-token"}) - tc := setup() + tc := setup(t) tc.oauthTokenService.passThruEnabled = true tc.oauthTokenService.token = token @@ -51,26 +63,29 @@ func TestQueryData(t *testing.T) { }) } -func setup() *testContext { +func setup(t *testing.T) *testContext { pc := &fakePluginClient{} - sc := &fakeSecretsService{} dc := &fakeDataSourceCache{ds: &models.DataSource{}} tc := &fakeOAuthTokenService{} rv := &fakePluginRequestValidator{} + ss := kvstore.SetupTestService(t) + ssvc := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) + ds := datasources.ProvideService(nil, ssvc, ss, nil, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) + return &testContext{ pluginContext: pc, - secretService: sc, + secretStore: ss, dataSourceCache: dc, oauthTokenService: tc, pluginRequestValidator: rv, - queryService: query.ProvideService(nil, dc, nil, rv, sc, pc, tc), + queryService: query.ProvideService(nil, dc, nil, rv, ds, pc, tc), } } type testContext struct { pluginContext *fakePluginClient - secretService *fakeSecretsService + secretStore kvstore.SecretsKVStore dataSourceCache *fakeDataSourceCache oauthTokenService *fakeOAuthTokenService pluginRequestValidator *fakePluginRequestValidator @@ -108,16 +123,6 @@ func (ts *fakeOAuthTokenService) IsOAuthPassThruEnabled(*models.DataSource) bool return ts.passThruEnabled } -type fakeSecretsService struct { - secrets.Service - - decryptedJson map[string]string -} - -func (s *fakeSecretsService) DecryptJsonData(ctx context.Context, sjd map[string][]byte) (map[string]string, error) { - return s.decryptedJson, nil -} - type fakeDataSourceCache struct { ds *models.DataSource } diff --git a/pkg/services/secrets/kvstore/helpers.go b/pkg/services/secrets/kvstore/helpers.go new file mode 100644 index 00000000000..c00923ca8ea --- /dev/null +++ b/pkg/services/secrets/kvstore/helpers.go @@ -0,0 +1,29 @@ +package kvstore + +import ( + "testing" + + "github.com/grafana/grafana/pkg/infra/log" + "github.com/grafana/grafana/pkg/services/secrets/database" + "github.com/grafana/grafana/pkg/services/secrets/manager" + "github.com/grafana/grafana/pkg/services/sqlstore" +) + +func SetupTestService(t *testing.T) SecretsKVStore { + t.Helper() + + sqlStore := sqlstore.InitTestDB(t) + store := database.ProvideSecretsStore(sqlstore.InitTestDB(t)) + secretsService := manager.SetupTestService(t, store) + + kv := &secretsKVStoreSQL{ + sqlStore: sqlStore, + log: log.New("secrets.kvstore"), + secretsService: secretsService, + decryptionCache: decryptionCache{ + cache: make(map[int64]cachedDecrypted), + }, + } + + return kv +} diff --git a/pkg/services/secrets/kvstore/kvstore.go b/pkg/services/secrets/kvstore/kvstore.go new file mode 100644 index 00000000000..b438aec3c41 --- /dev/null +++ b/pkg/services/secrets/kvstore/kvstore.go @@ -0,0 +1,77 @@ +package kvstore + +import ( + "context" + + "github.com/grafana/grafana/pkg/infra/log" + "github.com/grafana/grafana/pkg/services/secrets" + "github.com/grafana/grafana/pkg/services/sqlstore" +) + +const ( + // Wildcard to query all organizations + AllOrganizations = -1 +) + +func ProvideService(sqlStore sqlstore.Store, secretsService secrets.Service) SecretsKVStore { + return &secretsKVStoreSQL{ + sqlStore: sqlStore, + secretsService: secretsService, + log: log.New("secrets.kvstore"), + decryptionCache: decryptionCache{ + cache: make(map[int64]cachedDecrypted), + }, + } +} + +// SecretsKVStore is an interface for k/v store. +type SecretsKVStore interface { + Get(ctx context.Context, orgId int64, namespace string, typ string) (string, bool, error) + Set(ctx context.Context, orgId int64, namespace string, typ string, value string) error + Del(ctx context.Context, orgId int64, namespace string, typ string) error + Keys(ctx context.Context, orgId int64, namespace string, typ string) ([]Key, error) + Rename(ctx context.Context, orgId int64, namespace string, typ string, newNamespace string) error +} + +// WithType returns a kvstore wrapper with fixed orgId and type. +func With(kv SecretsKVStore, orgId int64, namespace string, typ string) *FixedKVStore { + return &FixedKVStore{ + kvStore: kv, + OrgId: orgId, + Namespace: namespace, + Type: typ, + } +} + +// FixedKVStore is a SecretsKVStore wrapper with fixed orgId, namespace and type. +type FixedKVStore struct { + kvStore SecretsKVStore + OrgId int64 + Namespace string + Type string +} + +func (kv *FixedKVStore) Get(ctx context.Context) (string, bool, error) { + return kv.kvStore.Get(ctx, kv.OrgId, kv.Namespace, kv.Type) +} + +func (kv *FixedKVStore) Set(ctx context.Context, value string) error { + return kv.kvStore.Set(ctx, kv.OrgId, kv.Namespace, kv.Type, value) +} + +func (kv *FixedKVStore) Del(ctx context.Context) error { + return kv.kvStore.Del(ctx, kv.OrgId, kv.Namespace, kv.Type) +} + +func (kv *FixedKVStore) Keys(ctx context.Context) ([]Key, error) { + return kv.kvStore.Keys(ctx, kv.OrgId, kv.Namespace, kv.Type) +} + +func (kv *FixedKVStore) Rename(ctx context.Context, newNamespace string) error { + err := kv.kvStore.Rename(ctx, kv.OrgId, kv.Namespace, kv.Type, newNamespace) + if err != nil { + return err + } + kv.Namespace = newNamespace + return nil +} diff --git a/pkg/services/secrets/kvstore/kvstore_test.go b/pkg/services/secrets/kvstore/kvstore_test.go new file mode 100644 index 00000000000..9b6bff4a371 --- /dev/null +++ b/pkg/services/secrets/kvstore/kvstore_test.go @@ -0,0 +1,226 @@ +package kvstore + +import ( + "context" + "fmt" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type TestCase struct { + OrgId int64 + Namespace string + Type string + Revision int64 +} + +func (t *TestCase) Value() string { + return fmt.Sprintf("%d:%s:%s:%d", t.OrgId, t.Namespace, t.Type, t.Revision) +} + +func TestKVStore(t *testing.T) { + kv := SetupTestService(t) + + ctx := context.Background() + + testCases := []*TestCase{ + { + OrgId: 0, + Namespace: "namespace1", + Type: "testing1", + }, + { + OrgId: 0, + Namespace: "namespace2", + Type: "testing2", + }, + { + OrgId: 1, + Namespace: "namespace1", + Type: "testing1", + }, + { + OrgId: 1, + Namespace: "namespace3", + Type: "testing3", + }, + } + + for _, tc := range testCases { + err := kv.Set(ctx, tc.OrgId, tc.Namespace, tc.Type, tc.Value()) + require.NoError(t, err) + } + + t.Run("get existing keys", func(t *testing.T) { + for _, tc := range testCases { + value, ok, err := kv.Get(ctx, tc.OrgId, tc.Namespace, tc.Type) + require.NoError(t, err) + require.True(t, ok) + require.Equal(t, tc.Value(), value) + } + }) + + t.Run("get nonexistent keys", func(t *testing.T) { + tcs := []*TestCase{ + { + OrgId: 0, + Namespace: "namespace3", + Type: "testing3", + }, + { + OrgId: 1, + Namespace: "namespace2", + Type: "testing2", + }, + { + OrgId: 2, + Namespace: "namespace1", + Type: "testing1", + }, + } + + for _, tc := range tcs { + value, ok, err := kv.Get(ctx, tc.OrgId, tc.Namespace, tc.Type) + require.Nil(t, err) + require.False(t, ok) + require.Equal(t, "", value) + } + }) + + t.Run("modify existing key", func(t *testing.T) { + tc := testCases[0] + + value, ok, err := kv.Get(ctx, tc.OrgId, tc.Namespace, tc.Type) + require.NoError(t, err) + require.True(t, ok) + assert.Equal(t, tc.Value(), value) + + tc.Revision += 1 + + err = kv.Set(ctx, tc.OrgId, tc.Namespace, tc.Type, tc.Value()) + require.NoError(t, err) + + value, ok, err = kv.Get(ctx, tc.OrgId, tc.Namespace, tc.Type) + require.NoError(t, err) + require.True(t, ok) + assert.Equal(t, tc.Value(), value) + }) + + t.Run("use fixed client", func(t *testing.T) { + tc := testCases[0] + + client := With(kv, tc.OrgId, tc.Namespace, tc.Type) + fmt.Println(client.Namespace, client.OrgId, client.Type) + + value, ok, err := client.Get(ctx) + require.NoError(t, err) + require.True(t, ok) + require.Equal(t, tc.Value(), value) + + tc.Revision += 1 + + err = client.Set(ctx, tc.Value()) + require.NoError(t, err) + + value, ok, err = client.Get(ctx) + require.NoError(t, err) + require.True(t, ok) + assert.Equal(t, tc.Value(), value) + }) + + t.Run("deleting keys", func(t *testing.T) { + var stillHasKeys bool + for _, tc := range testCases { + if _, ok, err := kv.Get(ctx, tc.OrgId, tc.Namespace, tc.Type); err == nil && ok { + stillHasKeys = true + break + } + } + require.True(t, stillHasKeys, + "we are going to test key deletion, but there are no keys to delete in the database") + for _, tc := range testCases { + err := kv.Del(ctx, tc.OrgId, tc.Namespace, tc.Type) + require.NoError(t, err) + } + for _, tc := range testCases { + _, ok, err := kv.Get(ctx, tc.OrgId, tc.Namespace, tc.Type) + require.NoError(t, err) + require.False(t, ok, "all keys should be deleted at this point") + } + }) + + t.Run("listing existing keys", func(t *testing.T) { + kv := SetupTestService(t) + + ctx := context.Background() + + namespace, typ := "listtest", "listtest" + + testCases := []*TestCase{ + { + OrgId: 1, + Type: typ, + Namespace: namespace, + }, + { + OrgId: 2, + Type: typ, + Namespace: namespace, + }, + { + OrgId: 3, + Type: typ, + Namespace: namespace, + }, + { + OrgId: 4, + Type: typ, + Namespace: namespace, + }, + { + OrgId: 1, + Type: typ, + Namespace: "other_key", + }, + { + OrgId: 4, + Type: typ, + Namespace: "another_one", + }, + } + + for _, tc := range testCases { + err := kv.Set(ctx, tc.OrgId, tc.Namespace, tc.Type, tc.Value()) + require.NoError(t, err) + } + + keys, err := kv.Keys(ctx, AllOrganizations, namespace, typ) + + require.NoError(t, err) + require.Len(t, keys, 4) + + found := 0 + + for _, key := range keys { + for _, tc := range testCases { + if key.OrgId == tc.OrgId && key.Namespace == tc.Namespace && key.Type == tc.Type { + found++ + break + } + } + } + + require.Equal(t, 4, found, "querying for all orgs should return 4 records") + + keys, err = kv.Keys(ctx, 1, namespace, typ) + + require.NoError(t, err) + require.Len(t, keys, 1, "querying for a specific org should return 1 record") + + keys, err = kv.Keys(ctx, AllOrganizations, "not_existing_namespace", "not_existing_type") + require.NoError(t, err, "querying a not existing namespace should not throw an error") + require.Len(t, keys, 0, "querying a not existing namespace should return an empty slice") + }) +} diff --git a/pkg/services/secrets/kvstore/model.go b/pkg/services/secrets/kvstore/model.go new file mode 100644 index 00000000000..6643de3a1a1 --- /dev/null +++ b/pkg/services/secrets/kvstore/model.go @@ -0,0 +1,31 @@ +package kvstore + +import ( + "time" +) + +// Item stored in k/v store. +type Item struct { + Id int64 + OrgId *int64 + Namespace *string + Type *string + Value string + + Created time.Time + Updated time.Time +} + +func (i *Item) TableName() string { + return "secrets" +} + +type Key struct { + OrgId int64 + Namespace string + Type string +} + +func (i *Key) TableName() string { + return "secrets" +} diff --git a/pkg/services/secrets/kvstore/sql.go b/pkg/services/secrets/kvstore/sql.go new file mode 100644 index 00000000000..08b1c9fe257 --- /dev/null +++ b/pkg/services/secrets/kvstore/sql.go @@ -0,0 +1,220 @@ +package kvstore + +import ( + "context" + "encoding/base64" + "sync" + "time" + + "github.com/grafana/grafana/pkg/infra/log" + "github.com/grafana/grafana/pkg/services/secrets" + "github.com/grafana/grafana/pkg/services/sqlstore" +) + +// secretsKVStoreSQL provides a key/value store backed by the Grafana database +type secretsKVStoreSQL struct { + log log.Logger + sqlStore sqlstore.Store + secretsService secrets.Service + decryptionCache decryptionCache +} + +type decryptionCache struct { + cache map[int64]cachedDecrypted + sync.Mutex +} + +type cachedDecrypted struct { + updated time.Time + value string +} + +var b64 = base64.RawStdEncoding + +// Get an item from the store +func (kv *secretsKVStoreSQL) Get(ctx context.Context, orgId int64, namespace string, typ string) (string, bool, error) { + item := Item{ + OrgId: &orgId, + Namespace: &namespace, + Type: &typ, + } + var isFound bool + var decryptedValue []byte + + err := kv.sqlStore.WithDbSession(ctx, func(dbSession *sqlstore.DBSession) error { + has, err := dbSession.Get(&item) + if err != nil { + kv.log.Debug("error getting secret value", "orgId", orgId, "type", typ, "namespace", namespace, "err", err) + return err + } + if !has { + kv.log.Debug("secret value not found", "orgId", orgId, "type", typ, "namespace", namespace) + return nil + } + isFound = true + kv.log.Debug("got secret value", "orgId", orgId, "type", typ, "namespace", namespace) + return nil + }) + + if err == nil && isFound { + kv.decryptionCache.Lock() + defer kv.decryptionCache.Unlock() + + if cache, present := kv.decryptionCache.cache[item.Id]; present && item.Updated.Equal(cache.updated) { + return cache.value, isFound, err + } + + decodedValue, err := b64.DecodeString(item.Value) + if err != nil { + kv.log.Debug("error decoding secret value", "orgId", orgId, "type", typ, "namespace", namespace, "err", err) + return string(decryptedValue), isFound, err + } + + decryptedValue, err = kv.secretsService.Decrypt(ctx, decodedValue) + if err != nil { + kv.log.Debug("error decrypting secret value", "orgId", orgId, "type", typ, "namespace", namespace, "err", err) + return string(decryptedValue), isFound, err + } + + kv.decryptionCache.cache[item.Id] = cachedDecrypted{ + updated: item.Updated, + value: string(decryptedValue), + } + } + + return string(decryptedValue), isFound, err +} + +// Set an item in the store +func (kv *secretsKVStoreSQL) Set(ctx context.Context, orgId int64, namespace string, typ string, value string) error { + encryptedValue, err := kv.secretsService.Encrypt(ctx, []byte(value), secrets.WithoutScope()) + if err != nil { + kv.log.Debug("error encrypting secret value", "orgId", orgId, "type", typ, "namespace", namespace, "err", err) + return err + } + encodedValue := b64.EncodeToString(encryptedValue) + return kv.sqlStore.WithTransactionalDbSession(ctx, func(dbSession *sqlstore.DBSession) error { + item := Item{ + OrgId: &orgId, + Namespace: &namespace, + Type: &typ, + } + + has, err := dbSession.Get(&item) + if err != nil { + kv.log.Debug("error checking secret value", "orgId", orgId, "type", typ, "namespace", namespace, "err", err) + return err + } + + if has && item.Value == encodedValue { + kv.log.Debug("secret value not changed", "orgId", orgId, "type", typ, "namespace", namespace) + return nil + } + + item.Value = encodedValue + item.Updated = time.Now() + + if has { + // if item already exists we update it + _, err = dbSession.ID(item.Id).Update(&item) + if err != nil { + kv.log.Debug("error updating secret value", "orgId", orgId, "type", typ, "namespace", namespace, "err", err) + } else { + kv.decryptionCache.cache[item.Id] = cachedDecrypted{ + updated: item.Updated, + value: value, + } + kv.log.Debug("secret value updated", "orgId", orgId, "type", typ, "namespace", namespace) + } + return err + } + + // if item doesn't exist we create it + item.Created = item.Updated + _, err = dbSession.Insert(&item) + if err != nil { + kv.log.Debug("error inserting secret value", "orgId", orgId, "type", typ, "namespace", namespace, "err", err) + } else { + kv.log.Debug("secret value inserted", "orgId", orgId, "type", typ, "namespace", namespace) + } + return err + }) +} + +// Del deletes an item from the store. +func (kv *secretsKVStoreSQL) Del(ctx context.Context, orgId int64, namespace string, typ string) error { + err := kv.sqlStore.WithDbSession(ctx, func(dbSession *sqlstore.DBSession) error { + item := Item{ + OrgId: &orgId, + Namespace: &namespace, + Type: &typ, + } + + has, err := dbSession.Get(&item) + if err != nil { + kv.log.Debug("error checking secret value", "orgId", orgId, "type", typ, "namespace", namespace, "err", err) + return err + } + + if has { + // if item exists we delete it + _, err = dbSession.ID(item.Id).Delete(&item) + if err != nil { + kv.log.Debug("error deleting secret value", "orgId", orgId, "type", typ, "namespace", namespace, "err", err) + } else { + delete(kv.decryptionCache.cache, item.Id) + kv.log.Debug("secret value deleted", "orgId", orgId, "type", typ, "namespace", namespace) + } + return err + } + return nil + }) + return err +} + +// Keys get all keys for a given namespace. To query for all +// organizations the constant 'kvstore.AllOrganizations' can be passed as orgId. +func (kv *secretsKVStoreSQL) Keys(ctx context.Context, orgId int64, namespace string, typ string) ([]Key, error) { + var keys []Key + err := kv.sqlStore.WithDbSession(ctx, func(dbSession *sqlstore.DBSession) error { + query := dbSession.Where("namespace = ?", namespace).And("type = ?", typ) + if orgId != AllOrganizations { + query.And("org_id = ?", orgId) + } + return query.Find(&keys) + }) + return keys, err +} + +// Rename an item in the store +func (kv *secretsKVStoreSQL) Rename(ctx context.Context, orgId int64, namespace string, typ string, newNamespace string) error { + return kv.sqlStore.WithTransactionalDbSession(ctx, func(dbSession *sqlstore.DBSession) error { + item := Item{ + OrgId: &orgId, + Namespace: &namespace, + Type: &typ, + } + + has, err := dbSession.Get(&item) + if err != nil { + kv.log.Debug("error checking secret value", "orgId", orgId, "type", typ, "namespace", namespace, "err", err) + return err + } + + item.Namespace = &newNamespace + item.Updated = time.Now() + + if has { + // if item already exists we update it + _, err = dbSession.ID(item.Id).Update(&item) + if err != nil { + kv.log.Debug("error updating secret namespace", "orgId", orgId, "type", typ, "namespace", namespace, "err", err) + } else { + kv.log.Debug("secret namespace updated", "orgId", orgId, "type", typ, "namespace", namespace) + } + return err + } + + return err + }) +} diff --git a/pkg/services/sqlstore/migrations/secrets_mig.go b/pkg/services/sqlstore/migrations/secrets_mig.go index 3c2012a999f..d6bc6f02c90 100644 --- a/pkg/services/sqlstore/migrations/secrets_mig.go +++ b/pkg/services/sqlstore/migrations/secrets_mig.go @@ -18,4 +18,24 @@ func addSecretsMigration(mg *migrator.Migrator) { } mg.AddMigration("create data_keys table", migrator.NewAddTableMigration(dataKeysV1)) + + secretsV1 := migrator.Table{ + Name: "secrets", + Columns: []*migrator.Column{ + {Name: "id", Type: migrator.DB_BigInt, IsPrimaryKey: true, IsAutoIncrement: true}, + {Name: "org_id", Type: migrator.DB_BigInt, Nullable: false}, + {Name: "namespace", Type: migrator.DB_NVarchar, Length: 255, Nullable: false}, + {Name: "type", Type: migrator.DB_NVarchar, Length: 255, Nullable: false}, + {Name: "value", Type: migrator.DB_Text, Nullable: true}, + {Name: "created", Type: migrator.DB_DateTime, Nullable: false}, + {Name: "updated", Type: migrator.DB_DateTime, Nullable: false}, + }, + Indices: []*migrator.Index{ + {Cols: []string{"org_id"}}, + {Cols: []string{"org_id", "namespace"}}, + {Cols: []string{"org_id", "namespace", "type"}, Type: migrator.UniqueIndex}, + }, + } + + mg.AddMigration("create secrets table", migrator.NewAddTableMigration(secretsV1)) } diff --git a/pkg/tsdb/legacydata/service/service.go b/pkg/tsdb/legacydata/service/service.go index b946d255252..5c7f5ac5b7a 100644 --- a/pkg/tsdb/legacydata/service/service.go +++ b/pkg/tsdb/legacydata/service/service.go @@ -40,6 +40,11 @@ func (h *Service) HandleRequest(ctx context.Context, ds *models.DataSource, quer return legacydata.DataResponse{}, err } + decryptedValues, err := h.dataSourcesService.DecryptedValues(ctx, ds) + if err != nil { + return legacydata.DataResponse{}, err + } + instanceSettings := &backend.DataSourceInstanceSettings{ ID: ds.Id, Name: ds.Name, @@ -49,7 +54,7 @@ func (h *Service) HandleRequest(ctx context.Context, ds *models.DataSource, quer BasicAuthEnabled: ds.BasicAuth, BasicAuthUser: ds.BasicAuthUser, JSONData: jsonDataBytes, - DecryptedSecureJSONData: h.dataSourcesService.DecryptedValues(ds), + DecryptedSecureJSONData: decryptedValues, Updated: ds.Updated, UID: ds.Uid, } diff --git a/pkg/tsdb/legacydata/service/service_test.go b/pkg/tsdb/legacydata/service/service_test.go index 7e895da093a..33bef6cf3b5 100644 --- a/pkg/tsdb/legacydata/service/service_test.go +++ b/pkg/tsdb/legacydata/service/service_test.go @@ -13,6 +13,7 @@ import ( "github.com/grafana/grafana/pkg/services/featuremgmt" "github.com/grafana/grafana/pkg/services/oauthtoken" "github.com/grafana/grafana/pkg/services/secrets/fakes" + "github.com/grafana/grafana/pkg/services/secrets/kvstore" secretsManager "github.com/grafana/grafana/pkg/services/secrets/manager" "github.com/grafana/grafana/pkg/setting" "github.com/grafana/grafana/pkg/tsdb/legacydata" @@ -38,8 +39,9 @@ func TestHandleRequest(t *testing.T) { actualReq = req return backend.NewQueryDataResponse(), nil } + secretsStore := kvstore.SetupTestService(t) secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) - dsService := datasourceservice.ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) + dsService := datasourceservice.ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) s := ProvideService(client, nil, dsService) ds := &models.DataSource{Id: 12, Type: "unregisteredType", JsonData: simplejson.New()}