Secrets: Implement basic unified secret store service (#45804)

* wip: Implement kvstore for secrets

* wip: Refactor kvstore for secrets

* wip: Add format key function to secrets kvstore sql

* wip: Add migration for secrets kvstore

* Remove unused Key field from secrets kvstore

* Remove secret values from debug logs

* Integrate unified secrets with datasources

* Fix minor issues and tests for kvstore

* Create test service helper for secret store

* Remove encryption tests from datasources

* Move secret operations after datasources

* Fix datasource proxy tests

* Fix legacy data tests

* Add Name to all delete data source commands

* Implement decryption cache on sql secret store

* Fix minor issue with cache and tests

* Use secret type on secret store datasource operations

* Add comments to make create and update clear

* Rename itemFound variable to isFound

* Improve secret deletion and cache management

* Add base64 encoding to sql secret store

* Move secret retrieval to decrypted values function

* Refactor decrypt secure json data functions

* Fix expr tests

* Fix datasource tests

* Fix plugin proxy tests

* Fix query tests

* Fix metrics api tests

* Remove unused fake secrets service from query tests

* Add rename function to secret store

* Add check for error renaming secret

* Remove bus from tests to fix merge conflicts

* Add background secrets migration to datasources

* Get datasource secure json fields from secrets

* Move migration to secret store

* Revert "Move migration to secret store"

This reverts commit 7c3f872072e9aff601fb9d639127d468c03f97ef.

* Add secret service to datasource service on tests

* Fix datasource tests

* Remove merge conflict on wire

* Add ctx to data source http transport on prometheus stats collector

* Add ctx to data source http transport on stats collector test
This commit is contained in:
Guilherme Caulada
2022-04-25 13:57:45 -03:00
committed by GitHub
parent 0ca32f0c61
commit a367ad730c
31 changed files with 1243 additions and 452 deletions

View File

@ -97,7 +97,7 @@ func (hs *HTTPServer) GetDataSourceById(c *models.ReqContext) response.Response
return response.Error(404, "Data source not found", err) return response.Error(404, "Data source not found", err)
} }
dto := convertModelToDtos(filtered[0]) dto := hs.convertModelToDtos(c.Req.Context(), filtered[0])
// Add accesscontrol metadata // Add accesscontrol metadata
dto.AccessControl = hs.getAccessControlMetadata(c, c.OrgId, datasources.ScopePrefix, dto.UID) dto.AccessControl = hs.getAccessControlMetadata(c, c.OrgId, datasources.ScopePrefix, dto.UID)
@ -128,7 +128,7 @@ func (hs *HTTPServer) DeleteDataSourceById(c *models.ReqContext) response.Respon
return response.Error(403, "Cannot delete read-only data source", nil) return response.Error(403, "Cannot delete read-only data source", nil)
} }
cmd := &models.DeleteDataSourceCommand{ID: id, OrgID: c.OrgId} cmd := &models.DeleteDataSourceCommand{ID: id, OrgID: c.OrgId, Name: ds.Name}
err = hs.DataSourcesService.DeleteDataSource(c.Req.Context(), cmd) err = hs.DataSourcesService.DeleteDataSource(c.Req.Context(), cmd)
if err != nil { if err != nil {
@ -156,7 +156,7 @@ func (hs *HTTPServer) GetDataSourceByUID(c *models.ReqContext) response.Response
return response.Error(404, "Data source not found", err) return response.Error(404, "Data source not found", err)
} }
dto := convertModelToDtos(filtered[0]) dto := hs.convertModelToDtos(c.Req.Context(), filtered[0])
// Add accesscontrol metadata // Add accesscontrol metadata
dto.AccessControl = hs.getAccessControlMetadata(c, c.OrgId, datasources.ScopePrefix, dto.UID) dto.AccessControl = hs.getAccessControlMetadata(c, c.OrgId, datasources.ScopePrefix, dto.UID)
@ -184,7 +184,7 @@ func (hs *HTTPServer) DeleteDataSourceByUID(c *models.ReqContext) response.Respo
return response.Error(403, "Cannot delete read-only data source", nil) return response.Error(403, "Cannot delete read-only data source", nil)
} }
cmd := &models.DeleteDataSourceCommand{UID: uid, OrgID: c.OrgId} cmd := &models.DeleteDataSourceCommand{UID: uid, OrgID: c.OrgId, Name: ds.Name}
err = hs.DataSourcesService.DeleteDataSource(c.Req.Context(), cmd) err = hs.DataSourcesService.DeleteDataSource(c.Req.Context(), cmd)
if err != nil { if err != nil {
@ -265,7 +265,7 @@ func (hs *HTTPServer) AddDataSource(c *models.ReqContext) response.Response {
return response.Error(500, "Failed to add datasource", err) return response.Error(500, "Failed to add datasource", err)
} }
ds := convertModelToDtos(cmd.Result) ds := hs.convertModelToDtos(c.Req.Context(), cmd.Result)
return response.JSON(http.StatusOK, util.DynMap{ return response.JSON(http.StatusOK, util.DynMap{
"message": "Datasource added", "message": "Datasource added",
"id": cmd.Result.Id, "id": cmd.Result.Id,
@ -327,7 +327,7 @@ func (hs *HTTPServer) UpdateDataSource(c *models.ReqContext) response.Response {
return response.Error(500, "Failed to query datasource", err) return response.Error(500, "Failed to query datasource", err)
} }
datasourceDTO := convertModelToDtos(query.Result) datasourceDTO := hs.convertModelToDtos(c.Req.Context(), query.Result)
hs.Live.HandleDatasourceUpdate(c.OrgId, datasourceDTO.UID) hs.Live.HandleDatasourceUpdate(c.OrgId, datasourceDTO.UID)
@ -408,7 +408,7 @@ func (hs *HTTPServer) GetDataSourceByName(c *models.ReqContext) response.Respons
return response.Error(404, "Data source not found", err) return response.Error(404, "Data source not found", err)
} }
dto := convertModelToDtos(filtered[0]) dto := hs.convertModelToDtos(c.Req.Context(), filtered[0])
return response.JSON(http.StatusOK, &dto) return response.JSON(http.StatusOK, &dto)
} }
@ -457,7 +457,7 @@ func (hs *HTTPServer) CallDatasourceResource(c *models.ReqContext) {
hs.callPluginResource(c, plugin.ID, ds.Uid) hs.callPluginResource(c, plugin.ID, ds.Uid)
} }
func convertModelToDtos(ds *models.DataSource) dtos.DataSource { func (hs *HTTPServer) convertModelToDtos(ctx context.Context, ds *models.DataSource) dtos.DataSource {
dto := dtos.DataSource{ dto := dtos.DataSource{
Id: ds.Id, Id: ds.Id,
UID: ds.Uid, UID: ds.Uid,
@ -480,9 +480,12 @@ func convertModelToDtos(ds *models.DataSource) dtos.DataSource {
ReadOnly: ds.ReadOnly, ReadOnly: ds.ReadOnly,
} }
for k, v := range ds.SecureJsonData { secrets, err := hs.DataSourcesService.DecryptedValues(ctx, ds)
if len(v) > 0 { if err == nil {
dto.SecureJsonFields[k] = true for k, v := range secrets {
if len(v) > 0 {
dto.SecureJsonFields[k] = true
}
} }
} }
@ -510,7 +513,7 @@ func (hs *HTTPServer) CheckDatasourceHealth(c *models.ReqContext) response.Respo
return response.Error(http.StatusInternalServerError, "Unable to find datasource plugin", err) return response.Error(http.StatusInternalServerError, "Unable to find datasource plugin", err)
} }
dsInstanceSettings, err := adapters.ModelToInstanceSettings(ds, hs.decryptSecureJsonDataFn()) dsInstanceSettings, err := adapters.ModelToInstanceSettings(ds, hs.decryptSecureJsonDataFn(c.Req.Context()))
if err != nil { if err != nil {
return response.Error(http.StatusInternalServerError, "Unable to get datasource model", err) return response.Error(http.StatusInternalServerError, "Unable to get datasource model", err)
} }
@ -561,9 +564,9 @@ func (hs *HTTPServer) CheckDatasourceHealth(c *models.ReqContext) response.Respo
return response.JSON(http.StatusOK, payload) return response.JSON(http.StatusOK, payload)
} }
func (hs *HTTPServer) decryptSecureJsonDataFn() func(map[string][]byte) map[string]string { func (hs *HTTPServer) decryptSecureJsonDataFn(ctx context.Context) func(ds *models.DataSource) map[string]string {
return func(m map[string][]byte) map[string]string { return func(ds *models.DataSource) map[string]string {
decryptedJsonData, err := hs.SecretsService.DecryptJsonData(context.Background(), m) decryptedJsonData, err := hs.DataSourcesService.DecryptedValues(ctx, ds)
if err != nil { if err != nil {
hs.log.Error("Failed to decrypt secure json data", "error", err) hs.log.Error("Failed to decrypt secure json data", "error", err)
} }

View File

@ -583,3 +583,8 @@ func (m *dataSourcesServiceMock) UpdateDataSource(ctx context.Context, cmd *mode
cmd.Result = m.expectedDatasource cmd.Result = m.expectedDatasource
return m.expectedError return m.expectedError
} }
func (m *dataSourcesServiceMock) DecryptedValues(ctx context.Context, ds *models.DataSource) (map[string]string, error) {
decryptedValues := make(map[string]string)
return decryptedValues, m.expectedError
}

View File

@ -248,9 +248,14 @@ func (hs *HTTPServer) getFSDataSources(c *models.ReqContext, enabledPlugins Enab
if ds.Access == models.DS_ACCESS_DIRECT { if ds.Access == models.DS_ACCESS_DIRECT {
if ds.BasicAuth { if ds.BasicAuth {
password, err := hs.DataSourcesService.DecryptedBasicAuthPassword(c.Req.Context(), ds)
if err != nil {
return nil, err
}
dsDTO.BasicAuth = util.GetBasicAuthHeader( dsDTO.BasicAuth = util.GetBasicAuthHeader(
ds.BasicAuthUser, ds.BasicAuthUser,
hs.DataSourcesService.DecryptedBasicAuthPassword(ds), password,
) )
} }
if ds.WithCredentials { if ds.WithCredentials {
@ -258,14 +263,24 @@ func (hs *HTTPServer) getFSDataSources(c *models.ReqContext, enabledPlugins Enab
} }
if ds.Type == models.DS_INFLUXDB_08 { if ds.Type == models.DS_INFLUXDB_08 {
password, err := hs.DataSourcesService.DecryptedPassword(c.Req.Context(), ds)
if err != nil {
return nil, err
}
dsDTO.Username = ds.User dsDTO.Username = ds.User
dsDTO.Password = hs.DataSourcesService.DecryptedPassword(ds) dsDTO.Password = password
dsDTO.URL = url + "/db/" + ds.Database dsDTO.URL = url + "/db/" + ds.Database
} }
if ds.Type == models.DS_INFLUXDB { if ds.Type == models.DS_INFLUXDB {
password, err := hs.DataSourcesService.DecryptedPassword(c.Req.Context(), ds)
if err != nil {
return nil, err
}
dsDTO.Username = ds.User dsDTO.Username = ds.User
dsDTO.Password = hs.DataSourcesService.DecryptedPassword(ds) dsDTO.Password = password
dsDTO.URL = url dsDTO.URL = url
} }
} }

View File

@ -9,6 +9,7 @@ import (
"strings" "strings"
"testing" "testing"
acmock "github.com/grafana/grafana/pkg/services/accesscontrol/mock"
"github.com/grafana/grafana/pkg/services/featuremgmt" "github.com/grafana/grafana/pkg/services/featuremgmt"
"github.com/grafana/grafana/pkg/services/sqlstore/mockstore" "github.com/grafana/grafana/pkg/services/sqlstore/mockstore"
@ -18,8 +19,11 @@ import (
"github.com/grafana/grafana/pkg/components/simplejson" "github.com/grafana/grafana/pkg/components/simplejson"
"github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/plugins" "github.com/grafana/grafana/pkg/plugins"
datasources "github.com/grafana/grafana/pkg/services/datasources/service"
"github.com/grafana/grafana/pkg/services/query" "github.com/grafana/grafana/pkg/services/query"
"github.com/grafana/grafana/pkg/services/secrets/fakes" "github.com/grafana/grafana/pkg/services/secrets/fakes"
"github.com/grafana/grafana/pkg/services/secrets/kvstore"
secretsManager "github.com/grafana/grafana/pkg/services/secrets/manager"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
) )
@ -193,13 +197,17 @@ type dashboardFakePluginClient struct {
func TestAPIEndpoint_Metrics_QueryMetricsFromDashboard(t *testing.T) { func TestAPIEndpoint_Metrics_QueryMetricsFromDashboard(t *testing.T) {
sc := setupHTTPServerWithMockDb(t, false, false) sc := setupHTTPServerWithMockDb(t, false, false)
secretsStore := kvstore.SetupTestService(t)
secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore())
ds := datasources.ProvideService(nil, secretsService, secretsStore, nil, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock())
setInitCtxSignedInViewer(sc.initCtx) setInitCtxSignedInViewer(sc.initCtx)
sc.hs.queryDataService = query.ProvideService( sc.hs.queryDataService = query.ProvideService(
nil, nil,
nil, nil,
nil, nil,
&fakePluginRequestValidator{}, &fakePluginRequestValidator{},
fakes.NewFakeSecretsService(), ds,
&dashboardFakePluginClient{}, &dashboardFakePluginClient{},
&fakeOAuthTokenService{}, &fakeOAuthTokenService{},
) )

View File

@ -19,7 +19,6 @@ import (
"github.com/grafana/grafana/pkg/plugins" "github.com/grafana/grafana/pkg/plugins"
"github.com/grafana/grafana/pkg/services/datasources" "github.com/grafana/grafana/pkg/services/datasources"
"github.com/grafana/grafana/pkg/services/oauthtoken" "github.com/grafana/grafana/pkg/services/oauthtoken"
"github.com/grafana/grafana/pkg/services/secrets"
"github.com/grafana/grafana/pkg/setting" "github.com/grafana/grafana/pkg/setting"
"github.com/grafana/grafana/pkg/util" "github.com/grafana/grafana/pkg/util"
"github.com/grafana/grafana/pkg/util/proxyutil" "github.com/grafana/grafana/pkg/util/proxyutil"
@ -43,7 +42,6 @@ type DataSourceProxy struct {
oAuthTokenService oauthtoken.OAuthTokenService oAuthTokenService oauthtoken.OAuthTokenService
dataSourcesService datasources.DataSourceService dataSourcesService datasources.DataSourceService
tracer tracing.Tracer tracer tracing.Tracer
secretsService secrets.Service
} }
type httpClient interface { type httpClient interface {
@ -54,7 +52,7 @@ type httpClient interface {
func NewDataSourceProxy(ds *models.DataSource, pluginRoutes []*plugins.Route, ctx *models.ReqContext, func NewDataSourceProxy(ds *models.DataSource, pluginRoutes []*plugins.Route, ctx *models.ReqContext,
proxyPath string, cfg *setting.Cfg, clientProvider httpclient.Provider, proxyPath string, cfg *setting.Cfg, clientProvider httpclient.Provider,
oAuthTokenService oauthtoken.OAuthTokenService, dsService datasources.DataSourceService, oAuthTokenService oauthtoken.OAuthTokenService, dsService datasources.DataSourceService,
tracer tracing.Tracer, secretsService secrets.Service) (*DataSourceProxy, error) { tracer tracing.Tracer) (*DataSourceProxy, error) {
targetURL, err := datasource.ValidateURL(ds.Type, ds.Url) targetURL, err := datasource.ValidateURL(ds.Type, ds.Url)
if err != nil { if err != nil {
return nil, err return nil, err
@ -71,7 +69,6 @@ func NewDataSourceProxy(ds *models.DataSource, pluginRoutes []*plugins.Route, ct
oAuthTokenService: oAuthTokenService, oAuthTokenService: oAuthTokenService,
dataSourcesService: dsService, dataSourcesService: dsService,
tracer: tracer, tracer: tracer,
secretsService: secretsService,
}, nil }, nil
} }
@ -97,7 +94,7 @@ func (proxy *DataSourceProxy) HandleRequest() {
"referer", proxy.ctx.Req.Referer(), "referer", proxy.ctx.Req.Referer(),
) )
transport, err := proxy.dataSourcesService.GetHTTPTransport(proxy.ds, proxy.clientProvider) transport, err := proxy.dataSourcesService.GetHTTPTransport(proxy.ctx.Req.Context(), proxy.ds, proxy.clientProvider)
if err != nil { if err != nil {
proxy.ctx.JsonApiErr(400, "Unable to load TLS certificate", err) proxy.ctx.JsonApiErr(400, "Unable to load TLS certificate", err)
return return
@ -169,17 +166,28 @@ func (proxy *DataSourceProxy) director(req *http.Request) {
switch proxy.ds.Type { switch proxy.ds.Type {
case models.DS_INFLUXDB_08: case models.DS_INFLUXDB_08:
password, err := proxy.dataSourcesService.DecryptedPassword(req.Context(), proxy.ds)
if err != nil {
logger.Error("Error interpolating proxy url", "error", err)
return
}
req.URL.RawPath = util.JoinURLFragments(proxy.targetUrl.Path, "db/"+proxy.ds.Database+"/"+proxy.proxyPath) req.URL.RawPath = util.JoinURLFragments(proxy.targetUrl.Path, "db/"+proxy.ds.Database+"/"+proxy.proxyPath)
reqQueryVals.Add("u", proxy.ds.User) reqQueryVals.Add("u", proxy.ds.User)
reqQueryVals.Add("p", proxy.dataSourcesService.DecryptedPassword(proxy.ds)) reqQueryVals.Add("p", password)
req.URL.RawQuery = reqQueryVals.Encode() req.URL.RawQuery = reqQueryVals.Encode()
case models.DS_INFLUXDB: case models.DS_INFLUXDB:
password, err := proxy.dataSourcesService.DecryptedPassword(req.Context(), proxy.ds)
if err != nil {
logger.Error("Error interpolating proxy url", "error", err)
return
}
req.URL.RawPath = util.JoinURLFragments(proxy.targetUrl.Path, proxy.proxyPath) req.URL.RawPath = util.JoinURLFragments(proxy.targetUrl.Path, proxy.proxyPath)
req.URL.RawQuery = reqQueryVals.Encode() req.URL.RawQuery = reqQueryVals.Encode()
if !proxy.ds.BasicAuth { if !proxy.ds.BasicAuth {
req.Header.Set( req.Header.Set(
"Authorization", "Authorization",
util.GetBasicAuthHeader(proxy.ds.User, proxy.dataSourcesService.DecryptedPassword(proxy.ds)), util.GetBasicAuthHeader(proxy.ds.User, password),
) )
} }
default: default:
@ -195,8 +203,13 @@ func (proxy *DataSourceProxy) director(req *http.Request) {
req.URL.Path = unescapedPath req.URL.Path = unescapedPath
if proxy.ds.BasicAuth { if proxy.ds.BasicAuth {
password, err := proxy.dataSourcesService.DecryptedBasicAuthPassword(req.Context(), proxy.ds)
if err != nil {
logger.Error("Error interpolating proxy url", "error", err)
return
}
req.Header.Set("Authorization", util.GetBasicAuthHeader(proxy.ds.BasicAuthUser, req.Header.Set("Authorization", util.GetBasicAuthHeader(proxy.ds.BasicAuthUser,
proxy.dataSourcesService.DecryptedBasicAuthPassword(proxy.ds))) password))
} }
dsAuth := req.Header.Get("X-DS-Authorization") dsAuth := req.Header.Get("X-DS-Authorization")
@ -226,23 +239,23 @@ func (proxy *DataSourceProxy) director(req *http.Request) {
} }
} }
secureJsonData, err := proxy.secretsService.DecryptJsonData(req.Context(), proxy.ds.SecureJsonData)
if err != nil {
logger.Error("Error interpolating proxy url", "error", err)
return
}
if proxy.matchedRoute != nil { if proxy.matchedRoute != nil {
ApplyRoute(proxy.ctx.Req.Context(), req, proxy.proxyPath, proxy.matchedRoute, DSInfo{ decryptedValues, err := proxy.dataSourcesService.DecryptedValues(req.Context(), proxy.ds)
if err != nil {
logger.Error("Error interpolating proxy url", "error", err)
return
}
ApplyRoute(req.Context(), req, proxy.proxyPath, proxy.matchedRoute, DSInfo{
ID: proxy.ds.Id, ID: proxy.ds.Id,
Updated: proxy.ds.Updated, Updated: proxy.ds.Updated,
JSONData: jsonData, JSONData: jsonData,
DecryptedSecureJSONData: secureJsonData, DecryptedSecureJSONData: decryptedValues,
}, proxy.cfg) }, proxy.cfg)
} }
if proxy.oAuthTokenService.IsOAuthPassThruEnabled(proxy.ds) { if proxy.oAuthTokenService.IsOAuthPassThruEnabled(proxy.ds) {
if token := proxy.oAuthTokenService.GetCurrentOAuthToken(proxy.ctx.Req.Context(), proxy.ctx.SignedInUser); token != nil { if token := proxy.oAuthTokenService.GetCurrentOAuthToken(req.Context(), proxy.ctx.SignedInUser); token != nil {
req.Header.Set("Authorization", fmt.Sprintf("%s %s", token.Type(), token.AccessToken)) req.Header.Set("Authorization", fmt.Sprintf("%s %s", token.Type(), token.AccessToken))
idToken, ok := token.Extra("id_token").(string) idToken, ok := token.Extra("id_token").(string)

View File

@ -3,6 +3,7 @@ package pluginproxy
import ( import (
"bytes" "bytes"
"context" "context"
"encoding/json"
"fmt" "fmt"
"io/ioutil" "io/ioutil"
"net/http" "net/http"
@ -24,6 +25,7 @@ import (
"github.com/grafana/grafana/pkg/services/oauthtoken" "github.com/grafana/grafana/pkg/services/oauthtoken"
"github.com/grafana/grafana/pkg/services/secrets" "github.com/grafana/grafana/pkg/services/secrets"
"github.com/grafana/grafana/pkg/services/secrets/fakes" "github.com/grafana/grafana/pkg/services/secrets/fakes"
"github.com/grafana/grafana/pkg/services/secrets/kvstore"
secretsManager "github.com/grafana/grafana/pkg/services/secrets/manager" secretsManager "github.com/grafana/grafana/pkg/services/secrets/manager"
"github.com/grafana/grafana/pkg/setting" "github.com/grafana/grafana/pkg/setting"
"github.com/grafana/grafana/pkg/web" "github.com/grafana/grafana/pkg/web"
@ -90,6 +92,7 @@ func TestDataSourceProxy_routeRule(t *testing.T) {
}) })
setting.SecretKey = "password" //nolint:goconst setting.SecretKey = "password" //nolint:goconst
secretsStore := kvstore.SetupTestService(t)
secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore())
key, err := secretsService.Encrypt(context.Background(), []byte("123"), secrets.WithoutScope()) key, err := secretsService.Encrypt(context.Background(), []byte("123"), secrets.WithoutScope())
require.NoError(t, err) require.NoError(t, err)
@ -128,9 +131,9 @@ func TestDataSourceProxy_routeRule(t *testing.T) {
t.Run("When matching route path", func(t *testing.T) { t.Run("When matching route path", func(t *testing.T) {
ctx, req := setUp() ctx, req := setUp()
dsService := datasourceservice.ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) dsService := datasourceservice.ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock())
proxy, err := NewDataSourceProxy(ds, routes, ctx, "api/v4/some/method", cfg, httpClientProvider, proxy, err := NewDataSourceProxy(ds, routes, ctx, "api/v4/some/method", cfg, httpClientProvider,
&oauthtoken.Service{}, dsService, tracer, secretsService) &oauthtoken.Service{}, dsService, tracer)
require.NoError(t, err) require.NoError(t, err)
proxy.matchedRoute = routes[0] proxy.matchedRoute = routes[0]
ApplyRoute(proxy.ctx.Req.Context(), req, proxy.proxyPath, proxy.matchedRoute, dsInfo, cfg) ApplyRoute(proxy.ctx.Req.Context(), req, proxy.proxyPath, proxy.matchedRoute, dsInfo, cfg)
@ -141,8 +144,8 @@ func TestDataSourceProxy_routeRule(t *testing.T) {
t.Run("When matching route path and has dynamic url", func(t *testing.T) { t.Run("When matching route path and has dynamic url", func(t *testing.T) {
ctx, req := setUp() ctx, req := setUp()
dsService := datasourceservice.ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) dsService := datasourceservice.ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock())
proxy, err := NewDataSourceProxy(ds, routes, ctx, "api/common/some/method", cfg, httpClientProvider, &oauthtoken.Service{}, dsService, tracer, secretsService) proxy, err := NewDataSourceProxy(ds, routes, ctx, "api/common/some/method", cfg, httpClientProvider, &oauthtoken.Service{}, dsService, tracer)
require.NoError(t, err) require.NoError(t, err)
proxy.matchedRoute = routes[3] proxy.matchedRoute = routes[3]
ApplyRoute(proxy.ctx.Req.Context(), req, proxy.proxyPath, proxy.matchedRoute, dsInfo, cfg) ApplyRoute(proxy.ctx.Req.Context(), req, proxy.proxyPath, proxy.matchedRoute, dsInfo, cfg)
@ -153,8 +156,8 @@ func TestDataSourceProxy_routeRule(t *testing.T) {
t.Run("When matching route path with no url", func(t *testing.T) { t.Run("When matching route path with no url", func(t *testing.T) {
ctx, req := setUp() ctx, req := setUp()
dsService := datasourceservice.ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) dsService := datasourceservice.ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock())
proxy, err := NewDataSourceProxy(ds, routes, ctx, "", cfg, httpClientProvider, &oauthtoken.Service{}, dsService, tracer, secretsService) proxy, err := NewDataSourceProxy(ds, routes, ctx, "", cfg, httpClientProvider, &oauthtoken.Service{}, dsService, tracer)
require.NoError(t, err) require.NoError(t, err)
proxy.matchedRoute = routes[4] proxy.matchedRoute = routes[4]
ApplyRoute(proxy.ctx.Req.Context(), req, proxy.proxyPath, proxy.matchedRoute, dsInfo, cfg) ApplyRoute(proxy.ctx.Req.Context(), req, proxy.proxyPath, proxy.matchedRoute, dsInfo, cfg)
@ -164,8 +167,8 @@ func TestDataSourceProxy_routeRule(t *testing.T) {
t.Run("When matching route path and has dynamic body", func(t *testing.T) { t.Run("When matching route path and has dynamic body", func(t *testing.T) {
ctx, req := setUp() ctx, req := setUp()
dsService := datasourceservice.ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) dsService := datasourceservice.ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock())
proxy, err := NewDataSourceProxy(ds, routes, ctx, "api/body", cfg, httpClientProvider, &oauthtoken.Service{}, dsService, tracer, secretsService) proxy, err := NewDataSourceProxy(ds, routes, ctx, "api/body", cfg, httpClientProvider, &oauthtoken.Service{}, dsService, tracer)
require.NoError(t, err) require.NoError(t, err)
proxy.matchedRoute = routes[5] proxy.matchedRoute = routes[5]
ApplyRoute(proxy.ctx.Req.Context(), req, proxy.proxyPath, proxy.matchedRoute, dsInfo, cfg) ApplyRoute(proxy.ctx.Req.Context(), req, proxy.proxyPath, proxy.matchedRoute, dsInfo, cfg)
@ -178,8 +181,8 @@ func TestDataSourceProxy_routeRule(t *testing.T) {
t.Run("Validating request", func(t *testing.T) { t.Run("Validating request", func(t *testing.T) {
t.Run("plugin route with valid role", func(t *testing.T) { t.Run("plugin route with valid role", func(t *testing.T) {
ctx, _ := setUp() ctx, _ := setUp()
dsService := datasourceservice.ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) dsService := datasourceservice.ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock())
proxy, err := NewDataSourceProxy(ds, routes, ctx, "api/v4/some/method", cfg, httpClientProvider, &oauthtoken.Service{}, dsService, tracer, secretsService) proxy, err := NewDataSourceProxy(ds, routes, ctx, "api/v4/some/method", cfg, httpClientProvider, &oauthtoken.Service{}, dsService, tracer)
require.NoError(t, err) require.NoError(t, err)
err = proxy.validateRequest() err = proxy.validateRequest()
require.NoError(t, err) require.NoError(t, err)
@ -187,8 +190,8 @@ func TestDataSourceProxy_routeRule(t *testing.T) {
t.Run("plugin route with admin role and user is editor", func(t *testing.T) { t.Run("plugin route with admin role and user is editor", func(t *testing.T) {
ctx, _ := setUp() ctx, _ := setUp()
dsService := datasourceservice.ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) dsService := datasourceservice.ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock())
proxy, err := NewDataSourceProxy(ds, routes, ctx, "api/admin", cfg, httpClientProvider, &oauthtoken.Service{}, dsService, tracer, secretsService) proxy, err := NewDataSourceProxy(ds, routes, ctx, "api/admin", cfg, httpClientProvider, &oauthtoken.Service{}, dsService, tracer)
require.NoError(t, err) require.NoError(t, err)
err = proxy.validateRequest() err = proxy.validateRequest()
require.Error(t, err) require.Error(t, err)
@ -197,8 +200,8 @@ func TestDataSourceProxy_routeRule(t *testing.T) {
t.Run("plugin route with admin role and user is admin", func(t *testing.T) { t.Run("plugin route with admin role and user is admin", func(t *testing.T) {
ctx, _ := setUp() ctx, _ := setUp()
ctx.SignedInUser.OrgRole = models.ROLE_ADMIN ctx.SignedInUser.OrgRole = models.ROLE_ADMIN
dsService := datasourceservice.ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) dsService := datasourceservice.ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock())
proxy, err := NewDataSourceProxy(ds, routes, ctx, "api/admin", cfg, httpClientProvider, &oauthtoken.Service{}, dsService, tracer, secretsService) proxy, err := NewDataSourceProxy(ds, routes, ctx, "api/admin", cfg, httpClientProvider, &oauthtoken.Service{}, dsService, tracer)
require.NoError(t, err) require.NoError(t, err)
err = proxy.validateRequest() err = proxy.validateRequest()
require.NoError(t, err) require.NoError(t, err)
@ -242,6 +245,7 @@ func TestDataSourceProxy_routeRule(t *testing.T) {
}) })
setting.SecretKey = "password" setting.SecretKey = "password"
secretsStore := kvstore.SetupTestService(t)
secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore())
key, err := secretsService.Encrypt(context.Background(), []byte("123"), secrets.WithoutScope()) key, err := secretsService.Encrypt(context.Background(), []byte("123"), secrets.WithoutScope())
require.NoError(t, err) require.NoError(t, err)
@ -286,8 +290,8 @@ func TestDataSourceProxy_routeRule(t *testing.T) {
}, },
} }
dsService := datasourceservice.ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) dsService := datasourceservice.ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock())
proxy, err := NewDataSourceProxy(ds, routes, ctx, "pathwithtoken1", cfg, httpClientProvider, &oauthtoken.Service{}, dsService, tracer, secretsService) proxy, err := NewDataSourceProxy(ds, routes, ctx, "pathwithtoken1", cfg, httpClientProvider, &oauthtoken.Service{}, dsService, tracer)
require.NoError(t, err) require.NoError(t, err)
ApplyRoute(proxy.ctx.Req.Context(), req, proxy.proxyPath, routes[0], dsInfo, cfg) ApplyRoute(proxy.ctx.Req.Context(), req, proxy.proxyPath, routes[0], dsInfo, cfg)
@ -302,8 +306,8 @@ func TestDataSourceProxy_routeRule(t *testing.T) {
req, err := http.NewRequest("GET", "http://localhost/asd", nil) req, err := http.NewRequest("GET", "http://localhost/asd", nil)
require.NoError(t, err) require.NoError(t, err)
client = newFakeHTTPClient(t, json2) client = newFakeHTTPClient(t, json2)
dsService := datasourceservice.ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) dsService := datasourceservice.ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock())
proxy, err := NewDataSourceProxy(ds, routes, ctx, "pathwithtoken2", cfg, httpClientProvider, &oauthtoken.Service{}, dsService, tracer, secretsService) proxy, err := NewDataSourceProxy(ds, routes, ctx, "pathwithtoken2", cfg, httpClientProvider, &oauthtoken.Service{}, dsService, tracer)
require.NoError(t, err) require.NoError(t, err)
ApplyRoute(proxy.ctx.Req.Context(), req, proxy.proxyPath, routes[1], dsInfo, cfg) ApplyRoute(proxy.ctx.Req.Context(), req, proxy.proxyPath, routes[1], dsInfo, cfg)
@ -319,8 +323,8 @@ func TestDataSourceProxy_routeRule(t *testing.T) {
require.NoError(t, err) require.NoError(t, err)
client = newFakeHTTPClient(t, []byte{}) client = newFakeHTTPClient(t, []byte{})
dsService := datasourceservice.ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) dsService := datasourceservice.ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock())
proxy, err := NewDataSourceProxy(ds, routes, ctx, "pathwithtoken1", cfg, httpClientProvider, &oauthtoken.Service{}, dsService, tracer, secretsService) proxy, err := NewDataSourceProxy(ds, routes, ctx, "pathwithtoken1", cfg, httpClientProvider, &oauthtoken.Service{}, dsService, tracer)
require.NoError(t, err) require.NoError(t, err)
ApplyRoute(proxy.ctx.Req.Context(), req, proxy.proxyPath, routes[0], dsInfo, cfg) ApplyRoute(proxy.ctx.Req.Context(), req, proxy.proxyPath, routes[0], dsInfo, cfg)
@ -340,9 +344,10 @@ func TestDataSourceProxy_routeRule(t *testing.T) {
ds := &models.DataSource{Url: "htttp://graphite:8080", Type: models.DS_GRAPHITE} ds := &models.DataSource{Url: "htttp://graphite:8080", Type: models.DS_GRAPHITE}
ctx := &models.ReqContext{} ctx := &models.ReqContext{}
secretsStore := kvstore.SetupTestService(t)
secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore())
dsService := datasourceservice.ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) dsService := datasourceservice.ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock())
proxy, err := NewDataSourceProxy(ds, routes, ctx, "/render", &setting.Cfg{BuildVersion: "5.3.0"}, httpClientProvider, &oauthtoken.Service{}, dsService, tracer, secretsService) proxy, err := NewDataSourceProxy(ds, routes, ctx, "/render", &setting.Cfg{BuildVersion: "5.3.0"}, httpClientProvider, &oauthtoken.Service{}, dsService, tracer)
require.NoError(t, err) require.NoError(t, err)
req, err := http.NewRequest(http.MethodGet, "http://grafana.com/sub", nil) req, err := http.NewRequest(http.MethodGet, "http://grafana.com/sub", nil)
require.NoError(t, err) require.NoError(t, err)
@ -366,9 +371,10 @@ func TestDataSourceProxy_routeRule(t *testing.T) {
ctx := &models.ReqContext{} ctx := &models.ReqContext{}
var routes []*plugins.Route var routes []*plugins.Route
secretsStore := kvstore.SetupTestService(t)
secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore())
dsService := datasourceservice.ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) dsService := datasourceservice.ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock())
proxy, err := NewDataSourceProxy(ds, routes, ctx, "", &setting.Cfg{}, httpClientProvider, &oauthtoken.Service{}, dsService, tracer, secretsService) proxy, err := NewDataSourceProxy(ds, routes, ctx, "", &setting.Cfg{}, httpClientProvider, &oauthtoken.Service{}, dsService, tracer)
require.NoError(t, err) require.NoError(t, err)
req, err := http.NewRequest(http.MethodGet, "http://grafana.com/sub", nil) req, err := http.NewRequest(http.MethodGet, "http://grafana.com/sub", nil)
@ -390,9 +396,10 @@ func TestDataSourceProxy_routeRule(t *testing.T) {
ctx := &models.ReqContext{} ctx := &models.ReqContext{}
var routes []*plugins.Route var routes []*plugins.Route
secretsStore := kvstore.SetupTestService(t)
secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore())
dsService := datasourceservice.ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) dsService := datasourceservice.ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock())
proxy, err := NewDataSourceProxy(ds, routes, ctx, "", &setting.Cfg{}, httpClientProvider, &oauthtoken.Service{}, dsService, tracer, secretsService) proxy, err := NewDataSourceProxy(ds, routes, ctx, "", &setting.Cfg{}, httpClientProvider, &oauthtoken.Service{}, dsService, tracer)
require.NoError(t, err) require.NoError(t, err)
requestURL, err := url.Parse("http://grafana.com/sub") requestURL, err := url.Parse("http://grafana.com/sub")
@ -418,9 +425,10 @@ func TestDataSourceProxy_routeRule(t *testing.T) {
ctx := &models.ReqContext{} ctx := &models.ReqContext{}
var pluginRoutes []*plugins.Route var pluginRoutes []*plugins.Route
secretsStore := kvstore.SetupTestService(t)
secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore())
dsService := datasourceservice.ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) dsService := datasourceservice.ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock())
proxy, err := NewDataSourceProxy(ds, pluginRoutes, ctx, "", &setting.Cfg{}, httpClientProvider, &oauthtoken.Service{}, dsService, tracer, secretsService) proxy, err := NewDataSourceProxy(ds, pluginRoutes, ctx, "", &setting.Cfg{}, httpClientProvider, &oauthtoken.Service{}, dsService, tracer)
require.NoError(t, err) require.NoError(t, err)
requestURL, err := url.Parse("http://grafana.com/sub") requestURL, err := url.Parse("http://grafana.com/sub")
@ -441,9 +449,10 @@ func TestDataSourceProxy_routeRule(t *testing.T) {
} }
ctx := &models.ReqContext{} ctx := &models.ReqContext{}
var routes []*plugins.Route var routes []*plugins.Route
secretsStore := kvstore.SetupTestService(t)
secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore())
dsService := datasourceservice.ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) dsService := datasourceservice.ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock())
proxy, err := NewDataSourceProxy(ds, routes, ctx, "/path/to/folder/", &setting.Cfg{}, httpClientProvider, &oauthtoken.Service{}, dsService, tracer, secretsService) proxy, err := NewDataSourceProxy(ds, routes, ctx, "/path/to/folder/", &setting.Cfg{}, httpClientProvider, &oauthtoken.Service{}, dsService, tracer)
require.NoError(t, err) require.NoError(t, err)
req, err := http.NewRequest(http.MethodGet, "http://grafana.com/sub", nil) req, err := http.NewRequest(http.MethodGet, "http://grafana.com/sub", nil)
req.Header.Set("Origin", "grafana.com") req.Header.Set("Origin", "grafana.com")
@ -490,9 +499,10 @@ func TestDataSourceProxy_routeRule(t *testing.T) {
} }
var routes []*plugins.Route var routes []*plugins.Route
secretsStore := kvstore.SetupTestService(t)
secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore())
dsService := datasourceservice.ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) dsService := datasourceservice.ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock())
proxy, err := NewDataSourceProxy(ds, routes, ctx, "/path/to/folder/", &setting.Cfg{}, httpClientProvider, &mockAuthToken, dsService, tracer, secretsService) proxy, err := NewDataSourceProxy(ds, routes, ctx, "/path/to/folder/", &setting.Cfg{}, httpClientProvider, &mockAuthToken, dsService, tracer)
require.NoError(t, err) require.NoError(t, err)
req, err = http.NewRequest(http.MethodGet, "http://grafana.com/sub", nil) req, err = http.NewRequest(http.MethodGet, "http://grafana.com/sub", nil)
require.NoError(t, err) require.NoError(t, err)
@ -543,24 +553,25 @@ func TestDataSourceProxy_routeRule(t *testing.T) {
}) })
t.Run("When proxying data source proxy should handle authentication", func(t *testing.T) { t.Run("When proxying data source proxy should handle authentication", func(t *testing.T) {
secretsStore := kvstore.SetupTestService(t)
secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore())
tests := []*testCase{ tests := []*testCase{
createAuthTest(t, secretsService, models.DS_INFLUXDB_08, "http://localhost:9090", authTypePassword, authCheckQuery, false), createAuthTest(t, secretsStore, models.DS_INFLUXDB_08, "http://localhost:9090", authTypePassword, authCheckQuery, false),
createAuthTest(t, secretsService, models.DS_INFLUXDB_08, "http://localhost:9090", authTypePassword, authCheckQuery, true), createAuthTest(t, secretsStore, models.DS_INFLUXDB_08, "http://localhost:9090", authTypePassword, authCheckQuery, true),
createAuthTest(t, secretsService, models.DS_INFLUXDB, "http://localhost:9090", authTypePassword, authCheckHeader, true), createAuthTest(t, secretsStore, models.DS_INFLUXDB, "http://localhost:9090", authTypePassword, authCheckHeader, true),
createAuthTest(t, secretsService, models.DS_INFLUXDB, "http://localhost:9090", authTypePassword, authCheckHeader, false), createAuthTest(t, secretsStore, models.DS_INFLUXDB, "http://localhost:9090", authTypePassword, authCheckHeader, false),
createAuthTest(t, secretsService, models.DS_INFLUXDB, "http://localhost:9090", authTypeBasic, authCheckHeader, true), createAuthTest(t, secretsStore, models.DS_INFLUXDB, "http://localhost:9090", authTypeBasic, authCheckHeader, true),
createAuthTest(t, secretsService, models.DS_INFLUXDB, "http://localhost:9090", authTypeBasic, authCheckHeader, false), createAuthTest(t, secretsStore, models.DS_INFLUXDB, "http://localhost:9090", authTypeBasic, authCheckHeader, false),
// These two should be enough for any other datasource at the moment. Proxy has special handling // These two should be enough for any other datasource at the moment. Proxy has special handling
// only for Influx, others have the same path and only BasicAuth. Non BasicAuth datasources // only for Influx, others have the same path and only BasicAuth. Non BasicAuth datasources
// do not go through proxy but through TSDB API which is not tested here. // do not go through proxy but through TSDB API which is not tested here.
createAuthTest(t, secretsService, models.DS_ES, "http://localhost:9200", authTypeBasic, authCheckHeader, false), createAuthTest(t, secretsStore, models.DS_ES, "http://localhost:9200", authTypeBasic, authCheckHeader, false),
createAuthTest(t, secretsService, models.DS_ES, "http://localhost:9200", authTypeBasic, authCheckHeader, true), createAuthTest(t, secretsStore, models.DS_ES, "http://localhost:9200", authTypeBasic, authCheckHeader, true),
} }
for _, test := range tests { for _, test := range tests {
runDatasourceAuthTest(t, secretsService, cfg, test) runDatasourceAuthTest(t, secretsService, secretsStore, cfg, test)
} }
}) })
} }
@ -624,9 +635,10 @@ func TestDataSourceProxy_requestHandling(t *testing.T) {
t.Run("When response header Set-Cookie is not set should remove proxied Set-Cookie header", func(t *testing.T) { t.Run("When response header Set-Cookie is not set should remove proxied Set-Cookie header", func(t *testing.T) {
ctx, ds := setUp(t) ctx, ds := setUp(t)
var routes []*plugins.Route var routes []*plugins.Route
secretsStore := kvstore.SetupTestService(t)
secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore())
dsService := datasourceservice.ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) dsService := datasourceservice.ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock())
proxy, err := NewDataSourceProxy(ds, routes, ctx, "/render", &setting.Cfg{}, httpClientProvider, &oauthtoken.Service{}, dsService, tracer, secretsService) proxy, err := NewDataSourceProxy(ds, routes, ctx, "/render", &setting.Cfg{}, httpClientProvider, &oauthtoken.Service{}, dsService, tracer)
require.NoError(t, err) require.NoError(t, err)
proxy.HandleRequest() proxy.HandleRequest()
@ -642,9 +654,10 @@ func TestDataSourceProxy_requestHandling(t *testing.T) {
}, },
}) })
var routes []*plugins.Route var routes []*plugins.Route
secretsStore := kvstore.SetupTestService(t)
secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore())
dsService := datasourceservice.ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) dsService := datasourceservice.ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock())
proxy, err := NewDataSourceProxy(ds, routes, ctx, "/render", &setting.Cfg{}, httpClientProvider, &oauthtoken.Service{}, dsService, tracer, secretsService) proxy, err := NewDataSourceProxy(ds, routes, ctx, "/render", &setting.Cfg{}, httpClientProvider, &oauthtoken.Service{}, dsService, tracer)
require.NoError(t, err) require.NoError(t, err)
proxy.HandleRequest() proxy.HandleRequest()
@ -656,9 +669,10 @@ func TestDataSourceProxy_requestHandling(t *testing.T) {
t.Run("When response should set Content-Security-Policy header", func(t *testing.T) { t.Run("When response should set Content-Security-Policy header", func(t *testing.T) {
ctx, ds := setUp(t) ctx, ds := setUp(t)
var routes []*plugins.Route var routes []*plugins.Route
secretsStore := kvstore.SetupTestService(t)
secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore())
dsService := datasourceservice.ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) dsService := datasourceservice.ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock())
proxy, err := NewDataSourceProxy(ds, routes, ctx, "/render", &setting.Cfg{}, httpClientProvider, &oauthtoken.Service{}, dsService, tracer, secretsService) proxy, err := NewDataSourceProxy(ds, routes, ctx, "/render", &setting.Cfg{}, httpClientProvider, &oauthtoken.Service{}, dsService, tracer)
require.NoError(t, err) require.NoError(t, err)
proxy.HandleRequest() proxy.HandleRequest()
@ -678,9 +692,10 @@ func TestDataSourceProxy_requestHandling(t *testing.T) {
}, },
}) })
var routes []*plugins.Route var routes []*plugins.Route
secretsStore := kvstore.SetupTestService(t)
secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore())
dsService := datasourceservice.ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) dsService := datasourceservice.ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock())
proxy, err := NewDataSourceProxy(ds, routes, ctx, "/render", &setting.Cfg{}, httpClientProvider, &oauthtoken.Service{}, dsService, tracer, secretsService) proxy, err := NewDataSourceProxy(ds, routes, ctx, "/render", &setting.Cfg{}, httpClientProvider, &oauthtoken.Service{}, dsService, tracer)
require.NoError(t, err) require.NoError(t, err)
proxy.HandleRequest() proxy.HandleRequest()
@ -703,9 +718,10 @@ func TestDataSourceProxy_requestHandling(t *testing.T) {
ctx.Req = httptest.NewRequest("GET", "/api/datasources/proxy/1/path/%2Ftest%2Ftest%2F?query=%2Ftest%2Ftest%2F", nil) ctx.Req = httptest.NewRequest("GET", "/api/datasources/proxy/1/path/%2Ftest%2Ftest%2F?query=%2Ftest%2Ftest%2F", nil)
var routes []*plugins.Route var routes []*plugins.Route
secretsStore := kvstore.SetupTestService(t)
secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore())
dsService := datasourceservice.ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) dsService := datasourceservice.ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock())
proxy, err := NewDataSourceProxy(ds, routes, ctx, "/path/%2Ftest%2Ftest%2F", &setting.Cfg{}, httpClientProvider, &oauthtoken.Service{}, dsService, tracer, secretsService) proxy, err := NewDataSourceProxy(ds, routes, ctx, "/path/%2Ftest%2Ftest%2F", &setting.Cfg{}, httpClientProvider, &oauthtoken.Service{}, dsService, tracer)
require.NoError(t, err) require.NoError(t, err)
proxy.HandleRequest() proxy.HandleRequest()
@ -727,9 +743,10 @@ func TestDataSourceProxy_requestHandling(t *testing.T) {
ctx.Req = httptest.NewRequest("GET", "/api/datasources/proxy/1/path/%2Ftest%2Ftest%2F?query=%2Ftest%2Ftest%2F", nil) ctx.Req = httptest.NewRequest("GET", "/api/datasources/proxy/1/path/%2Ftest%2Ftest%2F?query=%2Ftest%2Ftest%2F", nil)
var routes []*plugins.Route var routes []*plugins.Route
secretsStore := kvstore.SetupTestService(t)
secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore())
dsService := datasourceservice.ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) dsService := datasourceservice.ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock())
proxy, err := NewDataSourceProxy(ds, routes, ctx, "/path/%2Ftest%2Ftest%2F", &setting.Cfg{}, httpClientProvider, &oauthtoken.Service{}, dsService, tracer, secretsService) proxy, err := NewDataSourceProxy(ds, routes, ctx, "/path/%2Ftest%2Ftest%2F", &setting.Cfg{}, httpClientProvider, &oauthtoken.Service{}, dsService, tracer)
require.NoError(t, err) require.NoError(t, err)
proxy.HandleRequest() proxy.HandleRequest()
@ -752,9 +769,10 @@ func TestNewDataSourceProxy_InvalidURL(t *testing.T) {
tracer, err := tracing.InitializeTracerForTest() tracer, err := tracing.InitializeTracerForTest()
require.NoError(t, err) require.NoError(t, err)
var routes []*plugins.Route var routes []*plugins.Route
secretsStore := kvstore.SetupTestService(t)
secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore())
dsService := datasourceservice.ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) dsService := datasourceservice.ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock())
_, err = NewDataSourceProxy(&ds, routes, &ctx, "api/method", cfg, httpclient.NewProvider(), &oauthtoken.Service{}, dsService, tracer, secretsService) _, err = NewDataSourceProxy(&ds, routes, &ctx, "api/method", cfg, httpclient.NewProvider(), &oauthtoken.Service{}, dsService, tracer)
require.Error(t, err) require.Error(t, err)
assert.True(t, strings.HasPrefix(err.Error(), `validation of data source URL "://host/root" failed`)) assert.True(t, strings.HasPrefix(err.Error(), `validation of data source URL "://host/root" failed`))
} }
@ -773,9 +791,10 @@ func TestNewDataSourceProxy_ProtocolLessURL(t *testing.T) {
require.NoError(t, err) require.NoError(t, err)
var routes []*plugins.Route var routes []*plugins.Route
secretsStore := kvstore.SetupTestService(t)
secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore())
dsService := datasourceservice.ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) dsService := datasourceservice.ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock())
_, err = NewDataSourceProxy(&ds, routes, &ctx, "api/method", cfg, httpclient.NewProvider(), &oauthtoken.Service{}, dsService, tracer, secretsService) _, err = NewDataSourceProxy(&ds, routes, &ctx, "api/method", cfg, httpclient.NewProvider(), &oauthtoken.Service{}, dsService, tracer)
require.NoError(t, err) require.NoError(t, err)
} }
@ -816,9 +835,10 @@ func TestNewDataSourceProxy_MSSQL(t *testing.T) {
} }
var routes []*plugins.Route var routes []*plugins.Route
secretsStore := kvstore.SetupTestService(t)
secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore())
dsService := datasourceservice.ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) dsService := datasourceservice.ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock())
p, err := NewDataSourceProxy(&ds, routes, &ctx, "api/method", cfg, httpclient.NewProvider(), &oauthtoken.Service{}, dsService, tracer, secretsService) p, err := NewDataSourceProxy(&ds, routes, &ctx, "api/method", cfg, httpclient.NewProvider(), &oauthtoken.Service{}, dsService, tracer)
if tc.err == nil { if tc.err == nil {
require.NoError(t, err) require.NoError(t, err)
assert.Equal(t, &url.URL{ assert.Equal(t, &url.URL{
@ -843,9 +863,10 @@ func getDatasourceProxiedRequest(t *testing.T, ctx *models.ReqContext, cfg *sett
require.NoError(t, err) require.NoError(t, err)
var routes []*plugins.Route var routes []*plugins.Route
secretsStore := kvstore.SetupTestService(t)
secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore())
dsService := datasourceservice.ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) dsService := datasourceservice.ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock())
proxy, err := NewDataSourceProxy(ds, routes, ctx, "", cfg, httpclient.NewProvider(), &oauthtoken.Service{}, dsService, tracer, secretsService) proxy, err := NewDataSourceProxy(ds, routes, ctx, "", cfg, httpclient.NewProvider(), &oauthtoken.Service{}, dsService, tracer)
require.NoError(t, err) require.NoError(t, err)
req, err := http.NewRequest(http.MethodGet, "http://grafana.com/sub", nil) req, err := http.NewRequest(http.MethodGet, "http://grafana.com/sub", nil)
require.NoError(t, err) require.NoError(t, err)
@ -897,15 +918,15 @@ const (
authCheckHeader = "header" authCheckHeader = "header"
) )
func createAuthTest(t *testing.T, secretsService secrets.Service, dsType string, url string, authType string, authCheck string, useSecureJsonData bool) *testCase { func createAuthTest(t *testing.T, secretsStore kvstore.SecretsKVStore, dsType string, url string, authType string, authCheck string, useSecureJsonData bool) *testCase {
ctx := context.Background()
// Basic user:password // Basic user:password
base64AuthHeader := "Basic dXNlcjpwYXNzd29yZA==" base64AuthHeader := "Basic dXNlcjpwYXNzd29yZA=="
test := &testCase{ test := &testCase{
datasource: &models.DataSource{ datasource: &models.DataSource{
Id: 1, Id: 1,
OrgId: 1,
Name: fmt.Sprintf("%s,%s,%s,%s,%t", dsType, url, authType, authCheck, useSecureJsonData),
Type: dsType, Type: dsType,
JsonData: simplejson.New(), JsonData: simplejson.New(),
Url: url, Url: url,
@ -917,11 +938,13 @@ func createAuthTest(t *testing.T, secretsService secrets.Service, dsType string,
message = fmt.Sprintf("%v should add username and password", dsType) message = fmt.Sprintf("%v should add username and password", dsType)
test.datasource.User = "user" test.datasource.User = "user"
if useSecureJsonData { if useSecureJsonData {
test.datasource.SecureJsonData, err = secretsService.EncryptJsonData( secureJsonData, err := json.Marshal(map[string]string{
ctx, "password": "password",
map[string]string{ })
"password": "password", require.NoError(t, err)
}, secrets.WithoutScope())
err = secretsStore.Set(context.Background(), test.datasource.OrgId, test.datasource.Name, "datasource", string(secureJsonData))
require.NoError(t, err)
} else { } else {
test.datasource.Password = "password" test.datasource.Password = "password"
} }
@ -930,11 +953,13 @@ func createAuthTest(t *testing.T, secretsService secrets.Service, dsType string,
test.datasource.BasicAuth = true test.datasource.BasicAuth = true
test.datasource.BasicAuthUser = "user" test.datasource.BasicAuthUser = "user"
if useSecureJsonData { if useSecureJsonData {
test.datasource.SecureJsonData, err = secretsService.EncryptJsonData( secureJsonData, err := json.Marshal(map[string]string{
ctx, "basicAuthPassword": "password",
map[string]string{ })
"basicAuthPassword": "password", require.NoError(t, err)
}, secrets.WithoutScope())
err = secretsStore.Set(context.Background(), test.datasource.OrgId, test.datasource.Name, "datasource", string(secureJsonData))
require.NoError(t, err)
} else { } else {
test.datasource.BasicAuthPassword = "password" test.datasource.BasicAuthPassword = "password"
} }
@ -962,14 +987,14 @@ func createAuthTest(t *testing.T, secretsService secrets.Service, dsType string,
return test return test
} }
func runDatasourceAuthTest(t *testing.T, secretsService secrets.Service, cfg *setting.Cfg, test *testCase) { func runDatasourceAuthTest(t *testing.T, secretsService secrets.Service, secretsStore kvstore.SecretsKVStore, cfg *setting.Cfg, test *testCase) {
ctx := &models.ReqContext{} ctx := &models.ReqContext{}
tracer, err := tracing.InitializeTracerForTest() tracer, err := tracing.InitializeTracerForTest()
require.NoError(t, err) require.NoError(t, err)
var routes []*plugins.Route var routes []*plugins.Route
dsService := datasourceservice.ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) dsService := datasourceservice.ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock())
proxy, err := NewDataSourceProxy(test.datasource, routes, ctx, "", &setting.Cfg{}, httpclient.NewProvider(), &oauthtoken.Service{}, dsService, tracer, secretsService) proxy, err := NewDataSourceProxy(test.datasource, routes, ctx, "", &setting.Cfg{}, httpclient.NewProvider(), &oauthtoken.Service{}, dsService, tracer)
require.NoError(t, err) require.NoError(t, err)
req, err := http.NewRequest(http.MethodGet, "http://grafana.com/sub", nil) req, err := http.NewRequest(http.MethodGet, "http://grafana.com/sub", nil)
@ -1010,9 +1035,10 @@ func Test_PathCheck(t *testing.T) {
return ctx, req return ctx, req
} }
ctx, _ := setUp() ctx, _ := setUp()
secretsStore := kvstore.SetupTestService(t)
secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore())
dsService := datasourceservice.ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) dsService := datasourceservice.ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock())
proxy, err := NewDataSourceProxy(&models.DataSource{}, routes, ctx, "b", &setting.Cfg{}, httpclient.NewProvider(), &oauthtoken.Service{}, dsService, tracer, secretsService) proxy, err := NewDataSourceProxy(&models.DataSource{}, routes, ctx, "b", &setting.Cfg{}, httpclient.NewProvider(), &oauthtoken.Service{}, dsService, tracer)
require.NoError(t, err) require.NoError(t, err)
require.Nil(t, proxy.validateRequest()) require.Nil(t, proxy.validateRequest())

View File

@ -7,7 +7,7 @@ import (
"github.com/grafana/grafana/pkg/components/simplejson" "github.com/grafana/grafana/pkg/components/simplejson"
"github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/plugins" "github.com/grafana/grafana/pkg/plugins"
"github.com/grafana/grafana/pkg/services/secrets" "github.com/grafana/grafana/pkg/services/datasources"
"github.com/grafana/grafana/pkg/setting" "github.com/grafana/grafana/pkg/setting"
) )
@ -36,16 +36,16 @@ func IsDataSource(uid string) bool {
// Service is service representation for expression handling. // Service is service representation for expression handling.
type Service struct { type Service struct {
cfg *setting.Cfg cfg *setting.Cfg
dataService backend.QueryDataHandler dataService backend.QueryDataHandler
secretsService secrets.Service dataSourceService datasources.DataSourceService
} }
func ProvideService(cfg *setting.Cfg, pluginClient plugins.Client, secretsService secrets.Service) *Service { func ProvideService(cfg *setting.Cfg, pluginClient plugins.Client, dataSourceService datasources.DataSourceService) *Service {
return &Service{ return &Service{
cfg: cfg, cfg: cfg,
dataService: pluginClient, dataService: pluginClient,
secretsService: secretsService, dataSourceService: dataSourceService,
} }
} }

View File

@ -11,8 +11,7 @@ import (
"github.com/grafana/grafana-plugin-sdk-go/backend" "github.com/grafana/grafana-plugin-sdk-go/backend"
"github.com/grafana/grafana-plugin-sdk-go/data" "github.com/grafana/grafana-plugin-sdk-go/data"
"github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/services/secrets/fakes" datasources "github.com/grafana/grafana/pkg/services/datasources/fakes"
secretsManager "github.com/grafana/grafana/pkg/services/secrets/manager"
"github.com/grafana/grafana/pkg/setting" "github.com/grafana/grafana/pkg/setting"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
) )
@ -28,12 +27,10 @@ func TestService(t *testing.T) {
cfg := setting.NewCfg() cfg := setting.NewCfg()
secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore())
s := Service{ s := Service{
cfg: cfg, cfg: cfg,
dataService: me, dataService: me,
secretsService: secretsService, dataSourceService: &datasources.FakeDataSourceService{},
} }
queries := []Query{ queries := []Query{

View File

@ -126,9 +126,9 @@ func hiddenRefIDs(queries []Query) (map[string]struct{}, error) {
return hidden, nil return hidden, nil
} }
func (s *Service) decryptSecureJsonDataFn(ctx context.Context) func(map[string][]byte) map[string]string { func (s *Service) decryptSecureJsonDataFn(ctx context.Context) func(ds *models.DataSource) map[string]string {
return func(m map[string][]byte) map[string]string { return func(ds *models.DataSource) map[string]string {
decryptedJsonData, err := s.secretsService.DecryptJsonData(ctx, m) decryptedJsonData, err := s.dataSourceService.DecryptedValues(ctx, ds)
if err != nil { if err != nil {
logger.Error("Failed to decrypt secure json data", "error", err) logger.Error("Failed to decrypt secure json data", "error", err)
} }

View File

@ -60,7 +60,7 @@ func (s *Service) detectPrometheusVariant(ctx context.Context, ds *models.DataSo
} `json:"data"` } `json:"data"`
} }
c, err := s.datasources.GetHTTPTransport(ds, s.httpClientProvider) c, err := s.datasources.GetHTTPTransport(ctx, ds, s.httpClientProvider)
if err != nil { if err != nil {
s.log.Error("Failed to get HTTP client for Prometheus data source", "error", err) s.log.Error("Failed to get HTTP client for Prometheus data source", "error", err)
return "", err return "", err

View File

@ -395,6 +395,6 @@ func (s mockDatasourceService) GetDataSourcesByType(ctx context.Context, query *
return nil return nil
} }
func (s mockDatasourceService) GetHTTPTransport(ds *models.DataSource, provider httpclient.Provider, customMiddlewares ...sdkhttpclient.Middleware) (http.RoundTripper, error) { func (s mockDatasourceService) GetHTTPTransport(ctx context.Context, ds *models.DataSource, provider httpclient.Provider, customMiddlewares ...sdkhttpclient.Middleware) (http.RoundTripper, error) {
return provider.GetTransport() return provider.GetTransport()
} }

View File

@ -9,7 +9,7 @@ import (
) )
// ModelToInstanceSettings converts a models.DataSource to a backend.DataSourceInstanceSettings. // ModelToInstanceSettings converts a models.DataSource to a backend.DataSourceInstanceSettings.
func ModelToInstanceSettings(ds *models.DataSource, decryptFn func(map[string][]byte) map[string]string, func ModelToInstanceSettings(ds *models.DataSource, decryptFn func(ds *models.DataSource) map[string]string,
) (*backend.DataSourceInstanceSettings, error) { ) (*backend.DataSourceInstanceSettings, error) {
var jsonDataBytes json.RawMessage var jsonDataBytes json.RawMessage
if ds.JsonData != nil { if ds.JsonData != nil {
@ -30,7 +30,7 @@ func ModelToInstanceSettings(ds *models.DataSource, decryptFn func(map[string][]
BasicAuthEnabled: ds.BasicAuth, BasicAuthEnabled: ds.BasicAuth,
BasicAuthUser: ds.BasicAuthUser, BasicAuthUser: ds.BasicAuthUser,
JSONData: jsonDataBytes, JSONData: jsonDataBytes,
DecryptedSecureJSONData: decryptFn(ds.SecureJsonData), DecryptedSecureJSONData: decryptFn(ds),
Updated: ds.Updated, Updated: ds.Updated,
}, nil }, nil
} }

View File

@ -15,18 +15,17 @@ import (
"github.com/grafana/grafana/pkg/plugins/adapters" "github.com/grafana/grafana/pkg/plugins/adapters"
"github.com/grafana/grafana/pkg/services/datasources" "github.com/grafana/grafana/pkg/services/datasources"
"github.com/grafana/grafana/pkg/services/pluginsettings" "github.com/grafana/grafana/pkg/services/pluginsettings"
"github.com/grafana/grafana/pkg/services/secrets"
"github.com/grafana/grafana/pkg/util/errutil" "github.com/grafana/grafana/pkg/util/errutil"
) )
func ProvideService(cacheService *localcache.CacheService, pluginStore plugins.Store, func ProvideService(cacheService *localcache.CacheService, pluginStore plugins.Store,
dataSourceCache datasources.CacheService, secretsService secrets.Service, dataSourceCache datasources.CacheService, dataSourceService datasources.DataSourceService,
pluginSettingsService pluginsettings.Service) *Provider { pluginSettingsService pluginsettings.Service) *Provider {
return &Provider{ return &Provider{
cacheService: cacheService, cacheService: cacheService,
pluginStore: pluginStore, pluginStore: pluginStore,
dataSourceCache: dataSourceCache, dataSourceCache: dataSourceCache,
secretsService: secretsService, dataSourceService: dataSourceService,
pluginSettingsService: pluginSettingsService, pluginSettingsService: pluginSettingsService,
logger: log.New("plugincontext"), logger: log.New("plugincontext"),
} }
@ -36,7 +35,7 @@ type Provider struct {
cacheService *localcache.CacheService cacheService *localcache.CacheService
pluginStore plugins.Store pluginStore plugins.Store
dataSourceCache datasources.CacheService dataSourceCache datasources.CacheService
secretsService secrets.Service dataSourceService datasources.DataSourceService
pluginSettingsService pluginsettings.Service pluginSettingsService pluginsettings.Service
logger log.Logger logger log.Logger
} }
@ -87,7 +86,7 @@ func (p *Provider) Get(ctx context.Context, pluginID string, datasourceUID strin
if err != nil { if err != nil {
return pc, false, errutil.Wrap("Failed to get datasource", err) return pc, false, errutil.Wrap("Failed to get datasource", err)
} }
datasourceSettings, err := adapters.ModelToInstanceSettings(ds, p.decryptSecureJsonDataFn()) datasourceSettings, err := adapters.ModelToInstanceSettings(ds, p.decryptSecureJsonDataFn(ctx))
if err != nil { if err != nil {
return pc, false, errutil.Wrap("Failed to convert datasource", err) return pc, false, errutil.Wrap("Failed to convert datasource", err)
} }
@ -122,9 +121,9 @@ func (p *Provider) getCachedPluginSettings(ctx context.Context, pluginID string,
return ps, nil return ps, nil
} }
func (p *Provider) decryptSecureJsonDataFn() func(map[string][]byte) map[string]string { func (p *Provider) decryptSecureJsonDataFn(ctx context.Context) func(ds *models.DataSource) map[string]string {
return func(m map[string][]byte) map[string]string { return func(ds *models.DataSource) map[string]string {
decryptedJsonData, err := p.secretsService.DecryptJsonData(context.Background(), m) decryptedJsonData, err := p.dataSourceService.DecryptedValues(ctx, ds)
if err != nil { if err != nil {
p.logger.Error("Failed to decrypt secure json data", "error", err) p.logger.Error("Failed to decrypt secure json data", "error", err)
} }

View File

@ -79,6 +79,7 @@ import (
"github.com/grafana/grafana/pkg/services/searchV2" "github.com/grafana/grafana/pkg/services/searchV2"
"github.com/grafana/grafana/pkg/services/secrets" "github.com/grafana/grafana/pkg/services/secrets"
secretsDatabase "github.com/grafana/grafana/pkg/services/secrets/database" secretsDatabase "github.com/grafana/grafana/pkg/services/secrets/database"
secretsStore "github.com/grafana/grafana/pkg/services/secrets/kvstore"
secretsManager "github.com/grafana/grafana/pkg/services/secrets/manager" secretsManager "github.com/grafana/grafana/pkg/services/secrets/manager"
"github.com/grafana/grafana/pkg/services/serviceaccounts" "github.com/grafana/grafana/pkg/services/serviceaccounts"
serviceaccountsmanager "github.com/grafana/grafana/pkg/services/serviceaccounts/manager" serviceaccountsmanager "github.com/grafana/grafana/pkg/services/serviceaccounts/manager"
@ -239,6 +240,7 @@ var wireBasicSet = wire.NewSet(
wire.Bind(new(alerting.DashAlertExtractor), new(*alerting.DashAlertExtractorService)), wire.Bind(new(alerting.DashAlertExtractor), new(*alerting.DashAlertExtractorService)),
comments.ProvideService, comments.ProvideService,
guardian.ProvideService, guardian.ProvideService,
secretsStore.ProvideService,
avatar.ProvideAvatarCacheServer, avatar.ProvideAvatarCacheServer,
authproxy.ProvideAuthProxy, authproxy.ProvideAuthProxy,
statscollector.ProvideService, statscollector.ProvideService,

View File

@ -115,7 +115,7 @@ func (p *DataSourceProxyService) proxyDatasourceRequest(c *models.ReqContext, ds
proxyPath := getProxyPath(c) proxyPath := getProxyPath(c)
proxy, err := pluginproxy.NewDataSourceProxy(ds, plugin.Routes, c, proxyPath, p.Cfg, p.HTTPClientProvider, proxy, err := pluginproxy.NewDataSourceProxy(ds, plugin.Routes, c, proxyPath, p.Cfg, p.HTTPClientProvider,
p.OAuthTokenService, p.DataSourcesService, p.tracer, p.secretsService) p.OAuthTokenService, p.DataSourcesService, p.tracer)
if err != nil { if err != nil {
if errors.Is(err, datasource.URLValidationError{}) { if errors.Is(err, datasource.URLValidationError{}) {
c.JsonApiErr(http.StatusBadRequest, fmt.Sprintf("Invalid data source URL: %q", ds.Url), err) c.JsonApiErr(http.StatusBadRequest, fmt.Sprintf("Invalid data source URL: %q", ds.Url), err)

View File

@ -33,23 +33,23 @@ type DataSourceService interface {
GetDefaultDataSource(ctx context.Context, query *models.GetDefaultDataSourceQuery) error GetDefaultDataSource(ctx context.Context, query *models.GetDefaultDataSourceQuery) error
// GetHTTPTransport gets a datasource specific HTTP transport. // GetHTTPTransport gets a datasource specific HTTP transport.
GetHTTPTransport(ds *models.DataSource, provider httpclient.Provider, customMiddlewares ...sdkhttpclient.Middleware) (http.RoundTripper, error) GetHTTPTransport(ctx context.Context, ds *models.DataSource, provider httpclient.Provider, customMiddlewares ...sdkhttpclient.Middleware) (http.RoundTripper, error)
// DecryptedValues decrypts the encrypted secureJSONData of the provided datasource and // DecryptedValues decrypts the encrypted secureJSONData of the provided datasource and
// returns the decrypted values. // returns the decrypted values.
DecryptedValues(ds *models.DataSource) map[string]string DecryptedValues(ctx context.Context, ds *models.DataSource) (map[string]string, error)
// DecryptedValue decrypts the encrypted datasource secureJSONData identified by key // DecryptedValue decrypts the encrypted datasource secureJSONData identified by key
// and returns the decryped value. // and returns the decryped value.
DecryptedValue(ds *models.DataSource, key string) (string, bool) DecryptedValue(ctx context.Context, ds *models.DataSource, key string) (string, bool, error)
// DecryptedBasicAuthPassword decrypts the encrypted datasource basic authentication // DecryptedBasicAuthPassword decrypts the encrypted datasource basic authentication
// password and returns the decryped value. // password and returns the decryped value.
DecryptedBasicAuthPassword(ds *models.DataSource) string DecryptedBasicAuthPassword(ctx context.Context, ds *models.DataSource) (string, error)
// DecryptedPassword decrypts the encrypted datasource password and returns the // DecryptedPassword decrypts the encrypted datasource password and returns the
// decryped value. // decryped value.
DecryptedPassword(ds *models.DataSource) string DecryptedPassword(ctx context.Context, ds *models.DataSource) (string, error)
} }
// CacheService interface for retrieving a cached datasource. // CacheService interface for retrieving a cached datasource.

View File

@ -4,13 +4,14 @@ import (
"context" "context"
"github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/services/datasources"
) )
type FakeCacheService struct { type FakeCacheService struct {
DataSources []*models.DataSource DataSources []*models.DataSource
} }
var _ CacheService = &FakeCacheService{} var _ datasources.CacheService = &FakeCacheService{}
func (c *FakeCacheService) GetDatasource(ctx context.Context, datasourceID int64, user *models.SignedInUser, skipCache bool) (*models.DataSource, error) { func (c *FakeCacheService) GetDatasource(ctx context.Context, datasourceID int64, user *models.SignedInUser, skipCache bool) (*models.DataSource, error) {
for _, datasource := range c.DataSources { for _, datasource := range c.DataSources {

View File

@ -0,0 +1,124 @@
package datasources
import (
"context"
"net/http"
sdkhttpclient "github.com/grafana/grafana-plugin-sdk-go/backend/httpclient"
"github.com/grafana/grafana/pkg/infra/httpclient"
"github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/services/datasources"
)
type FakeDataSourceService struct {
lastId int64
DataSources []*models.DataSource
}
var _ datasources.DataSourceService = &FakeDataSourceService{}
func (s *FakeDataSourceService) GetDataSource(ctx context.Context, query *models.GetDataSourceQuery) error {
for _, datasource := range s.DataSources {
idMatch := query.Id != 0 && query.Id == datasource.Id
uidMatch := query.Uid != "" && query.Uid == datasource.Uid
nameMatch := query.Name != "" && query.Name == datasource.Name
if idMatch || nameMatch || uidMatch {
query.Result = datasource
return nil
}
}
return models.ErrDataSourceNotFound
}
func (s *FakeDataSourceService) GetDataSources(ctx context.Context, query *models.GetDataSourcesQuery) error {
for _, datasource := range s.DataSources {
orgMatch := query.OrgId != 0 && query.OrgId == datasource.OrgId
if orgMatch {
query.Result = append(query.Result, datasource)
}
}
return nil
}
func (s *FakeDataSourceService) GetDataSourcesByType(ctx context.Context, query *models.GetDataSourcesByTypeQuery) error {
for _, datasource := range s.DataSources {
typeMatch := query.Type != "" && query.Type == datasource.Type
if typeMatch {
query.Result = append(query.Result, datasource)
}
}
return nil
}
func (s *FakeDataSourceService) AddDataSource(ctx context.Context, cmd *models.AddDataSourceCommand) error {
if s.lastId == 0 {
s.lastId = int64(len(s.DataSources) - 1)
}
cmd.Result = &models.DataSource{
Id: s.lastId + 1,
Name: cmd.Name,
Type: cmd.Type,
Uid: cmd.Uid,
OrgId: cmd.OrgId,
}
s.DataSources = append(s.DataSources, cmd.Result)
return nil
}
func (s *FakeDataSourceService) DeleteDataSource(ctx context.Context, cmd *models.DeleteDataSourceCommand) error {
for i, datasource := range s.DataSources {
idMatch := cmd.ID != 0 && cmd.ID == datasource.Id
uidMatch := cmd.UID != "" && cmd.UID == datasource.Uid
nameMatch := cmd.Name != "" && cmd.Name == datasource.Name
if idMatch || nameMatch || uidMatch {
s.DataSources = append(s.DataSources[:i], s.DataSources[i+1:]...)
return nil
}
}
return models.ErrDataSourceNotFound
}
func (s *FakeDataSourceService) UpdateDataSource(ctx context.Context, cmd *models.UpdateDataSourceCommand) error {
for _, datasource := range s.DataSources {
idMatch := cmd.Id != 0 && cmd.Id == datasource.Id
uidMatch := cmd.Uid != "" && cmd.Uid == datasource.Uid
nameMatch := cmd.Name != "" && cmd.Name == datasource.Name
if idMatch || nameMatch || uidMatch {
if cmd.Name != "" {
datasource.Name = cmd.Name
}
return nil
}
}
return models.ErrDataSourceNotFound
}
func (s *FakeDataSourceService) GetDefaultDataSource(ctx context.Context, query *models.GetDefaultDataSourceQuery) error {
return nil
}
func (s *FakeDataSourceService) GetHTTPTransport(ctx context.Context, ds *models.DataSource, provider httpclient.Provider, customMiddlewares ...sdkhttpclient.Middleware) (http.RoundTripper, error) {
rt, err := provider.GetTransport(sdkhttpclient.Options{})
if err != nil {
return nil, err
}
return rt, nil
}
func (s *FakeDataSourceService) DecryptedValues(ctx context.Context, ds *models.DataSource) (map[string]string, error) {
values := make(map[string]string)
return values, nil
}
func (s *FakeDataSourceService) DecryptedValue(ctx context.Context, ds *models.DataSource, key string) (string, bool, error) {
return "", false, nil
}
func (s *FakeDataSourceService) DecryptedBasicAuthPassword(ctx context.Context, ds *models.DataSource) (string, error) {
return "", nil
}
func (s *FakeDataSourceService) DecryptedPassword(ctx context.Context, ds *models.DataSource) (string, error) {
return "", nil
}

View File

@ -3,6 +3,7 @@ package service
import ( import (
"context" "context"
"crypto/tls" "crypto/tls"
"encoding/json"
"fmt" "fmt"
"net/http" "net/http"
"net/url" "net/url"
@ -23,20 +24,21 @@ import (
"github.com/grafana/grafana/pkg/services/datasources" "github.com/grafana/grafana/pkg/services/datasources"
"github.com/grafana/grafana/pkg/services/featuremgmt" "github.com/grafana/grafana/pkg/services/featuremgmt"
"github.com/grafana/grafana/pkg/services/secrets" "github.com/grafana/grafana/pkg/services/secrets"
"github.com/grafana/grafana/pkg/services/secrets/kvstore"
"github.com/grafana/grafana/pkg/services/sqlstore" "github.com/grafana/grafana/pkg/services/sqlstore"
"github.com/grafana/grafana/pkg/setting" "github.com/grafana/grafana/pkg/setting"
) )
type Service struct { type Service struct {
SQLStore *sqlstore.SQLStore SQLStore *sqlstore.SQLStore
SecretsStore kvstore.SecretsKVStore
SecretsService secrets.Service SecretsService secrets.Service
cfg *setting.Cfg cfg *setting.Cfg
features featuremgmt.FeatureToggles features featuremgmt.FeatureToggles
permissionsService accesscontrol.PermissionsService permissionsService accesscontrol.PermissionsService
ac accesscontrol.AccessControl ac accesscontrol.AccessControl
ptc proxyTransportCache ptc proxyTransportCache
dsDecryptionCache secureJSONDecryptionCache
} }
type proxyTransportCache struct { type proxyTransportCache struct {
@ -49,29 +51,17 @@ type cachedRoundTripper struct {
roundTripper http.RoundTripper roundTripper http.RoundTripper
} }
type secureJSONDecryptionCache struct {
cache map[int64]cachedDecryptedJSON
sync.Mutex
}
type cachedDecryptedJSON struct {
updated time.Time
json map[string]string
}
func ProvideService( func ProvideService(
store *sqlstore.SQLStore, secretsService secrets.Service, cfg *setting.Cfg, features featuremgmt.FeatureToggles, store *sqlstore.SQLStore, secretsService secrets.Service, secretsStore kvstore.SecretsKVStore, cfg *setting.Cfg,
ac accesscontrol.AccessControl, permissionsServices accesscontrol.PermissionsServices, features featuremgmt.FeatureToggles, ac accesscontrol.AccessControl, permissionsServices accesscontrol.PermissionsServices,
) *Service { ) *Service {
s := &Service{ s := &Service{
SQLStore: store, SQLStore: store,
SecretsStore: secretsStore,
SecretsService: secretsService, SecretsService: secretsService,
ptc: proxyTransportCache{ ptc: proxyTransportCache{
cache: make(map[int64]cachedRoundTripper), cache: make(map[int64]cachedRoundTripper),
}, },
dsDecryptionCache: secureJSONDecryptionCache{
cache: make(map[int64]cachedDecryptedJSON),
},
cfg: cfg, cfg: cfg,
features: features, features: features,
permissionsService: permissionsServices.GetDataSourceService(), permissionsService: permissionsServices.GetDataSourceService(),
@ -90,6 +80,8 @@ type DataSourceRetriever interface {
GetDataSource(ctx context.Context, query *models.GetDataSourceQuery) error GetDataSource(ctx context.Context, query *models.GetDataSourceQuery) error
} }
const secretType = "datasource"
// NewNameScopeResolver provides an AttributeScopeResolver able to // NewNameScopeResolver provides an AttributeScopeResolver able to
// translate a scope prefixed with "datasources:name:" into an uid based scope. // translate a scope prefixed with "datasources:name:" into an uid based scope.
func NewNameScopeResolver(db DataSourceRetriever) (string, accesscontrol.AttributeScopeResolveFunc) { func NewNameScopeResolver(db DataSourceRetriever) (string, accesscontrol.AttributeScopeResolveFunc) {
@ -155,12 +147,17 @@ func (s *Service) GetDataSourcesByType(ctx context.Context, query *models.GetDat
func (s *Service) AddDataSource(ctx context.Context, cmd *models.AddDataSourceCommand) error { func (s *Service) AddDataSource(ctx context.Context, cmd *models.AddDataSourceCommand) error {
var err error var err error
cmd.EncryptedSecureJsonData, err = s.SecretsService.EncryptJsonData(ctx, cmd.SecureJsonData, secrets.WithoutScope()) if err := s.SQLStore.AddDataSource(ctx, cmd); err != nil {
return err
}
secret, err := json.Marshal(cmd.SecureJsonData)
if err != nil { if err != nil {
return err return err
} }
if err := s.SQLStore.AddDataSource(ctx, cmd); err != nil { err = s.SecretsStore.Set(ctx, cmd.OrgId, cmd.Name, secretType, string(secret))
if err != nil {
return err return err
} }
@ -186,25 +183,50 @@ func (s *Service) AddDataSource(ctx context.Context, cmd *models.AddDataSourceCo
} }
func (s *Service) DeleteDataSource(ctx context.Context, cmd *models.DeleteDataSourceCommand) error { func (s *Service) DeleteDataSource(ctx context.Context, cmd *models.DeleteDataSourceCommand) error {
return s.SQLStore.DeleteDataSource(ctx, cmd) err := s.SQLStore.DeleteDataSource(ctx, cmd)
if err != nil {
return err
}
return s.SecretsStore.Del(ctx, cmd.OrgID, cmd.Name, secretType)
} }
func (s *Service) UpdateDataSource(ctx context.Context, cmd *models.UpdateDataSourceCommand) error { func (s *Service) UpdateDataSource(ctx context.Context, cmd *models.UpdateDataSourceCommand) error {
var err error var err error
cmd.EncryptedSecureJsonData, err = s.SecretsService.EncryptJsonData(ctx, cmd.SecureJsonData, secrets.WithoutScope()) secret, err := json.Marshal(cmd.SecureJsonData)
if err != nil { if err != nil {
return err return err
} }
return s.SQLStore.UpdateDataSource(ctx, cmd) query := &models.GetDataSourceQuery{
Id: cmd.Id,
OrgId: cmd.OrgId,
}
err = s.SQLStore.GetDataSource(ctx, query)
if err != nil {
return err
}
err = s.SQLStore.UpdateDataSource(ctx, cmd)
if err != nil {
return err
}
if query.Result.Name != cmd.Name {
err = s.SecretsStore.Rename(ctx, cmd.OrgId, query.Result.Name, secretType, cmd.Name)
if err != nil {
return err
}
}
return s.SecretsStore.Set(ctx, cmd.OrgId, cmd.Name, secretType, string(secret))
} }
func (s *Service) GetDefaultDataSource(ctx context.Context, query *models.GetDefaultDataSourceQuery) error { func (s *Service) GetDefaultDataSource(ctx context.Context, query *models.GetDefaultDataSourceQuery) error {
return s.SQLStore.GetDefaultDataSource(ctx, query) return s.SQLStore.GetDefaultDataSource(ctx, query)
} }
func (s *Service) GetHTTPClient(ds *models.DataSource, provider httpclient.Provider) (*http.Client, error) { func (s *Service) GetHTTPClient(ctx context.Context, ds *models.DataSource, provider httpclient.Provider) (*http.Client, error) {
transport, err := s.GetHTTPTransport(ds, provider) transport, err := s.GetHTTPTransport(ctx, ds, provider)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -215,7 +237,7 @@ func (s *Service) GetHTTPClient(ds *models.DataSource, provider httpclient.Provi
}, nil }, nil
} }
func (s *Service) GetHTTPTransport(ds *models.DataSource, provider httpclient.Provider, func (s *Service) GetHTTPTransport(ctx context.Context, ds *models.DataSource, provider httpclient.Provider,
customMiddlewares ...sdkhttpclient.Middleware) (http.RoundTripper, error) { customMiddlewares ...sdkhttpclient.Middleware) (http.RoundTripper, error) {
s.ptc.Lock() s.ptc.Lock()
defer s.ptc.Unlock() defer s.ptc.Unlock()
@ -224,7 +246,7 @@ func (s *Service) GetHTTPTransport(ds *models.DataSource, provider httpclient.Pr
return t.roundTripper, nil return t.roundTripper, nil
} }
opts, err := s.httpClientOptions(ds) opts, err := s.httpClientOptions(ctx, ds)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -244,58 +266,84 @@ func (s *Service) GetHTTPTransport(ds *models.DataSource, provider httpclient.Pr
return rt, nil return rt, nil
} }
func (s *Service) GetTLSConfig(ds *models.DataSource, httpClientProvider httpclient.Provider) (*tls.Config, error) { func (s *Service) GetTLSConfig(ctx context.Context, ds *models.DataSource, httpClientProvider httpclient.Provider) (*tls.Config, error) {
opts, err := s.httpClientOptions(ds) opts, err := s.httpClientOptions(ctx, ds)
if err != nil { if err != nil {
return nil, err return nil, err
} }
return httpClientProvider.GetTLSConfig(*opts) return httpClientProvider.GetTLSConfig(*opts)
} }
func (s *Service) DecryptedValues(ds *models.DataSource) map[string]string { func (s *Service) DecryptedValues(ctx context.Context, ds *models.DataSource) (map[string]string, error) {
s.dsDecryptionCache.Lock() decryptedValues := make(map[string]string)
defer s.dsDecryptionCache.Unlock() secret, exist, err := s.SecretsStore.Get(ctx, ds.OrgId, ds.Name, secretType)
if item, present := s.dsDecryptionCache.cache[ds.Id]; present && ds.Updated.Equal(item.updated) {
return item.json
}
json, err := s.SecretsService.DecryptJsonData(context.Background(), ds.SecureJsonData)
if err != nil { if err != nil {
return map[string]string{} return nil, err
} }
s.dsDecryptionCache.cache[ds.Id] = cachedDecryptedJSON{ if exist {
updated: ds.Updated, err := json.Unmarshal([]byte(secret), &decryptedValues)
json: json, if err != nil {
return nil, err
}
} else if len(ds.SecureJsonData) > 0 {
decryptedValues, err = s.MigrateSecrets(ctx, ds)
if err != nil {
return nil, err
}
} }
return json return decryptedValues, nil
} }
func (s *Service) DecryptedValue(ds *models.DataSource, key string) (string, bool) { func (s *Service) MigrateSecrets(ctx context.Context, ds *models.DataSource) (map[string]string, error) {
value, exists := s.DecryptedValues(ds)[key] secureJsonData, err := s.SecretsService.DecryptJsonData(ctx, ds.SecureJsonData)
return value, exists if err != nil {
} return nil, err
func (s *Service) DecryptedBasicAuthPassword(ds *models.DataSource) string {
if value, ok := s.DecryptedValue(ds, "basicAuthPassword"); ok {
return value
} }
return ds.BasicAuthPassword jsonData, err := json.Marshal(secureJsonData)
} if err != nil {
return nil, err
func (s *Service) DecryptedPassword(ds *models.DataSource) string {
if value, ok := s.DecryptedValue(ds, "password"); ok {
return value
} }
return ds.Password err = s.SecretsStore.Set(ctx, ds.OrgId, ds.Name, secretType, string(jsonData))
return secureJsonData, err
} }
func (s *Service) httpClientOptions(ds *models.DataSource) (*sdkhttpclient.Options, error) { func (s *Service) DecryptedValue(ctx context.Context, ds *models.DataSource, key string) (string, bool, error) {
tlsOptions := s.dsTLSOptions(ds) values, err := s.DecryptedValues(ctx, ds)
if err != nil {
return "", false, err
}
value, exists := values[key]
return value, exists, nil
}
func (s *Service) DecryptedBasicAuthPassword(ctx context.Context, ds *models.DataSource) (string, error) {
value, ok, err := s.DecryptedValue(ctx, ds, "basicAuthPassword")
if ok {
return value, nil
}
return ds.BasicAuthPassword, err
}
func (s *Service) DecryptedPassword(ctx context.Context, ds *models.DataSource) (string, error) {
value, ok, err := s.DecryptedValue(ctx, ds, "password")
if ok {
return value, nil
}
return ds.Password, err
}
func (s *Service) httpClientOptions(ctx context.Context, ds *models.DataSource) (*sdkhttpclient.Options, error) {
tlsOptions, err := s.dsTLSOptions(ctx, ds)
if err != nil {
return nil, err
}
timeouts := &sdkhttpclient.TimeoutOptions{ timeouts := &sdkhttpclient.TimeoutOptions{
Timeout: s.getTimeout(ds), Timeout: s.getTimeout(ds),
DialTimeout: sdkhttpclient.DefaultTimeoutOptions.DialTimeout, DialTimeout: sdkhttpclient.DefaultTimeoutOptions.DialTimeout,
@ -307,9 +355,15 @@ func (s *Service) httpClientOptions(ds *models.DataSource) (*sdkhttpclient.Optio
MaxIdleConnsPerHost: sdkhttpclient.DefaultTimeoutOptions.MaxIdleConnsPerHost, MaxIdleConnsPerHost: sdkhttpclient.DefaultTimeoutOptions.MaxIdleConnsPerHost,
IdleConnTimeout: sdkhttpclient.DefaultTimeoutOptions.IdleConnTimeout, IdleConnTimeout: sdkhttpclient.DefaultTimeoutOptions.IdleConnTimeout,
} }
decryptedValues, err := s.DecryptedValues(ctx, ds)
if err != nil {
return nil, err
}
opts := &sdkhttpclient.Options{ opts := &sdkhttpclient.Options{
Timeouts: timeouts, Timeouts: timeouts,
Headers: s.getCustomHeaders(ds.JsonData, s.DecryptedValues(ds)), Headers: s.getCustomHeaders(ds.JsonData, decryptedValues),
Labels: map[string]string{ Labels: map[string]string{
"datasource_name": ds.Name, "datasource_name": ds.Name,
"datasource_uid": ds.Uid, "datasource_uid": ds.Uid,
@ -320,22 +374,30 @@ func (s *Service) httpClientOptions(ds *models.DataSource) (*sdkhttpclient.Optio
if ds.JsonData != nil { if ds.JsonData != nil {
opts.CustomOptions = ds.JsonData.MustMap() opts.CustomOptions = ds.JsonData.MustMap()
} }
if ds.BasicAuth { if ds.BasicAuth {
password, err := s.DecryptedBasicAuthPassword(ctx, ds)
if err != nil {
return opts, err
}
opts.BasicAuth = &sdkhttpclient.BasicAuthOptions{ opts.BasicAuth = &sdkhttpclient.BasicAuthOptions{
User: ds.BasicAuthUser, User: ds.BasicAuthUser,
Password: s.DecryptedBasicAuthPassword(ds), Password: password,
} }
} else if ds.User != "" { } else if ds.User != "" {
password, err := s.DecryptedPassword(ctx, ds)
if err != nil {
return opts, err
}
opts.BasicAuth = &sdkhttpclient.BasicAuthOptions{ opts.BasicAuth = &sdkhttpclient.BasicAuthOptions{
User: ds.User, User: ds.User,
Password: s.DecryptedPassword(ds), Password: password,
} }
} }
// Azure authentication
if ds.JsonData != nil && s.features.IsEnabled(featuremgmt.FlagHttpclientproviderAzureAuth) { if ds.JsonData != nil && s.features.IsEnabled(featuremgmt.FlagHttpclientproviderAzureAuth) {
credentials, err := azcredentials.FromDatasourceData(ds.JsonData.MustMap(), s.DecryptedValues(ds)) credentials, err := azcredentials.FromDatasourceData(ds.JsonData.MustMap(), decryptedValues)
if err != nil { if err != nil {
err = fmt.Errorf("invalid Azure credentials: %s", err) err = fmt.Errorf("invalid Azure credentials: %s", err)
return nil, err return nil, err
@ -371,19 +433,27 @@ func (s *Service) httpClientOptions(ds *models.DataSource) (*sdkhttpclient.Optio
Profile: ds.JsonData.Get("sigV4Profile").MustString(), Profile: ds.JsonData.Get("sigV4Profile").MustString(),
} }
if val, exists := s.DecryptedValue(ds, "sigV4AccessKey"); exists { if val, exists, err := s.DecryptedValue(ctx, ds, "sigV4AccessKey"); err == nil {
opts.SigV4.AccessKey = val if exists {
opts.SigV4.AccessKey = val
}
} else {
return opts, err
} }
if val, exists := s.DecryptedValue(ds, "sigV4SecretKey"); exists { if val, exists, err := s.DecryptedValue(ctx, ds, "sigV4SecretKey"); err == nil {
opts.SigV4.SecretKey = val if exists {
opts.SigV4.SecretKey = val
}
} else {
return opts, err
} }
} }
return opts, nil return opts, nil
} }
func (s *Service) dsTLSOptions(ds *models.DataSource) sdkhttpclient.TLSOptions { func (s *Service) dsTLSOptions(ctx context.Context, ds *models.DataSource) (sdkhttpclient.TLSOptions, error) {
var tlsSkipVerify, tlsClientAuth, tlsAuthWithCACert bool var tlsSkipVerify, tlsClientAuth, tlsAuthWithCACert bool
var serverName string var serverName string
@ -401,22 +471,35 @@ func (s *Service) dsTLSOptions(ds *models.DataSource) sdkhttpclient.TLSOptions {
if tlsClientAuth || tlsAuthWithCACert { if tlsClientAuth || tlsAuthWithCACert {
if tlsAuthWithCACert { if tlsAuthWithCACert {
if val, exists := s.DecryptedValue(ds, "tlsCACert"); exists && len(val) > 0 { if val, exists, err := s.DecryptedValue(ctx, ds, "tlsCACert"); err == nil {
opts.CACertificate = val if exists && len(val) > 0 {
opts.CACertificate = val
}
} else {
return opts, err
} }
} }
if tlsClientAuth { if tlsClientAuth {
if val, exists := s.DecryptedValue(ds, "tlsClientCert"); exists && len(val) > 0 { if val, exists, err := s.DecryptedValue(ctx, ds, "tlsClientCert"); err == nil {
opts.ClientCertificate = val fmt.Print("\n\n\n\n", val, exists, err, "\n\n\n\n")
if exists && len(val) > 0 {
opts.ClientCertificate = val
}
} else {
return opts, err
} }
if val, exists := s.DecryptedValue(ds, "tlsClientKey"); exists && len(val) > 0 { if val, exists, err := s.DecryptedValue(ctx, ds, "tlsClientKey"); err == nil {
opts.ClientKey = val if exists && len(val) > 0 {
opts.ClientKey = val
}
} else {
return opts, err
} }
} }
} }
return opts return opts, nil
} }
func (s *Service) getTimeout(ds *models.DataSource) time.Duration { func (s *Service) getTimeout(ds *models.DataSource) time.Duration {

View File

@ -2,6 +2,7 @@ package service
import ( import (
"context" "context"
encJson "encoding/json"
"io/ioutil" "io/ioutil"
"net/http" "net/http"
"net/http/httptest" "net/http/httptest"
@ -10,6 +11,7 @@ import (
"github.com/grafana/grafana-azure-sdk-go/azsettings" "github.com/grafana/grafana-azure-sdk-go/azsettings"
sdkhttpclient "github.com/grafana/grafana-plugin-sdk-go/backend/httpclient" sdkhttpclient "github.com/grafana/grafana-plugin-sdk-go/backend/httpclient"
secretsManager "github.com/grafana/grafana/pkg/services/secrets/manager"
"github.com/grafana/grafana/pkg/components/simplejson" "github.com/grafana/grafana/pkg/components/simplejson"
"github.com/grafana/grafana/pkg/infra/httpclient" "github.com/grafana/grafana/pkg/infra/httpclient"
@ -17,59 +19,13 @@ import (
"github.com/grafana/grafana/pkg/services/accesscontrol" "github.com/grafana/grafana/pkg/services/accesscontrol"
acmock "github.com/grafana/grafana/pkg/services/accesscontrol/mock" acmock "github.com/grafana/grafana/pkg/services/accesscontrol/mock"
"github.com/grafana/grafana/pkg/services/featuremgmt" "github.com/grafana/grafana/pkg/services/featuremgmt"
"github.com/grafana/grafana/pkg/services/secrets"
"github.com/grafana/grafana/pkg/services/secrets/database"
"github.com/grafana/grafana/pkg/services/secrets/fakes" "github.com/grafana/grafana/pkg/services/secrets/fakes"
secretsManager "github.com/grafana/grafana/pkg/services/secrets/manager" "github.com/grafana/grafana/pkg/services/secrets/kvstore"
"github.com/grafana/grafana/pkg/services/sqlstore"
"github.com/grafana/grafana/pkg/setting" "github.com/grafana/grafana/pkg/setting"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
) )
func TestService(t *testing.T) {
cfg := &setting.Cfg{}
sqlStore := sqlstore.InitTestDB(t)
origSecret := setting.SecretKey
setting.SecretKey = "datasources_service_test"
t.Cleanup(func() {
setting.SecretKey = origSecret
})
secretsService := secretsManager.SetupTestService(t, database.ProvideSecretsStore(sqlStore))
s := ProvideService(sqlStore, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New().WithDisabled(), acmock.NewPermissionsServicesMock())
var ds *models.DataSource
t.Run("create datasource should encrypt the secure json data", func(t *testing.T) {
ctx := context.Background()
sjd := map[string]string{"password": "12345"}
cmd := models.AddDataSourceCommand{SecureJsonData: sjd}
err := s.AddDataSource(ctx, &cmd)
require.NoError(t, err)
ds = cmd.Result
decrypted, err := s.SecretsService.DecryptJsonData(ctx, ds.SecureJsonData)
require.NoError(t, err)
require.Equal(t, sjd, decrypted)
})
t.Run("update datasource should encrypt the secure json data", func(t *testing.T) {
ctx := context.Background()
sjd := map[string]string{"password": "678910"}
cmd := models.UpdateDataSourceCommand{Id: ds.Id, OrgId: ds.OrgId, SecureJsonData: sjd}
err := s.UpdateDataSource(ctx, &cmd)
require.NoError(t, err)
decrypted, err := s.SecretsService.DecryptJsonData(ctx, cmd.Result.SecureJsonData)
require.NoError(t, err)
require.Equal(t, sjd, decrypted)
})
}
type dataSourceMockRetriever struct { type dataSourceMockRetriever struct {
res []*models.DataSource res []*models.DataSource
} }
@ -237,15 +193,16 @@ func TestService_GetHttpTransport(t *testing.T) {
Type: "Kubernetes", Type: "Kubernetes",
} }
secretsStore := kvstore.SetupTestService(t)
secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore())
dsService := ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) dsService := ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock())
rt1, err := dsService.GetHTTPTransport(&ds, provider) rt1, err := dsService.GetHTTPTransport(context.Background(), &ds, provider)
require.NoError(t, err) require.NoError(t, err)
require.NotNil(t, rt1) require.NotNil(t, rt1)
tr1 := configuredTransport tr1 := configuredTransport
rt2, err := dsService.GetHTTPTransport(&ds, provider) rt2, err := dsService.GetHTTPTransport(context.Background(), &ds, provider)
require.NoError(t, err) require.NoError(t, err)
require.NotNil(t, rt2) require.NotNil(t, rt2)
tr2 := configuredTransport tr2 := configuredTransport
@ -270,21 +227,19 @@ func TestService_GetHttpTransport(t *testing.T) {
json := simplejson.New() json := simplejson.New()
json.Set("tlsAuthWithCACert", true) json.Set("tlsAuthWithCACert", true)
secretsStore := kvstore.SetupTestService(t)
secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore())
dsService := ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) dsService := ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock())
tlsCaCert, err := secretsService.Encrypt(context.Background(), []byte(caCert), secrets.WithoutScope())
require.NoError(t, err)
ds := models.DataSource{ ds := models.DataSource{
Id: 1, Id: 1,
Url: "http://k8s:8001", Url: "http://k8s:8001",
Type: "Kubernetes", Type: "Kubernetes",
SecureJsonData: map[string][]byte{"tlsCACert": tlsCaCert}, SecureJsonData: map[string][]byte{"tlsCACert": []byte(caCert)},
Updated: time.Now().Add(-2 * time.Minute), Updated: time.Now().Add(-2 * time.Minute),
} }
rt1, err := dsService.GetHTTPTransport(&ds, provider) rt1, err := dsService.GetHTTPTransport(context.Background(), &ds, provider)
require.NotNil(t, rt1) require.NotNil(t, rt1)
require.NoError(t, err) require.NoError(t, err)
@ -298,7 +253,7 @@ func TestService_GetHttpTransport(t *testing.T) {
ds.SecureJsonData = map[string][]byte{} ds.SecureJsonData = map[string][]byte{}
ds.Updated = time.Now() ds.Updated = time.Now()
rt2, err := dsService.GetHTTPTransport(&ds, provider) rt2, err := dsService.GetHTTPTransport(context.Background(), &ds, provider)
require.NoError(t, err) require.NoError(t, err)
require.NotNil(t, rt2) require.NotNil(t, rt2)
tr2 := configuredTransport tr2 := configuredTransport
@ -320,27 +275,29 @@ func TestService_GetHttpTransport(t *testing.T) {
json := simplejson.New() json := simplejson.New()
json.Set("tlsAuth", true) json.Set("tlsAuth", true)
secretsStore := kvstore.SetupTestService(t)
secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore())
dsService := ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) dsService := ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock())
tlsClientCert, err := secretsService.Encrypt(context.Background(), []byte(clientCert), secrets.WithoutScope())
require.NoError(t, err)
tlsClientKey, err := secretsService.Encrypt(context.Background(), []byte(clientKey), secrets.WithoutScope())
require.NoError(t, err)
ds := models.DataSource{ ds := models.DataSource{
Id: 1, Id: 1,
OrgId: 1,
Name: "kubernetes",
Url: "http://k8s:8001", Url: "http://k8s:8001",
Type: "Kubernetes", Type: "Kubernetes",
JsonData: json, JsonData: json,
SecureJsonData: map[string][]byte{
"tlsClientCert": tlsClientCert,
"tlsClientKey": tlsClientKey,
},
} }
rt, err := dsService.GetHTTPTransport(&ds, provider) secureJsonData, err := encJson.Marshal(map[string]string{
"tlsClientCert": clientCert,
"tlsClientKey": clientKey,
})
require.NoError(t, err)
err = secretsStore.Set(context.Background(), ds.OrgId, ds.Name, secretType, string(secureJsonData))
require.NoError(t, err)
rt, err := dsService.GetHTTPTransport(context.Background(), &ds, provider)
require.NoError(t, err) require.NoError(t, err)
require.NotNil(t, rt) require.NotNil(t, rt)
tr := configuredTransport tr := configuredTransport
@ -363,23 +320,28 @@ func TestService_GetHttpTransport(t *testing.T) {
json.Set("tlsAuthWithCACert", true) json.Set("tlsAuthWithCACert", true)
json.Set("serverName", "server-name") json.Set("serverName", "server-name")
secretsStore := kvstore.SetupTestService(t)
secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore())
dsService := ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) dsService := ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock())
tlsCaCert, err := secretsService.Encrypt(context.Background(), []byte(caCert), secrets.WithoutScope())
require.NoError(t, err)
ds := models.DataSource{ ds := models.DataSource{
Id: 1, Id: 1,
OrgId: 1,
Name: "kubernetes",
Url: "http://k8s:8001", Url: "http://k8s:8001",
Type: "Kubernetes", Type: "Kubernetes",
JsonData: json, JsonData: json,
SecureJsonData: map[string][]byte{
"tlsCACert": tlsCaCert,
},
} }
rt, err := dsService.GetHTTPTransport(&ds, provider) secureJsonData, err := encJson.Marshal(map[string]string{
"tlsCACert": caCert,
})
require.NoError(t, err)
err = secretsStore.Set(context.Background(), ds.OrgId, ds.Name, secretType, string(secureJsonData))
require.NoError(t, err)
rt, err := dsService.GetHTTPTransport(context.Background(), &ds, provider)
require.NoError(t, err) require.NoError(t, err)
require.NotNil(t, rt) require.NotNil(t, rt)
tr := configuredTransport tr := configuredTransport
@ -400,8 +362,9 @@ func TestService_GetHttpTransport(t *testing.T) {
json := simplejson.New() json := simplejson.New()
json.Set("tlsSkipVerify", true) json.Set("tlsSkipVerify", true)
secretsStore := kvstore.SetupTestService(t)
secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore())
dsService := ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) dsService := ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock())
ds := models.DataSource{ ds := models.DataSource{
Id: 1, Id: 1,
@ -410,12 +373,12 @@ func TestService_GetHttpTransport(t *testing.T) {
JsonData: json, JsonData: json,
} }
rt1, err := dsService.GetHTTPTransport(&ds, provider) rt1, err := dsService.GetHTTPTransport(context.Background(), &ds, provider)
require.NoError(t, err) require.NoError(t, err)
require.NotNil(t, rt1) require.NotNil(t, rt1)
tr1 := configuredTransport tr1 := configuredTransport
rt2, err := dsService.GetHTTPTransport(&ds, provider) rt2, err := dsService.GetHTTPTransport(context.Background(), &ds, provider)
require.NoError(t, err) require.NoError(t, err)
require.NotNil(t, rt2) require.NotNil(t, rt2)
tr2 := configuredTransport tr2 := configuredTransport
@ -431,20 +394,27 @@ func TestService_GetHttpTransport(t *testing.T) {
"httpHeaderName1": "Authorization", "httpHeaderName1": "Authorization",
}) })
secretsStore := kvstore.SetupTestService(t)
secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore())
dsService := ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) dsService := ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock())
encryptedData, err := secretsService.Encrypt(context.Background(), []byte(`Bearer xf5yhfkpsnmgo`), secrets.WithoutScope())
require.NoError(t, err)
ds := models.DataSource{ ds := models.DataSource{
Id: 1, Id: 1,
Url: "http://k8s:8001", OrgId: 1,
Type: "Kubernetes", Name: "kubernetes",
JsonData: json, Url: "http://k8s:8001",
SecureJsonData: map[string][]byte{"httpHeaderValue1": encryptedData}, Type: "Kubernetes",
JsonData: json,
} }
secureJsonData, err := encJson.Marshal(map[string]string{
"httpHeaderValue1": "Bearer xf5yhfkpsnmgo",
})
require.NoError(t, err)
err = secretsStore.Set(context.Background(), ds.OrgId, ds.Name, secretType, string(secureJsonData))
require.NoError(t, err)
headers := dsService.getCustomHeaders(json, map[string]string{"httpHeaderValue1": "Bearer xf5yhfkpsnmgo"}) headers := dsService.getCustomHeaders(json, map[string]string{"httpHeaderValue1": "Bearer xf5yhfkpsnmgo"})
require.Equal(t, "Bearer xf5yhfkpsnmgo", headers["Authorization"]) require.Equal(t, "Bearer xf5yhfkpsnmgo", headers["Authorization"])
@ -465,7 +435,7 @@ func TestService_GetHttpTransport(t *testing.T) {
// 2. Get HTTP transport from datasource which uses the test server as backend // 2. Get HTTP transport from datasource which uses the test server as backend
ds.Url = backend.URL ds.Url = backend.URL
rt, err := dsService.GetHTTPTransport(&ds, provider) rt, err := dsService.GetHTTPTransport(context.Background(), &ds, provider)
require.NoError(t, err) require.NoError(t, err)
require.NotNil(t, rt) require.NotNil(t, rt)
@ -490,8 +460,9 @@ func TestService_GetHttpTransport(t *testing.T) {
"timeout": 19, "timeout": 19,
}) })
secretsStore := kvstore.SetupTestService(t)
secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore())
dsService := ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) dsService := ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock())
ds := models.DataSource{ ds := models.DataSource{
Id: 1, Id: 1,
@ -500,7 +471,7 @@ func TestService_GetHttpTransport(t *testing.T) {
JsonData: json, JsonData: json,
} }
client, err := dsService.GetHTTPClient(&ds, provider) client, err := dsService.GetHTTPClient(context.Background(), &ds, provider)
require.NoError(t, err) require.NoError(t, err)
require.NotNil(t, client) require.NotNil(t, client)
require.Equal(t, 19*time.Second, client.Timeout) require.Equal(t, 19*time.Second, client.Timeout)
@ -523,15 +494,16 @@ func TestService_GetHttpTransport(t *testing.T) {
json, err := simplejson.NewJson([]byte(`{ "sigV4Auth": true }`)) json, err := simplejson.NewJson([]byte(`{ "sigV4Auth": true }`))
require.NoError(t, err) require.NoError(t, err)
secretsStore := kvstore.SetupTestService(t)
secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore())
dsService := ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) dsService := ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock())
ds := models.DataSource{ ds := models.DataSource{
Type: models.DS_ES, Type: models.DS_ES,
JsonData: json, JsonData: json,
} }
_, err = dsService.GetHTTPTransport(&ds, provider) _, err = dsService.GetHTTPTransport(context.Background(), &ds, provider)
require.NoError(t, err) require.NoError(t, err)
require.NotNil(t, configuredOpts) require.NotNil(t, configuredOpts)
require.NotNil(t, configuredOpts.SigV4) require.NotNil(t, configuredOpts.SigV4)
@ -558,8 +530,9 @@ func TestService_getTimeout(t *testing.T) {
{jsonData: simplejson.NewFromAny(map[string]interface{}{"timeout": "2"}), expectedTimeout: 2 * time.Second}, {jsonData: simplejson.NewFromAny(map[string]interface{}{"timeout": "2"}), expectedTimeout: 2 * time.Second},
} }
secretsStore := kvstore.SetupTestService(t)
secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore())
dsService := ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) dsService := ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock())
for _, tc := range testCases { for _, tc := range testCases {
ds := &models.DataSource{ ds := &models.DataSource{
@ -569,86 +542,6 @@ func TestService_getTimeout(t *testing.T) {
} }
} }
func TestService_DecryptedValue(t *testing.T) {
cfg := &setting.Cfg{}
t.Run("When datasource hasn't been updated, encrypted JSON should be fetched from cache", func(t *testing.T) {
secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore())
dsService := ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock())
encryptedJsonData, err := secretsService.EncryptJsonData(
context.Background(),
map[string]string{
"password": "password",
}, secrets.WithoutScope())
require.NoError(t, err)
ds := models.DataSource{
Id: 1,
Type: models.DS_INFLUXDB_08,
JsonData: simplejson.New(),
User: "user",
SecureJsonData: encryptedJsonData,
}
// Populate cache
password, ok := dsService.DecryptedValue(&ds, "password")
require.True(t, ok)
require.Equal(t, "password", password)
encryptedJsonData, err = secretsService.EncryptJsonData(
context.Background(),
map[string]string{
"password": "",
}, secrets.WithoutScope())
require.NoError(t, err)
ds.SecureJsonData = encryptedJsonData
password, ok = dsService.DecryptedValue(&ds, "password")
require.True(t, ok)
require.Equal(t, "password", password)
})
t.Run("When datasource is updated, encrypted JSON should not be fetched from cache", func(t *testing.T) {
secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore())
encryptedJsonData, err := secretsService.EncryptJsonData(
context.Background(),
map[string]string{
"password": "password",
}, secrets.WithoutScope())
require.NoError(t, err)
ds := models.DataSource{
Id: 1,
Type: models.DS_INFLUXDB_08,
JsonData: simplejson.New(),
User: "user",
SecureJsonData: encryptedJsonData,
}
dsService := ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock())
// Populate cache
password, ok := dsService.DecryptedValue(&ds, "password")
require.True(t, ok)
require.Equal(t, "password", password)
ds.SecureJsonData, err = secretsService.EncryptJsonData(
context.Background(),
map[string]string{
"password": "",
}, secrets.WithoutScope())
ds.Updated = time.Now()
require.NoError(t, err)
password, ok = dsService.DecryptedValue(&ds, "password")
require.True(t, ok)
require.Empty(t, password)
})
}
func TestService_HTTPClientOptions(t *testing.T) { func TestService_HTTPClientOptions(t *testing.T) {
cfg := &setting.Cfg{ cfg := &setting.Cfg{
Azure: &azsettings.AzureSettings{}, Azure: &azsettings.AzureSettings{},
@ -678,10 +571,11 @@ func TestService_HTTPClientOptions(t *testing.T) {
"azureEndpointResourceId": "https://api.example.com/abd5c4ce-ca73-41e9-9cb2-bed39aa2adb5", "azureEndpointResourceId": "https://api.example.com/abd5c4ce-ca73-41e9-9cb2-bed39aa2adb5",
}) })
secretsStore := kvstore.SetupTestService(t)
secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore())
dsService := ProvideService(nil, secretsService, cfg, features, acmock.New(), acmock.NewPermissionsServicesMock()) dsService := ProvideService(nil, secretsService, secretsStore, cfg, features, acmock.New(), acmock.NewPermissionsServicesMock())
opts, err := dsService.httpClientOptions(&ds) opts, err := dsService.httpClientOptions(context.Background(), &ds)
require.NoError(t, err) require.NoError(t, err)
require.NotNil(t, opts.Middlewares) require.NotNil(t, opts.Middlewares)
@ -695,10 +589,11 @@ func TestService_HTTPClientOptions(t *testing.T) {
"httpMethod": "POST", "httpMethod": "POST",
}) })
secretsStore := kvstore.SetupTestService(t)
secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore())
dsService := ProvideService(nil, secretsService, cfg, features, acmock.New(), acmock.NewPermissionsServicesMock()) dsService := ProvideService(nil, secretsService, secretsStore, cfg, features, acmock.New(), acmock.NewPermissionsServicesMock())
opts, err := dsService.httpClientOptions(&ds) opts, err := dsService.httpClientOptions(context.Background(), &ds)
require.NoError(t, err) require.NoError(t, err)
if opts.Middlewares != nil { if opts.Middlewares != nil {
@ -714,10 +609,11 @@ func TestService_HTTPClientOptions(t *testing.T) {
"azureCredentials": "invalid", "azureCredentials": "invalid",
}) })
secretsStore := kvstore.SetupTestService(t)
secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore())
dsService := ProvideService(nil, secretsService, cfg, features, acmock.New(), acmock.NewPermissionsServicesMock()) dsService := ProvideService(nil, secretsService, secretsStore, cfg, features, acmock.New(), acmock.NewPermissionsServicesMock())
_, err := dsService.httpClientOptions(&ds) _, err := dsService.httpClientOptions(context.Background(), &ds)
assert.Error(t, err) assert.Error(t, err)
}) })
@ -732,10 +628,11 @@ func TestService_HTTPClientOptions(t *testing.T) {
"azureEndpointResourceId": "https://api.example.com/abd5c4ce-ca73-41e9-9cb2-bed39aa2adb5", "azureEndpointResourceId": "https://api.example.com/abd5c4ce-ca73-41e9-9cb2-bed39aa2adb5",
}) })
secretsStore := kvstore.SetupTestService(t)
secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore())
dsService := ProvideService(nil, secretsService, cfg, features, acmock.New(), acmock.NewPermissionsServicesMock()) dsService := ProvideService(nil, secretsService, secretsStore, cfg, features, acmock.New(), acmock.NewPermissionsServicesMock())
opts, err := dsService.httpClientOptions(&ds) opts, err := dsService.httpClientOptions(context.Background(), &ds)
require.NoError(t, err) require.NoError(t, err)
require.NotNil(t, opts.Middlewares) require.NotNil(t, opts.Middlewares)
@ -750,10 +647,11 @@ func TestService_HTTPClientOptions(t *testing.T) {
"azureEndpointResourceId": "https://api.example.com/abd5c4ce-ca73-41e9-9cb2-bed39aa2adb5", "azureEndpointResourceId": "https://api.example.com/abd5c4ce-ca73-41e9-9cb2-bed39aa2adb5",
}) })
secretsStore := kvstore.SetupTestService(t)
secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore())
dsService := ProvideService(nil, secretsService, cfg, features, acmock.New(), acmock.NewPermissionsServicesMock()) dsService := ProvideService(nil, secretsService, secretsStore, cfg, features, acmock.New(), acmock.NewPermissionsServicesMock())
opts, err := dsService.httpClientOptions(&ds) opts, err := dsService.httpClientOptions(context.Background(), &ds)
require.NoError(t, err) require.NoError(t, err)
if opts.Middlewares != nil { if opts.Middlewares != nil {
@ -772,10 +670,11 @@ func TestService_HTTPClientOptions(t *testing.T) {
"azureEndpointResourceId": "invalid", "azureEndpointResourceId": "invalid",
}) })
secretsStore := kvstore.SetupTestService(t)
secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore())
dsService := ProvideService(nil, secretsService, cfg, features, acmock.New(), acmock.NewPermissionsServicesMock()) dsService := ProvideService(nil, secretsService, secretsStore, cfg, features, acmock.New(), acmock.NewPermissionsServicesMock())
_, err := dsService.httpClientOptions(&ds) _, err := dsService.httpClientOptions(context.Background(), &ds)
assert.Error(t, err) assert.Error(t, err)
}) })
}) })
@ -792,10 +691,11 @@ func TestService_HTTPClientOptions(t *testing.T) {
"azureEndpointResourceId": "https://api.example.com/abd5c4ce-ca73-41e9-9cb2-bed39aa2adb5", "azureEndpointResourceId": "https://api.example.com/abd5c4ce-ca73-41e9-9cb2-bed39aa2adb5",
}) })
secretsStore := kvstore.SetupTestService(t)
secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore())
dsService := ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) dsService := ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock())
opts, err := dsService.httpClientOptions(&ds) opts, err := dsService.httpClientOptions(context.Background(), &ds)
require.NoError(t, err) require.NoError(t, err)
if opts.Middlewares != nil { if opts.Middlewares != nil {

View File

@ -13,6 +13,7 @@ import (
"github.com/grafana/grafana/pkg/services/accesscontrol" "github.com/grafana/grafana/pkg/services/accesscontrol"
acMock "github.com/grafana/grafana/pkg/services/accesscontrol/mock" acMock "github.com/grafana/grafana/pkg/services/accesscontrol/mock"
"github.com/grafana/grafana/pkg/services/datasources" "github.com/grafana/grafana/pkg/services/datasources"
fakes "github.com/grafana/grafana/pkg/services/datasources/fakes"
"github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions" "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
"github.com/grafana/grafana/pkg/services/ngalert/eval" "github.com/grafana/grafana/pkg/services/ngalert/eval"
"github.com/grafana/grafana/pkg/services/ngalert/models" "github.com/grafana/grafana/pkg/services/ngalert/models"
@ -61,7 +62,7 @@ func TestRouteTestGrafanaRuleConfig(t *testing.T) {
{Action: datasources.ActionQuery, Scope: datasources.ScopeProvider.GetResourceScopeUID(data2.DatasourceUID)}, {Action: datasources.ActionQuery, Scope: datasources.ScopeProvider.GetResourceScopeUID(data2.DatasourceUID)},
}) })
ds := &datasources.FakeCacheService{DataSources: []*models2.DataSource{ ds := &fakes.FakeCacheService{DataSources: []*models2.DataSource{
{Uid: data1.DatasourceUID}, {Uid: data1.DatasourceUID},
{Uid: data2.DatasourceUID}, {Uid: data2.DatasourceUID},
}} }}
@ -102,7 +103,7 @@ func TestRouteTestGrafanaRuleConfig(t *testing.T) {
t.Run("should require user to be signed in", func(t *testing.T) { t.Run("should require user to be signed in", func(t *testing.T) {
data1 := models.GenerateAlertQuery() data1 := models.GenerateAlertQuery()
ds := &datasources.FakeCacheService{DataSources: []*models2.DataSource{ ds := &fakes.FakeCacheService{DataSources: []*models2.DataSource{
{Uid: data1.DatasourceUID}, {Uid: data1.DatasourceUID},
}} }}
@ -182,7 +183,7 @@ func TestRouteEvalQueries(t *testing.T) {
{Action: datasources.ActionQuery, Scope: datasources.ScopeProvider.GetResourceScopeUID(data2.DatasourceUID)}, {Action: datasources.ActionQuery, Scope: datasources.ScopeProvider.GetResourceScopeUID(data2.DatasourceUID)},
}) })
ds := &datasources.FakeCacheService{DataSources: []*models2.DataSource{ ds := &fakes.FakeCacheService{DataSources: []*models2.DataSource{
{Uid: data1.DatasourceUID}, {Uid: data1.DatasourceUID},
{Uid: data2.DatasourceUID}, {Uid: data2.DatasourceUID},
}} }}
@ -226,7 +227,7 @@ func TestRouteEvalQueries(t *testing.T) {
t.Run("should require user to be signed in", func(t *testing.T) { t.Run("should require user to be signed in", func(t *testing.T) {
data1 := models.GenerateAlertQuery() data1 := models.GenerateAlertQuery()
ds := &datasources.FakeCacheService{DataSources: []*models2.DataSource{ ds := &fakes.FakeCacheService{DataSources: []*models2.DataSource{
{Uid: data1.DatasourceUID}, {Uid: data1.DatasourceUID},
}} }}
@ -265,7 +266,7 @@ func TestRouteEvalQueries(t *testing.T) {
}) })
} }
func createTestingApiSrv(ds *datasources.FakeCacheService, ac *acMock.Mock, evaluator *eval.FakeEvaluator) *TestingApiSrv { func createTestingApiSrv(ds *fakes.FakeCacheService, ac *acMock.Mock, evaluator *eval.FakeEvaluator) *TestingApiSrv {
if ac == nil { if ac == nil {
ac = acMock.New().WithDisabled() ac = acMock.New().WithDisabled()
} }

View File

@ -15,7 +15,6 @@ import (
"github.com/grafana/grafana/pkg/plugins/adapters" "github.com/grafana/grafana/pkg/plugins/adapters"
"github.com/grafana/grafana/pkg/services/datasources" "github.com/grafana/grafana/pkg/services/datasources"
"github.com/grafana/grafana/pkg/services/oauthtoken" "github.com/grafana/grafana/pkg/services/oauthtoken"
"github.com/grafana/grafana/pkg/services/secrets"
"github.com/grafana/grafana/pkg/setting" "github.com/grafana/grafana/pkg/setting"
"github.com/grafana/grafana/pkg/tsdb/grafanads" "github.com/grafana/grafana/pkg/tsdb/grafanads"
"github.com/grafana/grafana/pkg/tsdb/legacydata" "github.com/grafana/grafana/pkg/tsdb/legacydata"
@ -33,7 +32,7 @@ func ProvideService(
dataSourceCache datasources.CacheService, dataSourceCache datasources.CacheService,
expressionService *expr.Service, expressionService *expr.Service,
pluginRequestValidator models.PluginRequestValidator, pluginRequestValidator models.PluginRequestValidator,
SecretsService secrets.Service, dataSourceService datasources.DataSourceService,
pluginClient plugins.Client, pluginClient plugins.Client,
oAuthTokenService oauthtoken.OAuthTokenService, oAuthTokenService oauthtoken.OAuthTokenService,
) *Service { ) *Service {
@ -42,7 +41,7 @@ func ProvideService(
dataSourceCache: dataSourceCache, dataSourceCache: dataSourceCache,
expressionService: expressionService, expressionService: expressionService,
pluginRequestValidator: pluginRequestValidator, pluginRequestValidator: pluginRequestValidator,
secretsService: SecretsService, dataSourceService: dataSourceService,
pluginClient: pluginClient, pluginClient: pluginClient,
oAuthTokenService: oAuthTokenService, oAuthTokenService: oAuthTokenService,
log: log.New("query_data"), log: log.New("query_data"),
@ -56,7 +55,7 @@ type Service struct {
dataSourceCache datasources.CacheService dataSourceCache datasources.CacheService
expressionService *expr.Service expressionService *expr.Service
pluginRequestValidator models.PluginRequestValidator pluginRequestValidator models.PluginRequestValidator
secretsService secrets.Service dataSourceService datasources.DataSourceService
pluginClient plugins.Client pluginClient plugins.Client
oAuthTokenService oauthtoken.OAuthTokenService oAuthTokenService oauthtoken.OAuthTokenService
log log.Logger log log.Logger
@ -291,9 +290,9 @@ func (s *Service) getDataSourceFromQuery(ctx context.Context, user *models.Signe
return nil, NewErrBadQuery("missing data source ID/UID") return nil, NewErrBadQuery("missing data source ID/UID")
} }
func (s *Service) decryptSecureJsonDataFn(ctx context.Context) func(map[string][]byte) map[string]string { func (s *Service) decryptSecureJsonDataFn(ctx context.Context) func(ds *models.DataSource) map[string]string {
return func(m map[string][]byte) map[string]string { return func(ds *models.DataSource) map[string]string {
decryptedJsonData, err := s.secretsService.DecryptJsonData(ctx, m) decryptedJsonData, err := s.dataSourceService.DecryptedValues(ctx, ds)
if err != nil { if err != nil {
s.log.Error("Failed to decrypt secure json data", "error", err) s.log.Error("Failed to decrypt secure json data", "error", err)
} }

View File

@ -2,6 +2,7 @@ package query_test
import ( import (
"context" "context"
"encoding/json"
"net/http" "net/http"
"testing" "testing"
@ -12,18 +13,29 @@ import (
"github.com/grafana/grafana/pkg/components/simplejson" "github.com/grafana/grafana/pkg/components/simplejson"
"github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/plugins" "github.com/grafana/grafana/pkg/plugins"
acmock "github.com/grafana/grafana/pkg/services/accesscontrol/mock"
datasources "github.com/grafana/grafana/pkg/services/datasources/service"
"github.com/grafana/grafana/pkg/services/featuremgmt"
"github.com/grafana/grafana/pkg/services/query" "github.com/grafana/grafana/pkg/services/query"
"github.com/grafana/grafana/pkg/services/secrets" "github.com/grafana/grafana/pkg/services/secrets/fakes"
"github.com/grafana/grafana/pkg/services/secrets/kvstore"
secretsManager "github.com/grafana/grafana/pkg/services/secrets/manager"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
) )
func TestQueryData(t *testing.T) { func TestQueryData(t *testing.T) {
t.Run("it attaches custom headers to the request", func(t *testing.T) { t.Run("it attaches custom headers to the request", func(t *testing.T) {
tc := setup() tc := setup(t)
tc.dataSourceCache.ds.JsonData = simplejson.NewFromAny(map[string]interface{}{"httpHeaderName1": "foo", "httpHeaderName2": "bar"}) tc.dataSourceCache.ds.JsonData = simplejson.NewFromAny(map[string]interface{}{"httpHeaderName1": "foo", "httpHeaderName2": "bar"})
tc.secretService.decryptedJson = map[string]string{"httpHeaderValue1": "test-header", "httpHeaderValue2": "test-header2"}
_, err := tc.queryService.QueryData(context.Background(), nil, true, metricRequest(), false) secureJsonData, err := json.Marshal(map[string]string{"httpHeaderValue1": "test-header", "httpHeaderValue2": "test-header2"})
require.NoError(t, err)
err = tc.secretStore.Set(context.Background(), tc.dataSourceCache.ds.OrgId, tc.dataSourceCache.ds.Name, "datasource", string(secureJsonData))
require.NoError(t, err)
_, err = tc.queryService.QueryData(context.Background(), nil, true, metricRequest(), false)
require.Nil(t, err) require.Nil(t, err)
require.Equal(t, map[string]string{"foo": "test-header", "bar": "test-header2"}, tc.pluginContext.req.Headers) require.Equal(t, map[string]string{"foo": "test-header", "bar": "test-header2"}, tc.pluginContext.req.Headers)
@ -36,7 +48,7 @@ func TestQueryData(t *testing.T) {
} }
token = token.WithExtra(map[string]interface{}{"id_token": "id-token"}) token = token.WithExtra(map[string]interface{}{"id_token": "id-token"})
tc := setup() tc := setup(t)
tc.oauthTokenService.passThruEnabled = true tc.oauthTokenService.passThruEnabled = true
tc.oauthTokenService.token = token tc.oauthTokenService.token = token
@ -51,26 +63,29 @@ func TestQueryData(t *testing.T) {
}) })
} }
func setup() *testContext { func setup(t *testing.T) *testContext {
pc := &fakePluginClient{} pc := &fakePluginClient{}
sc := &fakeSecretsService{}
dc := &fakeDataSourceCache{ds: &models.DataSource{}} dc := &fakeDataSourceCache{ds: &models.DataSource{}}
tc := &fakeOAuthTokenService{} tc := &fakeOAuthTokenService{}
rv := &fakePluginRequestValidator{} rv := &fakePluginRequestValidator{}
ss := kvstore.SetupTestService(t)
ssvc := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore())
ds := datasources.ProvideService(nil, ssvc, ss, nil, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock())
return &testContext{ return &testContext{
pluginContext: pc, pluginContext: pc,
secretService: sc, secretStore: ss,
dataSourceCache: dc, dataSourceCache: dc,
oauthTokenService: tc, oauthTokenService: tc,
pluginRequestValidator: rv, pluginRequestValidator: rv,
queryService: query.ProvideService(nil, dc, nil, rv, sc, pc, tc), queryService: query.ProvideService(nil, dc, nil, rv, ds, pc, tc),
} }
} }
type testContext struct { type testContext struct {
pluginContext *fakePluginClient pluginContext *fakePluginClient
secretService *fakeSecretsService secretStore kvstore.SecretsKVStore
dataSourceCache *fakeDataSourceCache dataSourceCache *fakeDataSourceCache
oauthTokenService *fakeOAuthTokenService oauthTokenService *fakeOAuthTokenService
pluginRequestValidator *fakePluginRequestValidator pluginRequestValidator *fakePluginRequestValidator
@ -108,16 +123,6 @@ func (ts *fakeOAuthTokenService) IsOAuthPassThruEnabled(*models.DataSource) bool
return ts.passThruEnabled return ts.passThruEnabled
} }
type fakeSecretsService struct {
secrets.Service
decryptedJson map[string]string
}
func (s *fakeSecretsService) DecryptJsonData(ctx context.Context, sjd map[string][]byte) (map[string]string, error) {
return s.decryptedJson, nil
}
type fakeDataSourceCache struct { type fakeDataSourceCache struct {
ds *models.DataSource ds *models.DataSource
} }

View File

@ -0,0 +1,29 @@
package kvstore
import (
"testing"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/services/secrets/database"
"github.com/grafana/grafana/pkg/services/secrets/manager"
"github.com/grafana/grafana/pkg/services/sqlstore"
)
func SetupTestService(t *testing.T) SecretsKVStore {
t.Helper()
sqlStore := sqlstore.InitTestDB(t)
store := database.ProvideSecretsStore(sqlstore.InitTestDB(t))
secretsService := manager.SetupTestService(t, store)
kv := &secretsKVStoreSQL{
sqlStore: sqlStore,
log: log.New("secrets.kvstore"),
secretsService: secretsService,
decryptionCache: decryptionCache{
cache: make(map[int64]cachedDecrypted),
},
}
return kv
}

View File

@ -0,0 +1,77 @@
package kvstore
import (
"context"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/services/secrets"
"github.com/grafana/grafana/pkg/services/sqlstore"
)
const (
// Wildcard to query all organizations
AllOrganizations = -1
)
func ProvideService(sqlStore sqlstore.Store, secretsService secrets.Service) SecretsKVStore {
return &secretsKVStoreSQL{
sqlStore: sqlStore,
secretsService: secretsService,
log: log.New("secrets.kvstore"),
decryptionCache: decryptionCache{
cache: make(map[int64]cachedDecrypted),
},
}
}
// SecretsKVStore is an interface for k/v store.
type SecretsKVStore interface {
Get(ctx context.Context, orgId int64, namespace string, typ string) (string, bool, error)
Set(ctx context.Context, orgId int64, namespace string, typ string, value string) error
Del(ctx context.Context, orgId int64, namespace string, typ string) error
Keys(ctx context.Context, orgId int64, namespace string, typ string) ([]Key, error)
Rename(ctx context.Context, orgId int64, namespace string, typ string, newNamespace string) error
}
// WithType returns a kvstore wrapper with fixed orgId and type.
func With(kv SecretsKVStore, orgId int64, namespace string, typ string) *FixedKVStore {
return &FixedKVStore{
kvStore: kv,
OrgId: orgId,
Namespace: namespace,
Type: typ,
}
}
// FixedKVStore is a SecretsKVStore wrapper with fixed orgId, namespace and type.
type FixedKVStore struct {
kvStore SecretsKVStore
OrgId int64
Namespace string
Type string
}
func (kv *FixedKVStore) Get(ctx context.Context) (string, bool, error) {
return kv.kvStore.Get(ctx, kv.OrgId, kv.Namespace, kv.Type)
}
func (kv *FixedKVStore) Set(ctx context.Context, value string) error {
return kv.kvStore.Set(ctx, kv.OrgId, kv.Namespace, kv.Type, value)
}
func (kv *FixedKVStore) Del(ctx context.Context) error {
return kv.kvStore.Del(ctx, kv.OrgId, kv.Namespace, kv.Type)
}
func (kv *FixedKVStore) Keys(ctx context.Context) ([]Key, error) {
return kv.kvStore.Keys(ctx, kv.OrgId, kv.Namespace, kv.Type)
}
func (kv *FixedKVStore) Rename(ctx context.Context, newNamespace string) error {
err := kv.kvStore.Rename(ctx, kv.OrgId, kv.Namespace, kv.Type, newNamespace)
if err != nil {
return err
}
kv.Namespace = newNamespace
return nil
}

View File

@ -0,0 +1,226 @@
package kvstore
import (
"context"
"fmt"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
type TestCase struct {
OrgId int64
Namespace string
Type string
Revision int64
}
func (t *TestCase) Value() string {
return fmt.Sprintf("%d:%s:%s:%d", t.OrgId, t.Namespace, t.Type, t.Revision)
}
func TestKVStore(t *testing.T) {
kv := SetupTestService(t)
ctx := context.Background()
testCases := []*TestCase{
{
OrgId: 0,
Namespace: "namespace1",
Type: "testing1",
},
{
OrgId: 0,
Namespace: "namespace2",
Type: "testing2",
},
{
OrgId: 1,
Namespace: "namespace1",
Type: "testing1",
},
{
OrgId: 1,
Namespace: "namespace3",
Type: "testing3",
},
}
for _, tc := range testCases {
err := kv.Set(ctx, tc.OrgId, tc.Namespace, tc.Type, tc.Value())
require.NoError(t, err)
}
t.Run("get existing keys", func(t *testing.T) {
for _, tc := range testCases {
value, ok, err := kv.Get(ctx, tc.OrgId, tc.Namespace, tc.Type)
require.NoError(t, err)
require.True(t, ok)
require.Equal(t, tc.Value(), value)
}
})
t.Run("get nonexistent keys", func(t *testing.T) {
tcs := []*TestCase{
{
OrgId: 0,
Namespace: "namespace3",
Type: "testing3",
},
{
OrgId: 1,
Namespace: "namespace2",
Type: "testing2",
},
{
OrgId: 2,
Namespace: "namespace1",
Type: "testing1",
},
}
for _, tc := range tcs {
value, ok, err := kv.Get(ctx, tc.OrgId, tc.Namespace, tc.Type)
require.Nil(t, err)
require.False(t, ok)
require.Equal(t, "", value)
}
})
t.Run("modify existing key", func(t *testing.T) {
tc := testCases[0]
value, ok, err := kv.Get(ctx, tc.OrgId, tc.Namespace, tc.Type)
require.NoError(t, err)
require.True(t, ok)
assert.Equal(t, tc.Value(), value)
tc.Revision += 1
err = kv.Set(ctx, tc.OrgId, tc.Namespace, tc.Type, tc.Value())
require.NoError(t, err)
value, ok, err = kv.Get(ctx, tc.OrgId, tc.Namespace, tc.Type)
require.NoError(t, err)
require.True(t, ok)
assert.Equal(t, tc.Value(), value)
})
t.Run("use fixed client", func(t *testing.T) {
tc := testCases[0]
client := With(kv, tc.OrgId, tc.Namespace, tc.Type)
fmt.Println(client.Namespace, client.OrgId, client.Type)
value, ok, err := client.Get(ctx)
require.NoError(t, err)
require.True(t, ok)
require.Equal(t, tc.Value(), value)
tc.Revision += 1
err = client.Set(ctx, tc.Value())
require.NoError(t, err)
value, ok, err = client.Get(ctx)
require.NoError(t, err)
require.True(t, ok)
assert.Equal(t, tc.Value(), value)
})
t.Run("deleting keys", func(t *testing.T) {
var stillHasKeys bool
for _, tc := range testCases {
if _, ok, err := kv.Get(ctx, tc.OrgId, tc.Namespace, tc.Type); err == nil && ok {
stillHasKeys = true
break
}
}
require.True(t, stillHasKeys,
"we are going to test key deletion, but there are no keys to delete in the database")
for _, tc := range testCases {
err := kv.Del(ctx, tc.OrgId, tc.Namespace, tc.Type)
require.NoError(t, err)
}
for _, tc := range testCases {
_, ok, err := kv.Get(ctx, tc.OrgId, tc.Namespace, tc.Type)
require.NoError(t, err)
require.False(t, ok, "all keys should be deleted at this point")
}
})
t.Run("listing existing keys", func(t *testing.T) {
kv := SetupTestService(t)
ctx := context.Background()
namespace, typ := "listtest", "listtest"
testCases := []*TestCase{
{
OrgId: 1,
Type: typ,
Namespace: namespace,
},
{
OrgId: 2,
Type: typ,
Namespace: namespace,
},
{
OrgId: 3,
Type: typ,
Namespace: namespace,
},
{
OrgId: 4,
Type: typ,
Namespace: namespace,
},
{
OrgId: 1,
Type: typ,
Namespace: "other_key",
},
{
OrgId: 4,
Type: typ,
Namespace: "another_one",
},
}
for _, tc := range testCases {
err := kv.Set(ctx, tc.OrgId, tc.Namespace, tc.Type, tc.Value())
require.NoError(t, err)
}
keys, err := kv.Keys(ctx, AllOrganizations, namespace, typ)
require.NoError(t, err)
require.Len(t, keys, 4)
found := 0
for _, key := range keys {
for _, tc := range testCases {
if key.OrgId == tc.OrgId && key.Namespace == tc.Namespace && key.Type == tc.Type {
found++
break
}
}
}
require.Equal(t, 4, found, "querying for all orgs should return 4 records")
keys, err = kv.Keys(ctx, 1, namespace, typ)
require.NoError(t, err)
require.Len(t, keys, 1, "querying for a specific org should return 1 record")
keys, err = kv.Keys(ctx, AllOrganizations, "not_existing_namespace", "not_existing_type")
require.NoError(t, err, "querying a not existing namespace should not throw an error")
require.Len(t, keys, 0, "querying a not existing namespace should return an empty slice")
})
}

View File

@ -0,0 +1,31 @@
package kvstore
import (
"time"
)
// Item stored in k/v store.
type Item struct {
Id int64
OrgId *int64
Namespace *string
Type *string
Value string
Created time.Time
Updated time.Time
}
func (i *Item) TableName() string {
return "secrets"
}
type Key struct {
OrgId int64
Namespace string
Type string
}
func (i *Key) TableName() string {
return "secrets"
}

View File

@ -0,0 +1,220 @@
package kvstore
import (
"context"
"encoding/base64"
"sync"
"time"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/services/secrets"
"github.com/grafana/grafana/pkg/services/sqlstore"
)
// secretsKVStoreSQL provides a key/value store backed by the Grafana database
type secretsKVStoreSQL struct {
log log.Logger
sqlStore sqlstore.Store
secretsService secrets.Service
decryptionCache decryptionCache
}
type decryptionCache struct {
cache map[int64]cachedDecrypted
sync.Mutex
}
type cachedDecrypted struct {
updated time.Time
value string
}
var b64 = base64.RawStdEncoding
// Get an item from the store
func (kv *secretsKVStoreSQL) Get(ctx context.Context, orgId int64, namespace string, typ string) (string, bool, error) {
item := Item{
OrgId: &orgId,
Namespace: &namespace,
Type: &typ,
}
var isFound bool
var decryptedValue []byte
err := kv.sqlStore.WithDbSession(ctx, func(dbSession *sqlstore.DBSession) error {
has, err := dbSession.Get(&item)
if err != nil {
kv.log.Debug("error getting secret value", "orgId", orgId, "type", typ, "namespace", namespace, "err", err)
return err
}
if !has {
kv.log.Debug("secret value not found", "orgId", orgId, "type", typ, "namespace", namespace)
return nil
}
isFound = true
kv.log.Debug("got secret value", "orgId", orgId, "type", typ, "namespace", namespace)
return nil
})
if err == nil && isFound {
kv.decryptionCache.Lock()
defer kv.decryptionCache.Unlock()
if cache, present := kv.decryptionCache.cache[item.Id]; present && item.Updated.Equal(cache.updated) {
return cache.value, isFound, err
}
decodedValue, err := b64.DecodeString(item.Value)
if err != nil {
kv.log.Debug("error decoding secret value", "orgId", orgId, "type", typ, "namespace", namespace, "err", err)
return string(decryptedValue), isFound, err
}
decryptedValue, err = kv.secretsService.Decrypt(ctx, decodedValue)
if err != nil {
kv.log.Debug("error decrypting secret value", "orgId", orgId, "type", typ, "namespace", namespace, "err", err)
return string(decryptedValue), isFound, err
}
kv.decryptionCache.cache[item.Id] = cachedDecrypted{
updated: item.Updated,
value: string(decryptedValue),
}
}
return string(decryptedValue), isFound, err
}
// Set an item in the store
func (kv *secretsKVStoreSQL) Set(ctx context.Context, orgId int64, namespace string, typ string, value string) error {
encryptedValue, err := kv.secretsService.Encrypt(ctx, []byte(value), secrets.WithoutScope())
if err != nil {
kv.log.Debug("error encrypting secret value", "orgId", orgId, "type", typ, "namespace", namespace, "err", err)
return err
}
encodedValue := b64.EncodeToString(encryptedValue)
return kv.sqlStore.WithTransactionalDbSession(ctx, func(dbSession *sqlstore.DBSession) error {
item := Item{
OrgId: &orgId,
Namespace: &namespace,
Type: &typ,
}
has, err := dbSession.Get(&item)
if err != nil {
kv.log.Debug("error checking secret value", "orgId", orgId, "type", typ, "namespace", namespace, "err", err)
return err
}
if has && item.Value == encodedValue {
kv.log.Debug("secret value not changed", "orgId", orgId, "type", typ, "namespace", namespace)
return nil
}
item.Value = encodedValue
item.Updated = time.Now()
if has {
// if item already exists we update it
_, err = dbSession.ID(item.Id).Update(&item)
if err != nil {
kv.log.Debug("error updating secret value", "orgId", orgId, "type", typ, "namespace", namespace, "err", err)
} else {
kv.decryptionCache.cache[item.Id] = cachedDecrypted{
updated: item.Updated,
value: value,
}
kv.log.Debug("secret value updated", "orgId", orgId, "type", typ, "namespace", namespace)
}
return err
}
// if item doesn't exist we create it
item.Created = item.Updated
_, err = dbSession.Insert(&item)
if err != nil {
kv.log.Debug("error inserting secret value", "orgId", orgId, "type", typ, "namespace", namespace, "err", err)
} else {
kv.log.Debug("secret value inserted", "orgId", orgId, "type", typ, "namespace", namespace)
}
return err
})
}
// Del deletes an item from the store.
func (kv *secretsKVStoreSQL) Del(ctx context.Context, orgId int64, namespace string, typ string) error {
err := kv.sqlStore.WithDbSession(ctx, func(dbSession *sqlstore.DBSession) error {
item := Item{
OrgId: &orgId,
Namespace: &namespace,
Type: &typ,
}
has, err := dbSession.Get(&item)
if err != nil {
kv.log.Debug("error checking secret value", "orgId", orgId, "type", typ, "namespace", namespace, "err", err)
return err
}
if has {
// if item exists we delete it
_, err = dbSession.ID(item.Id).Delete(&item)
if err != nil {
kv.log.Debug("error deleting secret value", "orgId", orgId, "type", typ, "namespace", namespace, "err", err)
} else {
delete(kv.decryptionCache.cache, item.Id)
kv.log.Debug("secret value deleted", "orgId", orgId, "type", typ, "namespace", namespace)
}
return err
}
return nil
})
return err
}
// Keys get all keys for a given namespace. To query for all
// organizations the constant 'kvstore.AllOrganizations' can be passed as orgId.
func (kv *secretsKVStoreSQL) Keys(ctx context.Context, orgId int64, namespace string, typ string) ([]Key, error) {
var keys []Key
err := kv.sqlStore.WithDbSession(ctx, func(dbSession *sqlstore.DBSession) error {
query := dbSession.Where("namespace = ?", namespace).And("type = ?", typ)
if orgId != AllOrganizations {
query.And("org_id = ?", orgId)
}
return query.Find(&keys)
})
return keys, err
}
// Rename an item in the store
func (kv *secretsKVStoreSQL) Rename(ctx context.Context, orgId int64, namespace string, typ string, newNamespace string) error {
return kv.sqlStore.WithTransactionalDbSession(ctx, func(dbSession *sqlstore.DBSession) error {
item := Item{
OrgId: &orgId,
Namespace: &namespace,
Type: &typ,
}
has, err := dbSession.Get(&item)
if err != nil {
kv.log.Debug("error checking secret value", "orgId", orgId, "type", typ, "namespace", namespace, "err", err)
return err
}
item.Namespace = &newNamespace
item.Updated = time.Now()
if has {
// if item already exists we update it
_, err = dbSession.ID(item.Id).Update(&item)
if err != nil {
kv.log.Debug("error updating secret namespace", "orgId", orgId, "type", typ, "namespace", namespace, "err", err)
} else {
kv.log.Debug("secret namespace updated", "orgId", orgId, "type", typ, "namespace", namespace)
}
return err
}
return err
})
}

View File

@ -18,4 +18,24 @@ func addSecretsMigration(mg *migrator.Migrator) {
} }
mg.AddMigration("create data_keys table", migrator.NewAddTableMigration(dataKeysV1)) mg.AddMigration("create data_keys table", migrator.NewAddTableMigration(dataKeysV1))
secretsV1 := migrator.Table{
Name: "secrets",
Columns: []*migrator.Column{
{Name: "id", Type: migrator.DB_BigInt, IsPrimaryKey: true, IsAutoIncrement: true},
{Name: "org_id", Type: migrator.DB_BigInt, Nullable: false},
{Name: "namespace", Type: migrator.DB_NVarchar, Length: 255, Nullable: false},
{Name: "type", Type: migrator.DB_NVarchar, Length: 255, Nullable: false},
{Name: "value", Type: migrator.DB_Text, Nullable: true},
{Name: "created", Type: migrator.DB_DateTime, Nullable: false},
{Name: "updated", Type: migrator.DB_DateTime, Nullable: false},
},
Indices: []*migrator.Index{
{Cols: []string{"org_id"}},
{Cols: []string{"org_id", "namespace"}},
{Cols: []string{"org_id", "namespace", "type"}, Type: migrator.UniqueIndex},
},
}
mg.AddMigration("create secrets table", migrator.NewAddTableMigration(secretsV1))
} }

View File

@ -40,6 +40,11 @@ func (h *Service) HandleRequest(ctx context.Context, ds *models.DataSource, quer
return legacydata.DataResponse{}, err return legacydata.DataResponse{}, err
} }
decryptedValues, err := h.dataSourcesService.DecryptedValues(ctx, ds)
if err != nil {
return legacydata.DataResponse{}, err
}
instanceSettings := &backend.DataSourceInstanceSettings{ instanceSettings := &backend.DataSourceInstanceSettings{
ID: ds.Id, ID: ds.Id,
Name: ds.Name, Name: ds.Name,
@ -49,7 +54,7 @@ func (h *Service) HandleRequest(ctx context.Context, ds *models.DataSource, quer
BasicAuthEnabled: ds.BasicAuth, BasicAuthEnabled: ds.BasicAuth,
BasicAuthUser: ds.BasicAuthUser, BasicAuthUser: ds.BasicAuthUser,
JSONData: jsonDataBytes, JSONData: jsonDataBytes,
DecryptedSecureJSONData: h.dataSourcesService.DecryptedValues(ds), DecryptedSecureJSONData: decryptedValues,
Updated: ds.Updated, Updated: ds.Updated,
UID: ds.Uid, UID: ds.Uid,
} }

View File

@ -13,6 +13,7 @@ import (
"github.com/grafana/grafana/pkg/services/featuremgmt" "github.com/grafana/grafana/pkg/services/featuremgmt"
"github.com/grafana/grafana/pkg/services/oauthtoken" "github.com/grafana/grafana/pkg/services/oauthtoken"
"github.com/grafana/grafana/pkg/services/secrets/fakes" "github.com/grafana/grafana/pkg/services/secrets/fakes"
"github.com/grafana/grafana/pkg/services/secrets/kvstore"
secretsManager "github.com/grafana/grafana/pkg/services/secrets/manager" secretsManager "github.com/grafana/grafana/pkg/services/secrets/manager"
"github.com/grafana/grafana/pkg/setting" "github.com/grafana/grafana/pkg/setting"
"github.com/grafana/grafana/pkg/tsdb/legacydata" "github.com/grafana/grafana/pkg/tsdb/legacydata"
@ -38,8 +39,9 @@ func TestHandleRequest(t *testing.T) {
actualReq = req actualReq = req
return backend.NewQueryDataResponse(), nil return backend.NewQueryDataResponse(), nil
} }
secretsStore := kvstore.SetupTestService(t)
secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore())
dsService := datasourceservice.ProvideService(nil, secretsService, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock()) dsService := datasourceservice.ProvideService(nil, secretsService, secretsStore, cfg, featuremgmt.WithFeatures(), acmock.New(), acmock.NewPermissionsServicesMock())
s := ProvideService(client, nil, dsService) s := ProvideService(client, nil, dsService)
ds := &models.DataSource{Id: 12, Type: "unregisteredType", JsonData: simplejson.New()} ds := &models.DataSource{Id: 12, Type: "unregisteredType", JsonData: simplejson.New()}