diff --git a/devenv/datasources.yaml b/devenv/datasources.yaml index a8c68ee1bb1..c742eab6072 100644 --- a/devenv/datasources.yaml +++ b/devenv/datasources.yaml @@ -247,7 +247,15 @@ datasources: access: proxy url: http://localhost:3100 editable: false + correlations: + - targetUid: gdev-jaeger + label: "Jaeger traces" + description: "Related traces stored in Jaeger" + - targetUid: gdev-zipkin + label: "Zipkin traces" + description: "Related traces stored in Zipkin" jsonData: + something: here manageAlerts: false derivedFields: - name: "traceID" diff --git a/docs/sources/developers/http_api/correlations.md b/docs/sources/developers/http_api/correlations.md new file mode 100644 index 00000000000..d3dde170f02 --- /dev/null +++ b/docs/sources/developers/http_api/correlations.md @@ -0,0 +1,70 @@ +--- +aliases: + - /docs/grafana/latest/developers/http_api/correlations/ + - /docs/grafana/latest/http_api/correlations/ +description: Grafana Correlations HTTP API +keywords: + - grafana + - http + - documentation + - api + - correlations + - Glue +title: 'Correlations HTTP API ' +--- + +# Correlations API + +This API can be used to define correlations between data sources. + +## Create correlations + +`POST /api/datasources/uid/:sourceUid/correlations` + +Creates a correlation between two data sources - the source data source indicated by the path UID, and the target data source which is specified in the body. + +**Example request:** + +```http +POST /api/datasources/uid/uyBf2637k/correlations HTTP/1.1 +Accept: application/json +Content-Type: application/json +Authorization: Bearer eyJrIjoiT0tTcG1pUlY2RnVKZTFVaDFsNFZXdE9ZWmNrMkZYbk +{ + "targetUid": "PDDA8E780A17E7EF1", + "label": "My Label", + "description": "Logs to Traces", +} +``` + +JSON body schema: + +- **targetUid** – Target data source uid. +- **label** – A label for the correlation. +- **description** – A description for the correlation. + +**Example response:** + +```http +HTTP/1.1 200 +Content-Type: application/json +{ + "message": "Correlation created", + "result": { + "description": "Logs to Traces", + "label": "My Label", + "sourceUid": "uyBf2637k", + "targetUid": "PDDA8E780A17E7EF1", + "uid": "50xhMlg9k" + } +} +``` + +Status codes: + +- **200** – OK +- **400** - Errors (invalid JSON, missing or invalid fields) +- **401** – Unauthorized +- **403** – Forbidden, source data source is read-only +- **404** – Not found, either source or target data source could not be found +- **500** – Internal error diff --git a/pkg/api/docs/definitions/correlations.go b/pkg/api/docs/definitions/correlations.go new file mode 100644 index 00000000000..e1c5a6fce9c --- /dev/null +++ b/pkg/api/docs/definitions/correlations.go @@ -0,0 +1,33 @@ +package definitions + +import ( + "github.com/grafana/grafana/pkg/services/correlations" +) + +// swagger:route POST /datasources/uid/{uid}/correlations correlations createCorrelation +// +// Add correlation. +// +// Responses: +// 200: createCorrelationResponse +// 400: badRequestError +// 401: unauthorisedError +// 403: forbiddenError +// 404: notFoundError +// 500: internalServerError + +// swagger:parameters createCorrelation +type CreateCorrelationParams struct { + // in:body + // required:true + Body correlations.CreateCorrelationCommand `json:"body"` + // in:path + // required:true + SourceUID string `json:"uid"` +} + +//swagger:response createCorrelationResponse +type CreateCorrelationResponse struct { + // in: body + Body correlations.CreateCorrelationResponse `json:"body"` +} diff --git a/pkg/api/http_server.go b/pkg/api/http_server.go index c0d466fe2ab..26450733d22 100644 --- a/pkg/api/http_server.go +++ b/pkg/api/http_server.go @@ -65,6 +65,7 @@ import ( "github.com/grafana/grafana/pkg/services/provisioning" "github.com/grafana/grafana/pkg/services/quota" + "github.com/grafana/grafana/pkg/services/correlations" publicdashboardsApi "github.com/grafana/grafana/pkg/services/publicdashboards/api" "github.com/grafana/grafana/pkg/services/query" "github.com/grafana/grafana/pkg/services/queryhistory" @@ -122,6 +123,7 @@ type HTTPServer struct { SearchService search.Service ShortURLService shorturls.Service QueryHistoryService queryhistory.Service + CorrelationsService correlations.Service Live *live.GrafanaLive LivePushGateway *pushhttp.Gateway ThumbService thumbs.Service @@ -188,7 +190,7 @@ func ProvideHTTPServer(opts ServerOptions, cfg *setting.Cfg, routeRegister routi pluginDashboardService plugindashboards.Service, pluginStore plugins.Store, pluginClient plugins.Client, pluginErrorResolver plugins.ErrorResolver, pluginManager plugins.Manager, settingsProvider setting.Provider, dataSourceCache datasources.CacheService, userTokenService models.UserTokenService, - cleanUpService *cleanup.CleanUpService, shortURLService shorturls.Service, queryHistoryService queryhistory.Service, + cleanUpService *cleanup.CleanUpService, shortURLService shorturls.Service, queryHistoryService queryhistory.Service, correlationsService correlations.Service, thumbService thumbs.Service, remoteCache *remotecache.RemoteCache, provisioningService provisioning.ProvisioningService, loginService login.Service, authenticator loginpkg.Authenticator, accessControl accesscontrol.AccessControl, dataSourceProxy *datasourceproxy.DataSourceProxyService, searchService *search.SearchService, @@ -239,6 +241,7 @@ func ProvideHTTPServer(opts ServerOptions, cfg *setting.Cfg, routeRegister routi cleanUpService: cleanUpService, ShortURLService: shortURLService, QueryHistoryService: queryHistoryService, + CorrelationsService: correlationsService, Features: features, ThumbService: thumbService, StorageService: storageService, diff --git a/pkg/server/wire.go b/pkg/server/wire.go index 119ee1f2613..11eb0b5d7a1 100644 --- a/pkg/server/wire.go +++ b/pkg/server/wire.go @@ -45,6 +45,7 @@ import ( "github.com/grafana/grafana/pkg/services/comments" "github.com/grafana/grafana/pkg/services/contexthandler" "github.com/grafana/grafana/pkg/services/contexthandler/authproxy" + "github.com/grafana/grafana/pkg/services/correlations" "github.com/grafana/grafana/pkg/services/dashboardimport" dashboardimportservice "github.com/grafana/grafana/pkg/services/dashboardimport/service" "github.com/grafana/grafana/pkg/services/dashboards" @@ -183,6 +184,8 @@ var wireBasicSet = wire.NewSet( wire.Bind(new(shorturls.Service), new(*shorturls.ShortURLService)), queryhistory.ProvideService, wire.Bind(new(queryhistory.Service), new(*queryhistory.QueryHistoryService)), + correlations.ProvideService, + wire.Bind(new(correlations.Service), new(*correlations.CorrelationsService)), quotaimpl.ProvideService, remotecache.ProvideService, loginservice.ProvideService, diff --git a/pkg/services/correlations/api.go b/pkg/services/correlations/api.go new file mode 100644 index 00000000000..b95b1c79f50 --- /dev/null +++ b/pkg/services/correlations/api.go @@ -0,0 +1,49 @@ +package correlations + +import ( + "errors" + "net/http" + + "github.com/grafana/grafana/pkg/api/response" + "github.com/grafana/grafana/pkg/api/routing" + "github.com/grafana/grafana/pkg/middleware" + "github.com/grafana/grafana/pkg/models" + ac "github.com/grafana/grafana/pkg/services/accesscontrol" + "github.com/grafana/grafana/pkg/services/datasources" + + "github.com/grafana/grafana/pkg/web" +) + +func (s *CorrelationsService) registerAPIEndpoints() { + uidScope := datasources.ScopeProvider.GetResourceScopeUID(ac.Parameter(":uid")) + authorize := ac.Middleware(s.AccessControl) + + s.RouteRegister.Group("/api/datasources/uid/:uid/correlations", func(entities routing.RouteRegister) { + entities.Post("/", middleware.ReqSignedIn, authorize(ac.ReqOrgAdmin, ac.EvalPermission(datasources.ActionWrite, uidScope)), routing.Wrap(s.createHandler)) + }) +} + +// createHandler handles POST /datasources/uid/:uid/correlations +func (s *CorrelationsService) createHandler(c *models.ReqContext) response.Response { + cmd := CreateCorrelationCommand{} + if err := web.Bind(c.Req, &cmd); err != nil { + return response.Error(http.StatusBadRequest, "bad request data", err) + } + cmd.SourceUID = web.Params(c.Req)[":uid"] + cmd.OrgId = c.OrgId + + correlation, err := s.CreateCorrelation(c.Req.Context(), cmd) + if err != nil { + if errors.Is(err, ErrSourceDataSourceDoesNotExists) || errors.Is(err, ErrTargetDataSourceDoesNotExists) { + return response.Error(http.StatusNotFound, "Data source not found", err) + } + + if errors.Is(err, ErrSourceDataSourceReadOnly) { + return response.Error(http.StatusForbidden, "Data source is read only", err) + } + + return response.Error(http.StatusInternalServerError, "Failed to add correlation", err) + } + + return response.JSON(http.StatusOK, CreateCorrelationResponse{Result: correlation, Message: "Correlation created"}) +} diff --git a/pkg/services/correlations/correlations.go b/pkg/services/correlations/correlations.go new file mode 100644 index 00000000000..45b4c4036fa --- /dev/null +++ b/pkg/services/correlations/correlations.go @@ -0,0 +1,74 @@ +package correlations + +import ( + "context" + + "github.com/grafana/grafana/pkg/api/routing" + "github.com/grafana/grafana/pkg/bus" + "github.com/grafana/grafana/pkg/events" + + "github.com/grafana/grafana/pkg/infra/log" + "github.com/grafana/grafana/pkg/services/accesscontrol" + "github.com/grafana/grafana/pkg/services/datasources" + "github.com/grafana/grafana/pkg/services/sqlstore" +) + +func ProvideService(sqlStore *sqlstore.SQLStore, routeRegister routing.RouteRegister, ds datasources.DataSourceService, ac accesscontrol.AccessControl, bus bus.Bus) *CorrelationsService { + s := &CorrelationsService{ + SQLStore: sqlStore, + RouteRegister: routeRegister, + log: log.New("correlations"), + DataSourceService: ds, + AccessControl: ac, + } + + s.registerAPIEndpoints() + + bus.AddEventListener(s.handleDatasourceDeletion) + + return s +} + +type Service interface { + CreateCorrelation(ctx context.Context, cmd CreateCorrelationCommand) (Correlation, error) + DeleteCorrelationsBySourceUID(ctx context.Context, cmd DeleteCorrelationsBySourceUIDCommand) error + DeleteCorrelationsByTargetUID(ctx context.Context, cmd DeleteCorrelationsByTargetUIDCommand) error +} + +type CorrelationsService struct { + SQLStore *sqlstore.SQLStore + RouteRegister routing.RouteRegister + log log.Logger + DataSourceService datasources.DataSourceService + AccessControl accesscontrol.AccessControl +} + +func (s CorrelationsService) CreateCorrelation(ctx context.Context, cmd CreateCorrelationCommand) (Correlation, error) { + return s.createCorrelation(ctx, cmd) +} + +func (s CorrelationsService) DeleteCorrelationsBySourceUID(ctx context.Context, cmd DeleteCorrelationsBySourceUIDCommand) error { + return s.deleteCorrelationsBySourceUID(ctx, cmd) +} + +func (s CorrelationsService) DeleteCorrelationsByTargetUID(ctx context.Context, cmd DeleteCorrelationsByTargetUIDCommand) error { + return s.deleteCorrelationsByTargetUID(ctx, cmd) +} + +func (s CorrelationsService) handleDatasourceDeletion(ctx context.Context, event *events.DataSourceDeleted) error { + return s.SQLStore.InTransaction(ctx, func(ctx context.Context) error { + if err := s.deleteCorrelationsBySourceUID(ctx, DeleteCorrelationsBySourceUIDCommand{ + SourceUID: event.UID, + }); err != nil { + return err + } + + if err := s.deleteCorrelationsByTargetUID(ctx, DeleteCorrelationsByTargetUIDCommand{ + TargetUID: event.UID, + }); err != nil { + return err + } + + return nil + }) +} diff --git a/pkg/services/correlations/database.go b/pkg/services/correlations/database.go new file mode 100644 index 00000000000..b4b832b6179 --- /dev/null +++ b/pkg/services/correlations/database.go @@ -0,0 +1,70 @@ +package correlations + +import ( + "context" + + "github.com/grafana/grafana/pkg/services/datasources" + "github.com/grafana/grafana/pkg/services/sqlstore" + "github.com/grafana/grafana/pkg/util" +) + +// createCorrelation adds a correlation +func (s CorrelationsService) createCorrelation(ctx context.Context, cmd CreateCorrelationCommand) (Correlation, error) { + correlation := Correlation{ + UID: util.GenerateShortUID(), + SourceUID: cmd.SourceUID, + TargetUID: cmd.TargetUID, + Label: cmd.Label, + Description: cmd.Description, + } + + err := s.SQLStore.WithTransactionalDbSession(ctx, func(session *sqlstore.DBSession) error { + var err error + + query := &datasources.GetDataSourceQuery{ + OrgId: cmd.OrgId, + Uid: cmd.SourceUID, + } + if err = s.DataSourceService.GetDataSource(ctx, query); err != nil { + return ErrSourceDataSourceDoesNotExists + } + + if !cmd.SkipReadOnlyCheck && query.Result.ReadOnly { + return ErrSourceDataSourceReadOnly + } + + if err = s.DataSourceService.GetDataSource(ctx, &datasources.GetDataSourceQuery{ + OrgId: cmd.OrgId, + Uid: cmd.TargetUID, + }); err != nil { + return ErrTargetDataSourceDoesNotExists + } + + _, err = session.Insert(correlation) + if err != nil { + return err + } + + return nil + }) + + if err != nil { + return Correlation{}, err + } + + return correlation, nil +} + +func (s CorrelationsService) deleteCorrelationsBySourceUID(ctx context.Context, cmd DeleteCorrelationsBySourceUIDCommand) error { + return s.SQLStore.WithDbSession(ctx, func(session *sqlstore.DBSession) error { + _, err := session.Delete(&Correlation{SourceUID: cmd.SourceUID}) + return err + }) +} + +func (s CorrelationsService) deleteCorrelationsByTargetUID(ctx context.Context, cmd DeleteCorrelationsByTargetUIDCommand) error { + return s.SQLStore.WithDbSession(ctx, func(session *sqlstore.DBSession) error { + _, err := session.Delete(&Correlation{TargetUID: cmd.TargetUID}) + return err + }) +} diff --git a/pkg/services/correlations/models.go b/pkg/services/correlations/models.go new file mode 100644 index 00000000000..8a006c08699 --- /dev/null +++ b/pkg/services/correlations/models.go @@ -0,0 +1,66 @@ +package correlations + +import ( + "errors" +) + +var ( + ErrSourceDataSourceReadOnly = errors.New("source data source is read only") + ErrSourceDataSourceDoesNotExists = errors.New("source data source does not exist") + ErrTargetDataSourceDoesNotExists = errors.New("target data source does not exist") + ErrCorrelationFailedGenerateUniqueUid = errors.New("failed to generate unique correlation UID") + ErrCorrelationIdentifierNotSet = errors.New("source identifier and org id are needed to be able to edit correlations") +) + +// Correlation is the model for correlations definitions +type Correlation struct { + // Unique identifier of the correlation + // example: 50xhMlg9k + UID string `json:"uid" xorm:"pk 'uid'"` + // UID of the data source the correlation originates from + // example:d0oxYRg4z + SourceUID string `json:"sourceUid" xorm:"pk 'source_uid'"` + // UID of the data source the correlation points to + // example:PE1C5CBDA0504A6A3 + TargetUID string `json:"targetUid" xorm:"target_uid"` + // Label identifying the correlation + // example: My Label + Label string `json:"label" xorm:"label"` + // Description of the correlation + // example: Logs to Traces + Description string `json:"description" xorm:"description"` +} + +// CreateCorrelationResponse is the response struct for CreateCorrelationCommand +// swagger:model +type CreateCorrelationResponse struct { + Result Correlation `json:"result"` + // example: Correlation created + Message string `json:"message"` +} + +// CreateCorrelationCommand is the command for creating a correlation +// swagger:model +type CreateCorrelationCommand struct { + // UID of the data source for which correlation is created. + SourceUID string `json:"-"` + OrgId int64 `json:"-"` + SkipReadOnlyCheck bool `json:"-"` + // Target data source UID to which the correlation is created + // example:PE1C5CBDA0504A6A3 + TargetUID string `json:"targetUid" binding:"Required"` + // Optional label identifying the correlation + // example: My label + Label string `json:"label"` + // Optional description of the correlation + // example: Logs to Traces + Description string `json:"description"` +} + +type DeleteCorrelationsBySourceUIDCommand struct { + SourceUID string +} + +type DeleteCorrelationsByTargetUIDCommand struct { + TargetUID string +} diff --git a/pkg/services/datasources/errors.go b/pkg/services/datasources/errors.go index 14bd0db8580..b220f93982d 100644 --- a/pkg/services/datasources/errors.go +++ b/pkg/services/datasources/errors.go @@ -10,4 +10,5 @@ var ( ErrDataSourceAccessDenied = errors.New("data source access denied") ErrDataSourceFailedGenerateUniqueUid = errors.New("failed to generate unique datasource ID") ErrDataSourceIdentifierNotSet = errors.New("unique identifier and org id are needed to be able to get or delete a datasource") + ErrDatasourceIsReadOnly = errors.New("data source is readonly, can only be updated from configuration") ) diff --git a/pkg/services/provisioning/datasources/config_reader_test.go b/pkg/services/provisioning/datasources/config_reader_test.go index 0cea8f65f33..85e7355782d 100644 --- a/pkg/services/provisioning/datasources/config_reader_test.go +++ b/pkg/services/provisioning/datasources/config_reader_test.go @@ -9,6 +9,7 @@ import ( "github.com/grafana/grafana/pkg/infra/log" "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/services/correlations" "github.com/grafana/grafana/pkg/services/datasources" "github.com/grafana/grafana/pkg/util" ) @@ -26,13 +27,16 @@ var ( multipleOrgsWithDefault = "testdata/multiple-org-default" withoutDefaults = "testdata/appliedDefaults" invalidAccess = "testdata/invalid-access" + + oneDatasourceWithTwoCorrelations = "testdata/one-datasource-two-correlations" ) func TestDatasourceAsConfig(t *testing.T) { t.Run("when some values missing should apply default on insert", func(t *testing.T) { store := &spyStore{} orgStore := &mockOrgStore{ExpectedOrg: &models.Org{Id: 1}} - dc := newDatasourceProvisioner(logger, store, orgStore) + correlationsStore := &mockCorrelationsStore{} + dc := newDatasourceProvisioner(logger, store, correlationsStore, orgStore) err := dc.applyChanges(context.Background(), withoutDefaults) if err != nil { t.Fatalf("applyChanges return an error %v", err) @@ -50,7 +54,8 @@ func TestDatasourceAsConfig(t *testing.T) { items: []*datasources.DataSource{{Name: "My datasource name", OrgId: 1, Id: 1, Uid: util.GenerateShortUID()}}, } orgStore := &mockOrgStore{} - dc := newDatasourceProvisioner(logger, store, orgStore) + correlationsStore := &mockCorrelationsStore{} + dc := newDatasourceProvisioner(logger, store, correlationsStore, orgStore) err := dc.applyChanges(context.Background(), withoutDefaults) if err != nil { t.Fatalf("applyChanges return an error %v", err) @@ -65,7 +70,8 @@ func TestDatasourceAsConfig(t *testing.T) { t.Run("no datasource in database", func(t *testing.T) { store := &spyStore{} orgStore := &mockOrgStore{} - dc := newDatasourceProvisioner(logger, store, orgStore) + correlationsStore := &mockCorrelationsStore{} + dc := newDatasourceProvisioner(logger, store, correlationsStore, orgStore) err := dc.applyChanges(context.Background(), twoDatasourcesConfig) if err != nil { t.Fatalf("applyChanges return an error %v", err) @@ -79,7 +85,8 @@ func TestDatasourceAsConfig(t *testing.T) { t.Run("One datasource in database with same name should update one datasource", func(t *testing.T) { store := &spyStore{items: []*datasources.DataSource{{Name: "Graphite", OrgId: 1, Id: 1}}} orgStore := &mockOrgStore{} - dc := newDatasourceProvisioner(logger, store, orgStore) + correlationsStore := &mockCorrelationsStore{} + dc := newDatasourceProvisioner(logger, store, correlationsStore, orgStore) err := dc.applyChanges(context.Background(), twoDatasourcesConfig) if err != nil { t.Fatalf("applyChanges return an error %v", err) @@ -93,7 +100,8 @@ func TestDatasourceAsConfig(t *testing.T) { t.Run("Two datasources with is_default should raise error", func(t *testing.T) { store := &spyStore{} orgStore := &mockOrgStore{} - dc := newDatasourceProvisioner(logger, store, orgStore) + correlationsStore := &mockCorrelationsStore{} + dc := newDatasourceProvisioner(logger, store, correlationsStore, orgStore) err := dc.applyChanges(context.Background(), doubleDatasourcesConfig) require.Equal(t, err, ErrInvalidConfigToManyDefault) }) @@ -101,7 +109,8 @@ func TestDatasourceAsConfig(t *testing.T) { t.Run("Multiple datasources in different organizations with isDefault in each organization should not raise error", func(t *testing.T) { store := &spyStore{} orgStore := &mockOrgStore{} - dc := newDatasourceProvisioner(logger, store, orgStore) + correlationsStore := &mockCorrelationsStore{} + dc := newDatasourceProvisioner(logger, store, correlationsStore, orgStore) err := dc.applyChanges(context.Background(), multipleOrgsWithDefault) require.NoError(t, err) require.Equal(t, len(store.inserted), 4) @@ -114,7 +123,8 @@ func TestDatasourceAsConfig(t *testing.T) { t.Run("Remove one datasource should have removed old datasource", func(t *testing.T) { store := &spyStore{} orgStore := &mockOrgStore{} - dc := newDatasourceProvisioner(logger, store, orgStore) + correlationsStore := &mockCorrelationsStore{} + dc := newDatasourceProvisioner(logger, store, correlationsStore, orgStore) err := dc.applyChanges(context.Background(), deleteOneDatasource) if err != nil { t.Fatalf("applyChanges return an error %v", err) @@ -130,7 +140,8 @@ func TestDatasourceAsConfig(t *testing.T) { t.Run("Two configured datasource and purge others", func(t *testing.T) { store := &spyStore{items: []*datasources.DataSource{{Name: "old-graphite", OrgId: 1, Id: 1}, {Name: "old-graphite2", OrgId: 1, Id: 2}}} orgStore := &mockOrgStore{} - dc := newDatasourceProvisioner(logger, store, orgStore) + correlationsStore := &mockCorrelationsStore{} + dc := newDatasourceProvisioner(logger, store, correlationsStore, orgStore) err := dc.applyChanges(context.Background(), twoDatasourcesConfigPurgeOthers) if err != nil { t.Fatalf("applyChanges return an error %v", err) @@ -144,7 +155,8 @@ func TestDatasourceAsConfig(t *testing.T) { t.Run("Two configured datasource and purge others = false", func(t *testing.T) { store := &spyStore{items: []*datasources.DataSource{{Name: "Graphite", OrgId: 1, Id: 1}, {Name: "old-graphite2", OrgId: 1, Id: 2}}} orgStore := &mockOrgStore{} - dc := newDatasourceProvisioner(logger, store, orgStore) + correlationsStore := &mockCorrelationsStore{} + dc := newDatasourceProvisioner(logger, store, correlationsStore, orgStore) err := dc.applyChanges(context.Background(), twoDatasourcesConfig) if err != nil { t.Fatalf("applyChanges return an error %v", err) @@ -224,6 +236,53 @@ func TestDatasourceAsConfig(t *testing.T) { validateDatasource(t, dsCfg) validateDeleteDatasources(t, dsCfg) }) + + t.Run("Correlations", func(t *testing.T) { + t.Run("Creates two correlations", func(t *testing.T) { + store := &spyStore{} + orgStore := &mockOrgStore{} + correlationsStore := &mockCorrelationsStore{} + dc := newDatasourceProvisioner(logger, store, correlationsStore, orgStore) + err := dc.applyChanges(context.Background(), oneDatasourceWithTwoCorrelations) + if err != nil { + t.Fatalf("applyChanges return an error %v", err) + } + + require.Equal(t, 2, len(correlationsStore.created)) + require.Equal(t, 0, len(correlationsStore.deletedBySourceUID)) + require.Equal(t, 0, len(correlationsStore.deletedByTargetUID)) + }) + + t.Run("Updating existing datasource deletes existing correlations and creates two", func(t *testing.T) { + store := &spyStore{items: []*datasources.DataSource{{Name: "Graphite", OrgId: 1, Id: 1}}} + orgStore := &mockOrgStore{} + correlationsStore := &mockCorrelationsStore{} + dc := newDatasourceProvisioner(logger, store, correlationsStore, orgStore) + err := dc.applyChanges(context.Background(), oneDatasourceWithTwoCorrelations) + if err != nil { + t.Fatalf("applyChanges return an error %v", err) + } + + require.Equal(t, 2, len(correlationsStore.created)) + require.Equal(t, 1, len(correlationsStore.deletedBySourceUID)) + require.Equal(t, 0, len(correlationsStore.deletedByTargetUID)) + }) + + t.Run("Deleting datasource deletes existing correlations", func(t *testing.T) { + store := &spyStore{items: []*datasources.DataSource{{Name: "old-data-source", OrgId: 1, Id: 1, Uid: "some-uid"}}} + orgStore := &mockOrgStore{} + correlationsStore := &mockCorrelationsStore{items: []correlations.Correlation{{UID: "some-uid", SourceUID: "some-uid", TargetUID: "target-uid"}}} + dc := newDatasourceProvisioner(logger, store, correlationsStore, orgStore) + err := dc.applyChanges(context.Background(), deleteOneDatasource) + if err != nil { + t.Fatalf("applyChanges return an error %v", err) + } + + require.Equal(t, 0, len(correlationsStore.created)) + require.Equal(t, 1, len(correlationsStore.deletedBySourceUID)) + require.Equal(t, 1, len(correlationsStore.deletedByTargetUID)) + }) + }) } func validateDeleteDatasources(t *testing.T, dsCfg *configs) { @@ -249,6 +308,12 @@ func validateDatasource(t *testing.T, dsCfg *configs) { require.True(t, ds.Editable) require.Equal(t, ds.Version, 10) + require.Equal(t, []map[string]interface{}{{ + "targetUid": "a target", + "label": "a label", + "description": "a description", + }}, ds.Correlations) + require.Greater(t, len(ds.JSONData), 2) require.Equal(t, ds.JSONData["graphiteVersion"], "1.1") require.Equal(t, ds.JSONData["tlsAuth"], true) @@ -273,6 +338,28 @@ func (m *mockOrgStore) GetOrgById(c context.Context, cmd *models.GetOrgByIdQuery return nil } +type mockCorrelationsStore struct { + created []correlations.CreateCorrelationCommand + deletedBySourceUID []correlations.DeleteCorrelationsBySourceUIDCommand + deletedByTargetUID []correlations.DeleteCorrelationsByTargetUIDCommand + items []correlations.Correlation +} + +func (m *mockCorrelationsStore) CreateCorrelation(c context.Context, cmd correlations.CreateCorrelationCommand) (correlations.Correlation, error) { + m.created = append(m.created, cmd) + return correlations.Correlation{}, nil +} + +func (m *mockCorrelationsStore) DeleteCorrelationsBySourceUID(c context.Context, cmd correlations.DeleteCorrelationsBySourceUIDCommand) error { + m.deletedBySourceUID = append(m.deletedBySourceUID, cmd) + return nil +} + +func (m *mockCorrelationsStore) DeleteCorrelationsByTargetUID(c context.Context, cmd correlations.DeleteCorrelationsByTargetUIDCommand) error { + m.deletedByTargetUID = append(m.deletedByTargetUID, cmd) + return nil +} + type spyStore struct { inserted []*datasources.AddDataSourceCommand deleted []*datasources.DeleteDataSourceCommand @@ -292,11 +379,20 @@ func (s *spyStore) GetDataSource(ctx context.Context, query *datasources.GetData func (s *spyStore) DeleteDataSource(ctx context.Context, cmd *datasources.DeleteDataSourceCommand) error { s.deleted = append(s.deleted, cmd) + for _, v := range s.items { + if cmd.Name == v.Name && cmd.OrgID == v.OrgId { + cmd.DeletedDatasourcesCount = 1 + return nil + } + } return nil } func (s *spyStore) AddDataSource(ctx context.Context, cmd *datasources.AddDataSourceCommand) error { s.inserted = append(s.inserted, cmd) + cmd.Result = &datasources.DataSource{ + Uid: cmd.Uid, + } return nil } diff --git a/pkg/services/provisioning/datasources/datasources.go b/pkg/services/provisioning/datasources/datasources.go index a1d87dd8c60..93f64cc973a 100644 --- a/pkg/services/provisioning/datasources/datasources.go +++ b/pkg/services/provisioning/datasources/datasources.go @@ -3,8 +3,10 @@ package datasources import ( "context" "errors" + "fmt" "github.com/grafana/grafana/pkg/infra/log" + "github.com/grafana/grafana/pkg/services/correlations" "github.com/grafana/grafana/pkg/services/datasources" "github.com/grafana/grafana/pkg/services/provisioning/utils" ) @@ -16,6 +18,12 @@ type Store interface { DeleteDataSource(ctx context.Context, cmd *datasources.DeleteDataSourceCommand) error } +type CorrelationsStore interface { + DeleteCorrelationsByTargetUID(ctx context.Context, cmd correlations.DeleteCorrelationsByTargetUIDCommand) error + DeleteCorrelationsBySourceUID(ctx context.Context, cmd correlations.DeleteCorrelationsBySourceUIDCommand) error + CreateCorrelation(ctx context.Context, cmd correlations.CreateCorrelationCommand) (correlations.Correlation, error) +} + var ( // ErrInvalidConfigToManyDefault indicates that multiple datasource in the provisioning files // contains more than one datasource marked as default. @@ -24,24 +32,26 @@ var ( // Provision scans a directory for provisioning config files // and provisions the datasource in those files. -func Provision(ctx context.Context, configDirectory string, store Store, orgStore utils.OrgStore) error { - dc := newDatasourceProvisioner(log.New("provisioning.datasources"), store, orgStore) +func Provision(ctx context.Context, configDirectory string, store Store, correlationsStore CorrelationsStore, orgStore utils.OrgStore) error { + dc := newDatasourceProvisioner(log.New("provisioning.datasources"), store, correlationsStore, orgStore) return dc.applyChanges(ctx, configDirectory) } // DatasourceProvisioner is responsible for provisioning datasources based on // configuration read by the `configReader` type DatasourceProvisioner struct { - log log.Logger - cfgProvider *configReader - store Store + log log.Logger + cfgProvider *configReader + store Store + correlationsStore CorrelationsStore } -func newDatasourceProvisioner(log log.Logger, store Store, orgStore utils.OrgStore) DatasourceProvisioner { +func newDatasourceProvisioner(log log.Logger, store Store, correlationsStore CorrelationsStore, orgStore utils.OrgStore) DatasourceProvisioner { return DatasourceProvisioner{ - log: log, - cfgProvider: &configReader{log: log, orgStore: orgStore}, - store: store, + log: log, + cfgProvider: &configReader{log: log, orgStore: orgStore}, + store: store, + correlationsStore: correlationsStore, } } @@ -50,6 +60,8 @@ func (dc *DatasourceProvisioner) apply(ctx context.Context, cfg *configs) error return err } + correlationsToInsert := make([]correlations.CreateCorrelationCommand, 0) + for _, ds := range cfg.Datasources { cmd := &datasources.GetDataSourceQuery{OrgId: ds.OrgID, Name: ds.Name} err := dc.store.GetDataSource(ctx, cmd) @@ -63,12 +75,44 @@ func (dc *DatasourceProvisioner) apply(ctx context.Context, cfg *configs) error if err := dc.store.AddDataSource(ctx, insertCmd); err != nil { return err } + + for _, correlation := range ds.Correlations { + if insertCorrelationCmd, err := makeCreateCorrelationCommand(correlation, insertCmd.Result.Uid, insertCmd.OrgId); err == nil { + correlationsToInsert = append(correlationsToInsert, insertCorrelationCmd) + } else { + dc.log.Error("failed to parse correlation", "correlation", correlation) + return err + } + } } else { updateCmd := createUpdateCommand(ds, cmd.Result.Id) dc.log.Debug("updating datasource from configuration", "name", updateCmd.Name, "uid", updateCmd.Uid) if err := dc.store.UpdateDataSource(ctx, updateCmd); err != nil { return err } + + if len(ds.Correlations) > 0 { + if err := dc.correlationsStore.DeleteCorrelationsBySourceUID(ctx, correlations.DeleteCorrelationsBySourceUIDCommand{ + SourceUID: cmd.Result.Uid, + }); err != nil { + return err + } + } + + for _, correlation := range ds.Correlations { + if insertCorrelationCmd, err := makeCreateCorrelationCommand(correlation, cmd.Result.Uid, updateCmd.OrgId); err == nil { + correlationsToInsert = append(correlationsToInsert, insertCorrelationCmd) + } else { + dc.log.Error("failed to parse correlation", "correlation", correlation) + return err + } + } + } + } + + for _, createCorrelationCmd := range correlationsToInsert { + if _, err := dc.correlationsStore.CreateCorrelation(ctx, createCorrelationCmd); err != nil { + return fmt.Errorf("err=%s source=%s", err.Error(), createCorrelationCmd.SourceUID) } } @@ -90,13 +134,50 @@ func (dc *DatasourceProvisioner) applyChanges(ctx context.Context, configPath st return nil } +func makeCreateCorrelationCommand(correlation map[string]interface{}, SourceUid string, OrgId int64) (correlations.CreateCorrelationCommand, error) { + targetUid, ok := correlation["targetUid"].(string) + if !ok { + return correlations.CreateCorrelationCommand{}, fmt.Errorf("correlation missing targetUid") + } + + return correlations.CreateCorrelationCommand{ + SourceUID: SourceUid, + TargetUID: targetUid, + Label: correlation["label"].(string), + Description: correlation["description"].(string), + OrgId: OrgId, + SkipReadOnlyCheck: true, + }, nil +} + func (dc *DatasourceProvisioner) deleteDatasources(ctx context.Context, dsToDelete []*deleteDatasourceConfig) error { for _, ds := range dsToDelete { cmd := &datasources.DeleteDataSourceCommand{OrgID: ds.OrgID, Name: ds.Name} + getDsQuery := &datasources.GetDataSourceQuery{Name: ds.Name, OrgId: ds.OrgID} + if err := dc.store.GetDataSource(ctx, getDsQuery); err != nil && !errors.Is(err, datasources.ErrDataSourceNotFound) { + return err + } + if err := dc.store.DeleteDataSource(ctx, cmd); err != nil { return err } + if getDsQuery.Result != nil { + if err := dc.correlationsStore.DeleteCorrelationsBySourceUID(ctx, correlations.DeleteCorrelationsBySourceUIDCommand{ + SourceUID: getDsQuery.Result.Uid, + }); err != nil { + return err + } + + if err := dc.correlationsStore.DeleteCorrelationsByTargetUID(ctx, correlations.DeleteCorrelationsByTargetUIDCommand{ + TargetUID: getDsQuery.Result.Uid, + }); err != nil { + return err + } + + dc.log.Info("deleted correlations based on configuration", "ds_name", ds.Name) + } + if cmd.DeletedDatasourcesCount > 0 { dc.log.Info("deleted datasource based on configuration", "name", ds.Name) } diff --git a/pkg/services/provisioning/datasources/testdata/all-properties/all-properties.yaml b/pkg/services/provisioning/datasources/testdata/all-properties/all-properties.yaml index f5f9f6dbc3a..e0b59def681 100644 --- a/pkg/services/provisioning/datasources/testdata/all-properties/all-properties.yaml +++ b/pkg/services/provisioning/datasources/testdata/all-properties/all-properties.yaml @@ -12,6 +12,10 @@ datasources: basicAuthUser: basic_auth_user withCredentials: true isDefault: true + correlations: + - targetUid: a target + label: a label + description: a description jsonData: graphiteVersion: "1.1" tlsAuth: true diff --git a/pkg/services/provisioning/datasources/testdata/one-datasource-two-correlations/one-datasource-two-correlations.yaml b/pkg/services/provisioning/datasources/testdata/one-datasource-two-correlations/one-datasource-two-correlations.yaml new file mode 100644 index 00000000000..d6f257a46d3 --- /dev/null +++ b/pkg/services/provisioning/datasources/testdata/one-datasource-two-correlations/one-datasource-two-correlations.yaml @@ -0,0 +1,15 @@ +apiVersion: 1 + +datasources: + - name: Graphite + type: graphite + uid: graphite + access: proxy + url: http://localhost:8080 + correlations: + - targetUid: graphite + label: a label + description: a description + - targetUid: graphite + label: a second label + description: a second description \ No newline at end of file diff --git a/pkg/services/provisioning/datasources/testdata/version-0/version-0.yaml b/pkg/services/provisioning/datasources/testdata/version-0/version-0.yaml index 6a72bc9aa28..6b73357c72d 100644 --- a/pkg/services/provisioning/datasources/testdata/version-0/version-0.yaml +++ b/pkg/services/provisioning/datasources/testdata/version-0/version-0.yaml @@ -10,6 +10,10 @@ datasources: basic_auth_user: basic_auth_user with_credentials: true is_default: true + correlations: + - targetUid: a target + label: a label + description: a description json_data: graphiteVersion: "1.1" tlsAuth: true diff --git a/pkg/services/provisioning/datasources/types.go b/pkg/services/provisioning/datasources/types.go index 6f2c24c3493..c774252c39d 100644 --- a/pkg/services/provisioning/datasources/types.go +++ b/pkg/services/provisioning/datasources/types.go @@ -42,6 +42,7 @@ type upsertDataSourceFromConfig struct { BasicAuthUser string WithCredentials bool IsDefault bool + Correlations []map[string]interface{} JSONData map[string]interface{} SecureJSONData map[string]string Editable bool @@ -74,21 +75,22 @@ type deleteDatasourceConfigV1 struct { } type upsertDataSourceFromConfigV0 struct { - OrgID int64 `json:"org_id" yaml:"org_id"` - Version int `json:"version" yaml:"version"` - Name string `json:"name" yaml:"name"` - Type string `json:"type" yaml:"type"` - Access string `json:"access" yaml:"access"` - URL string `json:"url" yaml:"url"` - User string `json:"user" yaml:"user"` - Database string `json:"database" yaml:"database"` - BasicAuth bool `json:"basic_auth" yaml:"basic_auth"` - BasicAuthUser string `json:"basic_auth_user" yaml:"basic_auth_user"` - WithCredentials bool `json:"with_credentials" yaml:"with_credentials"` - IsDefault bool `json:"is_default" yaml:"is_default"` - JSONData map[string]interface{} `json:"json_data" yaml:"json_data"` - SecureJSONData map[string]string `json:"secure_json_data" yaml:"secure_json_data"` - Editable bool `json:"editable" yaml:"editable"` + OrgID int64 `json:"org_id" yaml:"org_id"` + Version int `json:"version" yaml:"version"` + Name string `json:"name" yaml:"name"` + Type string `json:"type" yaml:"type"` + Access string `json:"access" yaml:"access"` + URL string `json:"url" yaml:"url"` + User string `json:"user" yaml:"user"` + Database string `json:"database" yaml:"database"` + BasicAuth bool `json:"basic_auth" yaml:"basic_auth"` + BasicAuthUser string `json:"basic_auth_user" yaml:"basic_auth_user"` + WithCredentials bool `json:"with_credentials" yaml:"with_credentials"` + IsDefault bool `json:"is_default" yaml:"is_default"` + Correlations []map[string]interface{} `json:"correlations" yaml:"correlations"` + JSONData map[string]interface{} `json:"json_data" yaml:"json_data"` + SecureJSONData map[string]string `json:"secure_json_data" yaml:"secure_json_data"` + Editable bool `json:"editable" yaml:"editable"` } type upsertDataSourceFromConfigV1 struct { @@ -104,6 +106,7 @@ type upsertDataSourceFromConfigV1 struct { BasicAuthUser values.StringValue `json:"basicAuthUser" yaml:"basicAuthUser"` WithCredentials values.BoolValue `json:"withCredentials" yaml:"withCredentials"` IsDefault values.BoolValue `json:"isDefault" yaml:"isDefault"` + Correlations values.JSONSliceValue `json:"correlations" yaml:"correlations"` JSONData values.JSONValue `json:"jsonData" yaml:"jsonData"` SecureJSONData values.StringMapValue `json:"secureJsonData" yaml:"secureJsonData"` Editable values.BoolValue `json:"editable" yaml:"editable"` @@ -132,6 +135,7 @@ func (cfg *configsV1) mapToDatasourceFromConfig(apiVersion int64) *configs { BasicAuthUser: ds.BasicAuthUser.Value(), WithCredentials: ds.WithCredentials.Value(), IsDefault: ds.IsDefault.Value(), + Correlations: ds.Correlations.Value(), JSONData: ds.JSONData.Value(), SecureJSONData: ds.SecureJSONData.Value(), Editable: ds.Editable.Value(), @@ -172,6 +176,7 @@ func (cfg *configsV0) mapToDatasourceFromConfig(apiVersion int64) *configs { BasicAuthUser: ds.BasicAuthUser, WithCredentials: ds.WithCredentials, IsDefault: ds.IsDefault, + Correlations: ds.Correlations, JSONData: ds.JSONData, SecureJSONData: ds.SecureJSONData, Editable: ds.Editable, diff --git a/pkg/services/provisioning/provisioning.go b/pkg/services/provisioning/provisioning.go index 8f50c2c2b5a..e4b6959d607 100644 --- a/pkg/services/provisioning/provisioning.go +++ b/pkg/services/provisioning/provisioning.go @@ -11,6 +11,7 @@ import ( "github.com/grafana/grafana/pkg/registry" "github.com/grafana/grafana/pkg/services/accesscontrol" "github.com/grafana/grafana/pkg/services/alerting" + "github.com/grafana/grafana/pkg/services/correlations" dashboardservice "github.com/grafana/grafana/pkg/services/dashboards" datasourceservice "github.com/grafana/grafana/pkg/services/datasources" "github.com/grafana/grafana/pkg/services/encryption" @@ -39,6 +40,7 @@ func ProvideService( notificatonService *notifications.NotificationService, dashboardProvisioningService dashboardservice.DashboardProvisioningService, datasourceService datasourceservice.DataSourceService, + correlationsService correlations.Service, dashboardService dashboardservice.DashboardService, folderService dashboardservice.FolderService, alertingService *alerting.AlertNotificationService, @@ -60,6 +62,7 @@ func ProvideService( dashboardProvisioningService: dashboardProvisioningService, dashboardService: dashboardService, datasourceService: datasourceService, + correlationsService: correlationsService, alertingService: alertingService, pluginsSettings: pluginSettings, searchService: searchService, @@ -98,7 +101,7 @@ func NewProvisioningServiceImpl() *ProvisioningServiceImpl { func newProvisioningServiceImpl( newDashboardProvisioner dashboards.DashboardProvisionerFactory, provisionNotifiers func(context.Context, string, notifiers.Manager, notifiers.SQLStore, encryption.Internal, *notifications.NotificationService) error, - provisionDatasources func(context.Context, string, datasources.Store, utils.OrgStore) error, + provisionDatasources func(context.Context, string, datasources.Store, datasources.CorrelationsStore, utils.OrgStore) error, provisionPlugins func(context.Context, string, plugins.Store, plugifaces.Store, pluginsettings.Service) error, ) *ProvisioningServiceImpl { return &ProvisioningServiceImpl{ @@ -122,13 +125,14 @@ type ProvisioningServiceImpl struct { newDashboardProvisioner dashboards.DashboardProvisionerFactory dashboardProvisioner dashboards.DashboardProvisioner provisionNotifiers func(context.Context, string, notifiers.Manager, notifiers.SQLStore, encryption.Internal, *notifications.NotificationService) error - provisionDatasources func(context.Context, string, datasources.Store, utils.OrgStore) error + provisionDatasources func(context.Context, string, datasources.Store, datasources.CorrelationsStore, utils.OrgStore) error provisionPlugins func(context.Context, string, plugins.Store, plugifaces.Store, pluginsettings.Service) error provisionRules func(context.Context, string, dashboardservice.DashboardService, dashboardservice.DashboardProvisioningService, provisioning.AlertRuleService) error mutex sync.Mutex dashboardProvisioningService dashboardservice.DashboardProvisioningService dashboardService dashboardservice.DashboardService datasourceService datasourceservice.DataSourceService + correlationsService correlations.Service alertingService *alerting.AlertNotificationService pluginsSettings pluginsettings.Service searchService searchV2.SearchService @@ -193,7 +197,7 @@ func (ps *ProvisioningServiceImpl) Run(ctx context.Context) error { func (ps *ProvisioningServiceImpl) ProvisionDatasources(ctx context.Context) error { datasourcePath := filepath.Join(ps.Cfg.ProvisioningPath, "datasources") - if err := ps.provisionDatasources(ctx, datasourcePath, ps.datasourceService, ps.SQLStore); err != nil { + if err := ps.provisionDatasources(ctx, datasourcePath, ps.datasourceService, ps.correlationsService, ps.SQLStore); err != nil { err = fmt.Errorf("%v: %w", "Datasource provisioning error", err) ps.log.Error("Failed to provision data sources", "error", err) return err diff --git a/pkg/services/provisioning/values/values.go b/pkg/services/provisioning/values/values.go index ce771310799..b187286a457 100644 --- a/pkg/services/provisioning/values/values.go +++ b/pkg/services/provisioning/values/values.go @@ -188,6 +188,47 @@ func (val *StringMapValue) Value() map[string]string { return val.value } +// JSONSliceValue represents a slice value in a YAML +// config that can be overridden by environment variables + +type JSONSliceValue struct { + value []map[string]interface{} + Raw []map[string]interface{} +} + +// UnmarshalYAML converts YAML into an *JSONSliceValue +func (val *JSONSliceValue) UnmarshalYAML(unmarshal func(interface{}) error) error { + unmarshaled := make([]interface{}, 0) + err := unmarshal(&unmarshaled) + if err != nil { + return err + } + interpolated := make([]map[string]interface{}, 0) + raw := make([]map[string]interface{}, 0) + + for _, v := range unmarshaled { + i := make(map[string]interface{}) + r := make(map[string]interface{}) + for key, val := range v.(map[interface{}]interface{}) { + i[key.(string)], r[key.(string)], err = transformInterface(val) + if err != nil { + return err + } + } + interpolated = append(interpolated, i) + raw = append(raw, r) + } + + val.Raw = raw + val.value = interpolated + return err +} + +// Value returns the wrapped []interface{} value +func (val *JSONSliceValue) Value() []map[string]interface{} { + return val.value +} + // transformInterface tries to transform any interface type into proper value with env expansion. It traverses maps and // slices and the actual interpolation is done on all simple string values in the structure. It returns a copy of any // map or slice value instead of modifying them in place and also return value without interpolation but with converted diff --git a/pkg/services/provisioning/values/values_test.go b/pkg/services/provisioning/values/values_test.go index 3d7d3308e8f..8f5209fb1dc 100644 --- a/pkg/services/provisioning/values/values_test.go +++ b/pkg/services/provisioning/values/values_test.go @@ -220,6 +220,54 @@ func TestValues(t *testing.T) { }) }) + t.Run("JSONSliceValue", func(t *testing.T) { + type Data struct { + Val JSONSliceValue `yaml:"val"` + } + d := &Data{} + + t.Run("Should unmarshal top-level slices and nested structures", func(t *testing.T) { + doc := ` + val: + - interpolatedString: $STRING + interpolatedInt: $INT + string: "just a string" + - interpolatedString: $STRING + interpolatedInt: $INT + string: "just a string" + ` + unmarshalingTest(t, doc, d) + + type stringMap = map[string]interface{} + + require.Equal(t, []stringMap{ + { + "interpolatedString": "test", + "interpolatedInt": "1", + "string": "just a string", + }, + { + "interpolatedString": "test", + "interpolatedInt": "1", + "string": "just a string", + }, + }, d.Val.Value()) + + require.Equal(t, []stringMap{ + { + "interpolatedString": "$STRING", + "interpolatedInt": "$INT", + "string": "just a string", + }, + { + "interpolatedString": "$STRING", + "interpolatedInt": "$INT", + "string": "just a string", + }, + }, d.Val.Raw) + }) + }) + t.Run("StringMapValue", func(t *testing.T) { type Data struct { Val StringMapValue `yaml:"val"` diff --git a/pkg/services/sqlstore/datasource.go b/pkg/services/sqlstore/datasource.go index 504eb0feaa0..fab74f05f8e 100644 --- a/pkg/services/sqlstore/datasource.go +++ b/pkg/services/sqlstore/datasource.go @@ -134,13 +134,15 @@ func (ss *SQLStore) DeleteDataSource(ctx context.Context, cmd *datasources.Delet } // Publish data source deletion event - sess.publishAfterCommit(&events.DataSourceDeleted{ - Timestamp: time.Now(), - Name: cmd.Name, - ID: cmd.ID, - UID: cmd.UID, - OrgID: cmd.OrgID, - }) + if cmd.DeletedDatasourcesCount > 0 { + sess.publishAfterCommit(&events.DataSourceDeleted{ + Timestamp: time.Now(), + Name: ds.Name, + ID: ds.Id, + UID: ds.Uid, + OrgID: ds.OrgId, + }) + } return nil }) diff --git a/pkg/services/sqlstore/datasource_test.go b/pkg/services/sqlstore/datasource_test.go index d26ac84fae2..34169f799a9 100644 --- a/pkg/services/sqlstore/datasource_test.go +++ b/pkg/services/sqlstore/datasource_test.go @@ -251,7 +251,7 @@ func TestIntegrationDataAccess(t *testing.T) { }) err := sqlStore.DeleteDataSource(context.Background(), - &datasources.DeleteDataSourceCommand{ID: ds.Id, UID: "nisse-uid", Name: "nisse", OrgID: int64(123123)}) + &datasources.DeleteDataSourceCommand{ID: ds.Id, UID: ds.Uid, Name: ds.Name, OrgID: ds.OrgId}) require.NoError(t, err) require.Eventually(t, func() bool { @@ -259,9 +259,27 @@ func TestIntegrationDataAccess(t *testing.T) { }, time.Second, time.Millisecond) require.Equal(t, ds.Id, deleted.ID) - require.Equal(t, int64(123123), deleted.OrgID) - require.Equal(t, "nisse", deleted.Name) - require.Equal(t, "nisse-uid", deleted.UID) + require.Equal(t, ds.OrgId, deleted.OrgID) + require.Equal(t, ds.Name, deleted.Name) + require.Equal(t, ds.Uid, deleted.UID) + }) + + t.Run("does not fire an event when the datasource is not deleted", func(t *testing.T) { + sqlStore := InitTestDB(t) + + var called bool + sqlStore.bus.AddEventListener(func(ctx context.Context, e *events.DataSourceDeleted) error { + called = true + return nil + }) + + err := sqlStore.DeleteDataSource(context.Background(), + &datasources.DeleteDataSourceCommand{ID: 1, UID: "non-existing", Name: "non-existing", OrgID: int64(10)}) + require.NoError(t, err) + + require.Never(t, func() bool { + return called + }, time.Second, time.Millisecond) }) t.Run("DeleteDataSourceByName", func(t *testing.T) { diff --git a/pkg/services/sqlstore/migrations/correlations_mig.go b/pkg/services/sqlstore/migrations/correlations_mig.go new file mode 100644 index 00000000000..ccc2d49cf3e --- /dev/null +++ b/pkg/services/sqlstore/migrations/correlations_mig.go @@ -0,0 +1,21 @@ +package migrations + +import ( + . "github.com/grafana/grafana/pkg/services/sqlstore/migrator" +) + +func addCorrelationsMigrations(mg *Migrator) { + correlationsV1 := Table{ + Name: "correlation", + Columns: []*Column{ + {Name: "uid", Type: DB_NVarchar, Length: 40, Nullable: false, IsPrimaryKey: true}, + {Name: "source_uid", Type: DB_NVarchar, Length: 40, Nullable: false, IsPrimaryKey: true}, + // Nullable because in the future we want to have correlations to external resources + {Name: "target_uid", Type: DB_NVarchar, Length: 40, Nullable: true}, + {Name: "label", Type: DB_Text, Nullable: false}, + {Name: "description", Type: DB_Text, Nullable: false}, + }, + } + + mg.AddMigration("create correlation table v1", NewAddTableMigration(correlationsV1)) +} diff --git a/pkg/services/sqlstore/migrations/migrations.go b/pkg/services/sqlstore/migrations/migrations.go index 0e6ea36c180..ba5126e4d1b 100644 --- a/pkg/services/sqlstore/migrations/migrations.go +++ b/pkg/services/sqlstore/migrations/migrations.go @@ -75,6 +75,8 @@ func (*OSSMigrations) AddMigration(mg *Migrator) { addQueryHistoryStarMigrations(mg) + addCorrelationsMigrations(mg) + if mg.Cfg != nil && mg.Cfg.IsFeatureToggleEnabled != nil { if mg.Cfg.IsFeatureToggleEnabled(featuremgmt.FlagDashboardComments) || mg.Cfg.IsFeatureToggleEnabled(featuremgmt.FlagAnnotationComments) { addCommentGroupMigrations(mg) diff --git a/pkg/tests/api/correlations/common_test.go b/pkg/tests/api/correlations/common_test.go new file mode 100644 index 00000000000..e61523d961b --- /dev/null +++ b/pkg/tests/api/correlations/common_test.go @@ -0,0 +1,84 @@ +package correlations + +import ( + "bytes" + "context" + "fmt" + "net/http" + "testing" + + "github.com/grafana/grafana/pkg/server" + "github.com/grafana/grafana/pkg/services/datasources" + "github.com/grafana/grafana/pkg/services/user" + "github.com/grafana/grafana/pkg/tests/testinfra" + "github.com/stretchr/testify/require" +) + +type TestContext struct { + env server.TestEnv + t *testing.T +} + +func NewTestEnv(t *testing.T) TestContext { + t.Helper() + dir, path := testinfra.CreateGrafDir(t, testinfra.GrafanaOpts{ + DisableAnonymous: true, + }) + _, env := testinfra.StartGrafanaEnv(t, dir, path) + + return TestContext{ + env: *env, + t: t, + } +} + +type User struct { + username string + password string +} + +type PostParams struct { + url string + body string + user User +} + +func (c TestContext) Post(params PostParams) *http.Response { + c.t.Helper() + buf := bytes.NewReader([]byte(params.body)) + baseUrl := fmt.Sprintf("http://%s", c.env.Server.HTTPServer.Listener.Addr()) + if params.user.username != "" && params.user.password != "" { + baseUrl = fmt.Sprintf("http://%s:%s@%s", params.user.username, params.user.password, c.env.Server.HTTPServer.Listener.Addr()) + } + + // nolint:gosec + resp, err := http.Post( + fmt.Sprintf( + "%s%s", + baseUrl, + params.url, + ), + "application/json", + buf, + ) + require.NoError(c.t, err) + + return resp +} + +func (c TestContext) createUser(cmd user.CreateUserCommand) { + c.t.Helper() + + c.env.SQLStore.Cfg.AutoAssignOrg = true + c.env.SQLStore.Cfg.AutoAssignOrgId = 1 + + _, err := c.env.SQLStore.CreateUser(context.Background(), cmd) + require.NoError(c.t, err) +} + +func (c TestContext) createDs(cmd *datasources.AddDataSourceCommand) { + c.t.Helper() + + err := c.env.SQLStore.AddDataSource(context.Background(), cmd) + require.NoError(c.t, err) +} diff --git a/pkg/tests/api/correlations/correlations_test.go b/pkg/tests/api/correlations/correlations_test.go new file mode 100644 index 00000000000..8409440ef58 --- /dev/null +++ b/pkg/tests/api/correlations/correlations_test.go @@ -0,0 +1,248 @@ +package correlations + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "net/http" + "testing" + + "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/services/correlations" + "github.com/grafana/grafana/pkg/services/datasources" + "github.com/grafana/grafana/pkg/services/user" + "github.com/stretchr/testify/require" +) + +type errorResponseBody struct { + Message string `json:"message"` + Error string `json:"error"` +} + +func TestIntegrationCreateCorrelation(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test") + } + ctx := NewTestEnv(t) + + adminUser := User{ + username: "admin", + password: "admin", + } + editorUser := User{ + username: "editor", + password: "editor", + } + + ctx.createUser(user.CreateUserCommand{ + DefaultOrgRole: string(models.ROLE_EDITOR), + Password: editorUser.password, + Login: editorUser.username, + }) + ctx.createUser(user.CreateUserCommand{ + DefaultOrgRole: string(models.ROLE_ADMIN), + Password: adminUser.password, + Login: adminUser.username, + }) + + createDsCommand := &datasources.AddDataSourceCommand{ + Name: "read-only", + Type: "loki", + ReadOnly: true, + OrgId: 1, + } + ctx.createDs(createDsCommand) + readOnlyDS := createDsCommand.Result.Uid + + createDsCommand = &datasources.AddDataSourceCommand{ + Name: "writable", + Type: "loki", + OrgId: 1, + } + ctx.createDs(createDsCommand) + writableDs := createDsCommand.Result.Uid + + t.Run("Unauthenticated users shouldn't be able to create correlations", func(t *testing.T) { + res := ctx.Post(PostParams{ + url: fmt.Sprintf("/api/datasources/uid/%s/correlations", "some-ds-uid"), + body: ``, + }) + require.Equal(t, http.StatusUnauthorized, res.StatusCode) + + responseBody, err := ioutil.ReadAll(res.Body) + require.NoError(t, err) + + var response errorResponseBody + err = json.Unmarshal(responseBody, &response) + require.NoError(t, err) + + require.Equal(t, "Unauthorized", response.Message) + + require.NoError(t, res.Body.Close()) + }) + + t.Run("non org admin shouldn't be able to create correlations", func(t *testing.T) { + res := ctx.Post(PostParams{ + url: fmt.Sprintf("/api/datasources/uid/%s/correlations", "some-ds-uid"), + body: ``, + user: editorUser, + }) + require.Equal(t, http.StatusForbidden, res.StatusCode) + + responseBody, err := ioutil.ReadAll(res.Body) + require.NoError(t, err) + + var response errorResponseBody + err = json.Unmarshal(responseBody, &response) + require.NoError(t, err) + + require.Contains(t, response.Message, "Permissions needed: datasources:write") + + require.NoError(t, res.Body.Close()) + }) + + t.Run("missing source data source in body should result in a 400", func(t *testing.T) { + res := ctx.Post(PostParams{ + url: fmt.Sprintf("/api/datasources/uid/%s/correlations", "nonexistent-ds-uid"), + body: `{}`, + user: adminUser, + }) + require.Equal(t, http.StatusBadRequest, res.StatusCode) + + responseBody, err := ioutil.ReadAll(res.Body) + require.NoError(t, err) + + var response errorResponseBody + err = json.Unmarshal(responseBody, &response) + require.NoError(t, err) + + require.Equal(t, "bad request data", response.Message) + + require.NoError(t, res.Body.Close()) + }) + + t.Run("inexistent source data source should result in a 404", func(t *testing.T) { + res := ctx.Post(PostParams{ + url: fmt.Sprintf("/api/datasources/uid/%s/correlations", "nonexistent-ds-uid"), + body: fmt.Sprintf(`{ + "targetUid": "%s" + }`, writableDs), + user: adminUser, + }) + require.Equal(t, http.StatusNotFound, res.StatusCode) + + responseBody, err := ioutil.ReadAll(res.Body) + require.NoError(t, err) + + var response errorResponseBody + err = json.Unmarshal(responseBody, &response) + require.NoError(t, err) + + require.Equal(t, "Data source not found", response.Message) + require.Equal(t, correlations.ErrSourceDataSourceDoesNotExists.Error(), response.Error) + + require.NoError(t, res.Body.Close()) + }) + + t.Run("inexistent target data source should result in a 404", func(t *testing.T) { + res := ctx.Post(PostParams{ + url: fmt.Sprintf("/api/datasources/uid/%s/correlations", writableDs), + body: `{ + "targetUid": "nonexistent-uid-uid" + }`, + user: adminUser, + }) + require.Equal(t, http.StatusNotFound, res.StatusCode) + + responseBody, err := ioutil.ReadAll(res.Body) + require.NoError(t, err) + + var response errorResponseBody + err = json.Unmarshal(responseBody, &response) + require.NoError(t, err) + + require.Equal(t, "Data source not found", response.Message) + require.Equal(t, correlations.ErrTargetDataSourceDoesNotExists.Error(), response.Error) + + require.NoError(t, res.Body.Close()) + }) + + t.Run("creating a correlation originating from a read-only data source should result in a 403", func(t *testing.T) { + res := ctx.Post(PostParams{ + url: fmt.Sprintf("/api/datasources/uid/%s/correlations", readOnlyDS), + body: fmt.Sprintf(`{ + "targetUid": "%s" + }`, readOnlyDS), + user: adminUser, + }) + require.Equal(t, http.StatusForbidden, res.StatusCode) + + responseBody, err := ioutil.ReadAll(res.Body) + require.NoError(t, err) + + var response errorResponseBody + err = json.Unmarshal(responseBody, &response) + require.NoError(t, err) + + require.Equal(t, "Data source is read only", response.Message) + require.Equal(t, correlations.ErrSourceDataSourceReadOnly.Error(), response.Error) + + require.NoError(t, res.Body.Close()) + }) + + t.Run("creating a correlation pointing to a read-only data source should work", func(t *testing.T) { + res := ctx.Post(PostParams{ + url: fmt.Sprintf("/api/datasources/uid/%s/correlations", writableDs), + body: fmt.Sprintf(`{ + "targetUid": "%s" + }`, readOnlyDS), + user: adminUser, + }) + require.Equal(t, http.StatusOK, res.StatusCode) + + responseBody, err := ioutil.ReadAll(res.Body) + require.NoError(t, err) + + var response correlations.CreateCorrelationResponse + err = json.Unmarshal(responseBody, &response) + require.NoError(t, err) + + require.Equal(t, "Correlation created", response.Message) + require.Equal(t, writableDs, response.Result.SourceUID) + require.Equal(t, readOnlyDS, response.Result.TargetUID) + require.Equal(t, "", response.Result.Description) + require.Equal(t, "", response.Result.Label) + + require.NoError(t, res.Body.Close()) + }) + + t.Run("Should correctly create a correlation", func(t *testing.T) { + description := "a description" + label := "a label" + res := ctx.Post(PostParams{ + url: fmt.Sprintf("/api/datasources/uid/%s/correlations", writableDs), + body: fmt.Sprintf(`{ + "targetUid": "%s", + "description": "%s", + "label": "%s" + }`, writableDs, description, label), + user: adminUser, + }) + require.Equal(t, http.StatusOK, res.StatusCode) + + responseBody, err := ioutil.ReadAll(res.Body) + require.NoError(t, err) + + var response correlations.CreateCorrelationResponse + err = json.Unmarshal(responseBody, &response) + require.NoError(t, err) + + require.Equal(t, "Correlation created", response.Message) + require.Equal(t, writableDs, response.Result.SourceUID) + require.Equal(t, writableDs, response.Result.TargetUID) + require.Equal(t, description, response.Result.Description) + require.Equal(t, label, response.Result.Label) + + require.NoError(t, res.Body.Close()) + }) +} diff --git a/public/api-merged.json b/public/api-merged.json index 5d22cbe015a..734cd29b79d 100644 --- a/public/api-merged.json +++ b/public/api-merged.json @@ -4237,6 +4237,49 @@ } } }, + "/datasources/uid/{uid}/correlations": { + "post": { + "tags": ["correlations"], + "summary": "Add correlation.", + "operationId": "createCorrelation", + "parameters": [ + { + "name": "body", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/CreateCorrelationCommand" + } + }, + { + "type": "string", + "name": "uid", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "$ref": "#/responses/createCorrelationResponse" + }, + "400": { + "$ref": "#/responses/badRequestError" + }, + "401": { + "$ref": "#/responses/unauthorisedError" + }, + "403": { + "$ref": "#/responses/forbiddenError" + }, + "404": { + "$ref": "#/responses/notFoundError" + }, + "500": { + "$ref": "#/responses/internalServerError" + } + } + } + }, "/datasources/uid/{uid}/health": { "get": { "tags": ["datasources"], @@ -10462,6 +10505,37 @@ "$ref": "#/definitions/EmbeddedContactPoint" } }, + "Correlation": { + "description": "Correlation is the model for correlations definitions", + "type": "object", + "properties": { + "description": { + "description": "Description of the correlation", + "type": "string", + "example": "Logs to Traces" + }, + "label": { + "description": "Label identifying the correlation", + "type": "string", + "example": "My Label" + }, + "sourceUid": { + "description": "UID of the data source the correlation originates from", + "type": "string", + "example": "d0oxYRg4z" + }, + "targetUid": { + "description": "UID of the data source the correlation points to", + "type": "string", + "example": "PE1C5CBDA0504A6A3" + }, + "uid": { + "description": "Unique identifier of the correlation", + "type": "string", + "example": "50xhMlg9k" + } + } + }, "CreateAlertNotificationCommand": { "type": "object", "properties": { @@ -10497,6 +10571,40 @@ } } }, + "CreateCorrelationCommand": { + "description": "CreateCorrelationCommand is the command for creating a correlation", + "type": "object", + "properties": { + "description": { + "description": "Optional description of the correlation", + "type": "string", + "example": "Logs to Traces" + }, + "label": { + "description": "Optional label identifying the correlation", + "type": "string", + "example": "My label" + }, + "targetUid": { + "description": "Target data source UID to which the correlation is created", + "type": "string", + "example": "PE1C5CBDA0504A6A3" + } + } + }, + "CreateCorrelationResponse": { + "description": "CreateCorrelationResponse is the response struct for CreateCorrelationCommand", + "type": "object", + "properties": { + "message": { + "type": "string", + "example": "Correlation created" + }, + "result": { + "$ref": "#/definitions/Correlation" + } + } + }, "CreateDashboardSnapshotCommand": { "type": "object", "required": ["dashboard"], @@ -17539,6 +17647,12 @@ } } }, + "createCorrelationResponse": { + "description": "(empty)", + "schema": { + "$ref": "#/definitions/CreateCorrelationResponse" + } + }, "createOrUpdateDatasourceResponse": { "description": "(empty)", "schema": { diff --git a/public/api-spec.json b/public/api-spec.json index 0ece8019daf..3c7914e043e 100644 --- a/public/api-spec.json +++ b/public/api-spec.json @@ -3656,6 +3656,49 @@ } } }, + "/datasources/uid/{uid}/correlations": { + "post": { + "tags": ["correlations"], + "summary": "Add correlation.", + "operationId": "createCorrelation", + "parameters": [ + { + "name": "body", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/CreateCorrelationCommand" + } + }, + { + "type": "string", + "name": "uid", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "$ref": "#/responses/createCorrelationResponse" + }, + "400": { + "$ref": "#/responses/badRequestError" + }, + "401": { + "$ref": "#/responses/unauthorisedError" + }, + "403": { + "$ref": "#/responses/forbiddenError" + }, + "404": { + "$ref": "#/responses/notFoundError" + }, + "500": { + "$ref": "#/responses/internalServerError" + } + } + } + }, "/datasources/uid/{uid}/health": { "get": { "tags": ["datasources"], @@ -9494,6 +9537,37 @@ } } }, + "Correlation": { + "description": "Correlation is the model for correlations definitions", + "type": "object", + "properties": { + "description": { + "description": "Description of the correlation", + "type": "string", + "example": "Logs to Traces" + }, + "label": { + "description": "Label identifying the correlation", + "type": "string", + "example": "My Label" + }, + "sourceUid": { + "description": "UID of the data source the correlation originates from", + "type": "string", + "example": "d0oxYRg4z" + }, + "targetUid": { + "description": "UID of the data source the correlation points to", + "type": "string", + "example": "PE1C5CBDA0504A6A3" + }, + "uid": { + "description": "Unique identifier of the correlation", + "type": "string", + "example": "50xhMlg9k" + } + } + }, "CreateAlertNotificationCommand": { "type": "object", "properties": { @@ -9529,6 +9603,40 @@ } } }, + "CreateCorrelationCommand": { + "description": "CreateCorrelationCommand is the command for creating a correlation", + "type": "object", + "properties": { + "description": { + "description": "Optional description of the correlation", + "type": "string", + "example": "Logs to Traces" + }, + "label": { + "description": "Optional label identifying the correlation", + "type": "string", + "example": "My label" + }, + "targetUid": { + "description": "Target data source UID to which the correlation is created", + "type": "string", + "example": "PE1C5CBDA0504A6A3" + } + } + }, + "CreateCorrelationResponse": { + "description": "CreateCorrelationResponse is the response struct for CreateCorrelationCommand", + "type": "object", + "properties": { + "message": { + "type": "string", + "example": "Correlation created" + }, + "result": { + "$ref": "#/definitions/Correlation" + } + } + }, "CreateDashboardSnapshotCommand": { "type": "object", "required": ["dashboard"], @@ -13759,6 +13867,12 @@ } } }, + "createCorrelationResponse": { + "description": "", + "schema": { + "$ref": "#/definitions/CreateCorrelationResponse" + } + }, "createOrUpdateDatasourceResponse": { "description": "", "schema": {