Backend Plugins: Refactor backend plugin registration and start (#21452)

Moves the details of loading plugins into the backend
plugin manager from the respective plugin (datasource,
transform and renderer).

pkg/plugins/app_plugin.go

@@ -46,7 +46,7 @@ type JwtTokenAuth struct {
 }
 
 func (app *AppPlugin) Load(decoder *json.Decoder, pluginDir string) error {
-	if err := decoder.Decode(&app); err != nil {
+	if err := decoder.Decode(app); err != nil {
 		return err
 	}
 

pkg/plugins/backendplugin/client.go

@@ -41,17 +41,30 @@ func newClientConfig(executablePath string, logger log.Logger, versionedPlugins
 	}
 }
 
+// LegacyStartFunc callback function called when a plugin with old plugin protocol is started.
+type LegacyStartFunc func(pluginID string, client *LegacyClient, logger log.Logger) error
+
+// StartFunc callback function called when a plugin with current plugin protocol version is started.
+type StartFunc func(pluginID string, client *Client, logger log.Logger) error
+
+// PluginStartFuncs functions called for plugin when started.
+type PluginStartFuncs struct {
+	OnLegacyStart LegacyStartFunc
+	OnStart       StartFunc
+}
+
 // PluginDescriptor descriptor used for registering backend plugins.
 type PluginDescriptor struct {
 	pluginID         string
 	executablePath   string
 	managed          bool
 	versionedPlugins map[int]plugin.PluginSet
+	startFns         PluginStartFuncs
 }
 
 // NewBackendPluginDescriptor creates a new backend plugin descriptor
 // used for registering a backend datasource plugin.
-func NewBackendPluginDescriptor(pluginID, executablePath string) PluginDescriptor {
+func NewBackendPluginDescriptor(pluginID, executablePath string, startFns PluginStartFuncs) PluginDescriptor {
 	return PluginDescriptor{
 		pluginID:       pluginID,
 		executablePath: executablePath,
@@ -65,12 +78,13 @@ func NewBackendPluginDescriptor(pluginID, executablePath string) PluginDescripto
 				"transform": &backend.TransformGRPCPlugin{},
 			},
 		},
+		startFns: startFns,
 	}
 }
 
 // NewRendererPluginDescriptor creates a new renderer plugin descriptor
 // used for registering a backend renderer plugin.
-func NewRendererPluginDescriptor(pluginID, executablePath string) PluginDescriptor {
+func NewRendererPluginDescriptor(pluginID, executablePath string, startFns PluginStartFuncs) PluginDescriptor {
 	return PluginDescriptor{
 		pluginID:       pluginID,
 		executablePath: executablePath,
@@ -80,5 +94,18 @@ func NewRendererPluginDescriptor(pluginID, executablePath string) PluginDescript
 				pluginID: &rendererV1.RendererPluginImpl{},
 			},
 		},
+		startFns: startFns,
 	}
 }
+
+// LegacyClient client for communicating with a plugin using the old plugin protocol.
+type LegacyClient struct {
+	DatasourcePlugin datasourceV1.DatasourcePlugin
+	RendererPlugin   rendererV1.RendererPlugin
+}
+
+// Client client for communicating with a plugin using the current plugin protocol.
+type Client struct {
+	BackendPlugin   backend.BackendPlugin
+	TransformPlugin backend.TransformPlugin
+}

pkg/plugins/backendplugin/manager.go

@@ -6,6 +6,9 @@ import (
 	"sync"
 	"time"
 
+	datasourceV1 "github.com/grafana/grafana-plugin-model/go/datasource"
+	rendererV1 "github.com/grafana/grafana-plugin-model/go/renderer"
+	backend "github.com/grafana/grafana-plugin-sdk-go/backend"
 	"github.com/grafana/grafana/pkg/infra/log"
 	plugin "github.com/hashicorp/go-plugin"
 	"golang.org/x/xerrors"
@@ -17,8 +20,6 @@ var (
 	logger    = log.New("plugins.backend")
 )
 
-type BackendPluginCallbackFunc func(pluginID string, client *plugin.Client, logger log.Logger) error
-
 type BackendPlugin struct {
 	id              string
 	executablePath  string
@@ -26,26 +27,74 @@ type BackendPlugin struct {
 	clientFactory   func() *plugin.Client
 	client          *plugin.Client
 	logger          log.Logger
-	callbackFn      BackendPluginCallbackFunc
+	startFns        PluginStartFuncs
 	supportsMetrics bool
 	supportsHealth  bool
 }
 
 func (p *BackendPlugin) start(ctx context.Context) error {
 	p.client = p.clientFactory()
-	// rpcClient, err := p.client.Client()
+	rpcClient, err := p.client.Client()
-	// if err != nil {
+	if err != nil {
-	// 	return err
+		return err
-	// }
+	}
-	// if p.client.NegotiatedVersion() > 1 {
-	// 	_, err = rpcClient.Dispense("diagnostics")
-	// 	if err != nil {
-	// 		return err
-	// 	}
-	// }
 
-	if p.callbackFn != nil {
+	var legacyClient *LegacyClient
-		return p.callbackFn(p.id, p.client, p.logger)
+	var client *Client
+
+	if p.client.NegotiatedVersion() > 1 {
+		rawBackend, err := rpcClient.Dispense("backend")
+		if err != nil {
+			return err
+		}
+
+		rawTransform, err := rpcClient.Dispense("transform")
+		if err != nil {
+			return err
+		}
+
+		client = &Client{}
+		if rawBackend != nil {
+			if plugin, ok := rawBackend.(backend.BackendPlugin); ok {
+				client.BackendPlugin = plugin
+			}
+		}
+
+		if rawTransform != nil {
+			if plugin, ok := rawTransform.(backend.TransformPlugin); ok {
+				client.TransformPlugin = plugin
+			}
+		}
+	} else {
+		raw, err := rpcClient.Dispense(p.id)
+		if err != nil {
+			return err
+		}
+
+		legacyClient = &LegacyClient{}
+		if plugin, ok := raw.(datasourceV1.DatasourcePlugin); ok {
+			legacyClient.DatasourcePlugin = plugin
+		}
+
+		if plugin, ok := raw.(rendererV1.RendererPlugin); ok {
+			legacyClient.RendererPlugin = plugin
+		}
+	}
+
+	if legacyClient == nil && client == nil {
+		return errors.New("no compatible plugin implementation found")
+	}
+
+	if legacyClient != nil && p.startFns.OnLegacyStart != nil {
+		if err := p.startFns.OnLegacyStart(p.id, legacyClient, p.logger); err != nil {
+			return err
+		}
+	}
+
+	if client != nil && p.startFns.OnStart != nil {
+		if err := p.startFns.OnStart(p.id, client, p.logger); err != nil {
+			return err
+		}
 	}
 
 	return nil
@@ -71,7 +120,7 @@ func (p *BackendPlugin) checkHealth(ctx context.Context) {
 }
 
 // Register registers a backend plugin
-func Register(descriptor PluginDescriptor, callbackFn BackendPluginCallbackFunc) error {
+func Register(descriptor PluginDescriptor) error {
 	logger.Debug("Registering backend plugin", "pluginId", descriptor.pluginID, "executablePath", descriptor.executablePath)
 	pluginsMu.Lock()
 	defer pluginsMu.Unlock()
@@ -88,8 +137,8 @@ func Register(descriptor PluginDescriptor, callbackFn BackendPluginCallbackFunc)
 		clientFactory: func() *plugin.Client {
 			return plugin.NewClient(newClientConfig(descriptor.executablePath, pluginLogger, descriptor.versionedPlugins))
 		},
-		callbackFn: callbackFn,
+		startFns: descriptor.startFns,
-		logger:     pluginLogger,
+		logger:   pluginLogger,
 	}
 
 	plugins[descriptor.pluginID] = plugin

pkg/plugins/datasource_plugin.go

@@ -2,22 +2,16 @@ package plugins
 
 import (
 	"encoding/json"
-	"errors"
-	"fmt"
 	"path"
 
 	"github.com/grafana/grafana/pkg/plugins/backendplugin"
 
-	"github.com/grafana/grafana/pkg/setting"
 	"github.com/grafana/grafana/pkg/util/errutil"
 
-	datasourceV1 "github.com/grafana/grafana-plugin-model/go/datasource"
-	sdk "github.com/grafana/grafana-plugin-sdk-go/backend"
 	"github.com/grafana/grafana/pkg/infra/log"
 	"github.com/grafana/grafana/pkg/models"
 	"github.com/grafana/grafana/pkg/plugins/datasource/wrapper"
 	"github.com/grafana/grafana/pkg/tsdb"
-	plugin "github.com/hashicorp/go-plugin"
 )
 
 // DataSourcePlugin contains all metadata about a datasource plugin
@@ -45,10 +39,6 @@ func (p *DataSourcePlugin) Load(decoder *json.Decoder, pluginDir string) error {
 		return errutil.Wrapf(err, "Failed to decode datasource plugin")
 	}
 
-	if !p.isVersionOne() && !setting.IsExpressionsEnabled() {
-		return errors.New("A plugin version 2 was found, but expressions feature toggle is not enabled")
-	}
-
 	if err := p.registerPlugin(pluginDir); err != nil {
 		return errutil.Wrapf(err, "Failed to register plugin")
 	}
@@ -56,8 +46,11 @@ func (p *DataSourcePlugin) Load(decoder *json.Decoder, pluginDir string) error {
 	if p.Backend {
 		cmd := ComposePluginStartCommmand(p.Executable)
 		fullpath := path.Join(p.PluginDir, cmd)
-		descriptor := backendplugin.NewBackendPluginDescriptor(p.Id, fullpath)
+		descriptor := backendplugin.NewBackendPluginDescriptor(p.Id, fullpath, backendplugin.PluginStartFuncs{
-		if err := backendplugin.Register(descriptor, p.onPluginStart); err != nil {
+			OnLegacyStart: p.onLegacyPluginStart,
+			OnStart:       p.onPluginStart,
+		})
+		if err := backendplugin.Register(descriptor); err != nil {
 			return errutil.Wrapf(err, "Failed to register backend plugin")
 		}
 	}
@@ -66,41 +59,20 @@ func (p *DataSourcePlugin) Load(decoder *json.Decoder, pluginDir string) error {
 	return nil
 }
 
-func (p *DataSourcePlugin) isVersionOne() bool {
+func (p *DataSourcePlugin) onLegacyPluginStart(pluginID string, client *backendplugin.LegacyClient, logger log.Logger) error {
-	return !p.SDK
-}
-
-func (p *DataSourcePlugin) onPluginStart(pluginID string, client *plugin.Client, logger log.Logger) error {
-	rpcClient, err := client.Client()
-	if err != nil {
-		return err
-	}
-
-	if client.NegotiatedVersion() == 1 {
-		raw, err := rpcClient.Dispense(pluginID)
-		if err != nil {
-			return err
-		}
-		plugin := raw.(datasourceV1.DatasourcePlugin)
-
-		tsdb.RegisterTsdbQueryEndpoint(pluginID, func(dsInfo *models.DataSource) (tsdb.TsdbQueryEndpoint, error) {
-			return wrapper.NewDatasourcePluginWrapper(logger, plugin), nil
-		})
-		return nil
-	}
-
-	raw, err := rpcClient.Dispense("backend")
-	if err != nil {
-		return err
-	}
-	plugin, ok := raw.(sdk.BackendPlugin)
-	if !ok {
-		return fmt.Errorf("unexpected type %T, expected sdk.Plugin", raw)
-	}
-
 	tsdb.RegisterTsdbQueryEndpoint(pluginID, func(dsInfo *models.DataSource) (tsdb.TsdbQueryEndpoint, error) {
-		return wrapper.NewDatasourcePluginWrapperV2(logger, plugin), nil
+		return wrapper.NewDatasourcePluginWrapper(logger, client.DatasourcePlugin), nil
 	})
 
 	return nil
 }
+
+func (p *DataSourcePlugin) onPluginStart(pluginID string, client *backendplugin.Client, logger log.Logger) error {
+	if client.BackendPlugin != nil {
+		tsdb.RegisterTsdbQueryEndpoint(pluginID, func(dsInfo *models.DataSource) (tsdb.TsdbQueryEndpoint, error) {
+			return wrapper.NewDatasourcePluginWrapperV2(logger, client.BackendPlugin), nil
+		})
+	}
+
+	return nil
+}

pkg/plugins/datasource_plugin_test.go

@@ -1,72 +0,0 @@
-package plugins
-
-import (
-	"bytes"
-	"encoding/json"
-	"testing"
-
-	"github.com/grafana/grafana/pkg/setting"
-
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-func TestLoadDatasourceVersion(t *testing.T) {
-	t.Run("If plugin version is not set, it should be treated as plugin version one", func(t *testing.T) {
-		refPlug := DataSourcePlugin{}
-		pluginJSON, err := json.Marshal(refPlug)
-		require.NoError(t, err)
-
-		datasourcePlugin := DataSourcePlugin{}
-		err = datasourcePlugin.Load(json.NewDecoder(bytes.NewReader(pluginJSON)), "/tmp")
-		require.NoError(t, err)
-		delete(Plugins, refPlug.Id)
-		delete(DataSources, refPlug.Id)
-
-		assert.True(t, datasourcePlugin.isVersionOne())
-	})
-
-	t.Run("If plugin version is set to one, it should be treated as plugin version one", func(t *testing.T) {
-		refPlug := DataSourcePlugin{SDK: false}
-		pluginJSON, err := json.Marshal(refPlug)
-		require.NoError(t, err)
-
-		datasourcePlugin := DataSourcePlugin{}
-		err = datasourcePlugin.Load(json.NewDecoder(bytes.NewReader(pluginJSON)), "/tmp")
-		require.NoError(t, err)
-		delete(Plugins, refPlug.Id)
-		delete(DataSources, refPlug.Id)
-
-		assert.True(t, datasourcePlugin.isVersionOne())
-		assert.False(t, datasourcePlugin.SDK)
-	})
-
-	t.Run("If plugin version is set to two, it should not be treated as plugin version one", func(t *testing.T) {
-		refPlug := DataSourcePlugin{SDK: true}
-		pluginJSON, err := json.Marshal(refPlug)
-		require.NoError(t, err)
-
-		origToggles := setting.FeatureToggles
-		setting.FeatureToggles = map[string]bool{"expressions": true}
-		datasourcePlugin := DataSourcePlugin{}
-		err = datasourcePlugin.Load(json.NewDecoder(bytes.NewReader(pluginJSON)), "/tmp")
-		setting.FeatureToggles = origToggles
-		require.NoError(t, err)
-		delete(Plugins, refPlug.Id)
-		delete(DataSources, refPlug.Id)
-
-		assert.False(t, datasourcePlugin.isVersionOne())
-		assert.True(t, datasourcePlugin.SDK)
-	})
-
-	t.Run("Plugin version two requires expressions feature to be toggled", func(t *testing.T) {
-		refPlug := DataSourcePlugin{SDK: true}
-		pluginJSON, err := json.Marshal(refPlug)
-		require.NoError(t, err)
-
-		require.Nil(t, setting.FeatureToggles, "setting.FeatureToggles shouldn't be set")
-		datasourcePlugin := DataSourcePlugin{}
-		err = datasourcePlugin.Load(json.NewDecoder(bytes.NewReader(pluginJSON)), "/tmp")
-		require.EqualError(t, err, "A plugin version 2 was found, but expressions feature toggle is not enabled")
-	})
-}

pkg/plugins/panel_plugin.go

@@ -8,7 +8,7 @@ type PanelPlugin struct {
 }
 
 func (p *PanelPlugin) Load(decoder *json.Decoder, pluginDir string) error {
-	if err := decoder.Decode(&p); err != nil {
+	if err := decoder.Decode(p); err != nil {
 		return err
 	}
 

pkg/plugins/renderer_plugin.go

@@ -9,7 +9,6 @@ import (
 	"github.com/grafana/grafana/pkg/infra/log"
 	"github.com/grafana/grafana/pkg/plugins/backendplugin"
 	"github.com/grafana/grafana/pkg/util/errutil"
-	plugin "github.com/hashicorp/go-plugin"
 )
 
 type RendererPlugin struct {
@@ -20,7 +19,7 @@ type RendererPlugin struct {
 }
 
 func (r *RendererPlugin) Load(decoder *json.Decoder, pluginDir string) error {
-	if err := decoder.Decode(&r); err != nil {
+	if err := decoder.Decode(r); err != nil {
 		return err
 	}
 
@@ -30,8 +29,10 @@ func (r *RendererPlugin) Load(decoder *json.Decoder, pluginDir string) error {
 
 	cmd := ComposePluginStartCommmand("plugin_start")
 	fullpath := path.Join(r.PluginDir, cmd)
-	descriptor := backendplugin.NewRendererPluginDescriptor(r.Id, fullpath)
+	descriptor := backendplugin.NewRendererPluginDescriptor(r.Id, fullpath, backendplugin.PluginStartFuncs{
-	if err := backendplugin.Register(descriptor, r.onPluginStart); err != nil {
+		OnLegacyStart: r.onLegacyPluginStart,
+	})
+	if err := backendplugin.Register(descriptor); err != nil {
 		return errutil.Wrapf(err, "Failed to register backend plugin")
 	}
 
@@ -47,17 +48,7 @@ func (r *RendererPlugin) Start(ctx context.Context) error {
 	return nil
 }
 
-func (r *RendererPlugin) onPluginStart(pluginID string, client *plugin.Client, logger log.Logger) error {
+func (r *RendererPlugin) onLegacyPluginStart(pluginID string, client *backendplugin.LegacyClient, logger log.Logger) error {
-	rpcClient, err := client.Client()
+	r.GrpcPlugin = client.RendererPlugin
-	if err != nil {
-		return err
-	}
-
-	raw, err := rpcClient.Dispense(pluginID)
-	if err != nil {
-		return err
-	}
-
-	r.GrpcPlugin = raw.(pluginModel.RendererPlugin)
 	return nil
 }

pkg/plugins/transform_plugin.go

@@ -15,9 +15,9 @@ import (
 	"github.com/grafana/grafana/pkg/infra/log"
 	"github.com/grafana/grafana/pkg/models"
 	"github.com/grafana/grafana/pkg/plugins/backendplugin"
+	"github.com/grafana/grafana/pkg/plugins/datasource/wrapper"
 	"github.com/grafana/grafana/pkg/tsdb"
 	"github.com/grafana/grafana/pkg/util/errutil"
-	plugin "github.com/hashicorp/go-plugin"
 )
 
 type TransformPlugin struct {
@@ -28,45 +28,38 @@ type TransformPlugin struct {
 	*TransformWrapper
 }
 
-func (tp *TransformPlugin) Load(decoder *json.Decoder, pluginDir string) error {
+func (p *TransformPlugin) Load(decoder *json.Decoder, pluginDir string) error {
-	if err := decoder.Decode(&tp); err != nil {
+	if err := decoder.Decode(p); err != nil {
 		return err
 	}
 
-	if err := tp.registerPlugin(pluginDir); err != nil {
+	if err := p.registerPlugin(pluginDir); err != nil {
 		return err
 	}
 
-	cmd := ComposePluginStartCommmand(tp.Executable)
+	cmd := ComposePluginStartCommmand(p.Executable)
-	fullpath := path.Join(tp.PluginDir, cmd)
+	fullpath := path.Join(p.PluginDir, cmd)
-	descriptor := backendplugin.NewBackendPluginDescriptor(tp.Id, fullpath)
+	descriptor := backendplugin.NewBackendPluginDescriptor(p.Id, fullpath, backendplugin.PluginStartFuncs{
-	if err := backendplugin.Register(descriptor, tp.onPluginStart); err != nil {
+		OnStart: p.onPluginStart,
+	})
+	if err := backendplugin.Register(descriptor); err != nil {
 		return errutil.Wrapf(err, "Failed to register backend plugin")
 	}
 
-	Transform = tp
+	Transform = p
 
 	return nil
 }
 
-func (p *TransformPlugin) onPluginStart(pluginID string, client *plugin.Client, logger log.Logger) error {
+func (p *TransformPlugin) onPluginStart(pluginID string, client *backendplugin.Client, logger log.Logger) error {
-	rpcClient, err := client.Client()
+	p.TransformWrapper = NewTransformWrapper(logger, client.TransformPlugin)
-	if err != nil {
-		return err
-	}
 
-	raw, err := rpcClient.Dispense("transform")
+	if client.BackendPlugin != nil {
-	if err != nil {
+		tsdb.RegisterTsdbQueryEndpoint(pluginID, func(dsInfo *models.DataSource) (tsdb.TsdbQueryEndpoint, error) {
-		return err
+			return wrapper.NewDatasourcePluginWrapperV2(logger, client.BackendPlugin), nil
+		})
 	}
 
-	plugin, ok := raw.(backend.TransformPlugin)
-	if !ok {
-		return fmt.Errorf("unexpected type %T, expected *backend.TransformPlugin", raw)
-	}
-
-	p.TransformWrapper = NewTransformWrapper(logger, plugin)
-
 	return nil
 }