Plugins: Plugins loader pipeline (#71438)

* discovery

* flesh out

* add docs

* remove unused func

* bootstrap stage

* fix docs

* update docs

* undo unnecessary changes

* add end tag

* update doc

* fix linter

* fix

* tidy

* update docs

* add class to filter func

* apply PR feedback

* fix test

pkg/plugins/manager/pipeline/bootstrap/bootstrap.go

@@ -0,0 +1,78 @@
+package bootstrap
+
+import (
+	"context"
+
+	"github.com/grafana/grafana/pkg/plugins"
+	"github.com/grafana/grafana/pkg/plugins/config"
+	"github.com/grafana/grafana/pkg/plugins/log"
+	"github.com/grafana/grafana/pkg/plugins/manager/loader/assetpath"
+	"github.com/grafana/grafana/pkg/plugins/manager/signature"
+)
+
+// Bootstrapper is responsible for the Bootstrap stage of the plugin loader pipeline.
+type Bootstrapper interface {
+	Bootstrap(ctx context.Context, src plugins.PluginSource, bundles []*plugins.FoundBundle) ([]*plugins.Plugin, error)
+}
+
+// ConstructFunc is the function used for the Construct step of the Bootstrap stage.
+type ConstructFunc func(ctx context.Context, src plugins.PluginSource, bundles []*plugins.FoundBundle) ([]*plugins.Plugin, error)
+
+// DecorateFunc is the function used for the Decorate step of the Bootstrap stage.
+type DecorateFunc func(ctx context.Context, p *plugins.Plugin) (*plugins.Plugin, error)
+
+// Bootstrap implements the Bootstrapper interface.
+//
+// The Bootstrap stage is made up of the following steps (in order):
+// - Construct: Create the initial plugin structs based on the plugin(s) found in the Discovery stage.
+// - Decorate: Decorate the plugins with additional metadata.
+//
+// The Construct step is implemented by the ConstructFunc type.
+//
+// The Decorate step is implemented by the DecorateFunc type.
+type Bootstrap struct {
+	constructStep ConstructFunc
+	decorateSteps []DecorateFunc
+	log           log.Logger
+}
+
+type Opts struct {
+	ConstructFunc ConstructFunc
+	DecorateFuncs []DecorateFunc
+}
+
+// New returns a new Bootstrap stage.
+func New(cfg *config.Cfg, opts Opts) *Bootstrap {
+	if opts.ConstructFunc == nil {
+		opts.ConstructFunc = DefaultConstructFunc(signature.DefaultCalculator(), assetpath.DefaultService(cfg))
+	}
+
+	if len(opts.DecorateFuncs) == 0 {
+		opts.DecorateFuncs = DefaultDecorateFuncs
+	}
+
+	return &Bootstrap{
+		constructStep: opts.ConstructFunc,
+		decorateSteps: opts.DecorateFuncs,
+		log:           log.New("plugins.bootstrap"),
+	}
+}
+
+// Bootstrap will execute the Construct and Decorate steps of the Bootstrap stage.
+func (b *Bootstrap) Bootstrap(ctx context.Context, src plugins.PluginSource, found []*plugins.FoundBundle) ([]*plugins.Plugin, error) {
+	ps, err := b.constructStep(ctx, src, found)
+	if err != nil {
+		return nil, err
+	}
+
+	for _, p := range ps {
+		for _, decorator := range b.decorateSteps {
+			p, err = decorator(ctx, p)
+			if err != nil {
+				return nil, err
+			}
+		}
+	}
+
+	return ps, nil
+}

pkg/plugins/manager/pipeline/bootstrap/doc.go

@@ -0,0 +1,6 @@
+// Package bootstrap defines the second stage of the plugin loader pipeline.
+//
+// The Bootstrap stage must implement the Bootstrapper interface.
+// - Bootstrap(ctx context.Context, src plugins.PluginSource, bundles []*plugins.FoundBundle) ([]*plugins.Plugin, error)
+
+package bootstrap

pkg/plugins/manager/pipeline/bootstrap/factory.go

@@ -0,0 +1,76 @@
+package bootstrap
+
+import (
+	"fmt"
+
+	"github.com/grafana/grafana/pkg/plugins"
+	"github.com/grafana/grafana/pkg/plugins/log"
+	"github.com/grafana/grafana/pkg/plugins/manager/loader/assetpath"
+)
+
+type pluginFactoryFunc func(p plugins.FoundPlugin, pluginClass plugins.Class, sig plugins.Signature) (*plugins.Plugin, error)
+
+// DefaultPluginFactory is the default plugin factory used by the Construct step of the Bootstrap stage.
+//
+// It creates the plugin using plugin information found during the Discovery stage and makes use of the assetPath
+// service to set the plugin's BaseURL, Module, Logos and Screenshots fields.
+type DefaultPluginFactory struct {
+	assetPath *assetpath.Service
+}
+
+// NewDefaultPluginFactory returns a new DefaultPluginFactory.
+func NewDefaultPluginFactory(assetPath *assetpath.Service) *DefaultPluginFactory {
+	return &DefaultPluginFactory{assetPath: assetPath}
+}
+
+func (f *DefaultPluginFactory) createPlugin(p plugins.FoundPlugin, class plugins.Class,
+	sig plugins.Signature) (*plugins.Plugin, error) {
+	baseURL, err := f.assetPath.Base(p.JSONData, class, p.FS.Base())
+	if err != nil {
+		return nil, fmt.Errorf("base url: %w", err)
+	}
+	moduleURL, err := f.assetPath.Module(p.JSONData, class, p.FS.Base())
+	if err != nil {
+		return nil, fmt.Errorf("module url: %w", err)
+	}
+
+	plugin := &plugins.Plugin{
+		JSONData:      p.JSONData,
+		FS:            p.FS,
+		BaseURL:       baseURL,
+		Module:        moduleURL,
+		Class:         class,
+		Signature:     sig.Status,
+		SignatureType: sig.Type,
+		SignatureOrg:  sig.SigningOrg,
+	}
+	plugin.SetLogger(log.New(fmt.Sprintf("plugin.%s", plugin.ID)))
+
+	if err = setImages(plugin, f.assetPath); err != nil {
+		return nil, err
+	}
+
+	return plugin, nil
+}
+
+func setImages(p *plugins.Plugin, assetPath *assetpath.Service) error {
+	var err error
+	for _, dst := range []*string{&p.Info.Logos.Small, &p.Info.Logos.Large} {
+		*dst, err = assetPath.RelativeURL(p, *dst, defaultLogoPath(p.Type))
+		if err != nil {
+			return fmt.Errorf("logo: %w", err)
+		}
+	}
+	for i := 0; i < len(p.Info.Screenshots); i++ {
+		screenshot := &p.Info.Screenshots[i]
+		screenshot.Path, err = assetPath.RelativeURL(p, screenshot.Path, "")
+		if err != nil {
+			return fmt.Errorf("screenshot %d relative url: %w", i, err)
+		}
+	}
+	return nil
+}
+
+func defaultLogoPath(pluginType plugins.Type) string {
+	return fmt.Sprintf("public/img/icn-%s.svg", string(pluginType))
+}

pkg/plugins/manager/pipeline/bootstrap/steps.go

@@ -0,0 +1,146 @@
+package bootstrap
+
+import (
+	"context"
+	"path"
+	"strings"
+
+	"github.com/grafana/grafana/pkg/infra/slugify"
+	"github.com/grafana/grafana/pkg/plugins"
+	"github.com/grafana/grafana/pkg/plugins/log"
+	"github.com/grafana/grafana/pkg/plugins/manager/loader/assetpath"
+	"github.com/grafana/grafana/pkg/util"
+)
+
+// DefaultConstructor implements the default ConstructFunc used for the Construct step of the Bootstrap stage.
+//
+// It uses a pluginFactoryFunc to create plugins and the signatureCalculator to calculate the plugin's signature state.
+type DefaultConstructor struct {
+	pluginFactoryFunc   pluginFactoryFunc
+	signatureCalculator plugins.SignatureCalculator
+	log                 log.Logger
+}
+
+// DefaultConstructFunc is the default ConstructFunc used for the Construct step of the Bootstrap stage.
+func DefaultConstructFunc(signatureCalculator plugins.SignatureCalculator, assetPath *assetpath.Service) ConstructFunc {
+	return NewDefaultConstructor(signatureCalculator, assetPath).Construct
+}
+
+// DefaultDecorateFuncs are the default DecorateFuncs used for the Decorate step of the Bootstrap stage.
+var DefaultDecorateFuncs = []DecorateFunc{
+	AliasDecorateFunc,
+	AppDefaultNavURLDecorateFunc,
+	AppChildDecorateFunc,
+}
+
+// NewDefaultConstructor returns a new DefaultConstructor.
+func NewDefaultConstructor(signatureCalculator plugins.SignatureCalculator, assetPath *assetpath.Service) *DefaultConstructor {
+	return &DefaultConstructor{
+		pluginFactoryFunc:   NewDefaultPluginFactory(assetPath).createPlugin,
+		signatureCalculator: signatureCalculator,
+		log:                 log.New("plugins.construct"),
+	}
+}
+
+// Construct will calculate the plugin's signature state and create the plugin using the pluginFactoryFunc.
+func (c *DefaultConstructor) Construct(ctx context.Context, src plugins.PluginSource, bundles []*plugins.FoundBundle) ([]*plugins.Plugin, error) {
+	res := make([]*plugins.Plugin, 0, len(bundles))
+
+	for _, bundle := range bundles {
+		sig, err := c.signatureCalculator.Calculate(ctx, src, bundle.Primary)
+		if err != nil {
+			c.log.Warn("Could not calculate plugin signature state", "pluginID", bundle.Primary.JSONData.ID, "err", err)
+			continue
+		}
+		plugin, err := c.pluginFactoryFunc(bundle.Primary, src.PluginClass(ctx), sig)
+		if err != nil {
+			c.log.Error("Could not create primary plugin base", "pluginID", bundle.Primary.JSONData.ID, "err", err)
+			continue
+		}
+		res = append(res, plugin)
+
+		children := make([]*plugins.Plugin, 0, len(bundle.Children))
+		for _, child := range bundle.Children {
+			cp, err := c.pluginFactoryFunc(*child, plugin.Class, sig)
+			if err != nil {
+				c.log.Error("Could not create child plugin base", "pluginID", child.JSONData.ID, "err", err)
+				continue
+			}
+			cp.Parent = plugin
+			plugin.Children = append(plugin.Children, cp)
+
+			children = append(children, cp)
+		}
+		res = append(res, children...)
+	}
+
+	return res, nil
+}
+
+// AliasDecorateFunc is a DecorateFunc that sets the alias for the plugin.
+func AliasDecorateFunc(_ context.Context, p *plugins.Plugin) (*plugins.Plugin, error) {
+	switch p.ID {
+	case "grafana-pyroscope-datasource": // rebranding
+		p.Alias = "phlare"
+	case "debug": // panel plugin used for testing
+		p.Alias = "debugX"
+	}
+	return p, nil
+}
+
+// AppDefaultNavURLDecorateFunc is a DecorateFunc that sets the default nav URL for app plugins.
+func AppDefaultNavURLDecorateFunc(_ context.Context, p *plugins.Plugin) (*plugins.Plugin, error) {
+	if p.IsApp() {
+		setDefaultNavURL(p)
+	}
+	return p, nil
+}
+
+func setDefaultNavURL(p *plugins.Plugin) {
+	// slugify pages
+	for _, include := range p.Includes {
+		if include.Slug == "" {
+			include.Slug = slugify.Slugify(include.Name)
+		}
+
+		if !include.DefaultNav {
+			continue
+		}
+
+		if include.Type == "page" {
+			p.DefaultNavURL = path.Join("/plugins/", p.ID, "/page/", include.Slug)
+		}
+		if include.Type == "dashboard" {
+			dboardURL := include.DashboardURLPath()
+			if dboardURL == "" {
+				p.Logger().Warn("Included dashboard is missing a UID field")
+				continue
+			}
+
+			p.DefaultNavURL = dboardURL
+		}
+	}
+}
+
+// AppChildDecorateFunc is a DecorateFunc that configures child plugins of app plugins.
+func AppChildDecorateFunc(_ context.Context, p *plugins.Plugin) (*plugins.Plugin, error) {
+	if p.Parent != nil && p.Parent.IsApp() {
+		configureAppChildPlugin(p.Parent, p)
+	}
+	return p, nil
+}
+
+func configureAppChildPlugin(parent *plugins.Plugin, child *plugins.Plugin) {
+	if !parent.IsApp() {
+		return
+	}
+	appSubPath := strings.ReplaceAll(strings.Replace(child.FS.Base(), parent.FS.Base(), "", 1), "\\", "/")
+	child.IncludedInAppID = parent.ID
+	child.BaseURL = parent.BaseURL
+
+	if parent.IsCorePlugin() {
+		child.Module = util.JoinURLFragments("app/plugins/app/"+parent.ID, appSubPath) + "/module"
+	} else {
+		child.Module = util.JoinURLFragments("plugins/"+parent.ID, appSubPath) + "/module"
+	}
+}

pkg/plugins/manager/pipeline/bootstrap/steps_test.go

@@ -0,0 +1,89 @@
+package bootstrap
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/grafana/grafana/pkg/plugins"
+	"github.com/grafana/grafana/pkg/plugins/log"
+	"github.com/grafana/grafana/pkg/plugins/manager/fakes"
+)
+
+func TestSetDefaultNavURL(t *testing.T) {
+	t.Run("When including a dashboard with DefaultNav: true", func(t *testing.T) {
+		pluginWithDashboard := &plugins.Plugin{
+			JSONData: plugins.JSONData{Includes: []*plugins.Includes{
+				{
+					Type:       "dashboard",
+					DefaultNav: true,
+					UID:        "",
+				},
+			}},
+		}
+		logger := log.NewTestLogger()
+		pluginWithDashboard.SetLogger(logger)
+
+		t.Run("Default nav URL is not set if dashboard UID field not is set", func(t *testing.T) {
+			setDefaultNavURL(pluginWithDashboard)
+			require.Equal(t, "", pluginWithDashboard.DefaultNavURL)
+			require.NotZero(t, logger.WarnLogs.Calls)
+			require.Equal(t, "Included dashboard is missing a UID field", logger.WarnLogs.Message)
+		})
+
+		t.Run("Default nav URL is set if dashboard UID field is set", func(t *testing.T) {
+			pluginWithDashboard.Includes[0].UID = "a1b2c3"
+
+			setDefaultNavURL(pluginWithDashboard)
+			require.Equal(t, "/d/a1b2c3", pluginWithDashboard.DefaultNavURL)
+		})
+	})
+
+	t.Run("When including a page with DefaultNav: true", func(t *testing.T) {
+		pluginWithPage := &plugins.Plugin{
+			JSONData: plugins.JSONData{Includes: []*plugins.Includes{
+				{
+					Type:       "page",
+					DefaultNav: true,
+					Slug:       "testPage",
+				},
+			}},
+		}
+
+		t.Run("Default nav URL is set using slug", func(t *testing.T) {
+			setDefaultNavURL(pluginWithPage)
+			require.Equal(t, "/plugins/page/testPage", pluginWithPage.DefaultNavURL)
+		})
+
+		t.Run("Default nav URL is set using slugified Name field if Slug field is empty", func(t *testing.T) {
+			pluginWithPage.Includes[0].Slug = ""
+			pluginWithPage.Includes[0].Name = "My Test Page"
+
+			setDefaultNavURL(pluginWithPage)
+			require.Equal(t, "/plugins/page/my-test-page", pluginWithPage.DefaultNavURL)
+		})
+	})
+}
+
+func TestSetPathsBasedOnApp(t *testing.T) {
+	t.Run("When setting paths based on core plugin on Windows", func(t *testing.T) {
+		child := &plugins.Plugin{
+			FS: fakes.NewFakePluginFiles("c:\\grafana\\public\\app\\plugins\\app\\testdata-app\\datasources\\datasource"),
+		}
+		parent := &plugins.Plugin{
+			JSONData: plugins.JSONData{
+				Type: plugins.TypeApp,
+				ID:   "testdata-app",
+			},
+			Class:   plugins.ClassCore,
+			FS:      fakes.NewFakePluginFiles("c:\\grafana\\public\\app\\plugins\\app\\testdata-app"),
+			BaseURL: "public/app/plugins/app/testdata-app",
+		}
+
+		configureAppChildPlugin(parent, child)
+
+		require.Equal(t, "app/plugins/app/testdata-app/datasources/datasource/module", child.Module)
+		require.Equal(t, "testdata-app", child.IncludedInAppID)
+		require.Equal(t, "public/app/plugins/app/testdata-app", child.BaseURL)
+	})
+}

pkg/plugins/manager/pipeline/discovery/discovery.go

@@ -0,0 +1,74 @@
+package discovery
+
+import (
+	"context"
+
+	"github.com/grafana/grafana/pkg/plugins"
+	"github.com/grafana/grafana/pkg/plugins/config"
+	"github.com/grafana/grafana/pkg/plugins/log"
+)
+
+// Discoverer is responsible for the Discovery stage of the plugin loader pipeline.
+type Discoverer interface {
+	Discover(ctx context.Context, src plugins.PluginSource) ([]*plugins.FoundBundle, error)
+}
+
+// FindFunc is the function used for the Find step of the Discovery stage.
+type FindFunc func(ctx context.Context, src plugins.PluginSource) ([]*plugins.FoundBundle, error)
+
+// FindFilterFunc is the function used for the Filter step of the Discovery stage.
+type FindFilterFunc func(ctx context.Context, class plugins.Class, bundles []*plugins.FoundBundle) ([]*plugins.FoundBundle, error)
+
+// Discovery implements the Discoverer interface.
+//
+// The Discovery stage is made up of the following steps (in order):
+// - Find: Find plugins (from disk, remote, etc.)
+// - Filter: Filter the results based on some criteria.
+//
+// The Find step is implemented by the FindFunc type.
+//
+// The Filter step is implemented by the FindFilterFunc type.
+type Discovery struct {
+	findStep        FindFunc
+	findFilterSteps []FindFilterFunc
+	log             log.Logger
+}
+
+type Opts struct {
+	FindFunc        FindFunc
+	FindFilterFuncs []FindFilterFunc
+}
+
+// New returns a new Discovery stage.
+func New(cfg *config.Cfg, opts Opts) *Discovery {
+	if opts.FindFunc == nil {
+		opts.FindFunc = DefaultFindFunc(cfg)
+	}
+
+	if len(opts.FindFilterFuncs) == 0 {
+		opts.FindFilterFuncs = []FindFilterFunc{} // no filters by default
+	}
+
+	return &Discovery{
+		findStep:        opts.FindFunc,
+		findFilterSteps: opts.FindFilterFuncs,
+		log:             log.New("plugins.discovery"),
+	}
+}
+
+// Discover will execute the Find and Filter steps of the Discovery stage.
+func (d *Discovery) Discover(ctx context.Context, src plugins.PluginSource) ([]*plugins.FoundBundle, error) {
+	found, err := d.findStep(ctx, src)
+	if err != nil {
+		return nil, err
+	}
+
+	for _, filterStep := range d.findFilterSteps {
+		found, err = filterStep(ctx, src.PluginClass(ctx), found)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	return found, nil
+}

pkg/plugins/manager/pipeline/discovery/doc.go

@@ -0,0 +1,6 @@
+// Package discovery defines the first stage of the plugin loader pipeline.
+
+// The Discovery stage must implement the Discoverer interface.
+// - Discover(ctx context.Context, src plugins.PluginSource) ([]*plugins.FoundBundle, error)
+
+package discovery

pkg/plugins/manager/pipeline/discovery/steps.go

@@ -0,0 +1,55 @@
+package discovery
+
+import (
+	"context"
+
+	"github.com/grafana/grafana/pkg/plugins"
+	"github.com/grafana/grafana/pkg/plugins/config"
+	"github.com/grafana/grafana/pkg/plugins/log"
+	"github.com/grafana/grafana/pkg/plugins/manager/loader/finder"
+	"github.com/grafana/grafana/pkg/plugins/manager/registry"
+)
+
+// DefaultFindFunc is the default function used for the Find step of the Discovery stage. It will scan the local
+// filesystem for plugins.
+func DefaultFindFunc(cfg *config.Cfg) FindFunc {
+	return finder.NewLocalFinder(cfg.DevMode).Find
+}
+
+// DuplicatePluginValidation is a filter step that will filter out any plugins that are already registered with the
+// registry. This includes both the primary plugin and any child plugins, which are matched using the plugin ID field.
+type DuplicatePluginValidation struct {
+	registry registry.Service
+	log      log.Logger
+}
+
+// NewDuplicatePluginFilterStep returns a new DuplicatePluginValidation.
+func NewDuplicatePluginFilterStep(registry registry.Service) *DuplicatePluginValidation {
+	return &DuplicatePluginValidation{
+		registry: registry,
+		log:      log.New("plugins.dedupe"),
+	}
+}
+
+// Filter will filter out any plugins that are already registered with the registry.
+func (d *DuplicatePluginValidation) Filter(ctx context.Context, bundles []*plugins.FoundBundle) ([]*plugins.FoundBundle, error) {
+	res := make([]*plugins.FoundBundle, 0, len(bundles))
+	for _, b := range bundles {
+		_, exists := d.registry.Plugin(ctx, b.Primary.JSONData.ID)
+		if exists {
+			d.log.Warn("Skipping loading of plugin as it's a duplicate", "pluginID", b.Primary.JSONData.ID)
+			continue
+		}
+
+		for _, child := range b.Children {
+			_, exists = d.registry.Plugin(ctx, child.JSONData.ID)
+			if exists {
+				d.log.Warn("Skipping loading of child plugin as it's a duplicate", "pluginID", child.JSONData.ID)
+				continue
+			}
+		}
+		res = append(res, b)
+	}
+
+	return res, nil
+}

pkg/plugins/manager/pipeline/doc.go

@@ -0,0 +1,11 @@
+// Package pipeline defines a load pipeline for Grafana plugins.
+//
+// A pipeline is a sequence of stages that are executed in order. Each stage is made up of a series of steps.
+// A plugin loader pipeline is defined by the following stages:
+// 	 Discovery: Find plugins (e.g. from disk, remote, etc.), and [optionally] filter the results based on some criteria.
+// 	 Bootstrap: Create the plugins found in the discovery stage and enrich them with metadata.
+// 	 Verification: Verify the plugins based on some criteria (e.g. signature validation, angular detection, etc.)
+// 	 Initialization: Initialize the plugin for use (e.g. register with Grafana, etc.)
+// 	 Post-Initialization: Perform any post-initialization tasks (e.g. start the backend process, declare RBAC roles etc.)
+
+package pipeline