Plugins: Refactor Plugin Management (#40477)

* add core plugin flow

* add instrumentation

* move func

* remove cruft

* support external backend plugins

* refactor + clean up

* remove comments

* refactor loader

* simplify core plugin path arg

* cleanup loggers

* move signature validator to plugins package

* fix sig packaging

* cleanup plugin model

* remove unnecessary plugin field

* add start+stop for pm

* fix failures

* add decommissioned state

* export fields just to get things flowing

* fix comments

* set static routes

* make image loading idempotent

* merge with backend plugin manager

* re-use funcs

* reorder imports + remove unnecessary interface

* add some TODOs + remove unused func

* remove unused instrumentation func

* simplify client usage

* remove import alias

* re-use backendplugin.Plugin interface

* re order funcs

* improve var name

* fix log statements

* refactor data model

* add logic for dupe check during loading

* cleanup state setting

* refactor loader

* cleanup manager interface

* add rendering flow

* refactor loading + init

* add renderer support

* fix renderer plugin

* reformat imports

* track errors

* fix plugin signature inheritance

* name param in interface

* update func comment

* fix func arg name

* introduce class concept

* remove func

* fix external plugin check

* apply changes from pm-experiment

* fix core plugins

* fix imports

* rename interface

* comment API interface

* add support for testdata plugin

* enable alerting + use correct core plugin contracts

* slim manager API

* fix param name

* fix filter

* support static routes

* fix rendering

* tidy rendering

* get tests compiling

* fix install+uninstall

* start finder test

* add finder test coverage

* start loader tests

* add test for core plugins

* load core + bundled test

* add test for nested plugin loading

* add test files

* clean interface + fix registering some core plugins

* refactoring

* reformat and create sub packages

* simplify core plugin init

* fix ctx cancel scenario

* migrate initializer

* remove Init() funcs

* add test starter

* new logger

* flesh out initializer tests

* refactoring

* remove unused svc

* refactor rendering flow

* fixup loader tests

* add enabled helper func

* fix logger name

* fix data fetchers

* fix case where plugin dir doesn't exist

* improve coverage + move dupe checking to loader

* remove noisy debug logs

* register core plugins automagically

* add support for renderer in catalog

* make private func + fix req validation

* use interface

* re-add check for renderer in catalog

* tidy up from moving to auto reg core plugins

* core plugin registrar

* guards

* copy over core plugins for test infra

* all tests green

* renames

* propagate new interfaces

* kill old manager

* get compiling

* tidy up

* update naming

* refactor manager test + cleanup

* add more cases to finder test

* migrate validator to field

* more coverage

* refactor dupe checking

* add test for plugin class

* add coverage for initializer

* split out rendering

* move

* fixup tests

* fix uss test

* fix frontend settings

* fix grafanads test

* add check when checking sig errors

* fix enabled map

* fixup

* allow manual setup of CM

* rename to cloud-monitoring

* remove TODO

* add installer interface for testing

* loader interface returns

* tests passing

* refactor + add more coverage

* support 'stackdriver'

* fix frontend settings loading

* improve naming based on package name

* small tidy

* refactor test

* fix renderer start

* make cloud-monitoring plugin ID clearer

* add plugin update test

* add integration tests

* don't break all if sig can't be calculated

* add root URL check test

* add more signature verification tests

* update DTO name

* update enabled plugins comment

* update comments

* fix linter

* revert fe naming change

* fix errors endpoint

* reset error code field name

* re-order test to help verify

* assert -> require

* pm check

* add missing entry + re-order

* re-check

* dump icon log

* verify manager contents first

* reformat

* apply PR feedback

* apply style changes

* fix one vs all loading err

* improve log output

* only start when no signature error

* move log

* rework plugin update check

* fix test

* fix multi loading from cfg.PluginSettings

* improve log output #2

* add error abstraction to capture errors without registering a plugin

* add debug log

* add unsigned warning

* e2e test attempt

* fix logger

* set home path

* prevent panic

* alternate

* ugh.. fix home path

* return renderer even if not started

* make renderer plugin managed

* add fallback renderer icon, update renderer badge + prevent changes when renderer is installed

* fix icon loading

* rollback renderer changes

* use correct field

* remove unneccessary block

* remove newline

* remove unused func

* fix bundled plugins base + module fields

* remove unused field since refactor

* add authorizer abstraction

* loader only returns plugins expected to run

* fix multi log output

pkg/api/plugins.go

@@ -3,15 +3,19 @@ package api
 import (
 	"encoding/json"
 	"errors"
+	"fmt"
+	"io/ioutil"
 	"net/http"
 	"os"
 	"path/filepath"
 	"sort"
+	"strings"
 
 	"github.com/grafana/grafana-plugin-sdk-go/backend"
 	"github.com/grafana/grafana/pkg/api/dtos"
 	"github.com/grafana/grafana/pkg/api/response"
 	"github.com/grafana/grafana/pkg/bus"
+	"github.com/grafana/grafana/pkg/infra/fs"
 	"github.com/grafana/grafana/pkg/models"
 	"github.com/grafana/grafana/pkg/plugins"
 	"github.com/grafana/grafana/pkg/plugins/backendplugin"
@@ -31,48 +35,48 @@ func (hs *HTTPServer) GetPluginList(c *models.ReqContext) response.Response {
 		coreFilter = "1"
 	}
 
-	pluginSettingsMap, err := hs.PluginManager.GetPluginSettings(c.OrgId)
+	pluginSettingsMap, err := hs.pluginSettings(c.OrgId)
 	if err != nil {
 		return response.Error(500, "Failed to get list of plugins", err)
 	}
 
 	result := make(dtos.PluginList, 0)
-	for _, pluginDef := range hs.PluginManager.Plugins() {
+	for _, pluginDef := range hs.pluginStore.Plugins() {
 		// filter out app sub plugins
-		if embeddedFilter == "0" && pluginDef.IncludedInAppId != "" {
+		if embeddedFilter == "0" && pluginDef.IncludedInAppID != "" {
 			continue
 		}
 
 		// filter out core plugins
-		if (coreFilter == "0" && pluginDef.IsCorePlugin) || (coreFilter == "1" && !pluginDef.IsCorePlugin) {
+		if (coreFilter == "0" && pluginDef.IsCorePlugin()) || (coreFilter == "1" && !pluginDef.IsCorePlugin()) {
 			continue
 		}
 
 		// filter on type
-		if typeFilter != "" && typeFilter != pluginDef.Type {
+		if typeFilter != "" && typeFilter != string(pluginDef.Type) {
 			continue
 		}
 
-		if pluginDef.State == plugins.PluginStateAlpha && !hs.Cfg.PluginsEnableAlpha {
+		if pluginDef.State == plugins.AlphaRelease && !hs.Cfg.PluginsEnableAlpha {
 			continue
 		}
 
 		listItem := dtos.PluginListItem{
-			Id:            pluginDef.Id,
+			Id:            pluginDef.ID,
 			Name:          pluginDef.Name,
-			Type:          pluginDef.Type,
+			Type:          string(pluginDef.Type),
 			Category:      pluginDef.Category,
 			Info:          &pluginDef.Info,
-			LatestVersion: pluginDef.GrafanaNetVersion,
-			HasUpdate:     pluginDef.GrafanaNetHasUpdate,
-			DefaultNavUrl: pluginDef.DefaultNavUrl,
+			LatestVersion: pluginDef.GrafanaComVersion,
+			HasUpdate:     pluginDef.GrafanaComHasUpdate,
+			DefaultNavUrl: pluginDef.DefaultNavURL,
 			State:         pluginDef.State,
 			Signature:     pluginDef.Signature,
 			SignatureType: pluginDef.SignatureType,
 			SignatureOrg:  pluginDef.SignatureOrg,
 		}
 
-		if pluginSetting, exists := pluginSettingsMap[pluginDef.Id]; exists {
+		if pluginSetting, exists := pluginSettingsMap[pluginDef.ID]; exists {
 			listItem.Enabled = pluginSetting.Enabled
 			listItem.Pinned = pluginSetting.Pinned
 		}
@@ -86,11 +90,9 @@ func (hs *HTTPServer) GetPluginList(c *models.ReqContext) response.Response {
 			continue
 		}
 
-		// filter out built in data sources
-		if ds := hs.PluginManager.GetDataSource(pluginDef.Id); ds != nil {
-			if ds.BuiltIn {
-				continue
-			}
+		// filter out built in plugins
+		if pluginDef.BuiltIn {
+			continue
 		}
 
 		result = append(result, listItem)
@@ -103,32 +105,32 @@ func (hs *HTTPServer) GetPluginList(c *models.ReqContext) response.Response {
 func (hs *HTTPServer) GetPluginSettingByID(c *models.ReqContext) response.Response {
 	pluginID := web.Params(c.Req)[":pluginId"]
 
-	def := hs.PluginManager.GetPlugin(pluginID)
+	def := hs.pluginStore.Plugin(pluginID)
 	if def == nil {
 		return response.Error(404, "Plugin not found, no installed plugin with that id", nil)
 	}
 
 	dto := &dtos.PluginSetting{
-		Type:          def.Type,
-		Id:            def.Id,
+		Type:          string(def.Type),
+		Id:            def.ID,
 		Name:          def.Name,
 		Info:          &def.Info,
 		Dependencies:  &def.Dependencies,
 		Includes:      def.Includes,
-		BaseUrl:       def.BaseUrl,
+		BaseUrl:       def.BaseURL,
 		Module:        def.Module,
-		DefaultNavUrl: def.DefaultNavUrl,
-		LatestVersion: def.GrafanaNetVersion,
-		HasUpdate:     def.GrafanaNetHasUpdate,
+		DefaultNavUrl: def.DefaultNavURL,
+		LatestVersion: def.GrafanaComVersion,
+		HasUpdate:     def.GrafanaComHasUpdate,
 		State:         def.State,
 		Signature:     def.Signature,
 		SignatureType: def.SignatureType,
 		SignatureOrg:  def.SignatureOrg,
 	}
 
-	if app := hs.PluginManager.GetApp(def.Id); app != nil {
-		dto.Enabled = app.AutoEnabled
-		dto.Pinned = app.AutoEnabled
+	if def.IsApp() {
+		dto.Enabled = def.AutoEnabled
+		dto.Pinned = def.AutoEnabled
 	}
 
 	query := models.GetPluginSettingByIdQuery{PluginId: pluginID, OrgId: c.OrgId}
@@ -148,7 +150,7 @@ func (hs *HTTPServer) GetPluginSettingByID(c *models.ReqContext) response.Respon
 func (hs *HTTPServer) UpdatePluginSetting(c *models.ReqContext, cmd models.UpdatePluginSettingCmd) response.Response {
 	pluginID := web.Params(c.Req)[":pluginId"]
 
-	if app := hs.PluginManager.GetApp(pluginID); app == nil {
+	if app := hs.pluginStore.Plugin(pluginID); app == nil {
 		return response.Error(404, "Plugin not installed", nil)
 	}
 
@@ -164,9 +166,9 @@ func (hs *HTTPServer) UpdatePluginSetting(c *models.ReqContext, cmd models.Updat
 func (hs *HTTPServer) GetPluginDashboards(c *models.ReqContext) response.Response {
 	pluginID := web.Params(c.Req)[":pluginId"]
 
-	list, err := hs.PluginManager.GetPluginDashboards(c.OrgId, pluginID)
+	list, err := hs.pluginDashboardManager.GetPluginDashboards(c.OrgId, pluginID)
 	if err != nil {
-		var notFound plugins.PluginNotFoundError
+		var notFound plugins.NotFoundError
 		if errors.As(err, &notFound) {
 			return response.Error(404, notFound.Error(), nil)
 		}
@@ -181,9 +183,9 @@ func (hs *HTTPServer) GetPluginMarkdown(c *models.ReqContext) response.Response
 	pluginID := web.Params(c.Req)[":pluginId"]
 	name := web.Params(c.Req)[":name"]
 
-	content, err := hs.PluginManager.GetPluginMarkdown(pluginID, name)
+	content, err := hs.pluginMarkdown(pluginID, name)
 	if err != nil {
-		var notFound plugins.PluginNotFoundError
+		var notFound plugins.NotFoundError
 		if errors.As(err, &notFound) {
 			return response.Error(404, notFound.Error(), nil)
 		}
@@ -193,7 +195,7 @@ func (hs *HTTPServer) GetPluginMarkdown(c *models.ReqContext) response.Response
 
 	// fallback try readme
 	if len(content) == 0 {
-		content, err = hs.PluginManager.GetPluginMarkdown(pluginID, "readme")
+		content, err = hs.pluginMarkdown(pluginID, "readme")
 		if err != nil {
 			return response.Error(501, "Could not get markdown file", err)
 		}
@@ -218,8 +220,8 @@ func (hs *HTTPServer) ImportDashboard(c *models.ReqContext, apiCmd dtos.ImportDa
 		}
 	}
 
-	dashInfo, dash, err := hs.PluginManager.ImportDashboard(apiCmd.PluginId, apiCmd.Path, c.OrgId, apiCmd.FolderId,
-		apiCmd.Dashboard, apiCmd.Overwrite, apiCmd.Inputs, c.SignedInUser, hs.DataService)
+	dashInfo, dash, err := hs.pluginDashboardManager.ImportDashboard(apiCmd.PluginId, apiCmd.Path, c.OrgId, apiCmd.FolderId,
+		apiCmd.Dashboard, apiCmd.Overwrite, apiCmd.Inputs, c.SignedInUser)
 	if err != nil {
 		return hs.dashboardSaveErrorToApiResponse(err)
 	}
@@ -242,12 +244,12 @@ func (hs *HTTPServer) ImportDashboard(c *models.ReqContext, apiCmd dtos.ImportDa
 // /api/plugins/:pluginId/metrics
 func (hs *HTTPServer) CollectPluginMetrics(c *models.ReqContext) response.Response {
 	pluginID := web.Params(c.Req)[":pluginId"]
-	plugin := hs.PluginManager.GetPlugin(pluginID)
+	plugin := hs.pluginStore.Plugin(pluginID)
 	if plugin == nil {
 		return response.Error(404, "Plugin not found", nil)
 	}
 
-	resp, err := hs.BackendPluginManager.CollectMetrics(c.Req.Context(), plugin.Id)
+	resp, err := hs.pluginClient.CollectMetrics(c.Req.Context(), plugin.ID)
 	if err != nil {
 		return translatePluginRequestErrorToAPIError(err)
 	}
@@ -263,7 +265,7 @@ func (hs *HTTPServer) CollectPluginMetrics(c *models.ReqContext) response.Respon
 // /public/plugins/:pluginId/*
 func (hs *HTTPServer) getPluginAssets(c *models.ReqContext) {
 	pluginID := web.Params(c.Req)[":pluginId"]
-	plugin := hs.PluginManager.GetPlugin(pluginID)
+	plugin := hs.pluginStore.Plugin(pluginID)
 	if plugin == nil {
 		c.JsonApiErr(404, "Plugin not found", nil)
 		return
@@ -323,7 +325,9 @@ func (hs *HTTPServer) CheckHealth(c *models.ReqContext) response.Response {
 		return response.Error(404, "Plugin not found", nil)
 	}
 
-	resp, err := hs.BackendPluginManager.CheckHealth(c.Req.Context(), pCtx)
+	resp, err := hs.pluginClient.CheckHealth(c.Req.Context(), &backend.CheckHealthRequest{
+		PluginContext: pCtx,
+	})
 	if err != nil {
 		return translatePluginRequestErrorToAPIError(err)
 	}
@@ -366,19 +370,19 @@ func (hs *HTTPServer) CallResource(c *models.ReqContext) {
 		c.JsonApiErr(404, "Plugin not found", nil)
 		return
 	}
-	hs.BackendPluginManager.CallResource(pCtx, c, web.Params(c.Req)["*"])
+	hs.pluginClient.CallResource(pCtx, c, web.Params(c.Req)["*"])
 }
 
 func (hs *HTTPServer) GetPluginErrorsList(_ *models.ReqContext) response.Response {
-	return response.JSON(200, hs.PluginManager.ScanningErrors())
+	return response.JSON(200, hs.pluginErrorResolver.PluginErrors())
 }
 
 func (hs *HTTPServer) InstallPlugin(c *models.ReqContext, dto dtos.InstallPluginCommand) response.Response {
 	pluginID := web.Params(c.Req)[":pluginId"]
 
-	err := hs.PluginManager.Install(c.Req.Context(), pluginID, dto.Version)
+	err := hs.pluginStore.Add(c.Req.Context(), pluginID, dto.Version, plugins.AddOpts{})
 	if err != nil {
-		var dupeErr plugins.DuplicatePluginError
+		var dupeErr plugins.DuplicateError
 		if errors.As(err, &dupeErr) {
 			return response.Error(http.StatusConflict, "Plugin already installed", err)
 		}
@@ -407,7 +411,7 @@ func (hs *HTTPServer) InstallPlugin(c *models.ReqContext, dto dtos.InstallPlugin
 func (hs *HTTPServer) UninstallPlugin(c *models.ReqContext) response.Response {
 	pluginID := web.Params(c.Req)[":pluginId"]
 
-	err := hs.PluginManager.Uninstall(c.Req.Context(), pluginID)
+	err := hs.pluginStore.Remove(c.Req.Context(), pluginID)
 	if err != nil {
 		if errors.Is(err, plugins.ErrPluginNotInstalled) {
 			return response.Error(http.StatusNotFound, "Plugin not installed", err)
@@ -443,3 +447,39 @@ func translatePluginRequestErrorToAPIError(err error) response.Response {
 
 	return response.Error(500, "Plugin request failed", err)
 }
+
+func (hs *HTTPServer) pluginMarkdown(pluginId string, name string) ([]byte, error) {
+	plug := hs.pluginStore.Plugin(pluginId)
+	if plug == nil {
+		return nil, plugins.NotFoundError{PluginID: pluginId}
+	}
+
+	// nolint:gosec
+	// We can ignore the gosec G304 warning on this one because `plug.PluginDir` is based
+	// on plugin the folder structure on disk and not user input.
+	path := filepath.Join(plug.PluginDir, fmt.Sprintf("%s.md", strings.ToUpper(name)))
+	exists, err := fs.Exists(path)
+	if err != nil {
+		return nil, err
+	}
+	if !exists {
+		path = filepath.Join(plug.PluginDir, fmt.Sprintf("%s.md", strings.ToLower(name)))
+	}
+
+	exists, err = fs.Exists(path)
+	if err != nil {
+		return nil, err
+	}
+	if !exists {
+		return make([]byte, 0), nil
+	}
+
+	// nolint:gosec
+	// We can ignore the gosec G304 warning on this one because `plug.PluginDir` is based
+	// on plugin the folder structure on disk and not user input.
+	data, err := ioutil.ReadFile(path)
+	if err != nil {
+		return nil, err
+	}
+	return data, nil
+}