Schema: Refactor plugin code generation (#58901)

* wip

* wip

* almost there..

* wip - change so it can run.

* treelist is working.

* support CODEGEN_VERIFY env variable

* use log.fatal

* comment out old PluginTreeList code generation

* cleanup

* rename corelist package files

* fix makefile

* move pkg/codegen/pluggen.go to pkg/plugins/codegen

* copy and refactor files to pkg/plugins/codegen

* use pkg/plugins/codegen instead of pkg/codegen for core plugins code gen

* remove unneeded files

* remove unused code to resolve linting errors

* adapters first hack

* added flattener

* add back ignore build tags to go generate file

* cleaned up the code a bit.

* seems to work, needs to do some refactoring of the GoTypesJenns and TSTypesJenny.

* one more step, going to get upstream changes in this branch.

* working but need to run import tmpl in jenny_schemapath to have the proper imports.

* added header to generated files.

* added missing jenny.

* preventing plugins with multiple decls/schemas to insert multiple lines in corelist.

* fixed so we use Slot type from kindsys to detect if its group.

* adding a go jenny that only runs if the plugin has a backend.

* added version object to generated ts.

* generating the ts types with the same output as prior to this refactoring.

* removed code that is replaced by the jenny pattern.

* removed the go code that isn't used anymore.

* removed some more unused code and renamed pluggen to util_ts

* fixed linting issue.

* removed unused vars.

* use a jenny list postprocessor for header injection

* moved decl and decl_parser to pfs.

* removed the pre-pended header in the gotypes jenny since it is done in the postprocess.

* moved decl to pfs.

* removed unused template.

Co-authored-by: Marcus Efraimsson <marcus.efraimsson@gmail.com>

pkg/plugins/codegen/jenny_plugingotypes.go

@@ -0,0 +1,46 @@
+package codegen
+
+import (
+	"fmt"
+	"path/filepath"
+	"strings"
+
+	"github.com/grafana/codejen"
+	"github.com/grafana/grafana/pkg/plugins/pfs"
+)
+
+func PluginGoTypesJenny(root string, inner codejen.OneToOne[*pfs.PluginDecl]) codejen.OneToOne[*pfs.PluginDecl] {
+	return &pgoJenny{
+		inner: inner,
+		root:  root,
+	}
+}
+
+type pgoJenny struct {
+	inner codejen.OneToOne[*pfs.PluginDecl]
+	root  string
+}
+
+func (j *pgoJenny) JennyName() string {
+	return "PluginGoTypesJenny"
+}
+
+func (j *pgoJenny) Generate(decl *pfs.PluginDecl) (*codejen.File, error) {
+	b := decl.PluginMeta.Backend
+	if b == nil || !*b || !decl.HasSchema() {
+		return nil, nil
+	}
+
+	f, err := j.inner.Generate(decl)
+	if err != nil {
+		return nil, err
+	}
+
+	pluginfolder := filepath.Base(decl.PluginPath)
+	slotname := strings.ToLower(decl.Slot.Name())
+	filename := fmt.Sprintf("types_%s_gen.go", slotname)
+	f.RelativePath = filepath.Join(j.root, pluginfolder, filename)
+	f.From = append(f.From, j)
+
+	return f, nil
+}

pkg/plugins/codegen/jenny_plugintreelist.go

@@ -0,0 +1,100 @@
+package codegen
+
+import (
+	"bytes"
+	"fmt"
+	"path"
+	"path/filepath"
+	"strings"
+
+	"github.com/grafana/codejen"
+	"github.com/grafana/grafana/pkg/plugins/pfs"
+)
+
+const prefix = "github.com/grafana/grafana/public/app/plugins"
+
+// PluginTreeListJenny creates a [codejen.ManyToOne] that produces Go code
+// for loading a [pfs.TreeList] given [*kindsys.PluginDecl] as inputs.
+func PluginTreeListJenny() codejen.ManyToOne[*pfs.PluginDecl] {
+	outputFile := filepath.Join("pkg", "plugins", "pfs", "corelist", "corelist_load_gen.go")
+
+	return &ptlJenny{
+		outputFile: outputFile,
+		plugins:    make(map[string]bool, 0),
+	}
+}
+
+type ptlJenny struct {
+	outputFile string
+	plugins    map[string]bool
+}
+
+func (j *ptlJenny) JennyName() string {
+	return "PluginTreeListJenny"
+}
+
+func (j *ptlJenny) Generate(decls ...*pfs.PluginDecl) (*codejen.File, error) {
+	buf := new(bytes.Buffer)
+	vars := templateVars_plugin_registry{
+		Plugins: make([]struct {
+			PkgName, Path, ImportPath string
+			NoAlias                   bool
+		}, 0, len(decls)),
+	}
+
+	type tpl struct {
+		PkgName, Path, ImportPath string
+		NoAlias                   bool
+	}
+
+	for _, decl := range decls {
+		meta := decl.PluginMeta
+
+		if _, exists := j.plugins[meta.Id]; exists {
+			continue
+		}
+
+		pluginId := j.sanitizePluginId(meta.Id)
+		vars.Plugins = append(vars.Plugins, tpl{
+			PkgName:    pluginId,
+			NoAlias:    pluginId != filepath.Base(decl.PluginPath),
+			ImportPath: filepath.ToSlash(filepath.Join(prefix, decl.PluginPath)),
+			Path:       path.Join(append(strings.Split(prefix, "/")[3:], decl.PluginPath)...),
+		})
+
+		j.plugins[meta.Id] = true
+	}
+
+	if err := tmpls.Lookup("plugin_registry.tmpl").Execute(buf, vars); err != nil {
+		return nil, fmt.Errorf("failed executing plugin registry template: %w", err)
+	}
+
+	byt, err := postprocessGoFile(genGoFile{
+		path: j.outputFile,
+		in:   buf.Bytes(),
+	})
+	if err != nil {
+		return nil, fmt.Errorf("error postprocessing plugin registry: %w", err)
+	}
+
+	return codejen.NewFile(j.outputFile, byt, j), nil
+}
+
+func (j *ptlJenny) sanitizePluginId(s string) string {
+	return strings.Map(func(r rune) rune {
+		switch {
+		case r >= 'a' && r <= 'z':
+			fallthrough
+		case r >= 'A' && r <= 'Z':
+			fallthrough
+		case r >= '0' && r <= '9':
+			fallthrough
+		case r == '_':
+			return r
+		case r == '-':
+			return '_'
+		default:
+			return -1
+		}
+	}, s)
+}

pkg/plugins/codegen/jenny_plugintstypes.go

@@ -0,0 +1,64 @@
+package codegen
+
+import (
+	"fmt"
+	"path/filepath"
+
+	"github.com/grafana/codejen"
+	tsast "github.com/grafana/cuetsy/ts/ast"
+	"github.com/grafana/grafana/pkg/plugins/pfs"
+)
+
+func PluginTSTypesJenny(root string, inner codejen.OneToOne[*pfs.PluginDecl]) codejen.OneToOne[*pfs.PluginDecl] {
+	return &ptsJenny{
+		root:  root,
+		inner: inner,
+	}
+}
+
+type ptsJenny struct {
+	root  string
+	inner codejen.OneToOne[*pfs.PluginDecl]
+}
+
+func (j *ptsJenny) JennyName() string {
+	return "PluginTSTypesJenny"
+}
+
+func (j *ptsJenny) Generate(decl *pfs.PluginDecl) (*codejen.File, error) {
+	if !decl.HasSchema() {
+		return nil, nil
+	}
+
+	tsf := &tsast.File{}
+
+	for _, im := range decl.Imports {
+		if tsim, err := convertImport(im); err != nil {
+			return nil, err
+		} else if tsim.From.Value != "" {
+			tsf.Imports = append(tsf.Imports, tsim)
+		}
+	}
+
+	slotname := decl.Slot.Name()
+	v := decl.Lineage.Latest().Version()
+
+	tsf.Nodes = append(tsf.Nodes, tsast.Raw{
+		Data: fmt.Sprintf("export const %sModelVersion = Object.freeze([%v, %v]);", slotname, v[0], v[1]),
+	})
+
+	jf, err := j.inner.Generate(decl)
+	if err != nil {
+		return nil, err
+	}
+
+	tsf.Nodes = append(tsf.Nodes, tsast.Raw{
+		Data: string(jf.Data),
+	})
+
+	path := filepath.Join(j.root, decl.PluginPath, "models.gen.ts")
+	data := []byte(tsf.String())
+	data = data[:len(data)-1] // remove the additional line break added by the inner jenny
+
+	return codejen.NewFile(path, data, append(jf.From, j)...), nil
+}

pkg/plugins/codegen/tmpl.go

@@ -0,0 +1,33 @@
+package codegen
+
+import (
+	"embed"
+	"text/template"
+	"time"
+)
+
+// All the parsed templates in the tmpl subdirectory
+var tmpls *template.Template
+
+func init() {
+	base := template.New("codegen").Funcs(template.FuncMap{
+		"now": time.Now,
+	})
+	tmpls = template.Must(base.ParseFS(tmplFS, "tmpl/*.tmpl"))
+}
+
+//go:embed tmpl/*.tmpl
+var tmplFS embed.FS
+
+// The following group of types, beginning with templateVars_*, all contain the set
+// of variables expected by the corresponding named template file under tmpl/
+type (
+	templateVars_plugin_registry struct {
+		Plugins []struct {
+			PkgName    string
+			Path       string
+			ImportPath string
+			NoAlias    bool
+		}
+	}
+)

pkg/plugins/codegen/tmpl/plugin_registry.tmpl

@@ -0,0 +1,30 @@
+package corelist
+
+import (
+  "fmt"
+  "io/fs"
+  "sync"
+  "github.com/grafana/grafana"
+	"github.com/grafana/grafana/pkg/plugins/pfs"
+	"github.com/grafana/thema"
+)
+
+func makeTreeOrPanic(path string, pkgname string, rt *thema.Runtime) *pfs.Tree {
+  sub, err := fs.Sub(grafana.CueSchemaFS, path)
+  if err != nil {
+    panic("could not create fs sub to " + path)
+  }
+  tree, err := pfs.ParsePluginFS(sub, rt)
+  if err != nil {
+    panic(fmt.Sprintf("error parsing plugin metadata for %s: %s", pkgname, err))
+  }
+  return tree
+}
+
+func coreTreeList(rt *thema.Runtime) pfs.TreeList{
+  return pfs.TreeList{
+    {{- range .Plugins }}
+    makeTreeOrPanic("{{ .Path }}", "{{ .PkgName }}", rt),
+    {{- end }}
+  }
+}

pkg/plugins/codegen/util_go.go

@@ -0,0 +1,67 @@
+package codegen
+
+import (
+	"bytes"
+	"fmt"
+	"go/format"
+	"go/parser"
+	"go/token"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"golang.org/x/tools/go/ast/astutil"
+	"golang.org/x/tools/imports"
+)
+
+type genGoFile struct {
+	path   string
+	walker astutil.ApplyFunc
+	in     []byte
+}
+
+func postprocessGoFile(cfg genGoFile) ([]byte, error) {
+	fname := filepath.Base(cfg.path)
+	buf := new(bytes.Buffer)
+	fset := token.NewFileSet()
+	gf, err := parser.ParseFile(fset, fname, string(cfg.in), parser.ParseComments)
+	if err != nil {
+		return nil, fmt.Errorf("error parsing generated file: %w", err)
+	}
+
+	if cfg.walker != nil {
+		astutil.Apply(gf, cfg.walker, nil)
+
+		err = format.Node(buf, fset, gf)
+		if err != nil {
+			return nil, fmt.Errorf("error formatting Go AST: %w", err)
+		}
+	} else {
+		buf = bytes.NewBuffer(cfg.in)
+	}
+
+	byt, err := imports.Process(fname, buf.Bytes(), nil)
+	if err != nil {
+		return nil, fmt.Errorf("goimports processing failed: %w", err)
+	}
+
+	// Compare imports before and after; warn about performance if some were added
+	gfa, _ := parser.ParseFile(fset, fname, string(byt), parser.ParseComments)
+	imap := make(map[string]bool)
+	for _, im := range gf.Imports {
+		imap[im.Path.Value] = true
+	}
+	var added []string
+	for _, im := range gfa.Imports {
+		if !imap[im.Path.Value] {
+			added = append(added, im.Path.Value)
+		}
+	}
+
+	if len(added) != 0 {
+		// TODO improve the guidance in this error if/when we better abstract over imports to generate
+		fmt.Fprintf(os.Stderr, "The following imports were added by goimports while generating %s: \n\t%s\nRelying on goimports to find imports significantly slows down code generation. Consider adding these to the relevant template.\n", cfg.path, strings.Join(added, "\n\t"))
+	}
+
+	return byt, nil
+}

pkg/plugins/codegen/util_ts.go

@@ -0,0 +1,73 @@
+package codegen
+
+import (
+	"fmt"
+	"sort"
+	"strings"
+
+	"cuelang.org/go/cue/ast"
+	tsast "github.com/grafana/cuetsy/ts/ast"
+	"github.com/grafana/grafana/pkg/plugins/pfs"
+)
+
+// CUE import paths, mapped to corresponding TS import paths. An empty value
+// indicates the import path should be dropped in the conversion to TS. Imports
+// not present in the list are not not allowed, and code generation will fail.
+var importMap = map[string]string{
+	"github.com/grafana/thema":                                      "",
+	"github.com/grafana/grafana/packages/grafana-schema/src/schema": "@grafana/schema",
+}
+
+func init() {
+	allow := pfs.PermittedCUEImports()
+	strsl := make([]string, 0, len(importMap))
+	for p := range importMap {
+		strsl = append(strsl, p)
+	}
+
+	sort.Strings(strsl)
+	sort.Strings(allow)
+	if strings.Join(strsl, "") != strings.Join(allow, "") {
+		panic("CUE import map is not the same as permitted CUE import list - these files must be kept in sync!")
+	}
+}
+
+// mapCUEImportToTS maps the provided CUE import path to the corresponding
+// TypeScript import path in generated code.
+//
+// Providing an import path that is not allowed results in an error. If a nil
+// error and empty string are returned, the import path should be dropped in
+// generated code.
+func mapCUEImportToTS(path string) (string, error) {
+	i, has := importMap[path]
+	if !has {
+		return "", fmt.Errorf("import %q in models.cue is not allowed", path)
+	}
+	return i, nil
+}
+
+// TODO convert this to use cuetsy ts types, once import * form is supported
+func convertImport(im *ast.ImportSpec) (tsast.ImportSpec, error) {
+	tsim := tsast.ImportSpec{}
+	pkg, err := mapCUEImportToTS(strings.Trim(im.Path.Value, "\""))
+	if err != nil || pkg == "" {
+		// err should be unreachable if paths has been verified already
+		// Empty string mapping means skip it
+		return tsim, err
+	}
+
+	tsim.From = tsast.Str{Value: pkg}
+
+	if im.Name != nil && im.Name.String() != "" {
+		tsim.AsName = im.Name.String()
+	} else {
+		sl := strings.Split(im.Path.Value, "/")
+		final := sl[len(sl)-1]
+		if idx := strings.Index(final, ":"); idx != -1 {
+			tsim.AsName = final[idx:]
+		} else {
+			tsim.AsName = final
+		}
+	}
+	return tsim, nil
+}