mirror of
https://github.com/grafana/grafana.git
synced 2025-07-31 17:22:09 +08:00
Schemas: Replace registry generation and github workflow (#83490)
* Create small registries for core and composable kinds * Update workflow with new registries * Fix imports in plugin schemas and deleted old registry generation files * Remove verification and maturity * Modify registries and add missing composable information to make schemas in kind-registry work * Add missing aliases * Remove unused templates * Remove kinds verification * Format generated code * Add gen header * Delete unused code and clean path in composable template * Delete kind-registry loader * Delete unused code * Update License link * Update codeowners path * Sort imports * More cleanup * Remove verify-kinds.yml from codeowners * Fix lint * Update composable_kidns * Fix cue extension * Restore verify-kinds to avoid to push outdated kind's registry * Fix composable format * Restore code owners for verify-kinds * Remove verify check
This commit is contained in:
Selene
74
pkg/plugins/codegen/jenny_plugin_registry.go
Normal file
74
pkg/plugins/codegen/jenny_plugin_registry.go
Normal file
@ -0,0 +1,74 @@
|
||||
package codegen
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"go/format"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/grafana/codejen"
|
||||
)
|
||||
|
||||
var registryPath = filepath.Join("pkg", "registry", "schemas")
|
||||
|
||||
var renamedPlugins = map[string]string{
|
||||
"cloud-monitoring": "googlecloudmonitoring",
|
||||
"grafana-pyroscope-datasource": "grafanapyroscope",
|
||||
"annolist": "annotationslist",
|
||||
"grafanatestdatadatasource": "testdata",
|
||||
"dashlist": "dashboardlist",
|
||||
}
|
||||
|
||||
type PluginRegistryJenny struct {
|
||||
}
|
||||
|
||||
func (jenny *PluginRegistryJenny) JennyName() string {
|
||||
return "PluginRegistryJenny"
|
||||
}
|
||||
|
||||
func (jenny *PluginRegistryJenny) Generate(files []string) (*codejen.File, error) {
|
||||
if len(files) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
schemas := make([]Schema, len(files))
|
||||
for i, file := range files {
|
||||
name, err := getSchemaName(file)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("unable to find schema name: %s", err)
|
||||
}
|
||||
|
||||
schemas[i] = Schema{
|
||||
Name: name,
|
||||
Filename: filepath.Base(file),
|
||||
FilePath: file,
|
||||
}
|
||||
}
|
||||
|
||||
buf := new(bytes.Buffer)
|
||||
if err := tmpls.Lookup("composable_registry.tmpl").Execute(buf, tmpl_vars_plugin_registry{
|
||||
Schemas: schemas,
|
||||
}); err != nil {
|
||||
return nil, fmt.Errorf("failed executing kind registry template: %w", err)
|
||||
}
|
||||
|
||||
b, err := format.Source(buf.Bytes())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return codejen.NewFile(filepath.Join(registryPath, "composable_kind.go"), b, jenny), nil
|
||||
}
|
||||
|
||||
func getSchemaName(path string) (string, error) {
|
||||
parts := strings.Split(path, "/")
|
||||
if len(parts) < 2 {
|
||||
return "", fmt.Errorf("path should contain more than 2 elements")
|
||||
}
|
||||
folderName := parts[len(parts)-2]
|
||||
if renamed, ok := renamedPlugins[folderName]; ok {
|
||||
folderName = renamed
|
||||
}
|
||||
folderName = strings.ReplaceAll(folderName, "-", "")
|
||||
return strings.ToLower(folderName), nil
|
||||
}
|
||||
@ -1,100 +0,0 @@
|
||||
package codegen
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/grafana/codejen"
|
||||
"github.com/grafana/grafana/pkg/plugins/pfs"
|
||||
)
|
||||
|
||||
const prefix = "github.com/grafana/grafana/public/app/plugins"
|
||||
|
||||
// PluginTreeListJenny creates a [codejen.ManyToOne] that produces Go code
|
||||
// for loading a [pfs.PluginList] given [*kindsys.PluginDecl] as inputs.
|
||||
func PluginTreeListJenny() codejen.ManyToOne[*pfs.PluginDecl] {
|
||||
outputFile := filepath.Join("pkg", "plugins", "pfs", "corelist", "corelist_load_gen.go")
|
||||
|
||||
return &ptlJenny{
|
||||
outputFile: outputFile,
|
||||
plugins: make(map[string]bool, 0),
|
||||
}
|
||||
}
|
||||
|
||||
type ptlJenny struct {
|
||||
outputFile string
|
||||
plugins map[string]bool
|
||||
}
|
||||
|
||||
func (j *ptlJenny) JennyName() string {
|
||||
return "PluginTreeListJenny"
|
||||
}
|
||||
|
||||
func (j *ptlJenny) Generate(decls ...*pfs.PluginDecl) (*codejen.File, error) {
|
||||
buf := new(bytes.Buffer)
|
||||
vars := templateVars_plugin_registry{
|
||||
Plugins: make([]struct {
|
||||
PkgName, Path, ImportPath string
|
||||
NoAlias bool
|
||||
}, 0, len(decls)),
|
||||
}
|
||||
|
||||
type tpl struct {
|
||||
PkgName, Path, ImportPath string
|
||||
NoAlias bool
|
||||
}
|
||||
|
||||
for _, decl := range decls {
|
||||
meta := decl.PluginMeta
|
||||
|
||||
if _, exists := j.plugins[meta.Id]; exists {
|
||||
continue
|
||||
}
|
||||
|
||||
pluginId := j.sanitizePluginId(meta.Id)
|
||||
vars.Plugins = append(vars.Plugins, tpl{
|
||||
PkgName: pluginId,
|
||||
NoAlias: pluginId != filepath.Base(decl.PluginPath),
|
||||
ImportPath: filepath.ToSlash(filepath.Join(prefix, decl.PluginPath)),
|
||||
Path: path.Join(append(strings.Split(prefix, "/")[3:], decl.PluginPath)...),
|
||||
})
|
||||
|
||||
j.plugins[meta.Id] = true
|
||||
}
|
||||
|
||||
if err := tmpls.Lookup("plugin_registry.tmpl").Execute(buf, vars); err != nil {
|
||||
return nil, fmt.Errorf("failed executing plugin registry template: %w", err)
|
||||
}
|
||||
|
||||
byt, err := postprocessGoFile(genGoFile{
|
||||
path: j.outputFile,
|
||||
in: buf.Bytes(),
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error postprocessing plugin registry: %w", err)
|
||||
}
|
||||
|
||||
return codejen.NewFile(j.outputFile, byt, j), nil
|
||||
}
|
||||
|
||||
func (j *ptlJenny) sanitizePluginId(s string) string {
|
||||
return strings.Map(func(r rune) rune {
|
||||
switch {
|
||||
case r >= 'a' && r <= 'z':
|
||||
fallthrough
|
||||
case r >= 'A' && r <= 'Z':
|
||||
fallthrough
|
||||
case r >= '0' && r <= '9':
|
||||
fallthrough
|
||||
case r == '_':
|
||||
return r
|
||||
case r == '-':
|
||||
return '_'
|
||||
default:
|
||||
return -1
|
||||
}
|
||||
}, s)
|
||||
}
|
||||
@ -22,12 +22,13 @@ var tmplFS embed.FS
|
||||
// The following group of types, beginning with templateVars_*, all contain the set
|
||||
// of variables expected by the corresponding named template file under tmpl/
|
||||
type (
|
||||
templateVars_plugin_registry struct {
|
||||
Plugins []struct {
|
||||
PkgName string
|
||||
Path string
|
||||
ImportPath string
|
||||
NoAlias bool
|
||||
}
|
||||
tmpl_vars_plugin_registry struct {
|
||||
Schemas []Schema
|
||||
}
|
||||
|
||||
Schema struct {
|
||||
Name string
|
||||
Filename string
|
||||
FilePath string
|
||||
}
|
||||
)
|
||||
|
||||
132
pkg/plugins/codegen/tmpl/composable_registry.tmpl
Normal file
132
pkg/plugins/codegen/tmpl/composable_registry.tmpl
Normal file
@ -0,0 +1,132 @@
|
||||
package schemas
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"testing/fstest"
|
||||
|
||||
"cuelang.org/go/cue"
|
||||
"cuelang.org/go/cue/cuecontext"
|
||||
"cuelang.org/go/cue/load"
|
||||
)
|
||||
|
||||
var cueImportsPath = filepath.Join("packages", "grafana-schema", "src", "common")
|
||||
var importPath = "github.com/grafana/grafana/packages/grafana-schema/src/common"
|
||||
|
||||
type ComposableKind struct {
|
||||
Name string
|
||||
Filename string
|
||||
CueFile cue.Value
|
||||
}
|
||||
|
||||
func GetComposableKinds() ([]ComposableKind, error) {
|
||||
kinds := make([]ComposableKind, 0)
|
||||
|
||||
_, caller, _, _ := runtime.Caller(0)
|
||||
root := filepath.Join(caller, "../../../..")
|
||||
|
||||
{{- range .Schemas }}
|
||||
|
||||
{{ .Name }}Cue, err := loadCueFileWithCommon(root, filepath.Join(root, "{{ .FilePath }}"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
kinds = append(kinds, ComposableKind{
|
||||
Name: "{{ .Name }}",
|
||||
Filename: "{{ .Filename }}",
|
||||
CueFile: {{ .Name }}Cue,
|
||||
})
|
||||
{{- end }}
|
||||
|
||||
return kinds, nil
|
||||
}
|
||||
|
||||
func loadCueFileWithCommon(root string, entrypoint string) (cue.Value, error) {
|
||||
commonFS, err := mockCommonFS(root)
|
||||
if err != nil {
|
||||
fmt.Printf("cannot load common cue files: %s\n", err)
|
||||
return cue.Value{}, err
|
||||
}
|
||||
|
||||
overlay, err := buildOverlay(commonFS)
|
||||
if err != nil {
|
||||
fmt.Printf("Cannot build overlay: %s\n", err)
|
||||
return cue.Value{}, err
|
||||
}
|
||||
|
||||
bis := load.Instances([]string{entrypoint}, &load.Config{
|
||||
ModuleRoot: "/",
|
||||
Overlay: overlay,
|
||||
})
|
||||
|
||||
values, err := cuecontext.New().BuildInstances(bis)
|
||||
if err != nil {
|
||||
fmt.Printf("Cannot build instance: %s\n", err)
|
||||
return cue.Value{}, err
|
||||
}
|
||||
|
||||
return values[0], nil
|
||||
}
|
||||
|
||||
func mockCommonFS(root string) (fs.FS, error) {
|
||||
path := filepath.Join(root, cueImportsPath)
|
||||
dir, err := os.ReadDir(path)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cannot open common cue files directory: %s", err)
|
||||
}
|
||||
|
||||
prefix := "cue.mod/pkg/" + importPath
|
||||
|
||||
commonFS := fstest.MapFS{}
|
||||
for _, d := range dir {
|
||||
if d.IsDir() {
|
||||
continue
|
||||
}
|
||||
|
||||
readPath := filepath.Join(path, d.Name())
|
||||
b, err := os.ReadFile(filepath.Clean(readPath))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
commonFS[filepath.Join(prefix, d.Name())] = &fstest.MapFile{Data: b}
|
||||
}
|
||||
|
||||
return commonFS, nil
|
||||
}
|
||||
|
||||
// It loads common cue files into the schema to be able to make import works
|
||||
func buildOverlay(commonFS fs.FS) (map[string]load.Source, error) {
|
||||
overlay := make(map[string]load.Source)
|
||||
|
||||
err := fs.WalkDir(commonFS, ".", func(path string, d fs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if d.IsDir() {
|
||||
return nil
|
||||
}
|
||||
|
||||
f, err := commonFS.Open(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() { _ = f.Close() }()
|
||||
|
||||
b, err := io.ReadAll(f)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
overlay[filepath.Join("/", path)] = load.FromBytes(b)
|
||||
|
||||
return nil
|
||||
})
|
||||
|
||||
return overlay, err
|
||||
}
|
||||
@ -1,30 +0,0 @@
|
||||
package corelist
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"sync"
|
||||
"github.com/grafana/grafana"
|
||||
"github.com/grafana/grafana/pkg/plugins/pfs"
|
||||
"github.com/grafana/thema"
|
||||
)
|
||||
|
||||
func parsePluginOrPanic(path string, pkgname string, rt *thema.Runtime) pfs.ParsedPlugin {
|
||||
sub, err := fs.Sub(grafana.CueSchemaFS, path)
|
||||
if err != nil {
|
||||
panic("could not create fs sub to " + path)
|
||||
}
|
||||
pp, err := pfs.ParsePluginFS(sub, rt)
|
||||
if err != nil {
|
||||
panic(fmt.Sprintf("error parsing plugin metadata for %s: %s", pkgname, err))
|
||||
}
|
||||
return pp
|
||||
}
|
||||
|
||||
func corePlugins(rt *thema.Runtime) []pfs.ParsedPlugin{
|
||||
return []pfs.ParsedPlugin{
|
||||
{{- range .Plugins }}
|
||||
parsePluginOrPanic("{{ .Path }}", "{{ .PkgName }}", rt),
|
||||
{{- end }}
|
||||
}
|
||||
}
|
||||
@ -1,68 +0,0 @@
|
||||
package codegen
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"go/format"
|
||||
"go/parser"
|
||||
"go/token"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/dave/dst/decorator"
|
||||
"github.com/dave/dst/dstutil"
|
||||
"golang.org/x/tools/imports"
|
||||
)
|
||||
|
||||
type genGoFile struct {
|
||||
path string
|
||||
walker dstutil.ApplyFunc
|
||||
in []byte
|
||||
}
|
||||
|
||||
func postprocessGoFile(cfg genGoFile) ([]byte, error) {
|
||||
fname := filepath.Base(cfg.path)
|
||||
buf := new(bytes.Buffer)
|
||||
fset := token.NewFileSet()
|
||||
gf, err := decorator.ParseFile(fset, fname, string(cfg.in), parser.ParseComments)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error parsing generated file: %w", err)
|
||||
}
|
||||
|
||||
if cfg.walker != nil {
|
||||
dstutil.Apply(gf, cfg.walker, nil)
|
||||
|
||||
err = format.Node(buf, fset, gf)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error formatting Go AST: %w", err)
|
||||
}
|
||||
} else {
|
||||
buf = bytes.NewBuffer(cfg.in)
|
||||
}
|
||||
|
||||
byt, err := imports.Process(fname, buf.Bytes(), nil)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("goimports processing failed: %w", err)
|
||||
}
|
||||
|
||||
// Compare imports before and after; warn about performance if some were added
|
||||
gfa, _ := parser.ParseFile(fset, fname, string(byt), parser.ParseComments)
|
||||
imap := make(map[string]bool)
|
||||
for _, im := range gf.Imports {
|
||||
imap[im.Path.Value] = true
|
||||
}
|
||||
var added []string
|
||||
for _, im := range gfa.Imports {
|
||||
if !imap[im.Path.Value] {
|
||||
added = append(added, im.Path.Value)
|
||||
}
|
||||
}
|
||||
|
||||
if len(added) != 0 {
|
||||
// TODO improve the guidance in this error if/when we better abstract over imports to generate
|
||||
fmt.Fprintf(os.Stderr, "The following imports were added by goimports while generating %s: \n\t%s\nRelying on goimports to find imports significantly slows down code generation. Consider adding these to the relevant template.\n", cfg.path, strings.Join(added, "\n\t"))
|
||||
}
|
||||
|
||||
return byt, nil
|
||||
}
|
||||
Reference in New Issue
Block a user