Make validate-schema CLI command work vs. disk (#36863)

* Make validate-schema CLI command work vs. disk

Previously, `grafana-cli cue validate-schema` would only validate the
CUE files that had been included at compile time. Now, the command
requires passing a grafana checkout root path, and will actually check
against live files on disk.

* Exempt validateScuemata from sec linter

* Nit: validateScuemata is a simpler, better name

* Try to fix validate-scuemata build step

.drone.yml

@@ -98,7 +98,7 @@ steps:
 - name: validate-scuemata
   image: grafana/build-container:1.4.1
   commands:
-  - ./bin/linux-amd64/grafana-cli cue validate-schema
+  - ./bin/linux-amd64/grafana-cli cue validate-schema --grafana-root .
   depends_on:
   - build-backend
 
@@ -354,7 +354,7 @@ steps:
 - name: validate-scuemata
   image: grafana/build-container:1.4.1
   commands:
-  - ./bin/linux-amd64/grafana-cli cue validate-schema
+  - ./bin/linux-amd64/grafana-cli cue validate-schema --grafana-root .
   depends_on:
   - build-backend
 
@@ -811,7 +811,7 @@ steps:
 - name: validate-scuemata
   image: grafana/build-container:1.4.1
   commands:
-  - ./bin/linux-amd64/grafana-cli cue validate-schema
+  - ./bin/linux-amd64/grafana-cli cue validate-schema --grafana-root .
   depends_on:
   - build-backend
 
@@ -1186,7 +1186,7 @@ steps:
 - name: validate-scuemata
   image: grafana/build-container:1.4.1
   commands:
-  - ./bin/linux-amd64/grafana-cli cue validate-schema
+  - ./bin/linux-amd64/grafana-cli cue validate-schema --grafana-root .
   depends_on:
   - build-backend
 
@@ -1765,7 +1765,7 @@ steps:
 - name: validate-scuemata
   image: grafana/build-container:1.4.1
   commands:
-  - ./bin/linux-amd64/grafana-cli cue validate-schema
+  - ./bin/linux-amd64/grafana-cli cue validate-schema --grafana-root .
   depends_on:
   - build-backend
 
@@ -2129,7 +2129,7 @@ steps:
 - name: validate-scuemata
   image: grafana/build-container:1.4.1
   commands:
-  - ./bin/linux-amd64/grafana-cli cue validate-schema
+  - ./bin/linux-amd64/grafana-cli cue validate-schema --grafana-root .
   depends_on:
   - build-backend
 
@@ -2698,7 +2698,7 @@ steps:
 - name: validate-scuemata
   image: grafana/build-container:1.4.1
   commands:
-  - ./bin/linux-amd64/grafana-cli cue validate-schema
+  - ./bin/linux-amd64/grafana-cli cue validate-schema --grafana-root .
   depends_on:
   - build-backend
 
@@ -3033,7 +3033,7 @@ steps:
 - name: validate-scuemata
   image: grafana/build-container:1.4.1
   commands:
-  - ./bin/linux-amd64/grafana-cli cue validate-schema
+  - ./bin/linux-amd64/grafana-cli cue validate-schema --grafana-root .
   depends_on:
   - build-backend
 

pkg/cmd/grafana-cli/commands/commands.go

@@ -137,8 +137,18 @@ var adminCommands = []*cli.Command{
 var cueCommands = []*cli.Command{
 	{
 		Name:   "validate-schema",
-		Usage:  "validate *.cue files in the project",
+		Usage:  "validate known *.cue files in the Grafana project",
-		Action: runPluginCommand(cmd.validateScuemataBasics),
+		Action: runPluginCommand(cmd.validateScuemata),
+		Description: `validate-schema checks that all CUE schema files are valid with respect
+to basic standards - valid CUE, valid scuemata, etc. Note that this
+command checks only paths that existed when grafana-cli was compiled,
+so must be recompiled to validate newly-added CUE files.`,
+		Flags: []cli.Flag{
+			&cli.StringFlag{
+				Name:  "grafana-root",
+				Usage: "path to the root of a Grafana repository to validate",
+			},
+		},
 	},
 	{
 		Name:   "validate-resource",

pkg/cmd/grafana-cli/commands/scuemata_validation_command.go

@@ -3,8 +3,11 @@ package commands
 import (
 	gerrors "errors"
 	"fmt"
+	"io"
+	"io/fs"
 	"os"
 	"path/filepath"
+	"testing/fstest"
 
 	"cuelang.org/go/cue/errors"
 	"github.com/grafana/grafana/pkg/cmd/grafana-cli/utils"
@@ -14,13 +17,59 @@ import (
 
 var paths = load.GetDefaultLoadPaths()
 
-func (cmd Command) validateScuemataBasics(c utils.CommandLine) error {
+func (cmd Command) validateScuemata(c utils.CommandLine) error {
-	if err := validateScuemata(paths, load.BaseDashboardFamily); err != nil {
+	root := c.String("grafana-root")
+	if root == "" {
+		return gerrors.New("must provide path to the root of a Grafana repository checkout")
+	}
+
+	// Construct a MapFS with the same set of files as those embedded in
+	// /embed.go, but sourced straight through from disk instead of relying on
+	// what's compiled.  Not the greatest, because we're duplicating
+	// filesystem-loading logic with what's in /embed.go.
+
+	populate := func(in fs.FS, join string) (fs.FS, error) {
+		out := make(fstest.MapFS)
+		err := fs.WalkDir(in, ".", func(path string, d fs.DirEntry, err error) error {
+			if err != nil {
+				return err
+			}
+
+			if d.IsDir() {
+				return nil
+			}
+			// Ignore gosec warning G304. The input set here is necessarily
+			// constrained to files specified in embed.go
+			// nolint:gosec
+			b, err := os.Open(filepath.Join(root, join, path))
+			if err != nil {
+				return err
+			}
+			byt, err := io.ReadAll(b)
+			if err != nil {
+				return err
+			}
+
+			out[path] = &fstest.MapFile{Data: byt}
+			return nil
+		})
+		return out, err
+	}
+
+	var fspaths load.BaseLoadPaths
+	var err error
+
+	fspaths.BaseCueFS, err = populate(paths.BaseCueFS, "")
+	if err != nil {
+		return err
+	}
+	fspaths.DistPluginCueFS, err = populate(paths.DistPluginCueFS, "public/app/plugins")
+	if err != nil {
 		return err
 	}
 
-	if err := validateScuemata(paths, load.DistDashboardFamily); err != nil {
+	if err := validateScuemata(fspaths, load.DistDashboardFamily); err != nil {
-		return err
+		return schema.WrapCUEError(err)
 	}
 
 	return nil

scripts/lib.star

@@ -1076,6 +1076,6 @@ def validate_scuemata():
             'build-backend',
         ],
         'commands': [
-            './bin/linux-amd64/grafana-cli cue validate-schema',
+            './bin/linux-amd64/grafana-cli cue validate-schema --grafana-root .',
         ],
     }