QueryTypes: Add feature toggle to show query types in datasource apiservers (#88213)

* initial attempt

* show query types

* show query types

* with formatting

* with formatting

* more cleanup

* add feature toggle

* fix build

docs/sources/setup-grafana/configure-grafana/feature-toggles/index.md

@@ -156,6 +156,7 @@ Experimental features might be changed or removed without prior notice.
 | `idForwarding`                              | Generate signed id token for identity that can be forwarded to plugins and external services                                                                                                                                                                                      |
 | `enableNativeHTTPHistogram`                 | Enables native HTTP Histograms                                                                                                                                                                                                                                                    |
 | `kubernetesSnapshots`                       | Routes snapshot requests from /api to the /apis endpoint                                                                                                                                                                                                                          |
+| `datasourceQueryTypes`                      | Show query type endpoints in datasource API servers (currently hardcoded for testdata, expressions, and prometheus)                                                                                                                                                               |
 | `queryService`                              | Register /apis/query.grafana.app/ -- will eventually replace /api/ds/query                                                                                                                                                                                                        |
 | `queryServiceRewrite`                       | Rewrite requests targeting /ds/query to the query service                                                                                                                                                                                                                         |
 | `queryServiceFromUI`                        | Routes requests to the new query service                                                                                                                                                                                                                                          |

packages/grafana-data/src/types/featureToggles.gen.ts

@@ -122,6 +122,7 @@ export interface FeatureToggles {
   transformationsVariableSupport?: boolean;
   kubernetesPlaylists?: boolean;
   kubernetesSnapshots?: boolean;
+  datasourceQueryTypes?: boolean;
   queryService?: boolean;
   queryServiceRewrite?: boolean;
   queryServiceFromUI?: boolean;

pkg/expr/query.go

@@ -1,6 +1,8 @@
 package expr
 
 import (
+	"embed"
+
 	"github.com/grafana/grafana/pkg/expr/classic"
 	"github.com/grafana/grafana/pkg/expr/mathexp"
 )
@@ -100,3 +102,10 @@ const (
 	// Replace non-numbers
 	ReduceModeReplace ReduceMode = "replaceNN"
 )
+
+//go:embed query.types.json
+var f embed.FS
+
+func QueryTypeDefinitionListJSON() ([]byte, error) {
+	return f.ReadFile("query.types.json")
+}

pkg/expr/reader.go

@@ -1,7 +1,6 @@
 package expr
 
 import (
-	"embed"
 	"fmt"
 	"strings"
 
@@ -178,13 +177,6 @@ func (h *ExpressionQueryReader) ReadQuery(
 	return eq, err
 }
 
-//go:embed query.types.json
-var f embed.FS
-
-func (h *ExpressionQueryReader) QueryTypeDefinitionListJSON() ([]byte, error) {
-	return f.ReadFile("query.types.json")
-}
-
 func getReferenceVar(exp string, refId string) (string, error) {
 	exp = strings.TrimPrefix(exp, "$")
 	if exp == "" {

pkg/promlib/models/query.go

@@ -434,6 +434,6 @@ func AlignTimeRange(t time.Time, step time.Duration, offset int64) time.Time {
 var f embed.FS
 
 // QueryTypeDefinitionsJSON returns the query type definitions
-func QueryTypeDefinitionsJSON() (json.RawMessage, error) {
+func QueryTypeDefinitionListJSON() (json.RawMessage, error) {
 	return f.ReadFile("query.types.json")
 }

pkg/registry/apis/datasource/register.go

@@ -2,13 +2,11 @@ package datasource
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
-	"net/http"
 	"time"
 
 	"github.com/grafana/grafana-plugin-sdk-go/backend"
-	data "github.com/grafana/grafana-plugin-sdk-go/experimental/apis/data/v0alpha1"
-	"github.com/grafana/grafana-plugin-sdk-go/experimental/schemabuilder"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/runtime"
@@ -19,7 +17,6 @@ import (
 	genericapiserver "k8s.io/apiserver/pkg/server"
 	openapi "k8s.io/kube-openapi/pkg/common"
 	"k8s.io/kube-openapi/pkg/spec3"
-	"k8s.io/kube-openapi/pkg/validation/spec"
 	"k8s.io/utils/strings/slices"
 
 	common "github.com/grafana/grafana/pkg/apimachinery/apis/common/v0alpha1"
@@ -27,15 +24,15 @@ import (
 	query "github.com/grafana/grafana/pkg/apis/query/v0alpha1"
 	"github.com/grafana/grafana/pkg/apiserver/builder"
 	"github.com/grafana/grafana/pkg/plugins"
+	"github.com/grafana/grafana/pkg/promlib/models"
+	"github.com/grafana/grafana/pkg/registry/apis/query/queryschema"
 	"github.com/grafana/grafana/pkg/services/accesscontrol"
 	"github.com/grafana/grafana/pkg/services/apiserver/utils"
 	"github.com/grafana/grafana/pkg/services/featuremgmt"
 	"github.com/grafana/grafana/pkg/services/pluginsintegration/pluginstore"
+	"github.com/grafana/grafana/pkg/tsdb/grafana-testdata-datasource/kinds"
 )
 
-const QueryRequestSchemaKey = "QueryRequestSchema"
-const QueryPayloadSchemaKey = "QueryPayloadSchema"
-
 var _ builder.APIGroupBuilder = (*DataSourceAPIBuilder)(nil)
 
 // DataSourceAPIBuilder is used just so wire has something unique to return
@@ -83,6 +80,7 @@ func RegisterAPIService(
 			datasources.GetDatasourceProvider(ds.JSONData),
 			contextProvider,
 			accessControl,
+			features.IsEnabledGlobally(featuremgmt.FlagDatasourceQueryTypes),
 		)
 		if err != nil {
 			return nil, err
@@ -105,20 +103,48 @@ func NewDataSourceAPIBuilder(
 	client PluginClient,
 	datasources PluginDatasourceProvider,
 	contextProvider PluginContextWrapper,
-	accessControl accesscontrol.AccessControl) (*DataSourceAPIBuilder, error) {
+	accessControl accesscontrol.AccessControl,
+	loadQueryTypes bool,
+) (*DataSourceAPIBuilder, error) {
 	ri, err := resourceFromPluginID(plugin.ID)
 	if err != nil {
 		return nil, err
 	}
 
-	return &DataSourceAPIBuilder{
+	builder := &DataSourceAPIBuilder{
 		connectionResourceInfo: ri,
 		pluginJSON:             plugin,
 		client:                 client,
 		datasources:            datasources,
 		contextProvider:        contextProvider,
 		accessControl:          accessControl,
-	}, nil
+	}
+	if loadQueryTypes {
+		// In the future, this will somehow come from the plugin
+		builder.queryTypes, err = getHardcodedQueryTypes(ri.GroupResource().Group)
+	}
+	return builder, err
+}
+
+// TODO -- somehow get the list from the plugin -- not hardcoded
+func getHardcodedQueryTypes(group string) (*query.QueryTypeDefinitionList, error) {
+	var err error
+	var raw json.RawMessage
+	switch group {
+	case "testdata.datasource.grafana.app":
+		raw, err = kinds.QueryTypeDefinitionListJSON()
+	case "prometheus.datasource.grafana.app":
+		raw, err = models.QueryTypeDefinitionListJSON()
+	}
+	if err != nil {
+		return nil, err
+	}
+	if raw != nil {
+		types := &query.QueryTypeDefinitionList{}
+		err = json.Unmarshal(raw, types)
+		return types, err
+	}
+	return nil, err
 }
 
 func (b *DataSourceAPIBuilder) GetGroupVersion() schema.GroupVersion {
@@ -134,6 +160,8 @@ func addKnownTypes(scheme *runtime.Scheme, gv schema.GroupVersion) {
 		// Query handler
 		&query.QueryDataRequest{},
 		&query.QueryDataResponse{},
+		&query.QueryTypeDefinition{},
+		&query.QueryTypeDefinitionList{},
 		&metav1.Status{},
 	)
 }
@@ -211,12 +239,16 @@ func (b *DataSourceAPIBuilder) GetAPIGroupInfo(
 		storage[conn.StoragePath("proxy")] = &subProxyREST{pluginJSON: b.pluginJSON}
 	}
 
+	// Register hardcoded query schemas
+	err := queryschema.RegisterQueryTypes(b.queryTypes, storage)
+
+	// Create the group info
 	apiGroupInfo := genericapiserver.NewDefaultAPIGroupInfo(
 		conn.GroupResource().Group, scheme,
 		metav1.ParameterCodec, codecs)
 
 	apiGroupInfo.VersionedResourcesStorageMap[conn.GroupVersion().Version] = storage
-	return &apiGroupInfo, nil
+	return &apiGroupInfo, err
 }
 
 func (b *DataSourceAPIBuilder) getPluginContext(ctx context.Context, uid string) (backend.PluginContext, error) {
@@ -247,66 +279,19 @@ func (b *DataSourceAPIBuilder) PostProcessOpenAPI(oas *spec3.OpenAPI) (*spec3.Op
 	// Hide the ability to list all connections across tenants
 	delete(oas.Paths.Paths, root+b.connectionResourceInfo.GroupResource().Resource)
 
-	var err error
+	// Add queries to the request properties
-	opts := schemabuilder.QuerySchemaOptions{
+	// Add queries to the request properties
-		PluginID:   []string{b.pluginJSON.ID},
+	err := queryschema.AddQueriesToOpenAPI(queryschema.OASQueryOptions{
-		QueryTypes: []data.QueryTypeDefinition{},
+		Swagger:          oas,
-		Mode:       schemabuilder.SchemaTypeQueryPayload,
+		PluginJSON:       &b.pluginJSON,
-	}
+		QueryTypes:       b.queryTypes,
-	if b.pluginJSON.AliasIDs != nil {
+		Root:             root,
-		opts.PluginID = append(opts.PluginID, b.pluginJSON.AliasIDs...)
+		QueryPath:        "namespaces/{namespace}/connections/{name}/query",
-	}
+		QueryDescription: fmt.Sprintf("Query the %s datasources", b.pluginJSON.Name),
-	if b.queryTypes != nil {
+	})
-		for _, qt := range b.queryTypes.Items {
-			// The SDK type and api type are not the same so we recreate it here
-			opts.QueryTypes = append(opts.QueryTypes, data.QueryTypeDefinition{
-				ObjectMeta: data.ObjectMeta{
-					Name: qt.Name,
-				},
-				Spec: qt.Spec,
-			})
-		}
-	}
-	oas.Components.Schemas[QueryPayloadSchemaKey], err = schemabuilder.GetQuerySchema(opts)
-	if err != nil {
-		return oas, err
-	}
-	opts.Mode = schemabuilder.SchemaTypeQueryRequest
-	oas.Components.Schemas[QueryRequestSchemaKey], err = schemabuilder.GetQuerySchema(opts)
-	if err != nil {
-		return oas, err
-	}
-
-	// Update the request object
-	sub := oas.Paths.Paths[root+"namespaces/{namespace}/connections/{name}/query"]
-	if sub != nil && sub.Post != nil {
-		sub.Post.Description = "Execute queries"
-		sub.Post.RequestBody = &spec3.RequestBody{
-			RequestBodyProps: spec3.RequestBodyProps{
-				Required: true,
-				Content: map[string]*spec3.MediaType{
-					"application/json": {
-						MediaTypeProps: spec3.MediaTypeProps{
-							Schema:   spec.RefSchema("#/components/schemas/" + QueryRequestSchemaKey),
-							Examples: getExamples(b.queryTypes),
-						},
-					},
-				},
-			},
-		}
-		okrsp, ok := sub.Post.Responses.StatusCodeResponses[200]
-		if ok {
-			sub.Post.Responses.StatusCodeResponses[http.StatusMultiStatus] = &spec3.Response{
-				ResponseProps: spec3.ResponseProps{
-					Description: "Query executed, but errors may exist in the datasource.  See the payload for more details.",
-					Content:     okrsp.Content,
-				},
-			}
-		}
-	}
 
 	// The root API discovery list
-	sub = oas.Paths.Paths[root]
+	sub := oas.Paths.Paths[root]
 	if sub != nil && sub.Get != nil {
 		sub.Get.Tags = []string{"API Discovery"} // sorts first in the list
 	}
@@ -317,38 +302,3 @@ func (b *DataSourceAPIBuilder) PostProcessOpenAPI(oas *spec3.OpenAPI) (*spec3.Op
 func (b *DataSourceAPIBuilder) GetAPIRoutes() *builder.APIRoutes {
 	return nil
 }
-
-func getExamples(queryTypes *query.QueryTypeDefinitionList) map[string]*spec3.Example {
-	if queryTypes == nil {
-		return nil
-	}
-
-	tr := data.TimeRange{From: "now-1h", To: "now"}
-	examples := map[string]*spec3.Example{}
-	for _, queryType := range queryTypes.Items {
-		for idx, example := range queryType.Spec.Examples {
-			q := data.NewDataQuery(example.SaveModel.Object)
-			q.RefID = "A"
-			for _, dis := range queryType.Spec.Discriminators {
-				_ = q.Set(dis.Field, dis.Value)
-			}
-			if q.MaxDataPoints < 1 {
-				q.MaxDataPoints = 1000
-			}
-			if q.IntervalMS < 1 {
-				q.IntervalMS = 5000 // 5s
-			}
-			examples[fmt.Sprintf("%s-%d", example.Name, idx)] = &spec3.Example{
-				ExampleProps: spec3.ExampleProps{
-					Summary:     example.Name,
-					Description: example.Description,
-					Value: data.QueryDataRequest{
-						TimeRange: tr,
-						Queries:   []data.DataQuery{q},
-					},
-				},
-			}
-		}
-	}
-	return examples
-}

pkg/registry/apis/query/parser.go

@@ -186,7 +186,6 @@ func (p *queryParser) parseRequest(ctx context.Context, input *query.QueryDataRe
 			}
 		}
 	}
-
 	return rsp, nil
 }
 

pkg/registry/apis/query/queryschema/oas_helper.go

@@ -0,0 +1,235 @@
+package queryschema
+
+import (
+	"fmt"
+	"strings"
+
+	"k8s.io/kube-openapi/pkg/spec3"
+	"k8s.io/kube-openapi/pkg/validation/spec"
+
+	"github.com/grafana/grafana-plugin-sdk-go/experimental/schemabuilder"
+
+	data "github.com/grafana/grafana-plugin-sdk-go/experimental/apis/data/v0alpha1"
+
+	query "github.com/grafana/grafana/pkg/apis/query/v0alpha1"
+	"github.com/grafana/grafana/pkg/plugins"
+)
+
+const QueryRequestSchemaKey = "QueryRequestSchema"
+
+// const QueryPayloadSchemaKey = "QueryPayloadSchema"
+// const QuerySaveModelSchemaKey = "QuerySaveModelSchema"
+
+type OASQueryOptions struct {
+	Swagger    *spec3.OpenAPI
+	PluginJSON *plugins.JSONData
+	QueryTypes *query.QueryTypeDefinitionList
+
+	Root             string
+	QueryPath        string // eg "namespaces/{namespace}/query/{name}"
+	QueryDescription string
+	QueryExamples    map[string]*spec3.Example
+}
+
+func AddQueriesToOpenAPI(options OASQueryOptions) error {
+	oas := options.Swagger
+	root := options.Root
+	examples := options.QueryExamples
+	resourceName := query.QueryTypeDefinitionResourceInfo.GroupResource().Resource
+
+	builder := schemabuilder.QuerySchemaOptions{
+		PluginID:   []string{""},
+		QueryTypes: []data.QueryTypeDefinition{},
+	}
+	if options.PluginJSON != nil {
+		builder.PluginID = []string{options.PluginJSON.ID}
+		if options.PluginJSON.AliasIDs != nil {
+			builder.PluginID = append(builder.PluginID, options.PluginJSON.AliasIDs...)
+		}
+	}
+	if options.QueryTypes != nil {
+		// The SDK type and api type are not the same so we just recreate it here
+		for _, qt := range options.QueryTypes.Items {
+			builder.QueryTypes = append(builder.QueryTypes, data.QueryTypeDefinition{
+				ObjectMeta: data.ObjectMeta{
+					Name: qt.Name,
+				},
+				Spec: qt.Spec,
+			})
+		}
+
+		if examples == nil {
+			examples = getExamples(options.QueryTypes)
+		}
+	}
+
+	// Rewrite the query path
+	query := oas.Paths.Paths[root+options.QueryPath]
+	if query != nil && query.Post != nil {
+		query.Post.Tags = []string{"Query"}
+		query.Parameters = []*spec3.Parameter{
+			{
+				ParameterProps: spec3.ParameterProps{
+					Name:        "namespace",
+					In:          "path",
+					Description: "object name and auth scope, such as for teams and projects",
+					Example:     "default",
+					Required:    true,
+					Schema:      spec.StringProperty().UniqueValues(),
+				},
+			},
+		}
+		query.Post.Description = options.QueryDescription
+		query.Post.Parameters = nil //
+		query.Post.RequestBody = &spec3.RequestBody{
+			RequestBodyProps: spec3.RequestBodyProps{
+				Content: map[string]*spec3.MediaType{
+					"application/json": {
+						MediaTypeProps: spec3.MediaTypeProps{
+							Schema:   spec.RefSchema("#/components/schemas/" + QueryRequestSchemaKey),
+							Examples: examples,
+						},
+					},
+				},
+			},
+		}
+
+		// Remove the {name} hack from from the query
+		if strings.HasSuffix(options.QueryPath, "/{name}") {
+			delete(oas.Paths.Paths, root+options.QueryPath)
+			oas.Paths.Paths[root+strings.TrimSuffix(options.QueryPath, "/{name}")] = query
+		}
+	}
+
+	// Update the validate endpoint
+	validate := oas.Paths.Paths[root+resourceName+"/{name}/validate"]
+	if validate != nil && validate.Post != nil {
+		validate.Post.Description = "Verify if a query payload matches the expected value and return a clean version"
+		validate.Parameters = []*spec3.Parameter{
+			{
+				ParameterProps: spec3.ParameterProps{
+					Name:        "name",
+					In:          "path",
+					Description: "The query type name, or {any}",
+					Example:     "{any}",
+					Required:    true,
+					Schema:      spec.StringProperty().UniqueValues(),
+				},
+			},
+		}
+
+		// Accept the same payload as the query type
+		validate.Post.RequestBody = query.Post.RequestBody
+	}
+
+	// Query Request
+	builder.Mode = schemabuilder.SchemaTypeQueryRequest
+	s, err := schemabuilder.GetQuerySchema(builder)
+	if err != nil {
+		return err
+	}
+
+	// The schema requires some munging to pass validation
+	// This should likely be fixed in the upstream "GetQuerySchema" function
+	removeSchemaRefs(s)
+	s.Description = "Schema for a set of queries sent to the query method"
+	oas.Components.Schemas[QueryRequestSchemaKey] = s
+
+	// // Query Payload (is this useful?)
+
+	// opts.Mode = schemabuilder.SchemaTypeQueryPayload
+	// s, err = schemabuilder.GetQuerySchema(opts)
+	// if err != nil {
+	// 	return err
+	// }
+	// delete(s.ExtraProps, "$schema")
+	// s.Description = "Schema for a single query object including all runtime properties"
+	// oas.Components.Schemas[QueryPayloadSchemaKey] = s
+
+	// // Query Save Model
+	// opts.Mode = schemabuilder.SchemaTypeSaveModel
+	// s, err = schemabuilder.GetQuerySchema(opts)
+	// if err != nil {
+	// 	return err
+	// }
+	// s.Extensions = nil // remove the $schema
+	// s.Description = "Valid save model for a single query target"
+	// oas.Components.Schemas[QuerySaveModelSchemaKey] = s
+	return nil
+}
+
+func getExamples(queryTypes *query.QueryTypeDefinitionList) map[string]*spec3.Example {
+	if queryTypes == nil {
+		return nil
+	}
+
+	tr := data.TimeRange{From: "now-1h", To: "now"}
+	examples := map[string]*spec3.Example{}
+	for _, queryType := range queryTypes.Items {
+		for idx, example := range queryType.Spec.Examples {
+			q := data.NewDataQuery(example.SaveModel.Object)
+			q.RefID = "A"
+			for _, dis := range queryType.Spec.Discriminators {
+				_ = q.Set(dis.Field, dis.Value)
+			}
+			if q.MaxDataPoints < 1 {
+				q.MaxDataPoints = 1000
+			}
+			if q.IntervalMS < 1 {
+				q.IntervalMS = 5000 // 5s
+			}
+			examples[fmt.Sprintf("%s-%d", example.Name, idx)] = &spec3.Example{
+				ExampleProps: spec3.ExampleProps{
+					Summary:     example.Name,
+					Description: example.Description,
+					Value: data.QueryDataRequest{
+						TimeRange: tr,
+						Queries:   []data.DataQuery{q},
+					},
+				},
+			}
+		}
+	}
+	return examples
+}
+
+func removeSchemaRefs(s *spec.Schema) {
+	if s == nil {
+		return
+	}
+	if s.Schema != "" {
+		s.Schema = ""
+	}
+	// Examples is invalid -- only use the first example
+	examples, ok := s.ExtraProps["examples"]
+	if ok && examples != nil {
+		//fmt.Printf("TODO, use reflection to get first element from: %+v\n", examples)
+		//s.Example = examples[0]
+		delete(s.ExtraProps, "examples")
+	}
+
+	removeSchemaRefs(s.Not)
+	for idx := range s.AllOf {
+		removeSchemaRefs(&s.AllOf[idx])
+	}
+	for idx := range s.AnyOf {
+		removeSchemaRefs(&s.AnyOf[idx])
+	}
+	for k := range s.Properties {
+		v := s.Properties[k]
+		removeSchemaRefs(&v)
+		s.Properties[k] = v
+	}
+	if s.Items != nil {
+		removeSchemaRefs(s.Items.Schema)
+		for idx := range s.Items.Schemas {
+			removeSchemaRefs(&s.Items.Schemas[idx])
+		}
+	}
+	if s.AdditionalProperties != nil {
+		removeSchemaRefs(s.AdditionalProperties.Schema)
+	}
+	if s.AdditionalItems != nil {
+		removeSchemaRefs(s.AdditionalItems.Schema)
+	}
+}

pkg/registry/apis/query/queryschema/query_type_storage.go

@@ -0,0 +1,147 @@
+package queryschema
+
+import (
+	"context"
+	"net/http"
+
+	"k8s.io/apimachinery/pkg/apis/meta/internalversion"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apiserver/pkg/registry/rest"
+
+	"github.com/grafana/grafana-plugin-sdk-go/experimental/apis/data/v0alpha1"
+
+	common "github.com/grafana/grafana/pkg/apimachinery/apis/common/v0alpha1"
+	query "github.com/grafana/grafana/pkg/apis/query/v0alpha1"
+)
+
+var (
+	_ rest.Storage              = (*queryTypeStorage)(nil)
+	_ rest.Scoper               = (*queryTypeStorage)(nil)
+	_ rest.SingularNameProvider = (*queryTypeStorage)(nil)
+	_ rest.Lister               = (*queryTypeStorage)(nil)
+	_ rest.Getter               = (*queryTypeStorage)(nil)
+
+	// The connectors
+	_ = rest.Connecter(&queryValidationREST{})
+)
+
+type queryTypeStorage struct {
+	resourceInfo   *common.ResourceInfo
+	tableConverter rest.TableConvertor
+	registry       query.QueryTypeDefinitionList
+}
+
+type queryValidationREST struct {
+	qt *queryTypeStorage
+}
+
+func RegisterQueryTypes(queryTypes *query.QueryTypeDefinitionList, storage map[string]rest.Storage) error {
+	if queryTypes == nil {
+		return nil // NO error
+	}
+
+	resourceInfo := query.QueryTypeDefinitionResourceInfo
+	store := &queryTypeStorage{
+		resourceInfo:   &resourceInfo,
+		tableConverter: rest.NewDefaultTableConvertor(resourceInfo.GroupResource()),
+		registry:       *queryTypes,
+	}
+
+	// Supports list+get for all query types
+	storage[resourceInfo.StoragePath()] = store
+
+	// Adds a query validation endpoint for each query type
+	// We will also support the "<any>" or "*" name
+	storage[resourceInfo.StoragePath("validate")] = &queryValidationREST{store}
+
+	return nil
+}
+
+func (s *queryTypeStorage) New() runtime.Object {
+	return s.resourceInfo.NewFunc()
+}
+
+func (s *queryTypeStorage) Destroy() {}
+
+func (s *queryTypeStorage) NamespaceScoped() bool {
+	return false
+}
+
+func (s *queryTypeStorage) GetSingularName() string {
+	return s.resourceInfo.GetSingularName()
+}
+
+func (s *queryTypeStorage) NewList() runtime.Object {
+	return s.resourceInfo.NewListFunc()
+}
+
+func (s *queryTypeStorage) ConvertToTable(ctx context.Context, object runtime.Object, tableOptions runtime.Object) (*metav1.Table, error) {
+	return s.tableConverter.ConvertToTable(ctx, object, tableOptions)
+}
+
+func (s *queryTypeStorage) List(ctx context.Context, options *internalversion.ListOptions) (runtime.Object, error) {
+	return &s.registry, nil
+}
+
+func (s *queryTypeStorage) Get(ctx context.Context, name string, options *metav1.GetOptions) (runtime.Object, error) {
+	for idx, qt := range s.registry.Items {
+		if qt.Name == name {
+			return &s.registry.Items[idx], nil
+		}
+	}
+	return nil, s.resourceInfo.NewNotFound(name)
+}
+
+//----------------------------------------------------
+// The validation processor
+//----------------------------------------------------
+
+func (r *queryValidationREST) New() runtime.Object {
+	return &query.QueryDataRequest{}
+}
+
+func (r *queryValidationREST) Destroy() {
+}
+
+func (r *queryValidationREST) ConnectMethods() []string {
+	return []string{"POST"}
+}
+
+func (r *queryValidationREST) ProducesMIMETypes(verb string) []string {
+	return []string{"application/json"}
+}
+
+func (r *queryValidationREST) ProducesObject(verb string) interface{} {
+	return &query.QueryDataRequest{}
+}
+
+func (r *queryValidationREST) NewConnectOptions() (runtime.Object, bool, string) {
+	return nil, false, "" // true means you can use the trailing path as a variable
+}
+
+func (r *queryValidationREST) Connect(ctx context.Context, name string, opts runtime.Object, responder rest.Responder) (http.Handler, error) {
+	return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
+		// TODO -- validate/mutate the query
+		// should we return the DQR, or raw validation response?
+		qdr := &query.QueryDataRequest{
+			QueryDataRequest: v0alpha1.QueryDataRequest{
+				Debug: true,
+			},
+		}
+
+		if name == "*" || name == "{any}" {
+			qdr.Queries = []v0alpha1.DataQuery{
+				v0alpha1.NewDataQuery(
+					map[string]any{
+						"refId": "???",
+						"HELLO": "world",
+						"TODO":  "parse any query",
+					},
+				),
+			}
+		}
+
+		responder.Object(http.StatusOK, qdr)
+	}), nil
+}

pkg/registry/apis/query/register.go

@@ -1,8 +1,8 @@
 package query
 
 import (
-	data "github.com/grafana/grafana-plugin-sdk-go/experimental/apis/data/v0alpha1"
+	"encoding/json"
-	"github.com/grafana/grafana-plugin-sdk-go/experimental/schemabuilder"
+
 	"github.com/prometheus/client_golang/prometheus"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
@@ -14,16 +14,15 @@ import (
 	genericapiserver "k8s.io/apiserver/pkg/server"
 	common "k8s.io/kube-openapi/pkg/common"
 	"k8s.io/kube-openapi/pkg/spec3"
-	"k8s.io/kube-openapi/pkg/validation/spec"
 
-	example "github.com/grafana/grafana/pkg/apis/example/v0alpha1"
+	query "github.com/grafana/grafana/pkg/apis/query/v0alpha1"
-	"github.com/grafana/grafana/pkg/apis/query/v0alpha1"
 	"github.com/grafana/grafana/pkg/apiserver/builder"
 	"github.com/grafana/grafana/pkg/expr"
 	"github.com/grafana/grafana/pkg/infra/log"
 	"github.com/grafana/grafana/pkg/infra/tracing"
 	"github.com/grafana/grafana/pkg/plugins"
 	"github.com/grafana/grafana/pkg/registry/apis/query/client"
+	"github.com/grafana/grafana/pkg/registry/apis/query/queryschema"
 	"github.com/grafana/grafana/pkg/services/accesscontrol"
 	"github.com/grafana/grafana/pkg/services/datasources"
 	"github.com/grafana/grafana/pkg/services/datasources/service"
@@ -41,22 +40,39 @@ type QueryAPIBuilder struct {
 	returnMultiStatus      bool // from feature toggle
 	features               featuremgmt.FeatureToggles
 
-	tracer    tracing.Tracer
+	tracer     tracing.Tracer
-	metrics   *metrics
+	metrics    *metrics
-	parser    *queryParser
+	parser     *queryParser
-	client    DataSourceClientSupplier
+	client     DataSourceClientSupplier
-	registry  v0alpha1.DataSourceApiServerRegistry
+	registry   query.DataSourceApiServerRegistry
-	converter *expr.ResultConverter
+	converter  *expr.ResultConverter
+	queryTypes *query.QueryTypeDefinitionList
 }
 
 func NewQueryAPIBuilder(features featuremgmt.FeatureToggles,
 	client DataSourceClientSupplier,
-	registry v0alpha1.DataSourceApiServerRegistry,
+	registry query.DataSourceApiServerRegistry,
 	legacy service.LegacyDataSourceLookup,
 	registerer prometheus.Registerer,
 	tracer tracing.Tracer,
 ) (*QueryAPIBuilder, error) {
 	reader := expr.NewExpressionQueryReader(features)
+
+	// Include well typed query definitions
+	var queryTypes *query.QueryTypeDefinitionList
+	if features.IsEnabledGlobally(featuremgmt.FlagDatasourceQueryTypes) {
+		// Read the expression query definitions
+		raw, err := expr.QueryTypeDefinitionListJSON()
+		if err != nil {
+			return nil, err
+		}
+		queryTypes = &query.QueryTypeDefinitionList{}
+		err = json.Unmarshal(raw, queryTypes)
+		if err != nil {
+			return nil, err
+		}
+	}
+
 	return &QueryAPIBuilder{
 		concurrentQueryLimit: 4,
 		log:                  log.New("query_apiserver"),
@@ -67,6 +83,7 @@ func NewQueryAPIBuilder(features featuremgmt.FeatureToggles,
 		metrics:              newMetrics(registerer),
 		tracer:               tracer,
 		features:             features,
+		queryTypes:           queryTypes,
 		converter: &expr.ResultConverter{
 			Features: features,
 			Tracer:   tracer,
@@ -103,25 +120,24 @@ func RegisterAPIService(features featuremgmt.FeatureToggles,
 }
 
 func (b *QueryAPIBuilder) GetGroupVersion() schema.GroupVersion {
-	return v0alpha1.SchemeGroupVersion
+	return query.SchemeGroupVersion
 }
 
 func addKnownTypes(scheme *runtime.Scheme, gv schema.GroupVersion) {
 	scheme.AddKnownTypes(gv,
-		&v0alpha1.DataSourceApiServer{},
+		&query.DataSourceApiServer{},
-		&v0alpha1.DataSourceApiServerList{},
+		&query.DataSourceApiServerList{},
-		&v0alpha1.QueryDataRequest{},
+		&query.QueryDataRequest{},
-		&v0alpha1.QueryDataResponse{},
+		&query.QueryDataResponse{},
-		&v0alpha1.QueryTypeDefinition{},
+		&query.QueryTypeDefinition{},
-		&v0alpha1.QueryTypeDefinitionList{},
+		&query.QueryTypeDefinitionList{},
-		&example.DummySubresource{},
 	)
 }
 
 func (b *QueryAPIBuilder) InstallSchema(scheme *runtime.Scheme) error {
-	addKnownTypes(scheme, v0alpha1.SchemeGroupVersion)
+	addKnownTypes(scheme, query.SchemeGroupVersion)
-	metav1.AddToGroupVersion(scheme, v0alpha1.SchemeGroupVersion)
+	metav1.AddToGroupVersion(scheme, query.SchemeGroupVersion)
-	return scheme.SetVersionPriority(v0alpha1.SchemeGroupVersion)
+	return scheme.SetVersionPriority(query.SchemeGroupVersion)
 }
 
 func (b *QueryAPIBuilder) GetAPIGroupInfo(
@@ -130,7 +146,7 @@ func (b *QueryAPIBuilder) GetAPIGroupInfo(
 	optsGetter generic.RESTOptionsGetter,
 	_ bool,
 ) (*genericapiserver.APIGroupInfo, error) {
-	gv := v0alpha1.SchemeGroupVersion
+	gv := query.SchemeGroupVersion
 	apiGroupInfo := genericapiserver.NewDefaultAPIGroupInfo(gv.Group, scheme, metav1.ParameterCodec, codecs)
 
 	storage := map[string]rest.Storage{}
@@ -147,12 +163,15 @@ func (b *QueryAPIBuilder) GetAPIGroupInfo(
 	// The query endpoint -- NOTE, this uses a rewrite hack to allow requests without a name parameter
 	storage["query"] = newQueryREST(b)
 
+	// Register the expressions query schemas
+	err := queryschema.RegisterQueryTypes(b.queryTypes, storage)
+
 	apiGroupInfo.VersionedResourcesStorageMap[gv.Version] = storage
-	return &apiGroupInfo, nil
+	return &apiGroupInfo, err
 }
 
 func (b *QueryAPIBuilder) GetOpenAPIDefinitions() common.GetOpenAPIDefinitions {
-	return v0alpha1.GetOpenAPIDefinitions
+	return query.GetOpenAPIDefinitions
 }
 
 // Register additional routes with the server
@@ -164,9 +183,6 @@ func (b *QueryAPIBuilder) GetAuthorizer() authorizer.Authorizer {
 	return nil // default is OK
 }
 
-const QueryRequestSchemaKey = "QueryRequestSchema"
-const QueryPayloadSchemaKey = "QueryPayloadSchema"
-
 func (b *QueryAPIBuilder) PostProcessOpenAPI(oas *spec3.OpenAPI) (*spec3.OpenAPI, error) {
 	// The plugin description
 	oas.Info.Description = "Query service"
@@ -174,115 +190,83 @@ func (b *QueryAPIBuilder) PostProcessOpenAPI(oas *spec3.OpenAPI) (*spec3.OpenAPI
 	// The root api URL
 	root := "/apis/" + b.GetGroupVersion().String() + "/"
 
-	var err error
+	// Add queries to the request properties
-	opts := schemabuilder.QuerySchemaOptions{
+	err := queryschema.AddQueriesToOpenAPI(queryschema.OASQueryOptions{
-		PluginID:   []string{""},
+		Swagger: oas,
-		QueryTypes: []data.QueryTypeDefinition{},
+		PluginJSON: &plugins.JSONData{
-		Mode:       schemabuilder.SchemaTypeQueryPayload,
+			ID: expr.DatasourceType, // Not really a plugin, but identified the same way
-	}
+		},
-	oas.Components.Schemas[QueryPayloadSchemaKey], err = schemabuilder.GetQuerySchema(opts)
+		QueryTypes:       b.queryTypes,
-	if err != nil {
+		Root:             root,
-		return oas, err
+		QueryPath:        "namespaces/{namespace}/query/{name}",
-	}
+		QueryDescription: "Query any datasources (with expressions)",
-	opts.Mode = schemabuilder.SchemaTypeQueryRequest
-	oas.Components.Schemas[QueryRequestSchemaKey], err = schemabuilder.GetQuerySchema(opts)
-	if err != nil {
-		return oas, err
-	}
 
-	// Rewrite the query path
+		// An explicit set of examples (otherwise we iterate the query type examples)
-	sub := oas.Paths.Paths[root+"namespaces/{namespace}/query/{name}"]
+		QueryExamples: map[string]*spec3.Example{
-	if sub != nil && sub.Post != nil {
+			"A": {
-		sub.Post.Tags = []string{"Query"}
+				ExampleProps: spec3.ExampleProps{
-		sub.Parameters = []*spec3.Parameter{
+					Summary:     "Random walk (testdata)",
-			{
+					Description: "Use testdata to execute a random walk query",
-				ParameterProps: spec3.ParameterProps{
+					Value: `{
-					Name:        "namespace",
+						"queries": [
-					In:          "path",
+							{
-					Description: "object name and auth scope, such as for teams and projects",
+								"refId": "A",
-					Example:     "default",
+								"scenarioId": "random_walk_table",
-					Required:    true,
+								"seriesCount": 1,
+								"datasource": {
+								"type": "grafana-testdata-datasource",
+								"uid": "PD8C576611E62080A"
+								},
+								"intervalMs": 60000,
+								"maxDataPoints": 20
+							}
+						],
+						"from": "now-6h",
+						"to": "now"
+					}`,
 				},
 			},
-		}
+			"B": {
-		sub.Post.Description = "Query datasources (with expressions)"
+				ExampleProps: spec3.ExampleProps{
-		sub.Post.Parameters = nil //
+					Summary:     "With deprecated datasource name",
-		sub.Post.RequestBody = &spec3.RequestBody{
+					Description: "Includes an old style string for datasource reference",
-			RequestBodyProps: spec3.RequestBodyProps{
+					Value: `{
-				Content: map[string]*spec3.MediaType{
+						"queries": [
-					"application/json": {
+							{
-						MediaTypeProps: spec3.MediaTypeProps{
+								"refId": "A",
-							Schema: spec.RefSchema("#/components/schemas/" + QueryRequestSchemaKey),
+								"datasource": {
-							Examples: map[string]*spec3.Example{
+									"type": "grafana-googlesheets-datasource",
-								"A": {
+									"uid": "b1808c48-9fc9-4045-82d7-081781f8a553"
-									ExampleProps: spec3.ExampleProps{
-										Summary:     "Random walk (testdata)",
-										Description: "Use testdata to execute a random walk query",
-										Value: `{
-											"queries": [
-												{
-													"refId": "A",
-													"scenarioId": "random_walk_table",
-													"seriesCount": 1,
-													"datasource": {
-													"type": "grafana-testdata-datasource",
-													"uid": "PD8C576611E62080A"
-													},
-													"intervalMs": 60000,
-													"maxDataPoints": 20
-												}
-											],
-											"from": "now-6h",
-											"to": "now"
-										}`,
-									},
-								},
-								"B": {
-									ExampleProps: spec3.ExampleProps{
-										Summary:     "With deprecated datasource name",
-										Description: "Includes an old style string for datasource reference",
-										Value: `{
-											"queries": [
-												{
-													"refId": "A",
-													"datasource": {
-														"type": "grafana-googlesheets-datasource",
-														"uid": "b1808c48-9fc9-4045-82d7-081781f8a553"
-													},
-													"cacheDurationSeconds": 300,
-													"spreadsheet": "spreadsheetID",
-													"datasourceId": 4,
-													"intervalMs": 30000,
-													"maxDataPoints": 794
-												},
-												{
-													"refId": "Z",
-													"datasource": "old",
-													"maxDataPoints": 10,
-													"timeRange": {
-														"from": "100",
-														"to": "200"
-													}
-												}
-											],
-											"from": "now-6h",
-											"to": "now"
-										}`,
-									},
 								},
+								"cacheDurationSeconds": 300,
+								"spreadsheet": "spreadsheetID",
+								"datasourceId": 4,
+								"intervalMs": 30000,
+								"maxDataPoints": 794
 							},
-						},
+							{
-					},
+								"refId": "Z",
+								"datasource": "old",
+								"maxDataPoints": 10,
+								"timeRange": {
+									"from": "100",
+									"to": "200"
+								}
+							}
+						],
+						"from": "now-6h",
+						"to": "now"
+					}`,
 				},
 			},
-		}
+		},
-
+	})
-		delete(oas.Paths.Paths, root+"namespaces/{namespace}/query/{name}")
+	if err != nil {
-		oas.Paths.Paths[root+"namespaces/{namespace}/query"] = sub
+		return oas, nil
 	}
 
 	// The root API discovery list
-	sub = oas.Paths.Paths[root]
+	sub := oas.Paths.Paths[root]
 	if sub != nil && sub.Get != nil {
 		sub.Get.Tags = []string{"API Discovery"} // sorts first in the list
 	}

pkg/registry/apis/scope/register.go

@@ -13,6 +13,7 @@ import (
 	genericapiserver "k8s.io/apiserver/pkg/server"
 	"k8s.io/kube-openapi/pkg/common"
 	"k8s.io/kube-openapi/pkg/spec3"
+	"k8s.io/kube-openapi/pkg/validation/spec"
 
 	scope "github.com/grafana/grafana/pkg/apis/scope/v0alpha1"
 	"github.com/grafana/grafana/pkg/apiserver/builder"
@@ -177,6 +178,7 @@ func (b *ScopeAPIBuilder) PostProcessOpenAPI(oas *spec3.OpenAPI) (*spec3.OpenAPI
 					Description: "object name and auth scope, such as for teams and projects",
 					Example:     "default",
 					Required:    true,
+					Schema:      spec.StringProperty().UniqueValues(),
 				},
 			},
 		}

pkg/services/apiserver/standalone/factory.go

@@ -107,6 +107,7 @@ func (p *DummyAPIFactory) MakeAPIServer(_ context.Context, tracer tracing.Tracer
 			},
 			&pluginDatasourceImpl{}, // stub
 			&actest.FakeAccessControl{ExpectedEvaluate: true},
+			true, // show query types
 		)
 	}
 

pkg/services/featuremgmt/registry.go

@@ -775,6 +775,13 @@ var (
 			Owner:           grafanaAppPlatformSquad,
 			RequiresRestart: true, // changes the API routing
 		},
+		{
+			Name:            "datasourceQueryTypes",
+			Description:     "Show query type endpoints in datasource API servers (currently hardcoded for testdata, expressions, and prometheus)",
+			Stage:           FeatureStageExperimental,
+			Owner:           grafanaAppPlatformSquad,
+			RequiresRestart: true, // changes the API routing
+		},
 		{
 			Name:            "queryService",
 			Description:     "Register /apis/query.grafana.app/ -- will eventually replace /api/ds/query",

pkg/services/featuremgmt/toggles_gen.csv

@@ -103,6 +103,7 @@ formatString,preview,@grafana/dataviz-squad,false,false,true
 transformationsVariableSupport,preview,@grafana/dataviz-squad,false,false,true
 kubernetesPlaylists,GA,@grafana/grafana-app-platform-squad,false,true,false
 kubernetesSnapshots,experimental,@grafana/grafana-app-platform-squad,false,true,false
+datasourceQueryTypes,experimental,@grafana/grafana-app-platform-squad,false,true,false
 queryService,experimental,@grafana/grafana-app-platform-squad,false,true,false
 queryServiceRewrite,experimental,@grafana/grafana-app-platform-squad,false,true,false
 queryServiceFromUI,experimental,@grafana/grafana-app-platform-squad,false,false,true

pkg/services/featuremgmt/toggles_gen.go

@@ -423,6 +423,10 @@ const (
 	// Routes snapshot requests from /api to the /apis endpoint
 	FlagKubernetesSnapshots = "kubernetesSnapshots"
 
+	// FlagDatasourceQueryTypes
+	// Show query type endpoints in datasource API servers (currently hardcoded for testdata, expressions, and prometheus)
+	FlagDatasourceQueryTypes = "datasourceQueryTypes"
+
 	// FlagQueryService
 	// Register /apis/query.grafana.app/ -- will eventually replace /api/ds/query
 	FlagQueryService = "queryService"

pkg/services/featuremgmt/toggles_gen.json

@@ -2224,6 +2224,19 @@
         "stage": "experimental",
         "codeowner": "@grafana/observability-metrics"
       }
+    },
+    {
+      "metadata": {
+        "name": "datasourceQueryTypes",
+        "resourceVersion": "1716473268430",
+        "creationTimestamp": "2024-05-23T14:07:48Z"
+      },
+      "spec": {
+        "description": "Show query type endpoints in datasource API servers (currently hardcoded for testdata, expressions, and prometheus)",
+        "stage": "experimental",
+        "codeowner": "@grafana/grafana-app-platform-squad",
+        "requiresRestart": true
+      }
     }
   ]
 }

pkg/tsdb/grafana-testdata-datasource/kinds/query.go

@@ -172,7 +172,7 @@ type USAQuery struct {
 //go:embed query.types.json
 var f embed.FS
 
-// QueryTypeDefinitionsJSON returns the query type definitions
+// QueryTypeDefinitionListJSON returns the query type definitions
-func QueryTypeDefinitionsJSON() (json.RawMessage, error) {
+func QueryTypeDefinitionListJSON() (json.RawMessage, error) {
 	return f.ReadFile("query.types.json")
 }