From 42b0f802de31f3d42ece68544c193a69ea381bc0 Mon Sep 17 00:00:00 2001 From: Ryan McKinley Date: Thu, 23 May 2024 19:46:28 +0300 Subject: [PATCH] QueryTypes: Add feature toggle to show query types in datasource apiservers (#88213) * initial attempt * show query types * show query types * with formatting * with formatting * more cleanup * add feature toggle * fix build --- .../feature-toggles/index.md | 1 + .../src/types/featureToggles.gen.ts | 1 + pkg/expr/query.go | 9 + pkg/expr/reader.go | 8 - pkg/promlib/models/query.go | 2 +- pkg/registry/apis/datasource/register.go | 158 ++++-------- pkg/registry/apis/query/parser.go | 1 - .../apis/query/queryschema/oas_helper.go | 235 +++++++++++++++++ .../query/queryschema/query_type_storage.go | 147 +++++++++++ pkg/registry/apis/query/register.go | 240 ++++++++---------- pkg/registry/apis/scope/register.go | 2 + pkg/services/apiserver/standalone/factory.go | 1 + pkg/services/featuremgmt/registry.go | 7 + pkg/services/featuremgmt/toggles_gen.csv | 1 + pkg/services/featuremgmt/toggles_gen.go | 4 + pkg/services/featuremgmt/toggles_gen.json | 13 + .../kinds/query.go | 4 +- 17 files changed, 590 insertions(+), 244 deletions(-) create mode 100644 pkg/registry/apis/query/queryschema/oas_helper.go create mode 100644 pkg/registry/apis/query/queryschema/query_type_storage.go diff --git a/docs/sources/setup-grafana/configure-grafana/feature-toggles/index.md b/docs/sources/setup-grafana/configure-grafana/feature-toggles/index.md index 5384028fe64..127edc25de1 100644 --- a/docs/sources/setup-grafana/configure-grafana/feature-toggles/index.md +++ b/docs/sources/setup-grafana/configure-grafana/feature-toggles/index.md @@ -156,6 +156,7 @@ Experimental features might be changed or removed without prior notice. | `idForwarding` | Generate signed id token for identity that can be forwarded to plugins and external services | | `enableNativeHTTPHistogram` | Enables native HTTP Histograms | | `kubernetesSnapshots` | Routes snapshot requests from /api to the /apis endpoint | +| `datasourceQueryTypes` | Show query type endpoints in datasource API servers (currently hardcoded for testdata, expressions, and prometheus) | | `queryService` | Register /apis/query.grafana.app/ -- will eventually replace /api/ds/query | | `queryServiceRewrite` | Rewrite requests targeting /ds/query to the query service | | `queryServiceFromUI` | Routes requests to the new query service | diff --git a/packages/grafana-data/src/types/featureToggles.gen.ts b/packages/grafana-data/src/types/featureToggles.gen.ts index 67aeccd9332..8792e5b55b9 100644 --- a/packages/grafana-data/src/types/featureToggles.gen.ts +++ b/packages/grafana-data/src/types/featureToggles.gen.ts @@ -122,6 +122,7 @@ export interface FeatureToggles { transformationsVariableSupport?: boolean; kubernetesPlaylists?: boolean; kubernetesSnapshots?: boolean; + datasourceQueryTypes?: boolean; queryService?: boolean; queryServiceRewrite?: boolean; queryServiceFromUI?: boolean; diff --git a/pkg/expr/query.go b/pkg/expr/query.go index 6fffd7d116c..53f45186508 100644 --- a/pkg/expr/query.go +++ b/pkg/expr/query.go @@ -1,6 +1,8 @@ package expr import ( + "embed" + "github.com/grafana/grafana/pkg/expr/classic" "github.com/grafana/grafana/pkg/expr/mathexp" ) @@ -100,3 +102,10 @@ const ( // Replace non-numbers ReduceModeReplace ReduceMode = "replaceNN" ) + +//go:embed query.types.json +var f embed.FS + +func QueryTypeDefinitionListJSON() ([]byte, error) { + return f.ReadFile("query.types.json") +} diff --git a/pkg/expr/reader.go b/pkg/expr/reader.go index 98f29ce3fa1..71a75c6cb85 100644 --- a/pkg/expr/reader.go +++ b/pkg/expr/reader.go @@ -1,7 +1,6 @@ package expr import ( - "embed" "fmt" "strings" @@ -178,13 +177,6 @@ func (h *ExpressionQueryReader) ReadQuery( return eq, err } -//go:embed query.types.json -var f embed.FS - -func (h *ExpressionQueryReader) QueryTypeDefinitionListJSON() ([]byte, error) { - return f.ReadFile("query.types.json") -} - func getReferenceVar(exp string, refId string) (string, error) { exp = strings.TrimPrefix(exp, "$") if exp == "" { diff --git a/pkg/promlib/models/query.go b/pkg/promlib/models/query.go index 1eaf6c5da91..c8c93e7e9b2 100644 --- a/pkg/promlib/models/query.go +++ b/pkg/promlib/models/query.go @@ -434,6 +434,6 @@ func AlignTimeRange(t time.Time, step time.Duration, offset int64) time.Time { var f embed.FS // QueryTypeDefinitionsJSON returns the query type definitions -func QueryTypeDefinitionsJSON() (json.RawMessage, error) { +func QueryTypeDefinitionListJSON() (json.RawMessage, error) { return f.ReadFile("query.types.json") } diff --git a/pkg/registry/apis/datasource/register.go b/pkg/registry/apis/datasource/register.go index f54eabab17a..c880f2b0e14 100644 --- a/pkg/registry/apis/datasource/register.go +++ b/pkg/registry/apis/datasource/register.go @@ -2,13 +2,11 @@ package datasource import ( "context" + "encoding/json" "fmt" - "net/http" "time" "github.com/grafana/grafana-plugin-sdk-go/backend" - data "github.com/grafana/grafana-plugin-sdk-go/experimental/apis/data/v0alpha1" - "github.com/grafana/grafana-plugin-sdk-go/experimental/schemabuilder" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" @@ -19,7 +17,6 @@ import ( genericapiserver "k8s.io/apiserver/pkg/server" openapi "k8s.io/kube-openapi/pkg/common" "k8s.io/kube-openapi/pkg/spec3" - "k8s.io/kube-openapi/pkg/validation/spec" "k8s.io/utils/strings/slices" common "github.com/grafana/grafana/pkg/apimachinery/apis/common/v0alpha1" @@ -27,15 +24,15 @@ import ( query "github.com/grafana/grafana/pkg/apis/query/v0alpha1" "github.com/grafana/grafana/pkg/apiserver/builder" "github.com/grafana/grafana/pkg/plugins" + "github.com/grafana/grafana/pkg/promlib/models" + "github.com/grafana/grafana/pkg/registry/apis/query/queryschema" "github.com/grafana/grafana/pkg/services/accesscontrol" "github.com/grafana/grafana/pkg/services/apiserver/utils" "github.com/grafana/grafana/pkg/services/featuremgmt" "github.com/grafana/grafana/pkg/services/pluginsintegration/pluginstore" + "github.com/grafana/grafana/pkg/tsdb/grafana-testdata-datasource/kinds" ) -const QueryRequestSchemaKey = "QueryRequestSchema" -const QueryPayloadSchemaKey = "QueryPayloadSchema" - var _ builder.APIGroupBuilder = (*DataSourceAPIBuilder)(nil) // DataSourceAPIBuilder is used just so wire has something unique to return @@ -83,6 +80,7 @@ func RegisterAPIService( datasources.GetDatasourceProvider(ds.JSONData), contextProvider, accessControl, + features.IsEnabledGlobally(featuremgmt.FlagDatasourceQueryTypes), ) if err != nil { return nil, err @@ -105,20 +103,48 @@ func NewDataSourceAPIBuilder( client PluginClient, datasources PluginDatasourceProvider, contextProvider PluginContextWrapper, - accessControl accesscontrol.AccessControl) (*DataSourceAPIBuilder, error) { + accessControl accesscontrol.AccessControl, + loadQueryTypes bool, +) (*DataSourceAPIBuilder, error) { ri, err := resourceFromPluginID(plugin.ID) if err != nil { return nil, err } - return &DataSourceAPIBuilder{ + builder := &DataSourceAPIBuilder{ connectionResourceInfo: ri, pluginJSON: plugin, client: client, datasources: datasources, contextProvider: contextProvider, accessControl: accessControl, - }, nil + } + if loadQueryTypes { + // In the future, this will somehow come from the plugin + builder.queryTypes, err = getHardcodedQueryTypes(ri.GroupResource().Group) + } + return builder, err +} + +// TODO -- somehow get the list from the plugin -- not hardcoded +func getHardcodedQueryTypes(group string) (*query.QueryTypeDefinitionList, error) { + var err error + var raw json.RawMessage + switch group { + case "testdata.datasource.grafana.app": + raw, err = kinds.QueryTypeDefinitionListJSON() + case "prometheus.datasource.grafana.app": + raw, err = models.QueryTypeDefinitionListJSON() + } + if err != nil { + return nil, err + } + if raw != nil { + types := &query.QueryTypeDefinitionList{} + err = json.Unmarshal(raw, types) + return types, err + } + return nil, err } func (b *DataSourceAPIBuilder) GetGroupVersion() schema.GroupVersion { @@ -134,6 +160,8 @@ func addKnownTypes(scheme *runtime.Scheme, gv schema.GroupVersion) { // Query handler &query.QueryDataRequest{}, &query.QueryDataResponse{}, + &query.QueryTypeDefinition{}, + &query.QueryTypeDefinitionList{}, &metav1.Status{}, ) } @@ -211,12 +239,16 @@ func (b *DataSourceAPIBuilder) GetAPIGroupInfo( storage[conn.StoragePath("proxy")] = &subProxyREST{pluginJSON: b.pluginJSON} } + // Register hardcoded query schemas + err := queryschema.RegisterQueryTypes(b.queryTypes, storage) + + // Create the group info apiGroupInfo := genericapiserver.NewDefaultAPIGroupInfo( conn.GroupResource().Group, scheme, metav1.ParameterCodec, codecs) apiGroupInfo.VersionedResourcesStorageMap[conn.GroupVersion().Version] = storage - return &apiGroupInfo, nil + return &apiGroupInfo, err } func (b *DataSourceAPIBuilder) getPluginContext(ctx context.Context, uid string) (backend.PluginContext, error) { @@ -247,66 +279,19 @@ func (b *DataSourceAPIBuilder) PostProcessOpenAPI(oas *spec3.OpenAPI) (*spec3.Op // Hide the ability to list all connections across tenants delete(oas.Paths.Paths, root+b.connectionResourceInfo.GroupResource().Resource) - var err error - opts := schemabuilder.QuerySchemaOptions{ - PluginID: []string{b.pluginJSON.ID}, - QueryTypes: []data.QueryTypeDefinition{}, - Mode: schemabuilder.SchemaTypeQueryPayload, - } - if b.pluginJSON.AliasIDs != nil { - opts.PluginID = append(opts.PluginID, b.pluginJSON.AliasIDs...) - } - if b.queryTypes != nil { - for _, qt := range b.queryTypes.Items { - // The SDK type and api type are not the same so we recreate it here - opts.QueryTypes = append(opts.QueryTypes, data.QueryTypeDefinition{ - ObjectMeta: data.ObjectMeta{ - Name: qt.Name, - }, - Spec: qt.Spec, - }) - } - } - oas.Components.Schemas[QueryPayloadSchemaKey], err = schemabuilder.GetQuerySchema(opts) - if err != nil { - return oas, err - } - opts.Mode = schemabuilder.SchemaTypeQueryRequest - oas.Components.Schemas[QueryRequestSchemaKey], err = schemabuilder.GetQuerySchema(opts) - if err != nil { - return oas, err - } - - // Update the request object - sub := oas.Paths.Paths[root+"namespaces/{namespace}/connections/{name}/query"] - if sub != nil && sub.Post != nil { - sub.Post.Description = "Execute queries" - sub.Post.RequestBody = &spec3.RequestBody{ - RequestBodyProps: spec3.RequestBodyProps{ - Required: true, - Content: map[string]*spec3.MediaType{ - "application/json": { - MediaTypeProps: spec3.MediaTypeProps{ - Schema: spec.RefSchema("#/components/schemas/" + QueryRequestSchemaKey), - Examples: getExamples(b.queryTypes), - }, - }, - }, - }, - } - okrsp, ok := sub.Post.Responses.StatusCodeResponses[200] - if ok { - sub.Post.Responses.StatusCodeResponses[http.StatusMultiStatus] = &spec3.Response{ - ResponseProps: spec3.ResponseProps{ - Description: "Query executed, but errors may exist in the datasource. See the payload for more details.", - Content: okrsp.Content, - }, - } - } - } + // Add queries to the request properties + // Add queries to the request properties + err := queryschema.AddQueriesToOpenAPI(queryschema.OASQueryOptions{ + Swagger: oas, + PluginJSON: &b.pluginJSON, + QueryTypes: b.queryTypes, + Root: root, + QueryPath: "namespaces/{namespace}/connections/{name}/query", + QueryDescription: fmt.Sprintf("Query the %s datasources", b.pluginJSON.Name), + }) // The root API discovery list - sub = oas.Paths.Paths[root] + sub := oas.Paths.Paths[root] if sub != nil && sub.Get != nil { sub.Get.Tags = []string{"API Discovery"} // sorts first in the list } @@ -317,38 +302,3 @@ func (b *DataSourceAPIBuilder) PostProcessOpenAPI(oas *spec3.OpenAPI) (*spec3.Op func (b *DataSourceAPIBuilder) GetAPIRoutes() *builder.APIRoutes { return nil } - -func getExamples(queryTypes *query.QueryTypeDefinitionList) map[string]*spec3.Example { - if queryTypes == nil { - return nil - } - - tr := data.TimeRange{From: "now-1h", To: "now"} - examples := map[string]*spec3.Example{} - for _, queryType := range queryTypes.Items { - for idx, example := range queryType.Spec.Examples { - q := data.NewDataQuery(example.SaveModel.Object) - q.RefID = "A" - for _, dis := range queryType.Spec.Discriminators { - _ = q.Set(dis.Field, dis.Value) - } - if q.MaxDataPoints < 1 { - q.MaxDataPoints = 1000 - } - if q.IntervalMS < 1 { - q.IntervalMS = 5000 // 5s - } - examples[fmt.Sprintf("%s-%d", example.Name, idx)] = &spec3.Example{ - ExampleProps: spec3.ExampleProps{ - Summary: example.Name, - Description: example.Description, - Value: data.QueryDataRequest{ - TimeRange: tr, - Queries: []data.DataQuery{q}, - }, - }, - } - } - } - return examples -} diff --git a/pkg/registry/apis/query/parser.go b/pkg/registry/apis/query/parser.go index 45feaf71afd..66d43162818 100644 --- a/pkg/registry/apis/query/parser.go +++ b/pkg/registry/apis/query/parser.go @@ -186,7 +186,6 @@ func (p *queryParser) parseRequest(ctx context.Context, input *query.QueryDataRe } } } - return rsp, nil } diff --git a/pkg/registry/apis/query/queryschema/oas_helper.go b/pkg/registry/apis/query/queryschema/oas_helper.go new file mode 100644 index 00000000000..628dd56751a --- /dev/null +++ b/pkg/registry/apis/query/queryschema/oas_helper.go @@ -0,0 +1,235 @@ +package queryschema + +import ( + "fmt" + "strings" + + "k8s.io/kube-openapi/pkg/spec3" + "k8s.io/kube-openapi/pkg/validation/spec" + + "github.com/grafana/grafana-plugin-sdk-go/experimental/schemabuilder" + + data "github.com/grafana/grafana-plugin-sdk-go/experimental/apis/data/v0alpha1" + + query "github.com/grafana/grafana/pkg/apis/query/v0alpha1" + "github.com/grafana/grafana/pkg/plugins" +) + +const QueryRequestSchemaKey = "QueryRequestSchema" + +// const QueryPayloadSchemaKey = "QueryPayloadSchema" +// const QuerySaveModelSchemaKey = "QuerySaveModelSchema" + +type OASQueryOptions struct { + Swagger *spec3.OpenAPI + PluginJSON *plugins.JSONData + QueryTypes *query.QueryTypeDefinitionList + + Root string + QueryPath string // eg "namespaces/{namespace}/query/{name}" + QueryDescription string + QueryExamples map[string]*spec3.Example +} + +func AddQueriesToOpenAPI(options OASQueryOptions) error { + oas := options.Swagger + root := options.Root + examples := options.QueryExamples + resourceName := query.QueryTypeDefinitionResourceInfo.GroupResource().Resource + + builder := schemabuilder.QuerySchemaOptions{ + PluginID: []string{""}, + QueryTypes: []data.QueryTypeDefinition{}, + } + if options.PluginJSON != nil { + builder.PluginID = []string{options.PluginJSON.ID} + if options.PluginJSON.AliasIDs != nil { + builder.PluginID = append(builder.PluginID, options.PluginJSON.AliasIDs...) + } + } + if options.QueryTypes != nil { + // The SDK type and api type are not the same so we just recreate it here + for _, qt := range options.QueryTypes.Items { + builder.QueryTypes = append(builder.QueryTypes, data.QueryTypeDefinition{ + ObjectMeta: data.ObjectMeta{ + Name: qt.Name, + }, + Spec: qt.Spec, + }) + } + + if examples == nil { + examples = getExamples(options.QueryTypes) + } + } + + // Rewrite the query path + query := oas.Paths.Paths[root+options.QueryPath] + if query != nil && query.Post != nil { + query.Post.Tags = []string{"Query"} + query.Parameters = []*spec3.Parameter{ + { + ParameterProps: spec3.ParameterProps{ + Name: "namespace", + In: "path", + Description: "object name and auth scope, such as for teams and projects", + Example: "default", + Required: true, + Schema: spec.StringProperty().UniqueValues(), + }, + }, + } + query.Post.Description = options.QueryDescription + query.Post.Parameters = nil // + query.Post.RequestBody = &spec3.RequestBody{ + RequestBodyProps: spec3.RequestBodyProps{ + Content: map[string]*spec3.MediaType{ + "application/json": { + MediaTypeProps: spec3.MediaTypeProps{ + Schema: spec.RefSchema("#/components/schemas/" + QueryRequestSchemaKey), + Examples: examples, + }, + }, + }, + }, + } + + // Remove the {name} hack from from the query + if strings.HasSuffix(options.QueryPath, "/{name}") { + delete(oas.Paths.Paths, root+options.QueryPath) + oas.Paths.Paths[root+strings.TrimSuffix(options.QueryPath, "/{name}")] = query + } + } + + // Update the validate endpoint + validate := oas.Paths.Paths[root+resourceName+"/{name}/validate"] + if validate != nil && validate.Post != nil { + validate.Post.Description = "Verify if a query payload matches the expected value and return a clean version" + validate.Parameters = []*spec3.Parameter{ + { + ParameterProps: spec3.ParameterProps{ + Name: "name", + In: "path", + Description: "The query type name, or {any}", + Example: "{any}", + Required: true, + Schema: spec.StringProperty().UniqueValues(), + }, + }, + } + + // Accept the same payload as the query type + validate.Post.RequestBody = query.Post.RequestBody + } + + // Query Request + builder.Mode = schemabuilder.SchemaTypeQueryRequest + s, err := schemabuilder.GetQuerySchema(builder) + if err != nil { + return err + } + + // The schema requires some munging to pass validation + // This should likely be fixed in the upstream "GetQuerySchema" function + removeSchemaRefs(s) + s.Description = "Schema for a set of queries sent to the query method" + oas.Components.Schemas[QueryRequestSchemaKey] = s + + // // Query Payload (is this useful?) + + // opts.Mode = schemabuilder.SchemaTypeQueryPayload + // s, err = schemabuilder.GetQuerySchema(opts) + // if err != nil { + // return err + // } + // delete(s.ExtraProps, "$schema") + // s.Description = "Schema for a single query object including all runtime properties" + // oas.Components.Schemas[QueryPayloadSchemaKey] = s + + // // Query Save Model + // opts.Mode = schemabuilder.SchemaTypeSaveModel + // s, err = schemabuilder.GetQuerySchema(opts) + // if err != nil { + // return err + // } + // s.Extensions = nil // remove the $schema + // s.Description = "Valid save model for a single query target" + // oas.Components.Schemas[QuerySaveModelSchemaKey] = s + return nil +} + +func getExamples(queryTypes *query.QueryTypeDefinitionList) map[string]*spec3.Example { + if queryTypes == nil { + return nil + } + + tr := data.TimeRange{From: "now-1h", To: "now"} + examples := map[string]*spec3.Example{} + for _, queryType := range queryTypes.Items { + for idx, example := range queryType.Spec.Examples { + q := data.NewDataQuery(example.SaveModel.Object) + q.RefID = "A" + for _, dis := range queryType.Spec.Discriminators { + _ = q.Set(dis.Field, dis.Value) + } + if q.MaxDataPoints < 1 { + q.MaxDataPoints = 1000 + } + if q.IntervalMS < 1 { + q.IntervalMS = 5000 // 5s + } + examples[fmt.Sprintf("%s-%d", example.Name, idx)] = &spec3.Example{ + ExampleProps: spec3.ExampleProps{ + Summary: example.Name, + Description: example.Description, + Value: data.QueryDataRequest{ + TimeRange: tr, + Queries: []data.DataQuery{q}, + }, + }, + } + } + } + return examples +} + +func removeSchemaRefs(s *spec.Schema) { + if s == nil { + return + } + if s.Schema != "" { + s.Schema = "" + } + // Examples is invalid -- only use the first example + examples, ok := s.ExtraProps["examples"] + if ok && examples != nil { + //fmt.Printf("TODO, use reflection to get first element from: %+v\n", examples) + //s.Example = examples[0] + delete(s.ExtraProps, "examples") + } + + removeSchemaRefs(s.Not) + for idx := range s.AllOf { + removeSchemaRefs(&s.AllOf[idx]) + } + for idx := range s.AnyOf { + removeSchemaRefs(&s.AnyOf[idx]) + } + for k := range s.Properties { + v := s.Properties[k] + removeSchemaRefs(&v) + s.Properties[k] = v + } + if s.Items != nil { + removeSchemaRefs(s.Items.Schema) + for idx := range s.Items.Schemas { + removeSchemaRefs(&s.Items.Schemas[idx]) + } + } + if s.AdditionalProperties != nil { + removeSchemaRefs(s.AdditionalProperties.Schema) + } + if s.AdditionalItems != nil { + removeSchemaRefs(s.AdditionalItems.Schema) + } +} diff --git a/pkg/registry/apis/query/queryschema/query_type_storage.go b/pkg/registry/apis/query/queryschema/query_type_storage.go new file mode 100644 index 00000000000..4018161b4b8 --- /dev/null +++ b/pkg/registry/apis/query/queryschema/query_type_storage.go @@ -0,0 +1,147 @@ +package queryschema + +import ( + "context" + "net/http" + + "k8s.io/apimachinery/pkg/apis/meta/internalversion" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apiserver/pkg/registry/rest" + + "github.com/grafana/grafana-plugin-sdk-go/experimental/apis/data/v0alpha1" + + common "github.com/grafana/grafana/pkg/apimachinery/apis/common/v0alpha1" + query "github.com/grafana/grafana/pkg/apis/query/v0alpha1" +) + +var ( + _ rest.Storage = (*queryTypeStorage)(nil) + _ rest.Scoper = (*queryTypeStorage)(nil) + _ rest.SingularNameProvider = (*queryTypeStorage)(nil) + _ rest.Lister = (*queryTypeStorage)(nil) + _ rest.Getter = (*queryTypeStorage)(nil) + + // The connectors + _ = rest.Connecter(&queryValidationREST{}) +) + +type queryTypeStorage struct { + resourceInfo *common.ResourceInfo + tableConverter rest.TableConvertor + registry query.QueryTypeDefinitionList +} + +type queryValidationREST struct { + qt *queryTypeStorage +} + +func RegisterQueryTypes(queryTypes *query.QueryTypeDefinitionList, storage map[string]rest.Storage) error { + if queryTypes == nil { + return nil // NO error + } + + resourceInfo := query.QueryTypeDefinitionResourceInfo + store := &queryTypeStorage{ + resourceInfo: &resourceInfo, + tableConverter: rest.NewDefaultTableConvertor(resourceInfo.GroupResource()), + registry: *queryTypes, + } + + // Supports list+get for all query types + storage[resourceInfo.StoragePath()] = store + + // Adds a query validation endpoint for each query type + // We will also support the "" or "*" name + storage[resourceInfo.StoragePath("validate")] = &queryValidationREST{store} + + return nil +} + +func (s *queryTypeStorage) New() runtime.Object { + return s.resourceInfo.NewFunc() +} + +func (s *queryTypeStorage) Destroy() {} + +func (s *queryTypeStorage) NamespaceScoped() bool { + return false +} + +func (s *queryTypeStorage) GetSingularName() string { + return s.resourceInfo.GetSingularName() +} + +func (s *queryTypeStorage) NewList() runtime.Object { + return s.resourceInfo.NewListFunc() +} + +func (s *queryTypeStorage) ConvertToTable(ctx context.Context, object runtime.Object, tableOptions runtime.Object) (*metav1.Table, error) { + return s.tableConverter.ConvertToTable(ctx, object, tableOptions) +} + +func (s *queryTypeStorage) List(ctx context.Context, options *internalversion.ListOptions) (runtime.Object, error) { + return &s.registry, nil +} + +func (s *queryTypeStorage) Get(ctx context.Context, name string, options *metav1.GetOptions) (runtime.Object, error) { + for idx, qt := range s.registry.Items { + if qt.Name == name { + return &s.registry.Items[idx], nil + } + } + return nil, s.resourceInfo.NewNotFound(name) +} + +//---------------------------------------------------- +// The validation processor +//---------------------------------------------------- + +func (r *queryValidationREST) New() runtime.Object { + return &query.QueryDataRequest{} +} + +func (r *queryValidationREST) Destroy() { +} + +func (r *queryValidationREST) ConnectMethods() []string { + return []string{"POST"} +} + +func (r *queryValidationREST) ProducesMIMETypes(verb string) []string { + return []string{"application/json"} +} + +func (r *queryValidationREST) ProducesObject(verb string) interface{} { + return &query.QueryDataRequest{} +} + +func (r *queryValidationREST) NewConnectOptions() (runtime.Object, bool, string) { + return nil, false, "" // true means you can use the trailing path as a variable +} + +func (r *queryValidationREST) Connect(ctx context.Context, name string, opts runtime.Object, responder rest.Responder) (http.Handler, error) { + return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + // TODO -- validate/mutate the query + // should we return the DQR, or raw validation response? + qdr := &query.QueryDataRequest{ + QueryDataRequest: v0alpha1.QueryDataRequest{ + Debug: true, + }, + } + + if name == "*" || name == "{any}" { + qdr.Queries = []v0alpha1.DataQuery{ + v0alpha1.NewDataQuery( + map[string]any{ + "refId": "???", + "HELLO": "world", + "TODO": "parse any query", + }, + ), + } + } + + responder.Object(http.StatusOK, qdr) + }), nil +} diff --git a/pkg/registry/apis/query/register.go b/pkg/registry/apis/query/register.go index c91cfd4e469..df6d5fa4203 100644 --- a/pkg/registry/apis/query/register.go +++ b/pkg/registry/apis/query/register.go @@ -1,8 +1,8 @@ package query import ( - data "github.com/grafana/grafana-plugin-sdk-go/experimental/apis/data/v0alpha1" - "github.com/grafana/grafana-plugin-sdk-go/experimental/schemabuilder" + "encoding/json" + "github.com/prometheus/client_golang/prometheus" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -14,16 +14,15 @@ import ( genericapiserver "k8s.io/apiserver/pkg/server" common "k8s.io/kube-openapi/pkg/common" "k8s.io/kube-openapi/pkg/spec3" - "k8s.io/kube-openapi/pkg/validation/spec" - example "github.com/grafana/grafana/pkg/apis/example/v0alpha1" - "github.com/grafana/grafana/pkg/apis/query/v0alpha1" + query "github.com/grafana/grafana/pkg/apis/query/v0alpha1" "github.com/grafana/grafana/pkg/apiserver/builder" "github.com/grafana/grafana/pkg/expr" "github.com/grafana/grafana/pkg/infra/log" "github.com/grafana/grafana/pkg/infra/tracing" "github.com/grafana/grafana/pkg/plugins" "github.com/grafana/grafana/pkg/registry/apis/query/client" + "github.com/grafana/grafana/pkg/registry/apis/query/queryschema" "github.com/grafana/grafana/pkg/services/accesscontrol" "github.com/grafana/grafana/pkg/services/datasources" "github.com/grafana/grafana/pkg/services/datasources/service" @@ -41,22 +40,39 @@ type QueryAPIBuilder struct { returnMultiStatus bool // from feature toggle features featuremgmt.FeatureToggles - tracer tracing.Tracer - metrics *metrics - parser *queryParser - client DataSourceClientSupplier - registry v0alpha1.DataSourceApiServerRegistry - converter *expr.ResultConverter + tracer tracing.Tracer + metrics *metrics + parser *queryParser + client DataSourceClientSupplier + registry query.DataSourceApiServerRegistry + converter *expr.ResultConverter + queryTypes *query.QueryTypeDefinitionList } func NewQueryAPIBuilder(features featuremgmt.FeatureToggles, client DataSourceClientSupplier, - registry v0alpha1.DataSourceApiServerRegistry, + registry query.DataSourceApiServerRegistry, legacy service.LegacyDataSourceLookup, registerer prometheus.Registerer, tracer tracing.Tracer, ) (*QueryAPIBuilder, error) { reader := expr.NewExpressionQueryReader(features) + + // Include well typed query definitions + var queryTypes *query.QueryTypeDefinitionList + if features.IsEnabledGlobally(featuremgmt.FlagDatasourceQueryTypes) { + // Read the expression query definitions + raw, err := expr.QueryTypeDefinitionListJSON() + if err != nil { + return nil, err + } + queryTypes = &query.QueryTypeDefinitionList{} + err = json.Unmarshal(raw, queryTypes) + if err != nil { + return nil, err + } + } + return &QueryAPIBuilder{ concurrentQueryLimit: 4, log: log.New("query_apiserver"), @@ -67,6 +83,7 @@ func NewQueryAPIBuilder(features featuremgmt.FeatureToggles, metrics: newMetrics(registerer), tracer: tracer, features: features, + queryTypes: queryTypes, converter: &expr.ResultConverter{ Features: features, Tracer: tracer, @@ -103,25 +120,24 @@ func RegisterAPIService(features featuremgmt.FeatureToggles, } func (b *QueryAPIBuilder) GetGroupVersion() schema.GroupVersion { - return v0alpha1.SchemeGroupVersion + return query.SchemeGroupVersion } func addKnownTypes(scheme *runtime.Scheme, gv schema.GroupVersion) { scheme.AddKnownTypes(gv, - &v0alpha1.DataSourceApiServer{}, - &v0alpha1.DataSourceApiServerList{}, - &v0alpha1.QueryDataRequest{}, - &v0alpha1.QueryDataResponse{}, - &v0alpha1.QueryTypeDefinition{}, - &v0alpha1.QueryTypeDefinitionList{}, - &example.DummySubresource{}, + &query.DataSourceApiServer{}, + &query.DataSourceApiServerList{}, + &query.QueryDataRequest{}, + &query.QueryDataResponse{}, + &query.QueryTypeDefinition{}, + &query.QueryTypeDefinitionList{}, ) } func (b *QueryAPIBuilder) InstallSchema(scheme *runtime.Scheme) error { - addKnownTypes(scheme, v0alpha1.SchemeGroupVersion) - metav1.AddToGroupVersion(scheme, v0alpha1.SchemeGroupVersion) - return scheme.SetVersionPriority(v0alpha1.SchemeGroupVersion) + addKnownTypes(scheme, query.SchemeGroupVersion) + metav1.AddToGroupVersion(scheme, query.SchemeGroupVersion) + return scheme.SetVersionPriority(query.SchemeGroupVersion) } func (b *QueryAPIBuilder) GetAPIGroupInfo( @@ -130,7 +146,7 @@ func (b *QueryAPIBuilder) GetAPIGroupInfo( optsGetter generic.RESTOptionsGetter, _ bool, ) (*genericapiserver.APIGroupInfo, error) { - gv := v0alpha1.SchemeGroupVersion + gv := query.SchemeGroupVersion apiGroupInfo := genericapiserver.NewDefaultAPIGroupInfo(gv.Group, scheme, metav1.ParameterCodec, codecs) storage := map[string]rest.Storage{} @@ -147,12 +163,15 @@ func (b *QueryAPIBuilder) GetAPIGroupInfo( // The query endpoint -- NOTE, this uses a rewrite hack to allow requests without a name parameter storage["query"] = newQueryREST(b) + // Register the expressions query schemas + err := queryschema.RegisterQueryTypes(b.queryTypes, storage) + apiGroupInfo.VersionedResourcesStorageMap[gv.Version] = storage - return &apiGroupInfo, nil + return &apiGroupInfo, err } func (b *QueryAPIBuilder) GetOpenAPIDefinitions() common.GetOpenAPIDefinitions { - return v0alpha1.GetOpenAPIDefinitions + return query.GetOpenAPIDefinitions } // Register additional routes with the server @@ -164,9 +183,6 @@ func (b *QueryAPIBuilder) GetAuthorizer() authorizer.Authorizer { return nil // default is OK } -const QueryRequestSchemaKey = "QueryRequestSchema" -const QueryPayloadSchemaKey = "QueryPayloadSchema" - func (b *QueryAPIBuilder) PostProcessOpenAPI(oas *spec3.OpenAPI) (*spec3.OpenAPI, error) { // The plugin description oas.Info.Description = "Query service" @@ -174,115 +190,83 @@ func (b *QueryAPIBuilder) PostProcessOpenAPI(oas *spec3.OpenAPI) (*spec3.OpenAPI // The root api URL root := "/apis/" + b.GetGroupVersion().String() + "/" - var err error - opts := schemabuilder.QuerySchemaOptions{ - PluginID: []string{""}, - QueryTypes: []data.QueryTypeDefinition{}, - Mode: schemabuilder.SchemaTypeQueryPayload, - } - oas.Components.Schemas[QueryPayloadSchemaKey], err = schemabuilder.GetQuerySchema(opts) - if err != nil { - return oas, err - } - opts.Mode = schemabuilder.SchemaTypeQueryRequest - oas.Components.Schemas[QueryRequestSchemaKey], err = schemabuilder.GetQuerySchema(opts) - if err != nil { - return oas, err - } + // Add queries to the request properties + err := queryschema.AddQueriesToOpenAPI(queryschema.OASQueryOptions{ + Swagger: oas, + PluginJSON: &plugins.JSONData{ + ID: expr.DatasourceType, // Not really a plugin, but identified the same way + }, + QueryTypes: b.queryTypes, + Root: root, + QueryPath: "namespaces/{namespace}/query/{name}", + QueryDescription: "Query any datasources (with expressions)", - // Rewrite the query path - sub := oas.Paths.Paths[root+"namespaces/{namespace}/query/{name}"] - if sub != nil && sub.Post != nil { - sub.Post.Tags = []string{"Query"} - sub.Parameters = []*spec3.Parameter{ - { - ParameterProps: spec3.ParameterProps{ - Name: "namespace", - In: "path", - Description: "object name and auth scope, such as for teams and projects", - Example: "default", - Required: true, + // An explicit set of examples (otherwise we iterate the query type examples) + QueryExamples: map[string]*spec3.Example{ + "A": { + ExampleProps: spec3.ExampleProps{ + Summary: "Random walk (testdata)", + Description: "Use testdata to execute a random walk query", + Value: `{ + "queries": [ + { + "refId": "A", + "scenarioId": "random_walk_table", + "seriesCount": 1, + "datasource": { + "type": "grafana-testdata-datasource", + "uid": "PD8C576611E62080A" + }, + "intervalMs": 60000, + "maxDataPoints": 20 + } + ], + "from": "now-6h", + "to": "now" + }`, }, }, - } - sub.Post.Description = "Query datasources (with expressions)" - sub.Post.Parameters = nil // - sub.Post.RequestBody = &spec3.RequestBody{ - RequestBodyProps: spec3.RequestBodyProps{ - Content: map[string]*spec3.MediaType{ - "application/json": { - MediaTypeProps: spec3.MediaTypeProps{ - Schema: spec.RefSchema("#/components/schemas/" + QueryRequestSchemaKey), - Examples: map[string]*spec3.Example{ - "A": { - ExampleProps: spec3.ExampleProps{ - Summary: "Random walk (testdata)", - Description: "Use testdata to execute a random walk query", - Value: `{ - "queries": [ - { - "refId": "A", - "scenarioId": "random_walk_table", - "seriesCount": 1, - "datasource": { - "type": "grafana-testdata-datasource", - "uid": "PD8C576611E62080A" - }, - "intervalMs": 60000, - "maxDataPoints": 20 - } - ], - "from": "now-6h", - "to": "now" - }`, - }, - }, - "B": { - ExampleProps: spec3.ExampleProps{ - Summary: "With deprecated datasource name", - Description: "Includes an old style string for datasource reference", - Value: `{ - "queries": [ - { - "refId": "A", - "datasource": { - "type": "grafana-googlesheets-datasource", - "uid": "b1808c48-9fc9-4045-82d7-081781f8a553" - }, - "cacheDurationSeconds": 300, - "spreadsheet": "spreadsheetID", - "datasourceId": 4, - "intervalMs": 30000, - "maxDataPoints": 794 - }, - { - "refId": "Z", - "datasource": "old", - "maxDataPoints": 10, - "timeRange": { - "from": "100", - "to": "200" - } - } - ], - "from": "now-6h", - "to": "now" - }`, - }, + "B": { + ExampleProps: spec3.ExampleProps{ + Summary: "With deprecated datasource name", + Description: "Includes an old style string for datasource reference", + Value: `{ + "queries": [ + { + "refId": "A", + "datasource": { + "type": "grafana-googlesheets-datasource", + "uid": "b1808c48-9fc9-4045-82d7-081781f8a553" }, + "cacheDurationSeconds": 300, + "spreadsheet": "spreadsheetID", + "datasourceId": 4, + "intervalMs": 30000, + "maxDataPoints": 794 }, - }, - }, + { + "refId": "Z", + "datasource": "old", + "maxDataPoints": 10, + "timeRange": { + "from": "100", + "to": "200" + } + } + ], + "from": "now-6h", + "to": "now" + }`, }, }, - } - - delete(oas.Paths.Paths, root+"namespaces/{namespace}/query/{name}") - oas.Paths.Paths[root+"namespaces/{namespace}/query"] = sub + }, + }) + if err != nil { + return oas, nil } // The root API discovery list - sub = oas.Paths.Paths[root] + sub := oas.Paths.Paths[root] if sub != nil && sub.Get != nil { sub.Get.Tags = []string{"API Discovery"} // sorts first in the list } diff --git a/pkg/registry/apis/scope/register.go b/pkg/registry/apis/scope/register.go index 1d6347b8dab..f5bbbdf20e9 100644 --- a/pkg/registry/apis/scope/register.go +++ b/pkg/registry/apis/scope/register.go @@ -13,6 +13,7 @@ import ( genericapiserver "k8s.io/apiserver/pkg/server" "k8s.io/kube-openapi/pkg/common" "k8s.io/kube-openapi/pkg/spec3" + "k8s.io/kube-openapi/pkg/validation/spec" scope "github.com/grafana/grafana/pkg/apis/scope/v0alpha1" "github.com/grafana/grafana/pkg/apiserver/builder" @@ -177,6 +178,7 @@ func (b *ScopeAPIBuilder) PostProcessOpenAPI(oas *spec3.OpenAPI) (*spec3.OpenAPI Description: "object name and auth scope, such as for teams and projects", Example: "default", Required: true, + Schema: spec.StringProperty().UniqueValues(), }, }, } diff --git a/pkg/services/apiserver/standalone/factory.go b/pkg/services/apiserver/standalone/factory.go index 0af7bf0132c..6cca2a1a6a7 100644 --- a/pkg/services/apiserver/standalone/factory.go +++ b/pkg/services/apiserver/standalone/factory.go @@ -107,6 +107,7 @@ func (p *DummyAPIFactory) MakeAPIServer(_ context.Context, tracer tracing.Tracer }, &pluginDatasourceImpl{}, // stub &actest.FakeAccessControl{ExpectedEvaluate: true}, + true, // show query types ) } diff --git a/pkg/services/featuremgmt/registry.go b/pkg/services/featuremgmt/registry.go index 899507e70c2..d62367f9821 100644 --- a/pkg/services/featuremgmt/registry.go +++ b/pkg/services/featuremgmt/registry.go @@ -775,6 +775,13 @@ var ( Owner: grafanaAppPlatformSquad, RequiresRestart: true, // changes the API routing }, + { + Name: "datasourceQueryTypes", + Description: "Show query type endpoints in datasource API servers (currently hardcoded for testdata, expressions, and prometheus)", + Stage: FeatureStageExperimental, + Owner: grafanaAppPlatformSquad, + RequiresRestart: true, // changes the API routing + }, { Name: "queryService", Description: "Register /apis/query.grafana.app/ -- will eventually replace /api/ds/query", diff --git a/pkg/services/featuremgmt/toggles_gen.csv b/pkg/services/featuremgmt/toggles_gen.csv index 91c44196b3a..a2a8db91b7c 100644 --- a/pkg/services/featuremgmt/toggles_gen.csv +++ b/pkg/services/featuremgmt/toggles_gen.csv @@ -103,6 +103,7 @@ formatString,preview,@grafana/dataviz-squad,false,false,true transformationsVariableSupport,preview,@grafana/dataviz-squad,false,false,true kubernetesPlaylists,GA,@grafana/grafana-app-platform-squad,false,true,false kubernetesSnapshots,experimental,@grafana/grafana-app-platform-squad,false,true,false +datasourceQueryTypes,experimental,@grafana/grafana-app-platform-squad,false,true,false queryService,experimental,@grafana/grafana-app-platform-squad,false,true,false queryServiceRewrite,experimental,@grafana/grafana-app-platform-squad,false,true,false queryServiceFromUI,experimental,@grafana/grafana-app-platform-squad,false,false,true diff --git a/pkg/services/featuremgmt/toggles_gen.go b/pkg/services/featuremgmt/toggles_gen.go index 0d12f1a28f5..128f6d145cd 100644 --- a/pkg/services/featuremgmt/toggles_gen.go +++ b/pkg/services/featuremgmt/toggles_gen.go @@ -423,6 +423,10 @@ const ( // Routes snapshot requests from /api to the /apis endpoint FlagKubernetesSnapshots = "kubernetesSnapshots" + // FlagDatasourceQueryTypes + // Show query type endpoints in datasource API servers (currently hardcoded for testdata, expressions, and prometheus) + FlagDatasourceQueryTypes = "datasourceQueryTypes" + // FlagQueryService // Register /apis/query.grafana.app/ -- will eventually replace /api/ds/query FlagQueryService = "queryService" diff --git a/pkg/services/featuremgmt/toggles_gen.json b/pkg/services/featuremgmt/toggles_gen.json index 1f83741eaa3..de752352db1 100644 --- a/pkg/services/featuremgmt/toggles_gen.json +++ b/pkg/services/featuremgmt/toggles_gen.json @@ -2224,6 +2224,19 @@ "stage": "experimental", "codeowner": "@grafana/observability-metrics" } + }, + { + "metadata": { + "name": "datasourceQueryTypes", + "resourceVersion": "1716473268430", + "creationTimestamp": "2024-05-23T14:07:48Z" + }, + "spec": { + "description": "Show query type endpoints in datasource API servers (currently hardcoded for testdata, expressions, and prometheus)", + "stage": "experimental", + "codeowner": "@grafana/grafana-app-platform-squad", + "requiresRestart": true + } } ] } \ No newline at end of file diff --git a/pkg/tsdb/grafana-testdata-datasource/kinds/query.go b/pkg/tsdb/grafana-testdata-datasource/kinds/query.go index f160c486b6d..6b96c1a6ee9 100644 --- a/pkg/tsdb/grafana-testdata-datasource/kinds/query.go +++ b/pkg/tsdb/grafana-testdata-datasource/kinds/query.go @@ -172,7 +172,7 @@ type USAQuery struct { //go:embed query.types.json var f embed.FS -// QueryTypeDefinitionsJSON returns the query type definitions -func QueryTypeDefinitionsJSON() (json.RawMessage, error) { +// QueryTypeDefinitionListJSON returns the query type definitions +func QueryTypeDefinitionListJSON() (json.RawMessage, error) { return f.ReadFile("query.types.json") }