opensource/grafana
1
0
Fork 0
mirror of https://github.com/grafana/grafana.git synced 2025-07-31 12:12:35 +08:00
Code Issues Packages Projects Releases Wiki Activity

QueryTypes: Add feature toggle to show query types in datasource apiservers (#88213)

Browse Source 
* initial attempt

* show query types

* show query types

* with formatting

* with formatting

* more cleanup

* add feature toggle

* fix build
This commit is contained in:
Ryan McKinley
2024-05-23 19:46:28 +03:00
committed by GitHub
parent 006d0021e3
commit 42b0f802de
17 changed files with 590 additions and 244 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
docs/sources/setup-grafana/configure-grafana/feature-toggles/index.md
View File

@ -156,6 +156,7 @@ Experimental features might be changed or removed without prior notice.
| `idForwarding` | Generate signed id token for identity that can be forwarded to plugins and external services |
| `enableNativeHTTPHistogram` | Enables native HTTP Histograms |
| `kubernetesSnapshots` | Routes snapshot requests from /api to the /apis endpoint |
| `datasourceQueryTypes` | Show query type endpoints in datasource API servers (currently hardcoded for testdata, expressions, and prometheus) |
| `queryService` | Register /apis/query.grafana.app/ -- will eventually replace /api/ds/query |
| `queryServiceRewrite` | Rewrite requests targeting /ds/query to the query service |
| `queryServiceFromUI` | Routes requests to the new query service |

1
packages/grafana-data/src/types/featureToggles.gen.ts
View File

@ -122,6 +122,7 @@ export interface FeatureToggles {
transformationsVariableSupport?: boolean;
kubernetesPlaylists?: boolean;
kubernetesSnapshots?: boolean;
datasourceQueryTypes?: boolean;
queryService?: boolean;
queryServiceRewrite?: boolean;
queryServiceFromUI?: boolean;

9
pkg/expr/query.go
View File

@ -1,6 +1,8 @@
package expr
import (
"embed"
"github.com/grafana/grafana/pkg/expr/classic"
"github.com/grafana/grafana/pkg/expr/mathexp"
)
@ -100,3 +102,10 @@ const (
// Replace non-numbers
ReduceModeReplace ReduceMode = "replaceNN"
)
//go:embed query.types.json
var f embed.FS
func QueryTypeDefinitionListJSON() ([]byte, error) {
return f.ReadFile("query.types.json")
}

8
pkg/expr/reader.go
View File

@ -1,7 +1,6 @@
package expr
import (
"embed"
"fmt"
"strings"
@ -178,13 +177,6 @@ func (h *ExpressionQueryReader) ReadQuery(
return eq, err
}
//go:embed query.types.json
var f embed.FS
func (h *ExpressionQueryReader) QueryTypeDefinitionListJSON() ([]byte, error) {
return f.ReadFile("query.types.json")
}
func getReferenceVar(exp string, refId string) (string, error) {
exp = strings.TrimPrefix(exp, "$")
if exp == "" {

2
pkg/promlib/models/query.go
View File

@ -434,6 +434,6 @@ func AlignTimeRange(t time.Time, step time.Duration, offset int64) time.Time {
var f embed.FS
// QueryTypeDefinitionsJSON returns the query type definitions
func QueryTypeDefinitionsJSON() (json.RawMessage, error) {
func QueryTypeDefinitionListJSON() (json.RawMessage, error) {
return f.ReadFile("query.types.json")
}

158
pkg/registry/apis/datasource/register.go
View File

@ -2,13 +2,11 @@ package datasource
import (
"context"
"encoding/json"
"fmt"
"net/http"
"time"
"github.com/grafana/grafana-plugin-sdk-go/backend"
data "github.com/grafana/grafana-plugin-sdk-go/experimental/apis/data/v0alpha1"
"github.com/grafana/grafana-plugin-sdk-go/experimental/schemabuilder"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime"
@ -19,7 +17,6 @@ import (
genericapiserver "k8s.io/apiserver/pkg/server"
openapi "k8s.io/kube-openapi/pkg/common"
"k8s.io/kube-openapi/pkg/spec3"
"k8s.io/kube-openapi/pkg/validation/spec"
"k8s.io/utils/strings/slices"
common "github.com/grafana/grafana/pkg/apimachinery/apis/common/v0alpha1"
@ -27,15 +24,15 @@ import (
query "github.com/grafana/grafana/pkg/apis/query/v0alpha1"
"github.com/grafana/grafana/pkg/apiserver/builder"
"github.com/grafana/grafana/pkg/plugins"
"github.com/grafana/grafana/pkg/promlib/models"
"github.com/grafana/grafana/pkg/registry/apis/query/queryschema"
"github.com/grafana/grafana/pkg/services/accesscontrol"
"github.com/grafana/grafana/pkg/services/apiserver/utils"
"github.com/grafana/grafana/pkg/services/featuremgmt"
"github.com/grafana/grafana/pkg/services/pluginsintegration/pluginstore"
"github.com/grafana/grafana/pkg/tsdb/grafana-testdata-datasource/kinds"
)
const QueryRequestSchemaKey = "QueryRequestSchema"
const QueryPayloadSchemaKey = "QueryPayloadSchema"
var _ builder.APIGroupBuilder = (*DataSourceAPIBuilder)(nil)
// DataSourceAPIBuilder is used just so wire has something unique to return
@ -83,6 +80,7 @@ func RegisterAPIService(
datasources.GetDatasourceProvider(ds.JSONData),
contextProvider,
accessControl,
features.IsEnabledGlobally(featuremgmt.FlagDatasourceQueryTypes),
)
if err != nil {
return nil, err
@ -105,20 +103,48 @@ func NewDataSourceAPIBuilder(
client PluginClient,
datasources PluginDatasourceProvider,
contextProvider PluginContextWrapper,
accessControl accesscontrol.AccessControl) (*DataSourceAPIBuilder, error) {
accessControl accesscontrol.AccessControl,
loadQueryTypes bool,
) (*DataSourceAPIBuilder, error) {
ri, err := resourceFromPluginID(plugin.ID)
if err != nil {
return nil, err
}
return &DataSourceAPIBuilder{
builder := &DataSourceAPIBuilder{
connectionResourceInfo: ri,
pluginJSON: plugin,
client: client,
datasources: datasources,
contextProvider: contextProvider,
accessControl: accessControl,
}, nil
}
if loadQueryTypes {
// In the future, this will somehow come from the plugin
builder.queryTypes, err = getHardcodedQueryTypes(ri.GroupResource().Group)
}
return builder, err
}
// TODO -- somehow get the list from the plugin -- not hardcoded
func getHardcodedQueryTypes(group string) (*query.QueryTypeDefinitionList, error) {
var err error
var raw json.RawMessage
switch group {
case "testdata.datasource.grafana.app":
raw, err = kinds.QueryTypeDefinitionListJSON()
case "prometheus.datasource.grafana.app":
raw, err = models.QueryTypeDefinitionListJSON()
}
if err != nil {
return nil, err
}
if raw != nil {
types := &query.QueryTypeDefinitionList{}
err = json.Unmarshal(raw, types)
return types, err
}
return nil, err
}
func (b *DataSourceAPIBuilder) GetGroupVersion() schema.GroupVersion {
@ -134,6 +160,8 @@ func addKnownTypes(scheme *runtime.Scheme, gv schema.GroupVersion) {
// Query handler
&query.QueryDataRequest{},
&query.QueryDataResponse{},
&query.QueryTypeDefinition{},
&query.QueryTypeDefinitionList{},
&metav1.Status{},
)
}
@ -211,12 +239,16 @@ func (b *DataSourceAPIBuilder) GetAPIGroupInfo(
storage[conn.StoragePath("proxy")] = &subProxyREST{pluginJSON: b.pluginJSON}
}
// Register hardcoded query schemas
err := queryschema.RegisterQueryTypes(b.queryTypes, storage)
// Create the group info
apiGroupInfo := genericapiserver.NewDefaultAPIGroupInfo(
conn.GroupResource().Group, scheme,
metav1.ParameterCodec, codecs)
apiGroupInfo.VersionedResourcesStorageMap[conn.GroupVersion().Version] = storage
return &apiGroupInfo, nil
return &apiGroupInfo, err
}
func (b *DataSourceAPIBuilder) getPluginContext(ctx context.Context, uid string) (backend.PluginContext, error) {
@ -247,66 +279,19 @@ func (b *DataSourceAPIBuilder) PostProcessOpenAPI(oas *spec3.OpenAPI) (*spec3.Op
// Hide the ability to list all connections across tenants
delete(oas.Paths.Paths, root+b.connectionResourceInfo.GroupResource().Resource)
var err error
opts := schemabuilder.QuerySchemaOptions{
PluginID: []string{b.pluginJSON.ID},
QueryTypes: []data.QueryTypeDefinition{},
Mode: schemabuilder.SchemaTypeQueryPayload,
}
if b.pluginJSON.AliasIDs != nil {
opts.PluginID = append(opts.PluginID, b.pluginJSON.AliasIDs...)
}
if b.queryTypes != nil {
for _, qt := range b.queryTypes.Items {
// The SDK type and api type are not the same so we recreate it here
opts.QueryTypes = append(opts.QueryTypes, data.QueryTypeDefinition{
ObjectMeta: data.ObjectMeta{
Name: qt.Name,
},
Spec: qt.Spec,
})
}
}
oas.Components.Schemas[QueryPayloadSchemaKey], err = schemabuilder.GetQuerySchema(opts)
if err != nil {
return oas, err
}
opts.Mode = schemabuilder.SchemaTypeQueryRequest
oas.Components.Schemas[QueryRequestSchemaKey], err = schemabuilder.GetQuerySchema(opts)
if err != nil {
return oas, err
}
// Update the request object
sub := oas.Paths.Paths[root+"namespaces/{namespace}/connections/{name}/query"]
if sub != nil && sub.Post != nil {
sub.Post.Description = "Execute queries"
sub.Post.RequestBody = &spec3.RequestBody{
RequestBodyProps: spec3.RequestBodyProps{
Required: true,
Content: map[string]*spec3.MediaType{
"application/json": {
MediaTypeProps: spec3.MediaTypeProps{
Schema: spec.RefSchema("#/components/schemas/" + QueryRequestSchemaKey),
Examples: getExamples(b.queryTypes),
},
},
},
},
}
okrsp, ok := sub.Post.Responses.StatusCodeResponses[200]
if ok {
sub.Post.Responses.StatusCodeResponses[http.StatusMultiStatus] = &spec3.Response{
ResponseProps: spec3.ResponseProps{
Description: "Query executed, but errors may exist in the datasource. See the payload for more details.",
Content: okrsp.Content,
},
}
}
}
// Add queries to the request properties
// Add queries to the request properties
err := queryschema.AddQueriesToOpenAPI(queryschema.OASQueryOptions{
Swagger: oas,
PluginJSON: &b.pluginJSON,
QueryTypes: b.queryTypes,
Root: root,
QueryPath: "namespaces/{namespace}/connections/{name}/query",
QueryDescription: fmt.Sprintf("Query the %s datasources", b.pluginJSON.Name),
})
// The root API discovery list
sub = oas.Paths.Paths[root]
sub := oas.Paths.Paths[root]
if sub != nil && sub.Get != nil {
sub.Get.Tags = []string{"API Discovery"} // sorts first in the list
}
@ -317,38 +302,3 @@ func (b *DataSourceAPIBuilder) PostProcessOpenAPI(oas *spec3.OpenAPI) (*spec3.Op
func (b *DataSourceAPIBuilder) GetAPIRoutes() *builder.APIRoutes {
return nil
}
func getExamples(queryTypes *query.QueryTypeDefinitionList) map[string]*spec3.Example {
if queryTypes == nil {
return nil
}
tr := data.TimeRange{From: "now-1h", To: "now"}
examples := map[string]*spec3.Example{}
for _, queryType := range queryTypes.Items {
for idx, example := range queryType.Spec.Examples {
q := data.NewDataQuery(example.SaveModel.Object)
q.RefID = "A"
for _, dis := range queryType.Spec.Discriminators {
_ = q.Set(dis.Field, dis.Value)
}
if q.MaxDataPoints < 1 {
q.MaxDataPoints = 1000
}
if q.IntervalMS < 1 {
q.IntervalMS = 5000 // 5s
}
examples[fmt.Sprintf("%s-%d", example.Name, idx)] = &spec3.Example{
ExampleProps: spec3.ExampleProps{
Summary: example.Name,
Description: example.Description,
Value: data.QueryDataRequest{
TimeRange: tr,
Queries: []data.DataQuery{q},
},
},
}
}
}
return examples
}

1
pkg/registry/apis/query/parser.go
View File

@ -186,7 +186,6 @@ func (p *queryParser) parseRequest(ctx context.Context, input *query.QueryDataRe
}
}
}
return rsp, nil
}

235
pkg/registry/apis/query/queryschema/oas_helper.go Normal file
View File

@ -0,0 +1,235 @@
package queryschema
import (
"fmt"
"strings"
"k8s.io/kube-openapi/pkg/spec3"
"k8s.io/kube-openapi/pkg/validation/spec"
"github.com/grafana/grafana-plugin-sdk-go/experimental/schemabuilder"
data "github.com/grafana/grafana-plugin-sdk-go/experimental/apis/data/v0alpha1"
query "github.com/grafana/grafana/pkg/apis/query/v0alpha1"
"github.com/grafana/grafana/pkg/plugins"
)
const QueryRequestSchemaKey = "QueryRequestSchema"
// const QueryPayloadSchemaKey = "QueryPayloadSchema"
// const QuerySaveModelSchemaKey = "QuerySaveModelSchema"
type OASQueryOptions struct {
Swagger *spec3.OpenAPI
PluginJSON *plugins.JSONData
QueryTypes *query.QueryTypeDefinitionList
Root string
QueryPath string // eg "namespaces/{namespace}/query/{name}"
QueryDescription string
QueryExamples map[string]*spec3.Example
}
func AddQueriesToOpenAPI(options OASQueryOptions) error {
oas := options.Swagger
root := options.Root
examples := options.QueryExamples
resourceName := query.QueryTypeDefinitionResourceInfo.GroupResource().Resource
builder := schemabuilder.QuerySchemaOptions{
PluginID: []string{""},
QueryTypes: []data.QueryTypeDefinition{},
}
if options.PluginJSON != nil {
builder.PluginID = []string{options.PluginJSON.ID}
if options.PluginJSON.AliasIDs != nil {
builder.PluginID = append(builder.PluginID, options.PluginJSON.AliasIDs...)
}
}
if options.QueryTypes != nil {
// The SDK type and api type are not the same so we just recreate it here
for _, qt := range options.QueryTypes.Items {
builder.QueryTypes = append(builder.QueryTypes, data.QueryTypeDefinition{
ObjectMeta: data.ObjectMeta{
Name: qt.Name,
},
Spec: qt.Spec,
})
}
if examples == nil {
examples = getExamples(options.QueryTypes)
}
}
// Rewrite the query path
query := oas.Paths.Paths[root+options.QueryPath]
if query != nil && query.Post != nil {
query.Post.Tags = []string{"Query"}
query.Parameters = []*spec3.Parameter{
{
ParameterProps: spec3.ParameterProps{
Name: "namespace",
In: "path",
Description: "object name and auth scope, such as for teams and projects",
Example: "default",
Required: true,
Schema: spec.StringProperty().UniqueValues(),
},
},
}
query.Post.Description = options.QueryDescription
query.Post.Parameters = nil //
query.Post.RequestBody = &spec3.RequestBody{
RequestBodyProps: spec3.RequestBodyProps{
Content: map[string]*spec3.MediaType{
"application/json": {
MediaTypeProps: spec3.MediaTypeProps{
Schema: spec.RefSchema("#/components/schemas/" + QueryRequestSchemaKey),
Examples: examples,
},
},
},
},
}
// Remove the {name} hack from from the query
if strings.HasSuffix(options.QueryPath, "/{name}") {
delete(oas.Paths.Paths, root+options.QueryPath)
oas.Paths.Paths[root+strings.TrimSuffix(options.QueryPath, "/{name}")] = query
}
}
// Update the validate endpoint
validate := oas.Paths.Paths[root+resourceName+"/{name}/validate"]
if validate != nil && validate.Post != nil {
validate.Post.Description = "Verify if a query payload matches the expected value and return a clean version"
validate.Parameters = []*spec3.Parameter{
{
ParameterProps: spec3.ParameterProps{
Name: "name",
In: "path",
Description: "The query type name, or {any}",
Example: "{any}",
Required: true,
Schema: spec.StringProperty().UniqueValues(),
},
},
}
// Accept the same payload as the query type
validate.Post.RequestBody = query.Post.RequestBody
}
// Query Request
builder.Mode = schemabuilder.SchemaTypeQueryRequest
s, err := schemabuilder.GetQuerySchema(builder)
if err != nil {
return err
}
// The schema requires some munging to pass validation
// This should likely be fixed in the upstream "GetQuerySchema" function
removeSchemaRefs(s)
s.Description = "Schema for a set of queries sent to the query method"
oas.Components.Schemas[QueryRequestSchemaKey] = s
// // Query Payload (is this useful?)
// opts.Mode = schemabuilder.SchemaTypeQueryPayload
// s, err = schemabuilder.GetQuerySchema(opts)
// if err != nil {
// return err
// }
// delete(s.ExtraProps, "$schema")
// s.Description = "Schema for a single query object including all runtime properties"
// oas.Components.Schemas[QueryPayloadSchemaKey] = s
// // Query Save Model
// opts.Mode = schemabuilder.SchemaTypeSaveModel
// s, err = schemabuilder.GetQuerySchema(opts)
// if err != nil {
// return err
// }
// s.Extensions = nil // remove the $schema
// s.Description = "Valid save model for a single query target"
// oas.Components.Schemas[QuerySaveModelSchemaKey] = s
return nil
}
func getExamples(queryTypes *query.QueryTypeDefinitionList) map[string]*spec3.Example {
if queryTypes == nil {
return nil
}
tr := data.TimeRange{From: "now-1h", To: "now"}
examples := map[string]*spec3.Example{}
for _, queryType := range queryTypes.Items {
for idx, example := range queryType.Spec.Examples {
q := data.NewDataQuery(example.SaveModel.Object)
q.RefID = "A"
for _, dis := range queryType.Spec.Discriminators {
_ = q.Set(dis.Field, dis.Value)
}
if q.MaxDataPoints < 1 {
q.MaxDataPoints = 1000
}
if q.IntervalMS < 1 {
q.IntervalMS = 5000 // 5s
}
examples[fmt.Sprintf("%s-%d", example.Name, idx)] = &spec3.Example{
ExampleProps: spec3.ExampleProps{
Summary: example.Name,
Description: example.Description,
Value: data.QueryDataRequest{
TimeRange: tr,
Queries: []data.DataQuery{q},
},
},
}
}
}
return examples
}
func removeSchemaRefs(s *spec.Schema) {
if s == nil {
return
}
if s.Schema != "" {
s.Schema = ""
}
// Examples is invalid -- only use the first example
examples, ok := s.ExtraProps["examples"]
if ok && examples != nil {
//fmt.Printf("TODO, use reflection to get first element from: %+v\n", examples)
//s.Example = examples[0]
delete(s.ExtraProps, "examples")
}
removeSchemaRefs(s.Not)
for idx := range s.AllOf {
removeSchemaRefs(&s.AllOf[idx])
}
for idx := range s.AnyOf {
removeSchemaRefs(&s.AnyOf[idx])
}
for k := range s.Properties {
v := s.Properties[k]
removeSchemaRefs(&v)
s.Properties[k] = v
}
if s.Items != nil {
removeSchemaRefs(s.Items.Schema)
for idx := range s.Items.Schemas {
removeSchemaRefs(&s.Items.Schemas[idx])
}
}
if s.AdditionalProperties != nil {
removeSchemaRefs(s.AdditionalProperties.Schema)
}
if s.AdditionalItems != nil {
removeSchemaRefs(s.AdditionalItems.Schema)
}
}

147
pkg/registry/apis/query/queryschema/query_type_storage.go Normal file
View File

@ -0,0 +1,147 @@
package queryschema
import (
"context"
"net/http"
"k8s.io/apimachinery/pkg/apis/meta/internalversion"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apiserver/pkg/registry/rest"
"github.com/grafana/grafana-plugin-sdk-go/experimental/apis/data/v0alpha1"
common "github.com/grafana/grafana/pkg/apimachinery/apis/common/v0alpha1"
query "github.com/grafana/grafana/pkg/apis/query/v0alpha1"
)
var (
_ rest.Storage = (*queryTypeStorage)(nil)
_ rest.Scoper = (*queryTypeStorage)(nil)
_ rest.SingularNameProvider = (*queryTypeStorage)(nil)
_ rest.Lister = (*queryTypeStorage)(nil)
_ rest.Getter = (*queryTypeStorage)(nil)
// The connectors
_ = rest.Connecter(&queryValidationREST{})
)
type queryTypeStorage struct {
resourceInfo *common.ResourceInfo
tableConverter rest.TableConvertor
registry query.QueryTypeDefinitionList
}
type queryValidationREST struct {
qt *queryTypeStorage
}
func RegisterQueryTypes(queryTypes *query.QueryTypeDefinitionList, storage map[string]rest.Storage) error {
if queryTypes == nil {
return nil // NO error
}
resourceInfo := query.QueryTypeDefinitionResourceInfo
store := &queryTypeStorage{
resourceInfo: &resourceInfo,
tableConverter: rest.NewDefaultTableConvertor(resourceInfo.GroupResource()),
registry: *queryTypes,
}
// Supports list+get for all query types
storage[resourceInfo.StoragePath()] = store
// Adds a query validation endpoint for each query type
// We will also support the "<any>" or "*" name
storage[resourceInfo.StoragePath("validate")] = &queryValidationREST{store}
return nil
}
func (s *queryTypeStorage) New() runtime.Object {
return s.resourceInfo.NewFunc()
}
func (s *queryTypeStorage) Destroy() {}
func (s *queryTypeStorage) NamespaceScoped() bool {
return false
}
func (s *queryTypeStorage) GetSingularName() string {
return s.resourceInfo.GetSingularName()
}
func (s *queryTypeStorage) NewList() runtime.Object {
return s.resourceInfo.NewListFunc()
}
func (s *queryTypeStorage) ConvertToTable(ctx context.Context, object runtime.Object, tableOptions runtime.Object) (*metav1.Table, error) {
return s.tableConverter.ConvertToTable(ctx, object, tableOptions)
}
func (s *queryTypeStorage) List(ctx context.Context, options *internalversion.ListOptions) (runtime.Object, error) {
return &s.registry, nil
}
func (s *queryTypeStorage) Get(ctx context.Context, name string, options *metav1.GetOptions) (runtime.Object, error) {
for idx, qt := range s.registry.Items {
if qt.Name == name {
return &s.registry.Items[idx], nil
}
}
return nil, s.resourceInfo.NewNotFound(name)
}
//----------------------------------------------------
// The validation processor
//----------------------------------------------------
func (r *queryValidationREST) New() runtime.Object {
return &query.QueryDataRequest{}
}
func (r *queryValidationREST) Destroy() {
}
func (r *queryValidationREST) ConnectMethods() []string {
return []string{"POST"}
}
func (r *queryValidationREST) ProducesMIMETypes(verb string) []string {
return []string{"application/json"}
}
func (r *queryValidationREST) ProducesObject(verb string) interface{} {
return &query.QueryDataRequest{}
}
func (r *queryValidationREST) NewConnectOptions() (runtime.Object, bool, string) {
return nil, false, "" // true means you can use the trailing path as a variable
}
func (r *queryValidationREST) Connect(ctx context.Context, name string, opts runtime.Object, responder rest.Responder) (http.Handler, error) {
return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
// TODO -- validate/mutate the query
// should we return the DQR, or raw validation response?
qdr := &query.QueryDataRequest{
QueryDataRequest: v0alpha1.QueryDataRequest{
Debug: true,
},
}
if name == "*" || name == "{any}" {
qdr.Queries = []v0alpha1.DataQuery{
v0alpha1.NewDataQuery(
map[string]any{
"refId": "???",
"HELLO": "world",
"TODO": "parse any query",
},
),
}
}
responder.Object(http.StatusOK, qdr)
}), nil
}

240
pkg/registry/apis/query/register.go
View File

@ -1,8 +1,8 @@
package query
import (
data "github.com/grafana/grafana-plugin-sdk-go/experimental/apis/data/v0alpha1"
"github.com/grafana/grafana-plugin-sdk-go/experimental/schemabuilder"
"encoding/json"
"github.com/prometheus/client_golang/prometheus"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
@ -14,16 +14,15 @@ import (
genericapiserver "k8s.io/apiserver/pkg/server"
common "k8s.io/kube-openapi/pkg/common"
"k8s.io/kube-openapi/pkg/spec3"
"k8s.io/kube-openapi/pkg/validation/spec"
example "github.com/grafana/grafana/pkg/apis/example/v0alpha1"
"github.com/grafana/grafana/pkg/apis/query/v0alpha1"
query "github.com/grafana/grafana/pkg/apis/query/v0alpha1"
"github.com/grafana/grafana/pkg/apiserver/builder"
"github.com/grafana/grafana/pkg/expr"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/infra/tracing"
"github.com/grafana/grafana/pkg/plugins"
"github.com/grafana/grafana/pkg/registry/apis/query/client"
"github.com/grafana/grafana/pkg/registry/apis/query/queryschema"
"github.com/grafana/grafana/pkg/services/accesscontrol"
"github.com/grafana/grafana/pkg/services/datasources"
"github.com/grafana/grafana/pkg/services/datasources/service"
@ -41,22 +40,39 @@ type QueryAPIBuilder struct {
returnMultiStatus bool // from feature toggle
features featuremgmt.FeatureToggles
tracer tracing.Tracer
metrics *metrics
parser *queryParser
client DataSourceClientSupplier
registry v0alpha1.DataSourceApiServerRegistry
converter *expr.ResultConverter
tracer tracing.Tracer
metrics *metrics
parser *queryParser
client DataSourceClientSupplier
registry query.DataSourceApiServerRegistry
converter *expr.ResultConverter
queryTypes *query.QueryTypeDefinitionList
}
func NewQueryAPIBuilder(features featuremgmt.FeatureToggles,
client DataSourceClientSupplier,
registry v0alpha1.DataSourceApiServerRegistry,
registry query.DataSourceApiServerRegistry,
legacy service.LegacyDataSourceLookup,
registerer prometheus.Registerer,
tracer tracing.Tracer,
) (*QueryAPIBuilder, error) {
reader := expr.NewExpressionQueryReader(features)
// Include well typed query definitions
var queryTypes *query.QueryTypeDefinitionList
if features.IsEnabledGlobally(featuremgmt.FlagDatasourceQueryTypes) {
// Read the expression query definitions
raw, err := expr.QueryTypeDefinitionListJSON()
if err != nil {
return nil, err
}
queryTypes = &query.QueryTypeDefinitionList{}
err = json.Unmarshal(raw, queryTypes)
if err != nil {
return nil, err
}
}
return &QueryAPIBuilder{
concurrentQueryLimit: 4,
log: log.New("query_apiserver"),
@ -67,6 +83,7 @@ func NewQueryAPIBuilder(features featuremgmt.FeatureToggles,
metrics: newMetrics(registerer),
tracer: tracer,
features: features,
queryTypes: queryTypes,
converter: &expr.ResultConverter{
Features: features,
Tracer: tracer,
@ -103,25 +120,24 @@ func RegisterAPIService(features featuremgmt.FeatureToggles,
}
func (b *QueryAPIBuilder) GetGroupVersion() schema.GroupVersion {
return v0alpha1.SchemeGroupVersion
return query.SchemeGroupVersion
}
func addKnownTypes(scheme *runtime.Scheme, gv schema.GroupVersion) {
scheme.AddKnownTypes(gv,
&v0alpha1.DataSourceApiServer{},
&v0alpha1.DataSourceApiServerList{},
&v0alpha1.QueryDataRequest{},
&v0alpha1.QueryDataResponse{},
&v0alpha1.QueryTypeDefinition{},
&v0alpha1.QueryTypeDefinitionList{},
&example.DummySubresource{},
&query.DataSourceApiServer{},
&query.DataSourceApiServerList{},
&query.QueryDataRequest{},
&query.QueryDataResponse{},
&query.QueryTypeDefinition{},
&query.QueryTypeDefinitionList{},
)
}
func (b *QueryAPIBuilder) InstallSchema(scheme *runtime.Scheme) error {
addKnownTypes(scheme, v0alpha1.SchemeGroupVersion)
metav1.AddToGroupVersion(scheme, v0alpha1.SchemeGroupVersion)
return scheme.SetVersionPriority(v0alpha1.SchemeGroupVersion)
addKnownTypes(scheme, query.SchemeGroupVersion)
metav1.AddToGroupVersion(scheme, query.SchemeGroupVersion)
return scheme.SetVersionPriority(query.SchemeGroupVersion)
}
func (b *QueryAPIBuilder) GetAPIGroupInfo(
@ -130,7 +146,7 @@ func (b *QueryAPIBuilder) GetAPIGroupInfo(
optsGetter generic.RESTOptionsGetter,
_ bool,
) (*genericapiserver.APIGroupInfo, error) {
gv := v0alpha1.SchemeGroupVersion
gv := query.SchemeGroupVersion
apiGroupInfo := genericapiserver.NewDefaultAPIGroupInfo(gv.Group, scheme, metav1.ParameterCodec, codecs)
storage := map[string]rest.Storage{}
@ -147,12 +163,15 @@ func (b *QueryAPIBuilder) GetAPIGroupInfo(
// The query endpoint -- NOTE, this uses a rewrite hack to allow requests without a name parameter
storage["query"] = newQueryREST(b)
// Register the expressions query schemas
err := queryschema.RegisterQueryTypes(b.queryTypes, storage)
apiGroupInfo.VersionedResourcesStorageMap[gv.Version] = storage
return &apiGroupInfo, nil
return &apiGroupInfo, err
}
func (b *QueryAPIBuilder) GetOpenAPIDefinitions() common.GetOpenAPIDefinitions {
return v0alpha1.GetOpenAPIDefinitions
return query.GetOpenAPIDefinitions
}
// Register additional routes with the server
@ -164,9 +183,6 @@ func (b *QueryAPIBuilder) GetAuthorizer() authorizer.Authorizer {
return nil // default is OK
}
const QueryRequestSchemaKey = "QueryRequestSchema"
const QueryPayloadSchemaKey = "QueryPayloadSchema"
func (b *QueryAPIBuilder) PostProcessOpenAPI(oas *spec3.OpenAPI) (*spec3.OpenAPI, error) {
// The plugin description
oas.Info.Description = "Query service"
@ -174,115 +190,83 @@ func (b *QueryAPIBuilder) PostProcessOpenAPI(oas *spec3.OpenAPI) (*spec3.OpenAPI
// The root api URL
root := "/apis/" + b.GetGroupVersion().String() + "/"
var err error
opts := schemabuilder.QuerySchemaOptions{
PluginID: []string{""},
QueryTypes: []data.QueryTypeDefinition{},
Mode: schemabuilder.SchemaTypeQueryPayload,
}
oas.Components.Schemas[QueryPayloadSchemaKey], err = schemabuilder.GetQuerySchema(opts)
if err != nil {
return oas, err
}
opts.Mode = schemabuilder.SchemaTypeQueryRequest
oas.Components.Schemas[QueryRequestSchemaKey], err = schemabuilder.GetQuerySchema(opts)
if err != nil {
return oas, err
}
// Add queries to the request properties
err := queryschema.AddQueriesToOpenAPI(queryschema.OASQueryOptions{
Swagger: oas,
PluginJSON: &plugins.JSONData{
ID: expr.DatasourceType, // Not really a plugin, but identified the same way
},
QueryTypes: b.queryTypes,
Root: root,
QueryPath: "namespaces/{namespace}/query/{name}",
QueryDescription: "Query any datasources (with expressions)",
// Rewrite the query path
sub := oas.Paths.Paths[root+"namespaces/{namespace}/query/{name}"]
if sub != nil && sub.Post != nil {
sub.Post.Tags = []string{"Query"}
sub.Parameters = []*spec3.Parameter{
{
ParameterProps: spec3.ParameterProps{
Name: "namespace",
In: "path",
Description: "object name and auth scope, such as for teams and projects",
Example: "default",
Required: true,
// An explicit set of examples (otherwise we iterate the query type examples)
QueryExamples: map[string]*spec3.Example{
"A": {
ExampleProps: spec3.ExampleProps{
Summary: "Random walk (testdata)",
Description: "Use testdata to execute a random walk query",
Value: `{
"queries": [
{
"refId": "A",
"scenarioId": "random_walk_table",
"seriesCount": 1,
"datasource": {
"type": "grafana-testdata-datasource",
"uid": "PD8C576611E62080A"
},
"intervalMs": 60000,
"maxDataPoints": 20
}
],
"from": "now-6h",
"to": "now"
}`,
},
},
}
sub.Post.Description = "Query datasources (with expressions)"
sub.Post.Parameters = nil //
sub.Post.RequestBody = &spec3.RequestBody{
RequestBodyProps: spec3.RequestBodyProps{
Content: map[string]*spec3.MediaType{
"application/json": {
MediaTypeProps: spec3.MediaTypeProps{
Schema: spec.RefSchema("#/components/schemas/" + QueryRequestSchemaKey),
Examples: map[string]*spec3.Example{
"A": {
ExampleProps: spec3.ExampleProps{
Summary: "Random walk (testdata)",
Description: "Use testdata to execute a random walk query",
Value: `{
"queries": [
{
"refId": "A",
"scenarioId": "random_walk_table",
"seriesCount": 1,
"datasource": {
"type": "grafana-testdata-datasource",
"uid": "PD8C576611E62080A"
},
"intervalMs": 60000,
"maxDataPoints": 20
}
],
"from": "now-6h",
"to": "now"
}`,
},
},
"B": {
ExampleProps: spec3.ExampleProps{
Summary: "With deprecated datasource name",
Description: "Includes an old style string for datasource reference",
Value: `{
"queries": [
{
"refId": "A",
"datasource": {
"type": "grafana-googlesheets-datasource",
"uid": "b1808c48-9fc9-4045-82d7-081781f8a553"
},
"cacheDurationSeconds": 300,
"spreadsheet": "spreadsheetID",
"datasourceId": 4,
"intervalMs": 30000,
"maxDataPoints": 794
},
{
"refId": "Z",
"datasource": "old",
"maxDataPoints": 10,
"timeRange": {
"from": "100",
"to": "200"
}
}
],
"from": "now-6h",
"to": "now"
}`,
},
"B": {
ExampleProps: spec3.ExampleProps{
Summary: "With deprecated datasource name",
Description: "Includes an old style string for datasource reference",
Value: `{
"queries": [
{
"refId": "A",
"datasource": {
"type": "grafana-googlesheets-datasource",
"uid": "b1808c48-9fc9-4045-82d7-081781f8a553"
},
"cacheDurationSeconds": 300,
"spreadsheet": "spreadsheetID",
"datasourceId": 4,
"intervalMs": 30000,
"maxDataPoints": 794
},
},
},
{
"refId": "Z",
"datasource": "old",
"maxDataPoints": 10,
"timeRange": {
"from": "100",
"to": "200"
}
}
],
"from": "now-6h",
"to": "now"
}`,
},
},
}
delete(oas.Paths.Paths, root+"namespaces/{namespace}/query/{name}")
oas.Paths.Paths[root+"namespaces/{namespace}/query"] = sub
},
})
if err != nil {
return oas, nil
}
// The root API discovery list
sub = oas.Paths.Paths[root]
sub := oas.Paths.Paths[root]
if sub != nil && sub.Get != nil {
sub.Get.Tags = []string{"API Discovery"} // sorts first in the list
}

2
pkg/registry/apis/scope/register.go
View File

@ -13,6 +13,7 @@ import (
genericapiserver "k8s.io/apiserver/pkg/server"
"k8s.io/kube-openapi/pkg/common"
"k8s.io/kube-openapi/pkg/spec3"
"k8s.io/kube-openapi/pkg/validation/spec"
scope "github.com/grafana/grafana/pkg/apis/scope/v0alpha1"
"github.com/grafana/grafana/pkg/apiserver/builder"
@ -177,6 +178,7 @@ func (b *ScopeAPIBuilder) PostProcessOpenAPI(oas *spec3.OpenAPI) (*spec3.OpenAPI
Description: "object name and auth scope, such as for teams and projects",
Example: "default",
Required: true,
Schema: spec.StringProperty().UniqueValues(),
},
},
}

1
pkg/services/apiserver/standalone/factory.go
View File

@ -107,6 +107,7 @@ func (p *DummyAPIFactory) MakeAPIServer(_ context.Context, tracer tracing.Tracer
},
&pluginDatasourceImpl{}, // stub
&actest.FakeAccessControl{ExpectedEvaluate: true},
true, // show query types
)
}

7
pkg/services/featuremgmt/registry.go
View File

@ -775,6 +775,13 @@ var (
Owner: grafanaAppPlatformSquad,
RequiresRestart: true, // changes the API routing
},
{
Name: "datasourceQueryTypes",
Description: "Show query type endpoints in datasource API servers (currently hardcoded for testdata, expressions, and prometheus)",
Stage: FeatureStageExperimental,
Owner: grafanaAppPlatformSquad,
RequiresRestart: true, // changes the API routing
},
{
Name: "queryService",
Description: "Register /apis/query.grafana.app/ -- will eventually replace /api/ds/query",

1
pkg/services/featuremgmt/toggles_gen.csv
View File

@ -103,6 +103,7 @@ formatString,preview,@grafana/dataviz-squad,false,false,true
transformationsVariableSupport,preview,@grafana/dataviz-squad,false,false,true
kubernetesPlaylists,GA,@grafana/grafana-app-platform-squad,false,true,false
kubernetesSnapshots,experimental,@grafana/grafana-app-platform-squad,false,true,false
datasourceQueryTypes,experimental,@grafana/grafana-app-platform-squad,false,true,false
queryService,experimental,@grafana/grafana-app-platform-squad,false,true,false
queryServiceRewrite,experimental,@grafana/grafana-app-platform-squad,false,true,false
queryServiceFromUI,experimental,@grafana/grafana-app-platform-squad,false,false,true

1 Name Stage Owner requiresDevMode RequiresRestart FrontendOnly
103 transformationsVariableSupport preview @grafana/dataviz-squad false false true
104 kubernetesPlaylists GA @grafana/grafana-app-platform-squad false true false
105 kubernetesSnapshots experimental @grafana/grafana-app-platform-squad false true false
106 datasourceQueryTypes experimental @grafana/grafana-app-platform-squad false true false
107 queryService experimental @grafana/grafana-app-platform-squad false true false
108 queryServiceRewrite experimental @grafana/grafana-app-platform-squad false true false
109 queryServiceFromUI experimental @grafana/grafana-app-platform-squad false false true

4
pkg/services/featuremgmt/toggles_gen.go
View File

@ -423,6 +423,10 @@ const (
// Routes snapshot requests from /api to the /apis endpoint
FlagKubernetesSnapshots = "kubernetesSnapshots"
// FlagDatasourceQueryTypes
// Show query type endpoints in datasource API servers (currently hardcoded for testdata, expressions, and prometheus)
FlagDatasourceQueryTypes = "datasourceQueryTypes"
// FlagQueryService
// Register /apis/query.grafana.app/ -- will eventually replace /api/ds/query
FlagQueryService = "queryService"

13
pkg/services/featuremgmt/toggles_gen.json
View File

@ -2224,6 +2224,19 @@
"stage": "experimental",
"codeowner": "@grafana/observability-metrics"
}
},
{
"metadata": {
"name": "datasourceQueryTypes",
"resourceVersion": "1716473268430",
"creationTimestamp": "2024-05-23T14:07:48Z"
},
"spec": {
"description": "Show query type endpoints in datasource API servers (currently hardcoded for testdata, expressions, and prometheus)",
"stage": "experimental",
"codeowner": "@grafana/grafana-app-platform-squad",
"requiresRestart": true
}
}
]
}

4
pkg/tsdb/grafana-testdata-datasource/kinds/query.go
View File

@ -172,7 +172,7 @@ type USAQuery struct {
//go:embed query.types.json
var f embed.FS
// QueryTypeDefinitionsJSON returns the query type definitions
func QueryTypeDefinitionsJSON() (json.RawMessage, error) {
// QueryTypeDefinitionListJSON returns the query type definitions
func QueryTypeDefinitionListJSON() (json.RawMessage, error) {
return f.ReadFile("query.types.json")
}