mirror of
https://github.com/grafana/grafana.git
synced 2025-07-29 22:12:08 +08:00

* expr: Add row limit to SQL expressions Adds a configurable row limit to SQL expressions to prevent memory issues with large result sets. The limit is configured via the `sql_expression_row_limit` setting in the `[expressions]` section of grafana.ini, with a default of 100,000 rows. The limit is enforced by checking the total number of rows across all input tables before executing the SQL query. If the total exceeds the limit, the query fails with an error message indicating the limit was exceeded. * revert addition of newline * Switch to table-driven tests * Remove single-frame test-cases. We only need to test for the multi frame case. Single frame is a subset of the multi-frame case * Add helper function Simplify the way tests are set up and written * Support convention, that limit: 0 is no limit * Set the row-limit in one place only * Update default limit to 20k rows As per some discussion here: https://raintank-corp.slack.com/archives/C071A5XCFST/p1741611647001369?thread_ts=1740047619.804869&cid=C071A5XCFST * Test row-limit is applied from config Make sure we protect this from regressions This is perhaps a brittle test, somewhat coupled to the code here. But it's good enough to prevent regressions at least. * Add public documentation for the limit * Limit total number of cells instead of rows * Use named-return for totalRows As @kylebrandt requested during review of #101700 * Leave DF cells as zero values during limits tests When testing the cell limit we don't interact with the cell values at all, so we leave them at their zero values both to speed up tests, and to simplify and clarify that their values aren't used. * Set SQLCmd limit at object creation - don't mutate * Test that SQL node receives limit when built And that it receives it from the Grafana config * Improve TODO message for new Expression Parser * Fix failing test by always creating config on the Service
203 lines
5.7 KiB
Go
203 lines
5.7 KiB
Go
package expr
|
|
|
|
import (
|
|
"fmt"
|
|
"strings"
|
|
|
|
"github.com/grafana/grafana-plugin-sdk-go/backend/gtime"
|
|
"github.com/grafana/grafana-plugin-sdk-go/data/utils/jsoniter"
|
|
data "github.com/grafana/grafana-plugin-sdk-go/experimental/apis/data/v0alpha1"
|
|
|
|
"github.com/grafana/grafana/pkg/expr/classic"
|
|
"github.com/grafana/grafana/pkg/expr/mathexp"
|
|
"github.com/grafana/grafana/pkg/services/featuremgmt"
|
|
)
|
|
|
|
// Once we are comfortable with the parsing logic, this struct will
|
|
// be merged/replace the existing Query struct in grafana/pkg/expr/transform.go
|
|
type ExpressionQuery struct {
|
|
GraphID int64 `json:"id,omitempty"`
|
|
RefID string `json:"refId"`
|
|
QueryType QueryType `json:"type"`
|
|
|
|
// The typed query parameters
|
|
Properties any `json:"properties"`
|
|
|
|
// Hidden in debug JSON
|
|
Command Command `json:"-"`
|
|
}
|
|
|
|
// ID is used to identify nodes in the directed graph
|
|
func (q ExpressionQuery) ID() int64 {
|
|
return q.GraphID
|
|
}
|
|
|
|
type ExpressionQueryReader struct {
|
|
features featuremgmt.FeatureToggles
|
|
}
|
|
|
|
func NewExpressionQueryReader(features featuremgmt.FeatureToggles) *ExpressionQueryReader {
|
|
return &ExpressionQueryReader{
|
|
features: features,
|
|
}
|
|
}
|
|
|
|
// nolint:gocyclo
|
|
func (h *ExpressionQueryReader) ReadQuery(
|
|
// Properties that have been parsed off the same node
|
|
common data.DataQuery,
|
|
// An iterator with context for the full node (include common values)
|
|
iter *jsoniter.Iterator,
|
|
) (eq ExpressionQuery, err error) {
|
|
referenceVar := ""
|
|
eq.RefID = common.RefID
|
|
eq.QueryType = QueryType(common.GetString("type"))
|
|
if eq.QueryType == "" {
|
|
return eq, fmt.Errorf("missing type")
|
|
}
|
|
switch eq.QueryType {
|
|
case QueryTypeMath:
|
|
q := &MathQuery{}
|
|
err = iter.ReadVal(q)
|
|
if err == nil {
|
|
eq.Command, err = NewMathCommand(common.RefID, q.Expression)
|
|
eq.Properties = q
|
|
}
|
|
|
|
case QueryTypeReduce:
|
|
var mapper mathexp.ReduceMapper = nil
|
|
q := &ReduceQuery{}
|
|
err = iter.ReadVal(q)
|
|
if err == nil {
|
|
referenceVar, err = getReferenceVar(q.Expression, common.RefID)
|
|
eq.Properties = q
|
|
}
|
|
if err == nil && q.Settings != nil {
|
|
switch q.Settings.Mode {
|
|
case ReduceModeStrict:
|
|
mapper = nil
|
|
case ReduceModeDrop:
|
|
mapper = mathexp.DropNonNumber{}
|
|
case ReduceModeReplace:
|
|
if q.Settings.ReplaceWithValue == nil {
|
|
err = fmt.Errorf("setting replaceWithValue must be specified when mode is '%s'", q.Settings.Mode)
|
|
}
|
|
mapper = mathexp.ReplaceNonNumberWithValue{Value: *q.Settings.ReplaceWithValue}
|
|
default:
|
|
err = fmt.Errorf("unsupported reduce mode")
|
|
}
|
|
}
|
|
if err == nil {
|
|
eq.Properties = q
|
|
eq.Command, err = NewReduceCommand(common.RefID,
|
|
q.Reducer, referenceVar, mapper)
|
|
}
|
|
|
|
case QueryTypeResample:
|
|
q := &ResampleQuery{}
|
|
err = iter.ReadVal(q)
|
|
if err == nil && common.TimeRange == nil {
|
|
err = fmt.Errorf("missing time range in query")
|
|
}
|
|
if err == nil {
|
|
referenceVar, err = getReferenceVar(q.Expression, common.RefID)
|
|
}
|
|
if err == nil {
|
|
tr := gtime.NewTimeRange(common.TimeRange.From, common.TimeRange.To)
|
|
eq.Properties = q
|
|
eq.Command, err = NewResampleCommand(common.RefID,
|
|
q.Window,
|
|
referenceVar,
|
|
q.Downsampler,
|
|
q.Upsampler,
|
|
AbsoluteTimeRange{
|
|
From: tr.GetFromAsTimeUTC(),
|
|
To: tr.GetToAsTimeUTC(),
|
|
},
|
|
)
|
|
}
|
|
|
|
case QueryTypeClassic:
|
|
q := &ClassicQuery{}
|
|
err = iter.ReadVal(q)
|
|
if err == nil {
|
|
eq.Properties = q
|
|
eq.Command, err = classic.NewConditionCmd(common.RefID, q.Conditions)
|
|
}
|
|
|
|
case QueryTypeSQL:
|
|
enabled := enableSqlExpressions(h)
|
|
if !enabled {
|
|
return eq, fmt.Errorf("sqlExpressions is not implemented")
|
|
}
|
|
q := &SQLExpression{}
|
|
err = iter.ReadVal(q)
|
|
if err == nil {
|
|
eq.Properties = q
|
|
// TODO: Cascade limit from Grafana config in this (new Expression Parser) branch of the code
|
|
cellLimit := 0 // zero means no limit
|
|
eq.Command, err = NewSQLCommand(common.RefID, q.Expression, int64(cellLimit))
|
|
}
|
|
|
|
case QueryTypeThreshold:
|
|
q := &ThresholdQuery{}
|
|
err = iter.ReadVal(q)
|
|
if err == nil {
|
|
referenceVar, err = getReferenceVar(q.Expression, common.RefID)
|
|
}
|
|
if err == nil {
|
|
// we only support one condition for now, we might want to turn this in to "OR" expressions later
|
|
if len(q.Conditions) != 1 {
|
|
return eq, fmt.Errorf("threshold expression requires exactly one condition")
|
|
}
|
|
firstCondition := q.Conditions[0]
|
|
|
|
threshold, err := NewThresholdCommand(common.RefID, referenceVar, firstCondition.Evaluator.Type, firstCondition.Evaluator.Params)
|
|
if err != nil {
|
|
return eq, fmt.Errorf("invalid condition: %w", err)
|
|
}
|
|
eq.Command = threshold
|
|
eq.Properties = q
|
|
|
|
if firstCondition.UnloadEvaluator != nil && h.features.IsEnabledGlobally(featuremgmt.FlagRecoveryThreshold) {
|
|
unloading, err := NewThresholdCommand(common.RefID, referenceVar, firstCondition.UnloadEvaluator.Type, firstCondition.UnloadEvaluator.Params)
|
|
unloading.Invert = true
|
|
if err != nil {
|
|
return eq, fmt.Errorf("invalid unloadCondition: %w", err)
|
|
}
|
|
var d Fingerprints
|
|
if firstCondition.LoadedDimensions != nil {
|
|
d, err = FingerprintsFromFrame(firstCondition.LoadedDimensions)
|
|
if err != nil {
|
|
return eq, fmt.Errorf("failed to parse loaded dimensions: %w", err)
|
|
}
|
|
}
|
|
eq.Command, err = NewHysteresisCommand(common.RefID, referenceVar, *threshold, *unloading, d)
|
|
if err != nil {
|
|
return eq, err
|
|
}
|
|
}
|
|
}
|
|
|
|
default:
|
|
err = fmt.Errorf("unknown query type (%s)", common.QueryType)
|
|
}
|
|
return eq, err
|
|
}
|
|
|
|
func getReferenceVar(exp string, refId string) (string, error) {
|
|
exp = strings.TrimPrefix(exp, "$")
|
|
if exp == "" {
|
|
return "", fmt.Errorf("no variable specified to reference for refId %v", refId)
|
|
}
|
|
return exp, nil
|
|
}
|
|
|
|
func enableSqlExpressions(h *ExpressionQueryReader) bool {
|
|
enabled := !h.features.IsEnabledGlobally(featuremgmt.FlagSqlExpressions)
|
|
if enabled {
|
|
return false
|
|
}
|
|
return false
|
|
}
|