Files
grafana/pkg/expr/sql_command.go
Sam Jewell 7a3415148e SQL Expressions: Add cell-limit for input dataframes (#101700)
* expr: Add row limit to SQL expressions

Adds a configurable row limit to SQL expressions to prevent memory issues with large
result sets. The limit is configured via the `sql_expression_row_limit` setting in
the `[expressions]` section of grafana.ini, with a default of 100,000 rows.

The limit is enforced by checking the total number of rows across all input tables
before executing the SQL query. If the total exceeds the limit, the query fails
with an error message indicating the limit was exceeded.

* revert addition of newline

* Switch to table-driven tests

* Remove single-frame test-cases.

We only need to test for the multi frame case. Single frame is a subset
of the multi-frame case

* Add helper function

Simplify the way tests are set up and written

* Support convention, that limit: 0 is no limit

* Set the row-limit in one place only

* Update default limit to 20k rows

As per some discussion here:
https://raintank-corp.slack.com/archives/C071A5XCFST/p1741611647001369?thread_ts=1740047619.804869&cid=C071A5XCFST

* Test row-limit is applied from config

Make sure we protect this from regressions

This is perhaps a brittle test, somewhat coupled to the code here. But
it's good enough to prevent regressions at least.

* Add public documentation for the limit

* Limit total number of cells instead of rows

* Use named-return for totalRows

As @kylebrandt requested during review of #101700

* Leave DF cells as zero values during limits tests

When testing the cell limit we don't interact with the cell values at
all, so we leave them at their zero values both to speed up tests, and
to simplify and clarify that their values aren't used.

* Set SQLCmd limit at object creation - don't mutate

* Test that SQL node receives limit when built

And that it receives it from the Grafana config

* Improve TODO message for new Expression Parser

* Fix failing test by always creating config on the Service
2025-03-11 17:14:33 +00:00

150 lines
4.0 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

package expr
import (
"context"
"errors"
"fmt"
"time"
"github.com/grafana/grafana-plugin-sdk-go/data"
"github.com/grafana/grafana/pkg/apimachinery/errutil"
"github.com/grafana/grafana/pkg/expr/mathexp"
"github.com/grafana/grafana/pkg/expr/sql"
"github.com/grafana/grafana/pkg/infra/tracing"
)
// SQLCommand is an expression to run SQL over results
type SQLCommand struct {
query string
varsToQuery []string
refID string
limit int64
}
// NewSQLCommand creates a new SQLCommand.
func NewSQLCommand(refID, rawSQL string, limit int64) (*SQLCommand, error) {
if rawSQL == "" {
return nil, errutil.BadRequest("sql-missing-query",
errutil.WithPublicMessage("missing SQL query"))
}
tables, err := sql.TablesList(rawSQL)
if err != nil {
logger.Warn("invalid sql query", "sql", rawSQL, "error", err)
return nil, errutil.BadRequest("sql-invalid-sql",
errutil.WithPublicMessage(fmt.Sprintf("invalid SQL query: %s", err)),
)
}
if len(tables) == 0 {
logger.Warn("no tables found in SQL query", "sql", rawSQL)
}
if tables != nil {
logger.Debug("REF tables", "tables", tables, "sql", rawSQL)
}
return &SQLCommand{
query: rawSQL,
varsToQuery: tables,
refID: refID,
limit: limit,
}, nil
}
// UnmarshalSQLCommand creates a SQLCommand from Grafana's frontend query.
func UnmarshalSQLCommand(rn *rawNode, limit int64) (*SQLCommand, error) {
if rn.TimeRange == nil {
logger.Error("time range must be specified for refID", "refID", rn.RefID)
return nil, fmt.Errorf("time range must be specified for refID %s", rn.RefID)
}
expressionRaw, ok := rn.Query["expression"]
if !ok {
logger.Error("no expression in the query", "query", rn.Query)
return nil, errors.New("no expression in the query")
}
expression, ok := expressionRaw.(string)
if !ok {
logger.Error("expected sql expression to be type string", "expression", expressionRaw)
return nil, fmt.Errorf("expected sql expression to be type string, but got type %T", expressionRaw)
}
return NewSQLCommand(rn.RefID, expression, limit)
}
// NeedsVars returns the variable names (refIds) that are dependencies
// to execute the command and allows the command to fulfill the Command interface.
func (gr *SQLCommand) NeedsVars() []string {
return gr.varsToQuery
}
// Execute runs the command and returns the results or an error if the command
// failed to execute.
func (gr *SQLCommand) Execute(ctx context.Context, now time.Time, vars mathexp.Vars, tracer tracing.Tracer) (mathexp.Results, error) {
_, span := tracer.Start(ctx, "SSE.ExecuteSQL")
defer span.End()
allFrames := []*data.Frame{}
for _, ref := range gr.varsToQuery {
results, ok := vars[ref]
if !ok {
logger.Warn("no results found for", "ref", ref)
continue
}
frames := results.Values.AsDataFrames(ref)
allFrames = append(allFrames, frames...)
}
totalCells := totalCells(allFrames)
// limit of 0 or less means no limit (following convention)
if gr.limit > 0 && totalCells > gr.limit {
return mathexp.Results{},
fmt.Errorf(
"SQL expression: total cell count across all input tables exceeds limit of %d. Total cells: %d",
gr.limit,
totalCells,
)
}
logger.Debug("Executing query", "query", gr.query, "frames", len(allFrames))
db := sql.DB{}
frame, err := db.QueryFrames(ctx, gr.refID, gr.query, allFrames)
rsp := mathexp.Results{}
if err != nil {
logger.Error("Failed to query frames", "error", err.Error())
rsp.Error = err
return rsp, nil
}
logger.Debug("Done Executing query", "query", gr.query, "rows", frame.Rows())
if frame.Rows() == 0 {
rsp.Values = mathexp.Values{
mathexp.NoData{Frame: frame},
}
return rsp, nil
}
rsp.Values = mathexp.Values{
mathexp.TableData{Frame: frame},
}
return rsp, nil
}
func (gr *SQLCommand) Type() string {
return TypeSQL.String()
}
func totalCells(frames []*data.Frame) (total int64) {
for _, frame := range frames {
if frame != nil {
// Calculate cells as rows × columns
rows := int64(frame.Rows())
cols := int64(len(frame.Fields))
total += rows * cols
}
}
return
}