Files
ismail simsek c088d003f2 InfluxDB: Implement InfluxQL json streaming parser (#76934)
* Have the first iteration

* Prepare bench testing

* rename the test files

* Remove unnecessary test file

* Introduce influxqlStreamingParser feature flag

* Apply streaming parser feature flag

* Add new tests

* More tests

* return executedQueryString only in first frame

* add frame meta and config

* Update golden json files

* Support tags/labels

* more tests

* more tests

* Don't change original response_parser.go

* provide context

* create util package

* don't pass the row

* update converter with formatted frameName

* add executedQueryString info only to first frame

* update golden files

* rename

* update test file

* use pointer values

* update testdata

* update parsing

* update converter for null values

* prepare converter for table response

* clean up

* return timeField in fields

* handle no time column responses

* better nil field handling

* refactor the code

* add table tests

* fix config for table

* table response format

* fix value

* if there is no time column set name

* linting

* refactoring

* handle the status code

* add tracing

* Update pkg/tsdb/influxdb/influxql/converter/converter_test.go

Co-authored-by: İnanç Gümüş <m@inanc.io>

* fix import

* update test data

* sanity

* sanity

* linting

* simplicity

* return empty rsp

* rename to prevent confusion

* nullableJson field type for null values

* better handling null values

* remove duplicate test file

* fix healthcheck

* use util for pointer

* move bench test to root

* provide fake feature manager

* add more tests

* partial fix for null values in table response format

* handle partial null fields

* comments for easy testing

* move frameName allocation in readSeries

* one less append operation

* performance improvement by making string conversion once

pkg: github.com/grafana/grafana/pkg/tsdb/influxdb/influxql
             │ stream2.txt │            stream3.txt             │
             │   sec/op    │   sec/op     vs base               │
ParseJson-10   314.4m ± 1%   303.9m ± 1%  -3.34% (p=0.000 n=10)

             │ stream2.txt  │             stream3.txt              │
             │     B/op     │     B/op      vs base                │
ParseJson-10   425.2Mi ± 0%   382.7Mi ± 0%  -10.00% (p=0.000 n=10)

             │ stream2.txt │            stream3.txt             │
             │  allocs/op  │  allocs/op   vs base               │
ParseJson-10   7.224M ± 0%   6.689M ± 0%  -7.41% (p=0.000 n=10)

* add comment lines

---------

Co-authored-by: İnanç Gümüş <m@inanc.io>
2023-12-06 12:39:05 +01:00

141 lines
4.0 KiB
Go

package influxql
import (
"context"
"errors"
"net/http"
"net/url"
"path"
"strings"
"github.com/grafana/grafana-plugin-sdk-go/backend"
"go.opentelemetry.io/otel/trace"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/services/featuremgmt"
"github.com/grafana/grafana/pkg/setting"
"github.com/grafana/grafana/pkg/tsdb/influxdb/influxql/buffered"
"github.com/grafana/grafana/pkg/tsdb/influxdb/influxql/querydata"
"github.com/grafana/grafana/pkg/tsdb/influxdb/models"
"github.com/grafana/grafana/pkg/tsdb/prometheus/utils"
)
const defaultRetentionPolicy = "default"
var (
ErrInvalidHttpMode = errors.New("'httpMode' should be either 'GET' or 'POST'")
glog = log.New("tsdb.influx_influxql")
)
func Query(ctx context.Context, tracer trace.Tracer, dsInfo *models.DatasourceInfo, req *backend.QueryDataRequest, features featuremgmt.FeatureToggles) (*backend.QueryDataResponse, error) {
logger := glog.FromContext(ctx)
response := backend.NewQueryDataResponse()
for _, reqQuery := range req.Queries {
query, err := models.QueryParse(reqQuery)
if err != nil {
return &backend.QueryDataResponse{}, err
}
rawQuery, err := query.Build(req)
if err != nil {
return &backend.QueryDataResponse{}, err
}
query.RefID = reqQuery.RefID
query.RawQuery = rawQuery
if setting.Env == setting.Dev {
logger.Debug("Influxdb query", "raw query", rawQuery)
}
request, err := createRequest(ctx, logger, dsInfo, rawQuery, query.Policy)
if err != nil {
return &backend.QueryDataResponse{}, err
}
resp, err := execute(ctx, tracer, dsInfo, logger, query, request, features.IsEnabled(ctx, featuremgmt.FlagInfluxqlStreamingParser))
if err != nil {
response.Responses[query.RefID] = backend.DataResponse{Error: err}
} else {
response.Responses[query.RefID] = resp
}
}
return response, nil
}
func createRequest(ctx context.Context, logger log.Logger, dsInfo *models.DatasourceInfo, queryStr string, retentionPolicy string) (*http.Request, error) {
u, err := url.Parse(dsInfo.URL)
if err != nil {
return nil, err
}
u.Path = path.Join(u.Path, "query")
httpMode := dsInfo.HTTPMode
var req *http.Request
switch httpMode {
case "GET":
req, err = http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil)
if err != nil {
return nil, err
}
case "POST":
bodyValues := url.Values{}
bodyValues.Add("q", queryStr)
body := bodyValues.Encode()
req, err = http.NewRequestWithContext(ctx, http.MethodPost, u.String(), strings.NewReader(body))
if err != nil {
return nil, err
}
default:
return nil, ErrInvalidHttpMode
}
params := req.URL.Query()
params.Set("db", dsInfo.DbName)
params.Set("epoch", "ms")
// default is hardcoded default retention policy
// InfluxDB will use the default policy when it is not added to the request
if retentionPolicy != "" && retentionPolicy != "default" {
params.Set("rp", retentionPolicy)
}
if httpMode == "GET" {
params.Set("q", queryStr)
} else if httpMode == "POST" {
req.Header.Set("Content-type", "application/x-www-form-urlencoded")
}
req.URL.RawQuery = params.Encode()
logger.Debug("Influxdb request", "url", req.URL.String())
return req, nil
}
func execute(ctx context.Context, tracer trace.Tracer, dsInfo *models.DatasourceInfo, logger log.Logger, query *models.Query, request *http.Request, isStreamingParserEnabled bool) (backend.DataResponse, error) {
res, err := dsInfo.HTTPClient.Do(request)
if err != nil {
return backend.DataResponse{}, err
}
defer func() {
if err := res.Body.Close(); err != nil {
logger.Warn("Failed to close response body", "err", err)
}
}()
_, endSpan := utils.StartTrace(ctx, tracer, "datasource.influxdb.influxql.parseResponse")
defer endSpan()
var resp *backend.DataResponse
if isStreamingParserEnabled {
logger.Info("InfluxDB InfluxQL streaming parser enabled: ", "info")
resp = querydata.ResponseParse(res.Body, res.StatusCode, query)
} else {
resp = buffered.ResponseParse(res.Body, res.StatusCode, query)
}
return *resp, nil
}