From a1bc2272289ad453e196b78a434281eca925ca12 Mon Sep 17 00:00:00 2001 From: ismail simsek Date: Tue, 11 Apr 2023 22:04:32 +0200 Subject: [PATCH] Instrumentation: Report the size of plugin request (#66149) * Report the size of the plugin request * Remove middleware, report directly * PR review updates --- .../instrumentation/instrumentation.go | 31 +++++++++++++++---- pkg/plugins/manager/client/client.go | 11 +++++-- 2 files changed, 34 insertions(+), 8 deletions(-) diff --git a/pkg/plugins/backendplugin/instrumentation/instrumentation.go b/pkg/plugins/backendplugin/instrumentation/instrumentation.go index 87438e4bd6f..ae58d9cd593 100644 --- a/pkg/plugins/backendplugin/instrumentation/instrumentation.go +++ b/pkg/plugins/backendplugin/instrumentation/instrumentation.go @@ -29,12 +29,26 @@ var ( Help: "Plugin request duration", Buckets: []float64{.005, .01, .025, .05, .1, .25, .5, 1, 2.5, 5, 10, 25, 50, 100}, }, []string{"plugin_id", "endpoint", "target"}) + + pluginRequestSizeHistogram = promauto.NewHistogramVec( + prometheus.HistogramOpts{ + Namespace: "grafana", + Name: "plugin_request_size_bytes", + Help: "histogram of plugin request sizes returned", + Buckets: []float64{128, 256, 512, 1024, 2048, 4096, 8192, 16384, 32768, 65536, 131072, 262144, 524288, 1048576}, + }, []string{"source", "plugin_id", "endpoint", "target"}, + ) ) const ( statusOK = "ok" statusError = "error" statusCancelled = "cancelled" + + endpointCallResource = "callResource" + endpointCheckHealth = "checkHealth" + endpointCollectMetrics = "collectMetrics" + endpointQueryData = "queryData" ) var logger = plog.New("plugin.instrumentation") @@ -99,20 +113,25 @@ type Cfg struct { // InstrumentCollectMetrics instruments collectMetrics. func InstrumentCollectMetrics(ctx context.Context, req *backend.PluginContext, cfg Cfg, fn func() error) error { - return instrumentPluginRequest(ctx, cfg, req, "collectMetrics", fn) + return instrumentPluginRequest(ctx, cfg, req, endpointCollectMetrics, fn) } // InstrumentCheckHealthRequest instruments checkHealth. func InstrumentCheckHealthRequest(ctx context.Context, req *backend.PluginContext, cfg Cfg, fn func() error) error { - return instrumentPluginRequest(ctx, cfg, req, "checkHealth", fn) + return instrumentPluginRequest(ctx, cfg, req, endpointCheckHealth, fn) } // InstrumentCallResourceRequest instruments callResource. -func InstrumentCallResourceRequest(ctx context.Context, req *backend.PluginContext, cfg Cfg, fn func() error) error { - return instrumentPluginRequest(ctx, cfg, req, "callResource", fn) +func InstrumentCallResourceRequest(ctx context.Context, req *backend.PluginContext, cfg Cfg, requestSize float64, fn func() error) error { + pluginRequestSizeHistogram.WithLabelValues("grafana-backend", req.PluginID, endpointCallResource, + string(cfg.Target)).Observe(requestSize) + return instrumentPluginRequest(ctx, cfg, req, endpointCallResource, fn) } // InstrumentQueryDataRequest instruments success rate and latency of query data requests. -func InstrumentQueryDataRequest(ctx context.Context, req *backend.PluginContext, cfg Cfg, fn func() error) error { - return instrumentPluginRequest(ctx, cfg, req, "queryData", fn) +func InstrumentQueryDataRequest(ctx context.Context, req *backend.PluginContext, cfg Cfg, + requestSize float64, fn func() error) error { + pluginRequestSizeHistogram.WithLabelValues("grafana-backend", req.PluginID, endpointQueryData, + string(cfg.Target)).Observe(requestSize) + return instrumentPluginRequest(ctx, cfg, req, endpointQueryData, fn) } diff --git a/pkg/plugins/manager/client/client.go b/pkg/plugins/manager/client/client.go index 4663cddef02..a1bd1e864f6 100644 --- a/pkg/plugins/manager/client/client.go +++ b/pkg/plugins/manager/client/client.go @@ -40,11 +40,16 @@ func (s *Service) QueryData(ctx context.Context, req *backend.QueryDataRequest) return nil, plugins.ErrPluginNotRegistered.Errorf("%w", backendplugin.ErrPluginNotRegistered) } + var totalBytes float64 + for _, v := range req.Queries { + totalBytes += float64(len(v.JSON)) + } + var resp *backend.QueryDataResponse err := instrumentation.InstrumentQueryDataRequest(ctx, &req.PluginContext, instrumentation.Cfg{ LogDatasourceRequests: s.cfg.LogDatasourceRequests, Target: p.Target(), - }, func() (innerErr error) { + }, totalBytes, func() (innerErr error) { resp, innerErr = p.QueryData(ctx, req) return }) @@ -86,10 +91,12 @@ func (s *Service) CallResource(ctx context.Context, req *backend.CallResourceReq if !exists { return backendplugin.ErrPluginNotRegistered } + + totalBytes := float64(len(req.Body)) err := instrumentation.InstrumentCallResourceRequest(ctx, &req.PluginContext, instrumentation.Cfg{ LogDatasourceRequests: s.cfg.LogDatasourceRequests, Target: p.Target(), - }, func() error { + }, totalBytes, func() error { removeConnectionHeaders(req.Headers) removeHopByHopHeaders(req.Headers)