Instrumentation: Report the size of plugin request (#66149)

* Report the size of the plugin request

* Remove middleware, report directly

* PR review updates

pkg/plugins/backendplugin/instrumentation/instrumentation.go

@@ -29,12 +29,26 @@ var (
 		Help:      "Plugin request duration",
 		Buckets:   []float64{.005, .01, .025, .05, .1, .25, .5, 1, 2.5, 5, 10, 25, 50, 100},
 	}, []string{"plugin_id", "endpoint", "target"})
+
+	pluginRequestSizeHistogram = promauto.NewHistogramVec(
+		prometheus.HistogramOpts{
+			Namespace: "grafana",
+			Name:      "plugin_request_size_bytes",
+			Help:      "histogram of plugin request sizes returned",
+			Buckets:   []float64{128, 256, 512, 1024, 2048, 4096, 8192, 16384, 32768, 65536, 131072, 262144, 524288, 1048576},
+		}, []string{"source", "plugin_id", "endpoint", "target"},
+	)
 )
 
 const (
 	statusOK        = "ok"
 	statusError     = "error"
 	statusCancelled = "cancelled"
+
+	endpointCallResource   = "callResource"
+	endpointCheckHealth    = "checkHealth"
+	endpointCollectMetrics = "collectMetrics"
+	endpointQueryData      = "queryData"
 )
 
 var logger = plog.New("plugin.instrumentation")
@@ -99,20 +113,25 @@ type Cfg struct {
 
 // InstrumentCollectMetrics instruments collectMetrics.
 func InstrumentCollectMetrics(ctx context.Context, req *backend.PluginContext, cfg Cfg, fn func() error) error {
-	return instrumentPluginRequest(ctx, cfg, req, "collectMetrics", fn)
+	return instrumentPluginRequest(ctx, cfg, req, endpointCollectMetrics, fn)
 }
 
 // InstrumentCheckHealthRequest instruments checkHealth.
 func InstrumentCheckHealthRequest(ctx context.Context, req *backend.PluginContext, cfg Cfg, fn func() error) error {
-	return instrumentPluginRequest(ctx, cfg, req, "checkHealth", fn)
+	return instrumentPluginRequest(ctx, cfg, req, endpointCheckHealth, fn)
 }
 
 // InstrumentCallResourceRequest instruments callResource.
-func InstrumentCallResourceRequest(ctx context.Context, req *backend.PluginContext, cfg Cfg, fn func() error) error {
+func InstrumentCallResourceRequest(ctx context.Context, req *backend.PluginContext, cfg Cfg, requestSize float64, fn func() error) error {
-	return instrumentPluginRequest(ctx, cfg, req, "callResource", fn)
+	pluginRequestSizeHistogram.WithLabelValues("grafana-backend", req.PluginID, endpointCallResource,
+		string(cfg.Target)).Observe(requestSize)
+	return instrumentPluginRequest(ctx, cfg, req, endpointCallResource, fn)
 }
 
 // InstrumentQueryDataRequest instruments success rate and latency of query data requests.
-func InstrumentQueryDataRequest(ctx context.Context, req *backend.PluginContext, cfg Cfg, fn func() error) error {
+func InstrumentQueryDataRequest(ctx context.Context, req *backend.PluginContext, cfg Cfg,
-	return instrumentPluginRequest(ctx, cfg, req, "queryData", fn)
+	requestSize float64, fn func() error) error {
+	pluginRequestSizeHistogram.WithLabelValues("grafana-backend", req.PluginID, endpointQueryData,
+		string(cfg.Target)).Observe(requestSize)
+	return instrumentPluginRequest(ctx, cfg, req, endpointQueryData, fn)
 }

pkg/plugins/manager/client/client.go

@@ -40,11 +40,16 @@ func (s *Service) QueryData(ctx context.Context, req *backend.QueryDataRequest)
 		return nil, plugins.ErrPluginNotRegistered.Errorf("%w", backendplugin.ErrPluginNotRegistered)
 	}
 
+	var totalBytes float64
+	for _, v := range req.Queries {
+		totalBytes += float64(len(v.JSON))
+	}
+
 	var resp *backend.QueryDataResponse
 	err := instrumentation.InstrumentQueryDataRequest(ctx, &req.PluginContext, instrumentation.Cfg{
 		LogDatasourceRequests: s.cfg.LogDatasourceRequests,
 		Target:                p.Target(),
-	}, func() (innerErr error) {
+	}, totalBytes, func() (innerErr error) {
 		resp, innerErr = p.QueryData(ctx, req)
 		return
 	})
@@ -86,10 +91,12 @@ func (s *Service) CallResource(ctx context.Context, req *backend.CallResourceReq
 	if !exists {
 		return backendplugin.ErrPluginNotRegistered
 	}
+
+	totalBytes := float64(len(req.Body))
 	err := instrumentation.InstrumentCallResourceRequest(ctx, &req.PluginContext, instrumentation.Cfg{
 		LogDatasourceRequests: s.cfg.LogDatasourceRequests,
 		Target:                p.Target(),
-	}, func() error {
+	}, totalBytes, func() error {
 		removeConnectionHeaders(req.Headers)
 		removeHopByHopHeaders(req.Headers)