Files
Giuseppe Guerra a89202eab2 Plugins: Improve instrumentation by adding metrics and tracing (#61035)
* WIP: Plugins tracing

* Trace ID middleware

* Add prometheus metrics and tracing to plugins updater

* Add TODOs

* Add instrumented http client

* Add tracing to grafana update checker

* Goimports

* Moved plugins tracing to middleware

* goimports, fix tests

* Removed X-Trace-Id header

* Fix comment in NewTracingHeaderMiddleware

* Add metrics to instrumented http client

* Add instrumented http client options

* Removed unused function

* Switch to contextual logger

* Refactoring, fix tests

* Moved InstrumentedHTTPClient and PrometheusMetrics to their own package

* Tracing middleware: handle errors

* Report span status codes when recording errors

* Add tests for tracing middleware

* Moved fakeSpan and fakeTracer to pkg/infra/tracing

* Add TestHTTPClientTracing

* Lint

* Changes after PR review

* Tests: Made "ended" in FakeSpan private, allow calling End only once

* Testing: panic in FakeSpan if span already ended

* Refactoring: Simplify Grafana updater checks

* Refactoring: Simplify plugins updater error checks and logs

* Fix wrong call to checkForUpdates -> instrumentedCheckForUpdates

* Tests: Fix wrong call to checkForUpdates -> instrumentedCheckForUpdates

* Log update checks duration, use Info log level for check succeeded logs

* Add plugin context span attributes in tracing_middleware

* Refactor prometheus metrics as httpclient middleware

* Fix call to ProvidePluginsService in plugins_test.go

* Propagate context to update checker outgoing http requests

* Plugin client tracing middleware: Removed operation name in status

* Fix tests

* Goimports tracing_middleware.go

* Goimports

* Fix imports

* Changed span name to plugins client middleware

* Add span name assertion in TestTracingMiddleware

* Removed Prometheus metrics middleware from grafana and plugins updatechecker

* Add span attributes for ds name, type, uid, panel and dashboard ids

* Fix http header reading in tracing middlewares

* Use contexthandler.FromContext, add X-Query-Group-Id

* Add test for RunStream

* Fix imports

* Changes from PR review

* TestTracingMiddleware: Changed assert to require for didPanic assertion

* Lint

* Fix imports
2023-03-28 11:01:06 +02:00

65 lines
2.2 KiB
Go

package httpclientprovider
import (
"fmt"
"net/http"
"net/http/httptrace"
"strconv"
"github.com/grafana/grafana-plugin-sdk-go/backend/httpclient"
"go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/codes"
"go.opentelemetry.io/otel/trace"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/infra/tracing"
)
const (
TracingMiddlewareName = "tracing"
httpContentLengthTagKey = "http.content_length"
)
func TracingMiddleware(logger log.Logger, tracer tracing.Tracer) httpclient.Middleware {
return httpclient.NamedMiddlewareFunc(TracingMiddlewareName, func(opts httpclient.Options, next http.RoundTripper) http.RoundTripper {
return httpclient.RoundTripperFunc(func(req *http.Request) (*http.Response, error) {
ctx, span := tracer.Start(req.Context(), "HTTP Outgoing Request", trace.WithSpanKind(trace.SpanKindClient))
defer span.End()
ctx = httptrace.WithClientTrace(ctx, otelhttptrace.NewClientTrace(ctx, otelhttptrace.WithoutSubSpans(), otelhttptrace.WithoutHeaders()))
req = req.WithContext(ctx)
for k, v := range opts.Labels {
span.SetAttributes(k, v, attribute.Key(k).String(v))
}
tracer.Inject(ctx, req.Header, span)
res, err := next.RoundTrip(req)
span.SetAttributes("http.url", req.URL.String(), attribute.String("http.url", req.URL.String()))
span.SetAttributes("http.method", req.Method, attribute.String("http.method", req.Method))
// ext.SpanKind.Set(span, ext.SpanKindRPCClientEnum)
if err != nil {
span.RecordError(err)
return res, err
}
if res != nil {
// we avoid measuring contentlength less than zero because it indicates
// that the content size is unknown. https://godoc.org/github.com/badu/http#Response
if res.ContentLength > 0 {
span.SetAttributes(httpContentLengthTagKey, res.ContentLength, attribute.Key(httpContentLengthTagKey).Int64(res.ContentLength))
}
span.SetAttributes("http.status_code", res.StatusCode, attribute.Int("http.status_code", res.StatusCode))
if res.StatusCode >= 400 {
span.SetStatus(codes.Error, fmt.Sprintf("error with HTTP status code %s", strconv.Itoa(res.StatusCode)))
}
}
return res, err
})
})
}