From b8b85fbf477765f79fb61f15af724f60887cd11c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mustafa=20Sencer=20=C3=96zcan?= <32759850+mustafasencer@users.noreply.github.com> Date: Wed, 10 Sep 2025 17:33:21 +0200 Subject: [PATCH] fix: add intrumentation for auth server grpc client (#110875) --- apps/iam/go.mod | 1 + apps/iam/go.sum | 2 ++ pkg/services/authz/rbac.go | 29 ++++++++++++++++++++++------- pkg/storage/unified/sql/service.go | 2 +- 4 files changed, 26 insertions(+), 8 deletions(-) diff --git a/apps/iam/go.mod b/apps/iam/go.mod index dd7ea5260e6..6ad62e287e1 100644 --- a/apps/iam/go.mod +++ b/apps/iam/go.mod @@ -308,6 +308,7 @@ require ( github.com/openfga/api/proto v0.0.0-20250127102726-f9709139a369 // indirect github.com/openfga/language/pkg/go v0.2.0-beta.2.0.20250220223040-ed0cfba54336 // indirect github.com/openfga/openfga v1.8.13 // indirect + github.com/opentracing-contrib/go-grpc v0.1.1 // indirect github.com/opentracing-contrib/go-stdlib v1.0.0 // indirect github.com/opentracing/opentracing-go v1.2.0 // indirect github.com/patrickmn/go-cache v2.1.0+incompatible // indirect diff --git a/apps/iam/go.sum b/apps/iam/go.sum index 63e2512ddaf..f86d810f3f8 100644 --- a/apps/iam/go.sum +++ b/apps/iam/go.sum @@ -1100,6 +1100,8 @@ github.com/openfga/language/pkg/go v0.2.0-beta.2.0.20250220223040-ed0cfba54336 h github.com/openfga/language/pkg/go v0.2.0-beta.2.0.20250220223040-ed0cfba54336/go.mod h1:IWRgDIekw3UGSWINwmCALHpMmn6NEJzz6e7KZGm+xQ4= github.com/openfga/openfga v1.8.13 h1:ROURkotKhbmtyBX3188+cNElN8AOZmTl0CMkxUqwawo= github.com/openfga/openfga v1.8.13/go.mod h1:h1VGcVW81eY1YyDtFx5+gxxAIEhIiOGR9SRGgs/X/k8= +github.com/opentracing-contrib/go-grpc v0.1.1 h1:Ws7IN1zyiL1DFqKQPhRXuKe5pLYzMfdxnC1qtajE2PE= +github.com/opentracing-contrib/go-grpc v0.1.1/go.mod h1:Nu6sz+4zzgxXu8rvKfnwjBEmHsuhTigxRwV2RhELrS8= github.com/opentracing-contrib/go-stdlib v1.0.0 h1:TBS7YuVotp8myLon4Pv7BtCBzOTo1DeZCld0Z63mW2w= github.com/opentracing-contrib/go-stdlib v1.0.0/go.mod h1:qtI1ogk+2JhVPIXVc6q+NHziSmy2W5GbdQZFUHADCBU= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= diff --git a/pkg/services/authz/rbac.go b/pkg/services/authz/rbac.go index e15603b7461..e2210096976 100644 --- a/pkg/services/authz/rbac.go +++ b/pkg/services/authz/rbac.go @@ -10,6 +10,7 @@ import ( "github.com/fullstorydev/grpchan/inprocgrpc" grpcAuth "github.com/grpc-ecosystem/go-grpc-middleware/v2/interceptors/auth" "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" "go.opentelemetry.io/otel/trace" "google.golang.org/grpc" "google.golang.org/grpc/credentials" @@ -21,6 +22,7 @@ import ( authzv1 "github.com/grafana/authlib/authz/proto/v1" "github.com/grafana/authlib/cache" authlib "github.com/grafana/authlib/types" + "github.com/grafana/dskit/grpcclient" "github.com/grafana/grafana/pkg/infra/db" "github.com/grafana/grafana/pkg/infra/log" "github.com/grafana/grafana/pkg/infra/tracing" @@ -69,7 +71,7 @@ func ProvideAuthZClient( switch authCfg.mode { case clientModeCloud: - rbacClient, err := newRemoteRBACClient(authCfg, tracer) + rbacClient, err := newRemoteRBACClient(authCfg, tracer, reg) if features.IsEnabledGlobally(featuremgmt.FlagZanzana) { return zanzana.WithShadowClient(rbacClient, zanzanaClient, reg) } @@ -128,7 +130,7 @@ func ProvideAuthZClient( // ProvideStandaloneAuthZClient provides a standalone AuthZ client, without registering the AuthZ service. // You need to provide a remote address in the configuration func ProvideStandaloneAuthZClient( - cfg *setting.Cfg, features featuremgmt.FeatureToggles, tracer trace.Tracer, + cfg *setting.Cfg, features featuremgmt.FeatureToggles, tracer trace.Tracer, reg prometheus.Registerer, ) (authlib.AccessClient, error) { if !features.IsEnabledGlobally(featuremgmt.FlagAuthZGRPCServer) { return nil, nil @@ -139,10 +141,10 @@ func ProvideStandaloneAuthZClient( return nil, err } - return newRemoteRBACClient(authCfg, tracer) + return newRemoteRBACClient(authCfg, tracer, reg) } -func newRemoteRBACClient(clientCfg *authzClientSettings, tracer trace.Tracer) (authlib.AccessClient, error) { +func newRemoteRBACClient(clientCfg *authzClientSettings, tracer trace.Tracer, reg prometheus.Registerer) (authlib.AccessClient, error) { tokenClient, err := authnlib.NewTokenExchangeClient(authnlib.TokenExchangeConfig{ Token: clientCfg.token, TokenExchangeURL: clientCfg.tokenExchangeURL, @@ -159,13 +161,26 @@ func newRemoteRBACClient(clientCfg *authzClientSettings, tracer trace.Tracer) (a } } - conn, err := grpc.NewClient( - clientCfg.remoteAddress, + authzRequestDuration := promauto.With(reg).NewHistogramVec(prometheus.HistogramOpts{ + Name: "authz_server_client_request_duration_seconds", + Help: "Time spent executing requests to authz server.", + NativeHistogramBucketFactor: 1.1, + NativeHistogramMaxBucketNumber: 160, + NativeHistogramMinResetDuration: time.Hour, + }, []string{"operation", "status_code"}) + + unaryInterceptors, streamInterceptors := grpcclient.Instrument(authzRequestDuration) + + opts := []grpc.DialOption{ grpc.WithTransportCredentials(transportCreds), grpc.WithPerRPCCredentials( NewGRPCTokenAuth(AuthzServiceAudience, clientCfg.tokenNamespace, tokenClient), ), - ) + grpc.WithChainUnaryInterceptor(unaryInterceptors...), + grpc.WithChainStreamInterceptor(streamInterceptors...), + } + + conn, err := grpc.NewClient(clientCfg.remoteAddress, opts...) if err != nil { return nil, fmt.Errorf("failed to create authz client to remote server: %w", err) } diff --git a/pkg/storage/unified/sql/service.go b/pkg/storage/unified/sql/service.go index cdc6ee97ca3..50cc24c9a69 100644 --- a/pkg/storage/unified/sql/service.go +++ b/pkg/storage/unified/sql/service.go @@ -201,7 +201,7 @@ func (s *service) starting(ctx context.Context) error { } } - authzClient, err := authz.ProvideStandaloneAuthZClient(s.cfg, s.features, s.tracing) + authzClient, err := authz.ProvideStandaloneAuthZClient(s.cfg, s.features, s.tracing, s.reg) if err != nil { return err }