From 4efdef9b9c534d9e4cfbd6c0228f280f65deac12 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20=C5=A0tibran=C3=BD?= Date: Tue, 8 Apr 2025 16:36:37 +0200 Subject: [PATCH] Switch to using in-mem sequence generator only when explicitly configured in connection string. Move spanner-related functions to pkg/util/spanner. (#103363) * Switch to using in-mem sequence generator only when explicitly configured in connection string. Move spanner-related functions to pkg/util/spanner. --- .../sqlstore/migrator/spanner_dialect.go | 3 +- pkg/services/sqlstore/sqlutil/sqlutil.go | 2 +- pkg/util/spanner/spanner.go | 40 +++++++++++++++++++ pkg/util/xorm/dialect_spanner.go | 38 +----------------- pkg/util/xorm/go.mod | 2 +- 5 files changed, 46 insertions(+), 39 deletions(-) create mode 100644 pkg/util/spanner/spanner.go diff --git a/pkg/services/sqlstore/migrator/spanner_dialect.go b/pkg/services/sqlstore/migrator/spanner_dialect.go index 1bd540ea4b1..c7048e154f1 100644 --- a/pkg/services/sqlstore/migrator/spanner_dialect.go +++ b/pkg/services/sqlstore/migrator/spanner_dialect.go @@ -17,6 +17,7 @@ import ( "google.golang.org/grpc/codes" "xorm.io/core" + utilspanner "github.com/grafana/grafana/pkg/util/spanner" "xorm.io/xorm" _ "embed" @@ -290,7 +291,7 @@ func (s *SpannerDialect) executeDDLStatements(ctx context.Context, engine *xorm. return err } - opts := xorm.SpannerConnectorConfigToClientOptions(cfg) + opts := utilspanner.ConnectorConfigToClientOptions(cfg) databaseAdminClient, err := database.NewDatabaseAdminClient(ctx, opts...) if err != nil { diff --git a/pkg/services/sqlstore/sqlutil/sqlutil.go b/pkg/services/sqlstore/sqlutil/sqlutil.go index 2950aeabf76..3b6e449ff4b 100644 --- a/pkg/services/sqlstore/sqlutil/sqlutil.go +++ b/pkg/services/sqlstore/sqlutil/sqlutil.go @@ -191,7 +191,7 @@ func spannerTestDB() (*TestDB, error) { // $ curl "localhost:9020/v1/projects/grafanatest/instances/grafanatest/databases" --data '{"createStatement": "CREATE DATABASE `grafanatest`"}' return &TestDB{ DriverName: "spanner", - ConnStr: fmt.Sprintf("%s/projects/grafanatest/instances/grafanatest/databases/grafanatest;usePlainText=true", host), + ConnStr: fmt.Sprintf("%s/projects/grafanatest/instances/grafanatest/databases/grafanatest;usePlainText=true;inMemSequenceGenerator=true", host), Cleanup: func() {}, }, nil } diff --git a/pkg/util/spanner/spanner.go b/pkg/util/spanner/spanner.go new file mode 100644 index 00000000000..4e04f267be2 --- /dev/null +++ b/pkg/util/spanner/spanner.go @@ -0,0 +1,40 @@ +// Package spanner should only be used from tests, or from enterprise code (eg. protected by build tags). +package spanner + +import ( + "strconv" + + spannerdriver "github.com/googleapis/go-sql-spanner" + "google.golang.org/api/option" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" +) + +func UsePlainText(connectorConfig spannerdriver.ConnectorConfig) bool { + if strval, ok := connectorConfig.Params["useplaintext"]; ok { + if val, err := strconv.ParseBool(strval); err == nil { + return val + } + } + return false +} + +// ConnectorConfigToClientOptions is adapted from https://github.com/googleapis/go-sql-spanner/blob/main/driver.go#L341-L477, from version 1.11.1. +func ConnectorConfigToClientOptions(connectorConfig spannerdriver.ConnectorConfig) []option.ClientOption { + var opts []option.ClientOption + if connectorConfig.Host != "" { + opts = append(opts, option.WithEndpoint(connectorConfig.Host)) + } + if strval, ok := connectorConfig.Params["credentials"]; ok { + opts = append(opts, option.WithCredentialsFile(strval)) + } + if strval, ok := connectorConfig.Params["credentialsjson"]; ok { + opts = append(opts, option.WithCredentialsJSON([]byte(strval))) + } + if UsePlainText(connectorConfig) { + opts = append(opts, + option.WithGRPCDialOption(grpc.WithTransportCredentials(insecure.NewCredentials())), + option.WithoutAuthentication()) + } + return opts +} diff --git a/pkg/util/xorm/dialect_spanner.go b/pkg/util/xorm/dialect_spanner.go index 671376e88b4..0f18fb2e6f7 100644 --- a/pkg/util/xorm/dialect_spanner.go +++ b/pkg/util/xorm/dialect_spanner.go @@ -11,10 +11,7 @@ import ( spannerclient "cloud.google.com/go/spanner" _ "github.com/googleapis/go-sql-spanner" spannerdriver "github.com/googleapis/go-sql-spanner" - "google.golang.org/api/option" - "google.golang.org/grpc" "google.golang.org/grpc/codes" - "google.golang.org/grpc/credentials/insecure" "xorm.io/core" ) @@ -386,10 +383,8 @@ func (s *spanner) CreateSequenceGenerator(db *sql.DB) (SequenceGenerator, error) return nil, err } - if UsePlainText(connectorConfig) { - // Plain-text means we're either using spannertest or Spanner emulator. - // Switch to fake in-memory sequence number generator in that case. - // + if connectorConfig.Params["inMemSequenceGenerator"] == "true" { + // Switch to using in-memory sequence number generator. // Using database-based sequence generator doesn't work with emulator, as emulator // only supports single transaction. If there is already another transaction started // generating new ID via database-based sequence generator would always fail. @@ -399,35 +394,6 @@ func (s *spanner) CreateSequenceGenerator(db *sql.DB) (SequenceGenerator, error) return newSequenceGenerator(db), nil } -func UsePlainText(connectorConfig spannerdriver.ConnectorConfig) bool { - if strval, ok := connectorConfig.Params["useplaintext"]; ok { - if val, err := strconv.ParseBool(strval); err == nil { - return val - } - } - return false -} - -// SpannerConnectorConfigToClientOptions is adapted from https://github.com/googleapis/go-sql-spanner/blob/main/driver.go#L341-L477, from version 1.11.1. -func SpannerConnectorConfigToClientOptions(connectorConfig spannerdriver.ConnectorConfig) []option.ClientOption { - var opts []option.ClientOption - if connectorConfig.Host != "" { - opts = append(opts, option.WithEndpoint(connectorConfig.Host)) - } - if strval, ok := connectorConfig.Params["credentials"]; ok { - opts = append(opts, option.WithCredentialsFile(strval)) - } - if strval, ok := connectorConfig.Params["credentialsjson"]; ok { - opts = append(opts, option.WithCredentialsJSON([]byte(strval))) - } - if UsePlainText(connectorConfig) { - opts = append(opts, - option.WithGRPCDialOption(grpc.WithTransportCredentials(insecure.NewCredentials())), - option.WithoutAuthentication()) - } - return opts -} - func (s *spanner) RetryOnError(err error) bool { return err != nil && spannerclient.ErrCode(spannerclient.ToSpannerError(err)) == codes.Aborted } diff --git a/pkg/util/xorm/go.mod b/pkg/util/xorm/go.mod index 453763cb895..874ba1b15b9 100644 --- a/pkg/util/xorm/go.mod +++ b/pkg/util/xorm/go.mod @@ -7,7 +7,6 @@ require ( github.com/googleapis/go-sql-spanner v1.11.1 github.com/mattn/go-sqlite3 v1.14.22 github.com/stretchr/testify v1.10.0 - google.golang.org/api v0.220.0 google.golang.org/grpc v1.71.0 xorm.io/builder v0.3.6 xorm.io/core v0.7.3 @@ -58,6 +57,7 @@ require ( golang.org/x/sys v0.31.0 // indirect golang.org/x/text v0.23.0 // indirect golang.org/x/time v0.9.0 // indirect + google.golang.org/api v0.220.0 // indirect google.golang.org/genproto v0.0.0-20250122153221-138b5a5a4fd4 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20250324211829-b45e905df463 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20250324211829-b45e905df463 // indirect