Postgres: Switch the datasource plugin from lib/pq to pgx (#83768)

postgres: switch from lib/pq to pgx
This commit is contained in:
Gábor Farkas
2024-03-13 09:52:39 +01:00
committed by GitHub
parent 2acd48d1c2
commit ecd6de826a
17 changed files with 1082 additions and 947 deletions

View File

@ -2,31 +2,33 @@ package postgres
import (
"context"
"errors"
"fmt"
"math/rand"
"net"
"net/http"
"os"
"strings"
"testing"
"time"
"github.com/grafana/grafana-plugin-sdk-go/backend"
"github.com/grafana/grafana-plugin-sdk-go/backend/log"
backendproxy "github.com/grafana/grafana-plugin-sdk-go/backend/proxy"
"github.com/grafana/grafana-plugin-sdk-go/data"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"golang.org/x/net/proxy"
"github.com/grafana/grafana/pkg/setting"
"github.com/grafana/grafana/pkg/tsdb/grafana-postgresql-datasource/tls"
"github.com/grafana/grafana/pkg/tsdb/sqleng"
_ "github.com/lib/pq"
_ "github.com/jackc/pgx/v5/stdlib"
)
// Test generateConnectionString.
func TestIntegrationGenerateConnectionString(t *testing.T) {
if testing.Short() {
t.Skip("skipping integration test")
}
cfg := setting.NewCfg()
cfg.DataPath = t.TempDir()
func TestGenerateConnectionConfig(t *testing.T) {
rootCertBytes, err := tls.CreateRandomRootCertBytes()
require.NoError(t, err)
testCases := []struct {
desc string
@ -34,10 +36,15 @@ func TestIntegrationGenerateConnectionString(t *testing.T) {
user string
password string
database string
tlsSettings tlsSettings
expConnStr string
tlsMode string
tlsRootCert []byte
expErr string
uid string
expHost string
expPort uint16
expUser string
expPassword string
expDatabase string
expTLS bool
}{
{
desc: "Unix socket host",
@ -45,8 +52,11 @@ func TestIntegrationGenerateConnectionString(t *testing.T) {
user: "user",
password: "password",
database: "database",
tlsSettings: tlsSettings{Mode: "verify-full"},
expConnStr: "user='user' password='password' host='/var/run/postgresql' dbname='database' sslmode='verify-full'",
tlsMode: "disable",
expUser: "user",
expPassword: "password",
expHost: "/var/run/postgresql",
expDatabase: "database",
},
{
desc: "TCP host",
@ -54,8 +64,12 @@ func TestIntegrationGenerateConnectionString(t *testing.T) {
user: "user",
password: "password",
database: "database",
tlsSettings: tlsSettings{Mode: "verify-full"},
expConnStr: "user='user' password='password' host='host' dbname='database' sslmode='verify-full'",
tlsMode: "disable",
expUser: "user",
expPassword: "password",
expHost: "host",
expPort: 5432,
expDatabase: "database",
},
{
desc: "TCP/port host",
@ -63,8 +77,12 @@ func TestIntegrationGenerateConnectionString(t *testing.T) {
user: "user",
password: "password",
database: "database",
tlsSettings: tlsSettings{Mode: "verify-full"},
expConnStr: "user='user' password='password' host='host' dbname='database' port=1234 sslmode='verify-full'",
tlsMode: "disable",
expUser: "user",
expPassword: "password",
expHost: "host",
expPort: 1234,
expDatabase: "database",
},
{
desc: "Ipv6 host",
@ -72,8 +90,11 @@ func TestIntegrationGenerateConnectionString(t *testing.T) {
user: "user",
password: "password",
database: "database",
tlsSettings: tlsSettings{Mode: "verify-full"},
expConnStr: "user='user' password='password' host='::1' dbname='database' sslmode='verify-full'",
tlsMode: "disable",
expUser: "user",
expPassword: "password",
expHost: "::1",
expDatabase: "database",
},
{
desc: "Ipv6/port host",
@ -81,16 +102,20 @@ func TestIntegrationGenerateConnectionString(t *testing.T) {
user: "user",
password: "password",
database: "database",
tlsSettings: tlsSettings{Mode: "verify-full"},
expConnStr: "user='user' password='password' host='::1' dbname='database' port=1234 sslmode='verify-full'",
tlsMode: "disable",
expUser: "user",
expPassword: "password",
expHost: "::1",
expPort: 1234,
expDatabase: "database",
},
{
desc: "Invalid port",
host: "host:invalid",
user: "user",
database: "database",
tlsSettings: tlsSettings{},
expErr: "invalid port in host specifier",
desc: "Invalid port",
host: "host:invalid",
user: "user",
database: "database",
tlsMode: "disable",
expErr: "invalid port in host specifier",
},
{
desc: "Password with single quote and backslash",
@ -98,8 +123,11 @@ func TestIntegrationGenerateConnectionString(t *testing.T) {
user: "user",
password: `p'\assword`,
database: "database",
tlsSettings: tlsSettings{Mode: "verify-full"},
expConnStr: `user='user' password='p\'\\assword' host='host' dbname='database' sslmode='verify-full'`,
tlsMode: "disable",
expUser: "user",
expPassword: `p'\assword`,
expHost: "host",
expDatabase: "database",
},
{
desc: "User/DB with single quote and backslash",
@ -107,8 +135,11 @@ func TestIntegrationGenerateConnectionString(t *testing.T) {
user: `u'\ser`,
password: `password`,
database: `d'\atabase`,
tlsSettings: tlsSettings{Mode: "verify-full"},
expConnStr: `user='u\'\\ser' password='password' host='host' dbname='d\'\\atabase' sslmode='verify-full'`,
tlsMode: "disable",
expUser: `u'\ser`,
expPassword: "password",
expDatabase: `d'\atabase`,
expHost: "host",
},
{
desc: "Custom TLS mode disabled",
@ -116,45 +147,55 @@ func TestIntegrationGenerateConnectionString(t *testing.T) {
user: "user",
password: "password",
database: "database",
tlsSettings: tlsSettings{Mode: "disable"},
expConnStr: "user='user' password='password' host='host' dbname='database' sslmode='disable'",
tlsMode: "disable",
expUser: "user",
expPassword: "password",
expHost: "host",
expDatabase: "database",
},
{
desc: "Custom TLS mode verify-full with certificate files",
host: "host",
user: "user",
password: "password",
database: "database",
tlsSettings: tlsSettings{
Mode: "verify-full",
RootCertFile: "i/am/coding/ca.crt",
CertFile: "i/am/coding/client.crt",
CertKeyFile: "i/am/coding/client.key",
},
expConnStr: "user='user' password='password' host='host' dbname='database' sslmode='verify-full' " +
"sslrootcert='i/am/coding/ca.crt' sslcert='i/am/coding/client.crt' sslkey='i/am/coding/client.key'",
desc: "Custom TLS mode verify-full with certificate files",
host: "host",
user: "user",
password: "password",
database: "database",
tlsMode: "verify-full",
tlsRootCert: rootCertBytes,
expUser: "user",
expPassword: "password",
expDatabase: "database",
expHost: "host",
expTLS: true,
},
}
for _, tt := range testCases {
t.Run(tt.desc, func(t *testing.T) {
svc := Service{
tlsManager: &tlsTestManager{settings: tt.tlsSettings},
logger: backend.NewLoggerWith("logger", "tsdb.postgres"),
}
ds := sqleng.DataSourceInfo{
URL: tt.host,
User: tt.user,
DecryptedSecureJSONData: map[string]string{"password": tt.password},
Database: tt.database,
UID: tt.uid,
URL: tt.host,
User: tt.user,
DecryptedSecureJSONData: map[string]string{
"password": tt.password,
"tlsCACert": string(tt.tlsRootCert),
},
Database: tt.database,
JsonData: sqleng.JsonData{
Mode: tt.tlsMode,
ConfigurationMethod: "file-content",
},
}
connStr, err := svc.generateConnectionString(ds)
c, err := generateConnectionConfig(ds)
if tt.expErr == "" {
require.NoError(t, err, tt.desc)
assert.Equal(t, tt.expConnStr, connStr)
assert.Equal(t, tt.expHost, c.Host)
if tt.expPort != 0 {
assert.Equal(t, tt.expPort, c.Port)
}
assert.Equal(t, tt.expUser, c.User)
assert.Equal(t, tt.expDatabase, c.Database)
assert.Equal(t, tt.expPassword, c.Password)
require.Equal(t, tt.expTLS, c.TLSConfig != nil)
} else {
require.Error(t, err, tt.desc)
assert.True(t, strings.HasPrefix(err.Error(), tt.expErr),
@ -191,24 +232,40 @@ func TestIntegrationPostgres(t *testing.T) {
return sql
}
host := os.Getenv("POSTGRES_HOST")
if host == "" {
host = "localhost"
}
port := os.Getenv("POSTGRES_PORT")
if port == "" {
port = "5432"
}
jsonData := sqleng.JsonData{
MaxOpenConns: 0,
MaxIdleConns: 2,
ConnMaxLifetime: 14400,
Timescaledb: false,
ConfigurationMethod: "file-path",
Mode: "disable",
}
dsInfo := sqleng.DataSourceInfo{
JsonData: jsonData,
DecryptedSecureJSONData: map[string]string{},
JsonData: jsonData,
DecryptedSecureJSONData: map[string]string{
"password": "grafanatest",
},
URL: host + ":" + port,
Database: "grafanadstest",
User: "grafanatest",
}
logger := backend.NewLoggerWith("logger", "postgres.test")
cnnstr := postgresTestDBConnString()
db, exe, err := newPostgres(context.Background(), "error", 10000, dsInfo, cnnstr, logger, backend.DataSourceInstanceSettings{})
settings := backend.DataSourceInstanceSettings{}
proxyClient, err := settings.ProxyClient(context.Background())
require.NoError(t, err)
db, exe, err := newPostgres("error", 10000, dsInfo, logger, proxyClient)
require.NoError(t, err)
@ -1261,8 +1318,10 @@ func TestIntegrationPostgres(t *testing.T) {
})
t.Run("When row limit set to 1", func(t *testing.T) {
dsInfo := sqleng.DataSourceInfo{}
_, handler, err := newPostgres(context.Background(), "error", 1, dsInfo, cnnstr, logger, backend.DataSourceInstanceSettings{})
settings := backend.DataSourceInstanceSettings{}
proxyClient, err := settings.ProxyClient(context.Background())
require.NoError(t, err)
_, handler, err := newPostgres("error", 1, dsInfo, logger, proxyClient)
require.NoError(t, err)
@ -1377,14 +1436,6 @@ func genTimeRangeByInterval(from time.Time, duration time.Duration, interval tim
return timeRange
}
type tlsTestManager struct {
settings tlsSettings
}
func (m *tlsTestManager) getTLSSettings(dsInfo sqleng.DataSourceInfo) (tlsSettings, error) {
return m.settings, nil
}
func isTestDbPostgres() bool {
if db, present := os.LookupEnv("GRAFANA_TEST_DB"); present {
return db == "postgres"
@ -1393,15 +1444,59 @@ func isTestDbPostgres() bool {
return false
}
func postgresTestDBConnString() string {
host := os.Getenv("POSTGRES_HOST")
if host == "" {
host = "localhost"
}
port := os.Getenv("POSTGRES_PORT")
if port == "" {
port = "5432"
}
return fmt.Sprintf("user=grafanatest password=grafanatest host=%s port=%s dbname=grafanadstest sslmode=disable",
host, port)
type testNoResolveDialer struct {
}
func (d *testNoResolveDialer) Dial(network, addr string) (c net.Conn, err error) {
return nil, fmt.Errorf("test-dialer: dialing to '%s'. not implemented", addr)
}
var _ proxy.Dialer = (&testNoResolveDialer{})
type testNoResolveProxyClient struct {
}
var _ backendproxy.Client = (&testNoResolveProxyClient{})
func (p *testNoResolveProxyClient) SecureSocksProxyEnabled() bool {
return true
}
func (p *testNoResolveProxyClient) ConfigureSecureSocksHTTPProxy(transport *http.Transport) error {
return errors.New("testNoResolveProxyClient.ConfigureSecureSocksHTTPProxy not implemented")
}
func (p *testNoResolveProxyClient) NewSecureSocksProxyContextDialer() (proxy.Dialer, error) {
return &testNoResolveDialer{}, nil
}
// we must make sure that pgx does not resolve hostnames:
// if we say the hostname is `localhost`, then it should
// instruct the socks-proxy to connect to `localhost`, not to `127.0.0.1`
// this is important, becase some other socks-proxy-code relies on this behavior.
func TestNoResolve(t *testing.T) {
jsonData := sqleng.JsonData{
MaxOpenConns: 0,
MaxIdleConns: 2,
ConnMaxLifetime: 14400,
Timescaledb: false,
ConfigurationMethod: "file-path",
}
dsInfo := sqleng.DataSourceInfo{
JsonData: jsonData,
DecryptedSecureJSONData: map[string]string{
"password": "password",
},
URL: "localhost:5432",
Database: "db",
User: "user",
}
db, _, err := newPostgres("error", 10000, dsInfo, log.New(), &testNoResolveProxyClient{})
require.NoError(t, err)
require.NotNil(t, db)
err = db.Ping()
require.Error(t, err)
require.Contains(t, err.Error(), "test-dialer: dialing to 'localhost:5432'. not implemented")
}