mirror of
https://github.com/grafana/grafana.git
synced 2025-07-31 14:42:20 +08:00
Postgres: Switch the datasource plugin from lib/pq to pgx (#103961)
* Create libpqToPGX feature toggle * Refactor PostgreSQL datasource to support PGX with feature toggle - Updated `ProvideService` to accept feature toggles for enabling PGX. - Modified integration tests to use the new PGX connection method. - Introduced new functions for handling PGX connections and queries. - Enhanced TLS configuration handling for PostgreSQL connections. - Updated existing tests to ensure compatibility with PGX and new connection methods. * Update PostgreSQL datasource to enhance connection pooling and error handling - Increased `MaxOpenConns` to 10 in integration tests for improved connection management. - Refactored connection handling in `newPostgresPGX` to return a connection pool instead of a single connection. - Updated health check error handling to utilize context and feature toggles for better error reporting. - Adjusted `DisposePGX` method to close the connection pool properly. - Enhanced query execution to acquire connections from the pool, ensuring efficient resource usage. * Cleanup * Revert postgres_test unnecessary changes * Rename feature toggle from `libpqToPGX` to `postgresDSUsePGX` * Add null check to dispose method * Fix lint issues * Refactor connection string generation * Address comment in health check file * Rename p to pool * Refactor executeQueryPGX and split into multiple functions * Fix lint issues * The returning error message from PGX is enough no need to separate the error code. * Move TLS handling to newPostgresPGX function * Disable ssl for integration tests * Use MaxIdleConns option * Remove old feature toggle * Rename`generateConnectionConfigPGX` to `generateConnectionStringPGX` * Add back part of the error messages * Don't show max idle connections option when PGX enabled * Address comments from Sriram * Add back Sriram's changes * PostgreSQL: Rework tls manager to use temporary files instead (#105330) * Rework tls manager to use temporary files instead * Lint and test fixes * Update pkg/tsdb/grafana-postgresql-datasource/postgres.go Co-authored-by: Ivana Huckova <30407135+ivanahuckova@users.noreply.github.com> * Update betterer --------- Co-authored-by: Ivana Huckova <30407135+ivanahuckova@users.noreply.github.com> --------- Co-authored-by: Ivana Huckova <30407135+ivanahuckova@users.noreply.github.com>
This commit is contained in:
Zoltán Bedi
@ -3,47 +3,20 @@ package postgres
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/grafana/grafana-plugin-sdk-go/backend/log"
|
||||
"github.com/grafana/grafana/pkg/tsdb/grafana-postgresql-datasource/sqleng"
|
||||
)
|
||||
|
||||
var validateCertFunc = validateCertFilePaths
|
||||
var writeCertFileFunc = writeCertFile
|
||||
|
||||
type certFileType int
|
||||
|
||||
const (
|
||||
rootCert = iota
|
||||
clientCert
|
||||
clientKey
|
||||
)
|
||||
|
||||
type tlsSettingsProvider interface {
|
||||
getTLSSettings(dsInfo sqleng.DataSourceInfo) (tlsSettings, error)
|
||||
}
|
||||
|
||||
type datasourceCacheManager struct {
|
||||
locker *locker
|
||||
cache sync.Map
|
||||
}
|
||||
|
||||
type tlsManager struct {
|
||||
logger log.Logger
|
||||
dsCacheInstance datasourceCacheManager
|
||||
dataPath string
|
||||
logger log.Logger
|
||||
}
|
||||
|
||||
func newTLSManager(logger log.Logger, dataPath string) tlsSettingsProvider {
|
||||
func newTLSManager(logger log.Logger) *tlsManager {
|
||||
return &tlsManager{
|
||||
logger: logger,
|
||||
dataPath: dataPath,
|
||||
dsCacheInstance: datasourceCacheManager{locker: newLocker()},
|
||||
logger: logger,
|
||||
}
|
||||
}
|
||||
|
||||
@ -55,178 +28,116 @@ type tlsSettings struct {
|
||||
CertKeyFile string
|
||||
}
|
||||
|
||||
// getTLSSettings retrieves TLS settings and handles certificate file creation if needed.
|
||||
func (m *tlsManager) getTLSSettings(dsInfo sqleng.DataSourceInfo) (tlsSettings, error) {
|
||||
tlsconfig := tlsSettings{
|
||||
tlsConfig := tlsSettings{
|
||||
Mode: dsInfo.JsonData.Mode,
|
||||
}
|
||||
|
||||
isTLSDisabled := (tlsconfig.Mode == "disable")
|
||||
|
||||
if isTLSDisabled {
|
||||
if tlsConfig.Mode == "disable" {
|
||||
m.logger.Debug("Postgres TLS/SSL is disabled")
|
||||
return tlsconfig, nil
|
||||
return tlsConfig, nil
|
||||
}
|
||||
|
||||
m.logger.Debug("Postgres TLS/SSL is enabled", "tlsMode", tlsconfig.Mode)
|
||||
tlsConfig.ConfigurationMethod = dsInfo.JsonData.ConfigurationMethod
|
||||
tlsConfig.RootCertFile = dsInfo.JsonData.RootCertFile
|
||||
tlsConfig.CertFile = dsInfo.JsonData.CertFile
|
||||
tlsConfig.CertKeyFile = dsInfo.JsonData.CertKeyFile
|
||||
|
||||
tlsconfig.ConfigurationMethod = dsInfo.JsonData.ConfigurationMethod
|
||||
tlsconfig.RootCertFile = dsInfo.JsonData.RootCertFile
|
||||
tlsconfig.CertFile = dsInfo.JsonData.CertFile
|
||||
tlsconfig.CertKeyFile = dsInfo.JsonData.CertKeyFile
|
||||
|
||||
if tlsconfig.ConfigurationMethod == "file-content" {
|
||||
if err := m.writeCertFiles(dsInfo, &tlsconfig); err != nil {
|
||||
return tlsconfig, err
|
||||
if tlsConfig.ConfigurationMethod == "file-content" {
|
||||
if err := m.createCertFiles(dsInfo, &tlsConfig); err != nil {
|
||||
return tlsConfig, fmt.Errorf("failed to create TLS certificate files: %w", err)
|
||||
}
|
||||
} else {
|
||||
if err := validateCertFunc(tlsconfig.RootCertFile, tlsconfig.CertFile, tlsconfig.CertKeyFile); err != nil {
|
||||
return tlsconfig, err
|
||||
if err := validateCertFunc(tlsConfig.RootCertFile, tlsConfig.CertFile, tlsConfig.CertKeyFile); err != nil {
|
||||
return tlsConfig, fmt.Errorf("invalid TLS certificate file paths: %w", err)
|
||||
}
|
||||
}
|
||||
return tlsconfig, nil
|
||||
|
||||
return tlsConfig, nil
|
||||
}
|
||||
|
||||
func (t certFileType) String() string {
|
||||
switch t {
|
||||
case rootCert:
|
||||
return "root certificate"
|
||||
case clientCert:
|
||||
return "client certificate"
|
||||
case clientKey:
|
||||
return "client key"
|
||||
default:
|
||||
panic(fmt.Sprintf("Unrecognized certFileType %d", t))
|
||||
}
|
||||
}
|
||||
// createCertFiles writes certificate files to temporary locations.
|
||||
func (m *tlsManager) createCertFiles(dsInfo sqleng.DataSourceInfo, tlsConfig *tlsSettings) error {
|
||||
m.logger.Debug("Writing TLS certificate files to temporary locations")
|
||||
|
||||
func getFileName(dataDir string, fileType certFileType) string {
|
||||
var filename string
|
||||
switch fileType {
|
||||
case rootCert:
|
||||
filename = "root.crt"
|
||||
case clientCert:
|
||||
filename = "client.crt"
|
||||
case clientKey:
|
||||
filename = "client.key"
|
||||
default:
|
||||
panic(fmt.Sprintf("unrecognized certFileType %s", fileType.String()))
|
||||
}
|
||||
generatedFilePath := filepath.Join(dataDir, filename)
|
||||
return generatedFilePath
|
||||
}
|
||||
|
||||
// writeCertFile writes a certificate file.
|
||||
func writeCertFile(logger log.Logger, fileContent string, generatedFilePath string) error {
|
||||
fileContent = strings.TrimSpace(fileContent)
|
||||
if fileContent != "" {
|
||||
logger.Debug("Writing cert file", "path", generatedFilePath)
|
||||
if err := os.WriteFile(generatedFilePath, []byte(fileContent), 0600); err != nil {
|
||||
return err
|
||||
}
|
||||
// Make sure the file has the permissions expected by the Postgresql driver, otherwise it will bail
|
||||
if err := os.Chmod(generatedFilePath, 0600); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
logger.Debug("Deleting cert file since no content is provided", "path", generatedFilePath)
|
||||
exists, err := fileExists(generatedFilePath)
|
||||
if err != nil {
|
||||
var err error
|
||||
if tlsConfig.RootCertFile, err = m.writeCertFile("root-*.crt", dsInfo.DecryptedSecureJSONData["tlsCACert"]); err != nil {
|
||||
return err
|
||||
}
|
||||
if exists {
|
||||
if err := os.Remove(generatedFilePath); err != nil {
|
||||
return fmt.Errorf("failed to remove %q: %w", generatedFilePath, err)
|
||||
}
|
||||
if tlsConfig.CertFile, err = m.writeCertFile("client-*.crt", dsInfo.DecryptedSecureJSONData["tlsClientCert"]); err != nil {
|
||||
return err
|
||||
}
|
||||
if tlsConfig.CertKeyFile, err = m.writeCertFile("client-*.key", dsInfo.DecryptedSecureJSONData["tlsClientKey"]); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *tlsManager) writeCertFiles(dsInfo sqleng.DataSourceInfo, tlsconfig *tlsSettings) error {
|
||||
m.logger.Debug("Writing TLS certificate files to disk")
|
||||
tlsRootCert := dsInfo.DecryptedSecureJSONData["tlsCACert"]
|
||||
tlsClientCert := dsInfo.DecryptedSecureJSONData["tlsClientCert"]
|
||||
tlsClientKey := dsInfo.DecryptedSecureJSONData["tlsClientKey"]
|
||||
if tlsRootCert == "" && tlsClientCert == "" && tlsClientKey == "" {
|
||||
m.logger.Debug("No TLS/SSL certificates provided")
|
||||
// writeCertFile writes a single certificate file to a temporary location.
|
||||
func (m *tlsManager) writeCertFile(pattern, content string) (string, error) {
|
||||
if content == "" {
|
||||
return "", nil
|
||||
}
|
||||
|
||||
// Calculate all files path
|
||||
workDir := filepath.Join(m.dataPath, "tls", dsInfo.UID+"generatedTLSCerts")
|
||||
tlsconfig.RootCertFile = getFileName(workDir, rootCert)
|
||||
tlsconfig.CertFile = getFileName(workDir, clientCert)
|
||||
tlsconfig.CertKeyFile = getFileName(workDir, clientKey)
|
||||
|
||||
// Find datasource in the cache, if found, skip writing files
|
||||
cacheKey := strconv.Itoa(int(dsInfo.ID))
|
||||
m.dsCacheInstance.locker.RLock(cacheKey)
|
||||
item, ok := m.dsCacheInstance.cache.Load(cacheKey)
|
||||
m.dsCacheInstance.locker.RUnlock(cacheKey)
|
||||
if ok {
|
||||
if !item.(time.Time).Before(dsInfo.Updated) {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
m.dsCacheInstance.locker.Lock(cacheKey)
|
||||
defer m.dsCacheInstance.locker.Unlock(cacheKey)
|
||||
|
||||
item, ok = m.dsCacheInstance.cache.Load(cacheKey)
|
||||
if ok {
|
||||
if !item.(time.Time).Before(dsInfo.Updated) {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// Write certification directory and files
|
||||
exists, err := fileExists(workDir)
|
||||
m.logger.Debug("Writing certificate file", "pattern", pattern)
|
||||
file, err := os.CreateTemp("", pattern)
|
||||
if err != nil {
|
||||
return err
|
||||
return "", fmt.Errorf("failed to create temporary file: %w", err)
|
||||
}
|
||||
if !exists {
|
||||
if err := os.MkdirAll(workDir, 0700); err != nil {
|
||||
return err
|
||||
defer func() {
|
||||
if err := file.Close(); err != nil {
|
||||
m.logger.Error("Failed to close file", "error", err)
|
||||
}
|
||||
}()
|
||||
|
||||
if _, err := file.WriteString(content); err != nil {
|
||||
return "", fmt.Errorf("failed to write to temporary file: %w", err)
|
||||
}
|
||||
|
||||
if err = writeCertFileFunc(m.logger, tlsRootCert, tlsconfig.RootCertFile); err != nil {
|
||||
return err
|
||||
}
|
||||
if err = writeCertFileFunc(m.logger, tlsClientCert, tlsconfig.CertFile); err != nil {
|
||||
return err
|
||||
}
|
||||
if err = writeCertFileFunc(m.logger, tlsClientKey, tlsconfig.CertKeyFile); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// we do not want to point to cert-files that do not exist
|
||||
if tlsRootCert == "" {
|
||||
tlsconfig.RootCertFile = ""
|
||||
}
|
||||
|
||||
if tlsClientCert == "" {
|
||||
tlsconfig.CertFile = ""
|
||||
}
|
||||
|
||||
if tlsClientKey == "" {
|
||||
tlsconfig.CertKeyFile = ""
|
||||
}
|
||||
|
||||
// Update datasource cache
|
||||
m.dsCacheInstance.cache.Store(cacheKey, dsInfo.Updated)
|
||||
return nil
|
||||
return file.Name(), nil
|
||||
}
|
||||
|
||||
// validateCertFilePaths validates configured certificate file paths.
|
||||
func validateCertFilePaths(rootCert, clientCert, clientKey string) error {
|
||||
for _, fpath := range []string{rootCert, clientCert, clientKey} {
|
||||
if fpath == "" {
|
||||
// cleanupCertFiles removes temporary certificate files.
|
||||
func (m *tlsManager) cleanupCertFiles(tlsConfig tlsSettings) {
|
||||
// Only clean up if the configuration method is "file-content"
|
||||
if tlsConfig.ConfigurationMethod != "file-content" {
|
||||
m.logger.Debug("Skipping cleanup of TLS certificate files")
|
||||
return
|
||||
}
|
||||
m.logger.Debug("Cleaning up TLS certificate files")
|
||||
|
||||
files := []struct {
|
||||
path string
|
||||
name string
|
||||
}{
|
||||
{tlsConfig.RootCertFile, "root certificate"},
|
||||
{tlsConfig.CertFile, "client certificate"},
|
||||
{tlsConfig.CertKeyFile, "client key"},
|
||||
}
|
||||
|
||||
for _, file := range files {
|
||||
if file.path == "" {
|
||||
continue
|
||||
}
|
||||
exists, err := fileExists(fpath)
|
||||
if err := os.Remove(file.path); err != nil {
|
||||
m.logger.Error("Failed to remove file", "type", file.name, "path", file.path, "error", err)
|
||||
} else {
|
||||
m.logger.Debug("Successfully removed file", "type", file.name, "path", file.path)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// validateCertFilePaths validates the existence of configured certificate file paths.
|
||||
func validateCertFilePaths(rootCert, clientCert, clientKey string) error {
|
||||
for _, path := range []string{rootCert, clientCert, clientKey} {
|
||||
if path == "" {
|
||||
continue
|
||||
}
|
||||
exists, err := fileExists(path)
|
||||
if err != nil {
|
||||
return err
|
||||
return fmt.Errorf("error checking file existence: %w", err)
|
||||
}
|
||||
if !exists {
|
||||
return sqleng.ErrCertFileNotExist
|
||||
@ -235,15 +146,14 @@ func validateCertFilePaths(rootCert, clientCert, clientKey string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Exists determines whether a file/directory exists or not.
|
||||
func fileExists(fpath string) (bool, error) {
|
||||
_, err := os.Stat(fpath)
|
||||
// fileExists checks if a file exists at the given path.
|
||||
func fileExists(path string) (bool, error) {
|
||||
_, err := os.Stat(path)
|
||||
if err != nil {
|
||||
if !os.IsNotExist(err) {
|
||||
return false, err
|
||||
if os.IsNotExist(err) {
|
||||
return false, nil
|
||||
}
|
||||
return false, nil
|
||||
return false, err
|
||||
}
|
||||
|
||||
return true, nil
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user