mirror of
https://github.com/grafana/grafana.git
synced 2025-08-01 14:12:15 +08:00
Postgres Data Source (#9475)
* add postgresql datasource * add rest of files for postgres datasource * fix timeseries query, remove unused code * consistent naming, refactoring * s/mysql/postgres/ * s/mysql/postgres/ * couple more tests * tests for more datatypes * fix macros for postgres * add __timeSec macro * add frontend for postgres datasource * adjust documentation * fix formatting * add proper plugin description * merge editor changes from mysql * port changes from mysql datasource * set proper defaultQuery for postgres * add time_sec to timeseries query accept int for value for timeseries query * revert allowing time_sec and handle int or float values as unix timestamp for "time" column * fix tslint error * handle decimal values in timeseries query * allow setting sslmode for postgres datasource * use type switch for handling data types * fix value for timeseries query * refactor timeseries queries to make them more flexible * remove debug statement from inner loop in type conversion * use plain for loop in getTypedRowData * fix timeseries queries * adjust postgres datasource to tsdb refactoring * adjust postgres datasource to frontend changes * update lib/pq to latest version * move type conversion to getTypedRowData * handle address types cidr, inet and macaddr * adjust response parser and docs for annotations * convert unknown types to string * add documentation for postgres datasource * add another example query with metric column * set more helpful default query * update help text in query editor * handle NULL in value column of timeseries query * add __timeGroup macro * add test for __timeGroup macro * document __timeGroup and set proper default query for annotations * fix typos in docs * add postgres to list of datasources * add postgres to builtInPlugins * mysql: refactoring as prep for merging postgres Refactors out the initialization of the xorm engine and the query logic for an sql data source. * mysql: rename refactoring + test update * postgres:refactor to use SqlEngine(same as mysql) Refactored to use a common base class with the MySql data source. Other changes from the original PR: - Changed time column to be time_sec to allow other time units in the future and to be the same as MySQL - Changed integration test to test the main Query method rather than the private transformToTable method - Changed the __timeSec macro name to __timeEpoch - Renamed PostgresExecutor to PostgresQueryEndpoint Fixes #9209 (the original PR) * postgres: encrypt password on config page With some other cosmetic changes to the config page: - placeholder texts - reset button for the password after it has been encrypted. - default value for the sslmode field. * postgres: change back col name to time from time_sec * postgres mysql: remove annotation title Title has been removed from annotations * postgres: fix images for docs page * postgres mysql: fix specs
This commit is contained in:
Daniel Lee
245
pkg/tsdb/postgres/postgres.go
Normal file
245
pkg/tsdb/postgres/postgres.go
Normal file
@ -0,0 +1,245 @@
|
||||
package postgres
|
||||
|
||||
import (
|
||||
"container/list"
|
||||
"context"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/go-xorm/core"
|
||||
"github.com/grafana/grafana/pkg/components/null"
|
||||
"github.com/grafana/grafana/pkg/log"
|
||||
"github.com/grafana/grafana/pkg/models"
|
||||
"github.com/grafana/grafana/pkg/tsdb"
|
||||
)
|
||||
|
||||
type PostgresQueryEndpoint struct {
|
||||
sqlEngine tsdb.SqlEngine
|
||||
log log.Logger
|
||||
}
|
||||
|
||||
func init() {
|
||||
tsdb.RegisterTsdbQueryEndpoint("postgres", NewPostgresQueryEndpoint)
|
||||
}
|
||||
|
||||
func NewPostgresQueryEndpoint(datasource *models.DataSource) (tsdb.TsdbQueryEndpoint, error) {
|
||||
endpoint := &PostgresQueryEndpoint{
|
||||
log: log.New("tsdb.postgres"),
|
||||
}
|
||||
|
||||
endpoint.sqlEngine = &tsdb.DefaultSqlEngine{
|
||||
MacroEngine: NewPostgresMacroEngine(),
|
||||
}
|
||||
|
||||
cnnstr := generateConnectionString(datasource)
|
||||
endpoint.log.Debug("getEngine", "connection", cnnstr)
|
||||
|
||||
if err := endpoint.sqlEngine.InitEngine("postgres", datasource, cnnstr); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return endpoint, nil
|
||||
}
|
||||
|
||||
func generateConnectionString(datasource *models.DataSource) string {
|
||||
password := ""
|
||||
for key, value := range datasource.SecureJsonData.Decrypt() {
|
||||
if key == "password" {
|
||||
password = value
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
sslmode := datasource.JsonData.Get("sslmode").MustString("require")
|
||||
return fmt.Sprintf("postgres://%s:%s@%s/%s?sslmode=%s", datasource.User, password, datasource.Url, datasource.Database, sslmode)
|
||||
}
|
||||
|
||||
func (e *PostgresQueryEndpoint) Query(ctx context.Context, dsInfo *models.DataSource, tsdbQuery *tsdb.TsdbQuery) (*tsdb.Response, error) {
|
||||
return e.sqlEngine.Query(ctx, dsInfo, tsdbQuery, e.transformToTimeSeries, e.transformToTable)
|
||||
}
|
||||
|
||||
func (e PostgresQueryEndpoint) transformToTable(query *tsdb.Query, rows *core.Rows, result *tsdb.QueryResult) error {
|
||||
|
||||
columnNames, err := rows.Columns()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
table := &tsdb.Table{
|
||||
Columns: make([]tsdb.TableColumn, len(columnNames)),
|
||||
Rows: make([]tsdb.RowValues, 0),
|
||||
}
|
||||
|
||||
for i, name := range columnNames {
|
||||
table.Columns[i].Text = name
|
||||
}
|
||||
|
||||
rowLimit := 1000000
|
||||
rowCount := 0
|
||||
|
||||
for ; rows.Next(); rowCount++ {
|
||||
if rowCount > rowLimit {
|
||||
return fmt.Errorf("PostgreSQL query row limit exceeded, limit %d", rowLimit)
|
||||
}
|
||||
|
||||
values, err := e.getTypedRowData(rows)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
table.Rows = append(table.Rows, values)
|
||||
}
|
||||
|
||||
result.Tables = append(result.Tables, table)
|
||||
result.Meta.Set("rowCount", rowCount)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (e PostgresQueryEndpoint) getTypedRowData(rows *core.Rows) (tsdb.RowValues, error) {
|
||||
|
||||
types, err := rows.ColumnTypes()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
values := make([]interface{}, len(types))
|
||||
valuePtrs := make([]interface{}, len(types))
|
||||
|
||||
for i := 0; i < len(types); i++ {
|
||||
valuePtrs[i] = &values[i]
|
||||
}
|
||||
|
||||
if err := rows.Scan(valuePtrs...); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// convert types not handled by lib/pq
|
||||
// unhandled types are returned as []byte
|
||||
for i := 0; i < len(types); i++ {
|
||||
if value, ok := values[i].([]byte); ok == true {
|
||||
switch types[i].DatabaseTypeName() {
|
||||
case "NUMERIC":
|
||||
if v, err := strconv.ParseFloat(string(value), 64); err == nil {
|
||||
values[i] = v
|
||||
} else {
|
||||
e.log.Debug("Rows", "Error converting numeric to float", value)
|
||||
}
|
||||
case "UNKNOWN", "CIDR", "INET", "MACADDR":
|
||||
// char literals have type UNKNOWN
|
||||
values[i] = string(value)
|
||||
default:
|
||||
e.log.Debug("Rows", "Unknown database type", types[i].DatabaseTypeName(), "value", value)
|
||||
values[i] = string(value)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return values, nil
|
||||
}
|
||||
|
||||
func (e PostgresQueryEndpoint) transformToTimeSeries(query *tsdb.Query, rows *core.Rows, result *tsdb.QueryResult) error {
|
||||
pointsBySeries := make(map[string]*tsdb.TimeSeries)
|
||||
seriesByQueryOrder := list.New()
|
||||
columnNames, err := rows.Columns()
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
rowLimit := 1000000
|
||||
rowCount := 0
|
||||
timeIndex := -1
|
||||
metricIndex := -1
|
||||
|
||||
// check columns of resultset
|
||||
for i, col := range columnNames {
|
||||
switch col {
|
||||
case "time":
|
||||
timeIndex = i
|
||||
case "metric":
|
||||
metricIndex = i
|
||||
}
|
||||
}
|
||||
|
||||
if timeIndex == -1 {
|
||||
return fmt.Errorf("Found no column named time")
|
||||
}
|
||||
|
||||
for rows.Next() {
|
||||
var timestamp float64
|
||||
var value null.Float
|
||||
var metric string
|
||||
|
||||
if rowCount > rowLimit {
|
||||
return fmt.Errorf("PostgreSQL query row limit exceeded, limit %d", rowLimit)
|
||||
}
|
||||
|
||||
values, err := e.getTypedRowData(rows)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
switch columnValue := values[timeIndex].(type) {
|
||||
case int64:
|
||||
timestamp = float64(columnValue * 1000)
|
||||
case float64:
|
||||
timestamp = columnValue * 1000
|
||||
case time.Time:
|
||||
timestamp = float64(columnValue.Unix() * 1000)
|
||||
default:
|
||||
return fmt.Errorf("Invalid type for column time, must be of type timestamp or unix timestamp")
|
||||
}
|
||||
|
||||
if metricIndex >= 0 {
|
||||
if columnValue, ok := values[metricIndex].(string); ok == true {
|
||||
metric = columnValue
|
||||
} else {
|
||||
return fmt.Errorf("Column metric must be of type char,varchar or text")
|
||||
}
|
||||
}
|
||||
|
||||
for i, col := range columnNames {
|
||||
if i == timeIndex || i == metricIndex {
|
||||
continue
|
||||
}
|
||||
|
||||
switch columnValue := values[i].(type) {
|
||||
case int64:
|
||||
value = null.FloatFrom(float64(columnValue))
|
||||
case float64:
|
||||
value = null.FloatFrom(columnValue)
|
||||
case nil:
|
||||
value.Valid = false
|
||||
default:
|
||||
return fmt.Errorf("Value column must have numeric datatype, column: %s type: %T value: %v", col, columnValue, columnValue)
|
||||
}
|
||||
if metricIndex == -1 {
|
||||
metric = col
|
||||
}
|
||||
e.appendTimePoint(pointsBySeries, seriesByQueryOrder, metric, timestamp, value)
|
||||
rowCount++
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
for elem := seriesByQueryOrder.Front(); elem != nil; elem = elem.Next() {
|
||||
key := elem.Value.(string)
|
||||
result.Series = append(result.Series, pointsBySeries[key])
|
||||
}
|
||||
|
||||
result.Meta.Set("rowCount", rowCount)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (e PostgresQueryEndpoint) appendTimePoint(pointsBySeries map[string]*tsdb.TimeSeries, seriesByQueryOrder *list.List, metric string, timestamp float64, value null.Float) {
|
||||
if series, exist := pointsBySeries[metric]; exist {
|
||||
series.Points = append(series.Points, tsdb.TimePoint{value, null.FloatFrom(timestamp)})
|
||||
} else {
|
||||
series := &tsdb.TimeSeries{Name: metric}
|
||||
series.Points = append(series.Points, tsdb.TimePoint{value, null.FloatFrom(timestamp)})
|
||||
pointsBySeries[metric] = series
|
||||
seriesByQueryOrder.PushBack(metric)
|
||||
}
|
||||
e.log.Debug("Rows", "metric", metric, "time", timestamp, "value", value)
|
||||
}
|
||||
Reference in New Issue
Block a user