diff --git a/docs/sources/panels-visualizations/query-transform-data/sql-expressions/index.md b/docs/sources/panels-visualizations/query-transform-data/sql-expressions/index.md index 7d891f8e9b6..629eaece47f 100644 --- a/docs/sources/panels-visualizations/query-transform-data/sql-expressions/index.md +++ b/docs/sources/panels-visualizations/query-transform-data/sql-expressions/index.md @@ -150,6 +150,7 @@ The SQL conversion path: - Currently, only one SQL expression is supported per panel or alert. - Grafana supports certain data sources. Refer to [compatible data sources](#compatible-data-sources) for a current list. +- Autocomplete is available, but column/field autocomplete is only available after enabling the `sqlExpressionsColumnAutoComplete` feature toggle, which is provided on an experimental basis. ## Supported data source formats diff --git a/packages/grafana-data/src/types/featureToggles.gen.ts b/packages/grafana-data/src/types/featureToggles.gen.ts index 9582065043f..3316e91792d 100644 --- a/packages/grafana-data/src/types/featureToggles.gen.ts +++ b/packages/grafana-data/src/types/featureToggles.gen.ts @@ -477,6 +477,10 @@ export interface FeatureToggles { */ sqlExpressions?: boolean; /** + * Enables column autocomplete for SQL Expressions + */ + sqlExpressionsColumnAutoComplete?: boolean; + /** * Enables the group to nested table transformation * @default true */ diff --git a/pkg/services/featuremgmt/registry.go b/pkg/services/featuremgmt/registry.go index a5881618107..8958d488a3c 100644 --- a/pkg/services/featuremgmt/registry.go +++ b/pkg/services/featuremgmt/registry.go @@ -803,6 +803,13 @@ var ( FrontendOnly: false, Owner: grafanaDatasourcesCoreServicesSquad, }, + { + Name: "sqlExpressionsColumnAutoComplete", + Description: "Enables column autocomplete for SQL Expressions", + Stage: FeatureStageExperimental, + FrontendOnly: true, + Owner: grafanaDataProSquad, + }, { Name: "groupToNestedTableTransformation", Description: "Enables the group to nested table transformation", diff --git a/pkg/services/featuremgmt/toggles_gen.csv b/pkg/services/featuremgmt/toggles_gen.csv index 8bf9cca2706..b4cc812bf54 100644 --- a/pkg/services/featuremgmt/toggles_gen.csv +++ b/pkg/services/featuremgmt/toggles_gen.csv @@ -105,6 +105,7 @@ scopeApi,experimental,@grafana/grafana-app-platform-squad,false,false,false promQLScope,GA,@grafana/oss-big-tent,false,false,false logQLScope,privatePreview,@grafana/observability-logs,false,false,false sqlExpressions,privatePreview,@grafana/grafana-datasources-core-services,false,false,false +sqlExpressionsColumnAutoComplete,experimental,@grafana/datapro,false,false,true groupToNestedTableTransformation,GA,@grafana/dataviz-squad,false,false,true newPDFRendering,GA,@grafana/grafana-operator-experience-squad,false,false,false tlsMemcached,GA,@grafana/grafana-operator-experience-squad,false,false,false diff --git a/pkg/services/featuremgmt/toggles_gen.go b/pkg/services/featuremgmt/toggles_gen.go index 5e8ccbb5615..43a51188253 100644 --- a/pkg/services/featuremgmt/toggles_gen.go +++ b/pkg/services/featuremgmt/toggles_gen.go @@ -431,6 +431,10 @@ const ( // Enables SQL Expressions, which can execute SQL queries against data source results. FlagSqlExpressions = "sqlExpressions" + // FlagSqlExpressionsColumnAutoComplete + // Enables column autocomplete for SQL Expressions + FlagSqlExpressionsColumnAutoComplete = "sqlExpressionsColumnAutoComplete" + // FlagGroupToNestedTableTransformation // Enables the group to nested table transformation FlagGroupToNestedTableTransformation = "groupToNestedTableTransformation" diff --git a/pkg/services/featuremgmt/toggles_gen.json b/pkg/services/featuremgmt/toggles_gen.json index 88ab6c38de0..c1915403da5 100644 --- a/pkg/services/featuremgmt/toggles_gen.json +++ b/pkg/services/featuremgmt/toggles_gen.json @@ -3111,6 +3111,19 @@ "codeowner": "@grafana/grafana-datasources-core-services" } }, + { + "metadata": { + "name": "sqlExpressionsColumnAutoComplete", + "resourceVersion": "1751471729972", + "creationTimestamp": "2025-07-02T15:55:29Z" + }, + "spec": { + "description": "Enables column autocomplete for SQL Expressions", + "stage": "experimental", + "codeowner": "@grafana/datapro", + "frontend": true + } + }, { "metadata": { "name": "sseGroupByDatasource", diff --git a/public/app/features/alerting/unified/components/expressions/Expression.tsx b/public/app/features/alerting/unified/components/expressions/Expression.tsx index 9b3d6a2a289..032feeabd5d 100644 --- a/public/app/features/alerting/unified/components/expressions/Expression.tsx +++ b/public/app/features/alerting/unified/components/expressions/Expression.tsx @@ -133,7 +133,15 @@ export const Expression: FC = ({ ); case ExpressionQueryType.sql: - return onChangeQuery(query)} query={query} refIds={availableRefIds} alerting />; + return ( + onChangeQuery(query)} + query={query} + refIds={availableRefIds} + alerting + queries={[]} + /> + ); default: return ( diff --git a/public/app/features/expressions/ExpressionDatasource.ts b/public/app/features/expressions/ExpressionDatasource.ts index 57f4b45ffbf..ded8c75d89f 100644 --- a/public/app/features/expressions/ExpressionDatasource.ts +++ b/public/app/features/expressions/ExpressionDatasource.ts @@ -1,15 +1,27 @@ -import { from, mergeMap, Observable } from 'rxjs'; +import { from, lastValueFrom, map, mergeMap, Observable } from 'rxjs'; import { + DataFrame, DataQueryRequest, DataQueryResponse, DataSourceInstanceSettings, DataSourcePluginMeta, PluginType, ScopedVars, + TimeRange, } from '@grafana/data'; -import { DataSourceWithBackend, getDataSourceSrv, getTemplateSrv } from '@grafana/runtime'; +import { SQLQuery } from '@grafana/plugin-ui'; +import { + BackendDataSourceResponse, + DataSourceWithBackend, + FetchResponse, + getBackendSrv, + getDataSourceSrv, + getTemplateSrv, + toDataQueryResponse, +} from '@grafana/runtime'; import { ExpressionDatasourceRef } from '@grafana/runtime/internal'; +import { DataQuery } from '@grafana/schema/dist/esm/index'; import icnDatasourceSvg from 'img/icn-datasource.svg'; import { ExpressionQueryEditor } from './ExpressionQueryEditor'; @@ -59,6 +71,38 @@ export class ExpressionDatasourceApi extends DataSourceWithBackend, range: TimeRange, queries: DataQuery[]): Promise { + const refId = request.refId || 'meta'; + const metaSqlExpressionQuery: ExpressionQuery = { + window: '', + hide: false, + expression: request.rawSql, + datasource: ExpressionDatasourceRef, + refId, + type: ExpressionQueryType.sql, + }; + return lastValueFrom( + getBackendSrv() + .fetch({ + url: '/api/ds/query', + method: 'POST', + headers: this.getRequestHeaders(), + data: { + from: range.from.valueOf().toString(), + to: range.to.valueOf().toString(), + queries: [...queries, metaSqlExpressionQuery], + }, + requestId: refId, + }) + .pipe( + map((res: FetchResponse) => { + const rsp = toDataQueryResponse(res, queries); + return rsp.data[0] ?? { fields: [] }; + }) + ) + ); + } } export const instanceSettings: DataSourceInstanceSettings = { diff --git a/public/app/features/expressions/ExpressionQueryEditor.tsx b/public/app/features/expressions/ExpressionQueryEditor.tsx index b1f07c253bb..e49d3932c48 100644 --- a/public/app/features/expressions/ExpressionQueryEditor.tsx +++ b/public/app/features/expressions/ExpressionQueryEditor.tsx @@ -118,7 +118,7 @@ export function ExpressionQueryEditor(props: Props) { return ; case ExpressionQueryType.sql: - return ; + return ; } }; diff --git a/public/app/features/expressions/components/SqlExpr.test.tsx b/public/app/features/expressions/components/SqlExpr.test.tsx index 566e180fabf..e63a80133d6 100644 --- a/public/app/features/expressions/components/SqlExpr.test.tsx +++ b/public/app/features/expressions/components/SqlExpr.test.tsx @@ -19,7 +19,7 @@ describe('SqlExpr', () => { const refIds = [{ value: 'A' }]; const query = { refId: 'expr1', type: 'sql', expression: '' } as ExpressionQuery; - render(); + render(); // Verify onChange was called expect(onChange).toHaveBeenCalled(); @@ -35,7 +35,7 @@ describe('SqlExpr', () => { const existingExpression = 'SELECT 1 AS foo'; const query = { refId: 'expr1', type: 'sql', expression: existingExpression } as ExpressionQuery; - render(); + render(); // Check if onChange was called if (onChange.mock.calls.length > 0) { @@ -53,7 +53,7 @@ describe('SqlExpr', () => { const refIds = [{ value: 'A' }]; const query = { refId: 'expr1', type: 'sql' } as ExpressionQuery; - render(); + render(); const updatedQuery = onChange.mock.calls[0][0]; expect(updatedQuery.format).toBe('alerting'); diff --git a/public/app/features/expressions/components/SqlExpr.tsx b/public/app/features/expressions/components/SqlExpr.tsx index 1f61a558471..ac2917b1c40 100644 --- a/public/app/features/expressions/components/SqlExpr.tsx +++ b/public/app/features/expressions/components/SqlExpr.tsx @@ -2,34 +2,44 @@ import { css } from '@emotion/css'; import { useMemo, useRef, useEffect, useState } from 'react'; import { SelectableValue } from '@grafana/data'; -import { SQLEditor, LanguageDefinition } from '@grafana/plugin-ui'; +import { SQLEditor, CompletionItemKind, LanguageDefinition, TableIdentifier } from '@grafana/plugin-ui'; +import { DataQuery } from '@grafana/schema/dist/esm/index'; import { useStyles2 } from '@grafana/ui'; import { SqlExpressionQuery } from '../types'; +import { fetchSQLFields } from '../utils/metaSqlExpr'; + +import { getSqlCompletionProvider } from './sqlCompletionProvider'; // Account for Monaco editor's border to prevent clipping const EDITOR_BORDER_ADJUSTMENT = 2; // 1px border on top and bottom -// Define the language definition for MySQL syntax highlighting and autocomplete -const EDITOR_LANGUAGE_DEFINITION: LanguageDefinition = { - id: 'mysql', - // Additional properties could be added here in the future if needed - // eg: - // completionProvider: to autocomplete field (ie column) names when given - // a table name (dataframe reference) - // formatter: to format the SQL query and dashboard variables -}; - interface Props { refIds: Array>; query: SqlExpressionQuery; + queries: DataQuery[] | undefined; onChange: (query: SqlExpressionQuery) => void; /** Should the `format` property be set to `alerting`? */ alerting?: boolean; } -export const SqlExpr = ({ onChange, refIds, query, alerting = false }: Props) => { +export const SqlExpr = ({ onChange, refIds, query, alerting = false, queries }: Props) => { const vars = useMemo(() => refIds.map((v) => v.value!), [refIds]); + const completionProvider = useMemo( + () => + getSqlCompletionProvider({ + getFields: (identifier: TableIdentifier) => fetchFields(identifier, queries || []), + refIds, + }), + [queries, refIds] + ); + + // Define the language definition for MySQL syntax highlighting and autocomplete + const EDITOR_LANGUAGE_DEFINITION: LanguageDefinition = { + id: 'mysql', + completionProvider, + }; + const initialQuery = `SELECT * FROM ${vars[0]} LIMIT 10`; @@ -90,3 +100,8 @@ const getStyles = () => ({ minHeight: '100px', }), }); + +async function fetchFields(identifier: TableIdentifier, queries: DataQuery[]) { + const fields = await fetchSQLFields({ table: identifier.table }, queries); + return fields.map((t) => ({ name: t.name, completion: t.value, kind: CompletionItemKind.Field })); +} diff --git a/public/app/features/expressions/components/sqlCompletionProvider.ts b/public/app/features/expressions/components/sqlCompletionProvider.ts new file mode 100644 index 00000000000..013088aa770 --- /dev/null +++ b/public/app/features/expressions/components/sqlCompletionProvider.ts @@ -0,0 +1,46 @@ +import { SelectableValue } from '@grafana/data'; +import { ColumnDefinition, LanguageCompletionProvider, TableDefinition, TableIdentifier } from '@grafana/plugin-ui'; +import { config } from '@grafana/runtime'; + +import { ALLOWED_FUNCTIONS } from '../utils/metaSqlExpr'; + +interface CompletionProviderGetterArgs { + getFields: (t: TableIdentifier) => Promise; + refIds: Array>; +} + +export const getSqlCompletionProvider: (args: CompletionProviderGetterArgs) => LanguageCompletionProvider = + (args) => (monaco, language) => ({ + ...language, + triggerCharacters: [' '], + tables: { + resolve: async () => { + const refIdsToTableDefs = args.refIds.map((refId) => { + const tableDef: TableDefinition = { + name: refId.label || refId.value || '', + completion: refId.label || refId.value || '', + }; + return tableDef; + }); + return refIdsToTableDefs; + }, + }, + columns: { + resolve: async (t?: TableIdentifier) => { + if (config.featureToggles.sqlExpressionsColumnAutoComplete) { + try { + return await args.getFields({ table: t?.table }); + } catch { + return []; + } + } else { + return []; + } + }, + }, + supportedFunctions: () => { + return ALLOWED_FUNCTIONS.map((func) => { + return { id: func, name: func }; + }); + }, + }); diff --git a/public/app/features/expressions/utils/metaSqlExpr.ts b/public/app/features/expressions/utils/metaSqlExpr.ts new file mode 100644 index 00000000000..349d2abb8a3 --- /dev/null +++ b/public/app/features/expressions/utils/metaSqlExpr.ts @@ -0,0 +1,155 @@ +import { v4 as uuidv4 } from 'uuid'; + +import { getDefaultTimeRange, DataFrameView } from '@grafana/data'; +import { QueryFormat, SQLQuery, SQLSelectableValue } from '@grafana/plugin-ui'; +import { DataQuery } from '@grafana/schema'; +import { mapFieldsToTypes } from 'app/plugins/datasource/mysql/fields'; +import { quoteIdentifierIfNecessary } from 'app/plugins/datasource/mysql/sqlUtil'; + +import { dataSource } from '../ExpressionDatasource'; + +export async function fetchSQLFields(query: Partial, queries: DataQuery[]): Promise { + const datasource = dataSource; + if (!query.table) { + return []; + } + + const queryString = `SELECT * FROM ${query.table} LIMIT 1`; + + const queryResponse = await datasource.runMetaSQLExprQuery( + { rawSql: queryString, format: QueryFormat.Table, refId: `fields-${uuidv4()}` }, + getDefaultTimeRange(), + queries.filter((q) => q.refId === query.table) + ); + const frame = new DataFrameView(queryResponse); + + const fields = Object.values(frame.fields).map(({ name, type }) => { + return { + name, + text: name, + label: name, + value: quoteIdentifierIfNecessary(name), + type, + }; + }); + + return mapFieldsToTypes(fields); +} + +// based off https://github.com/grafana/grafana/blob/main/pkg/expr/sql/parser_allow.go +export const ALLOWED_FUNCTIONS = [ + 'if', + 'coalesce', + 'ifnull', + 'nullif', + 'sum', + 'avg', + 'count', + 'min', + 'max', + 'stddev', + 'std', + 'stddev_pop', + 'variance', + 'var_pop', + 'group_concat', + 'row_number', + 'rank', + 'dense_rank', + 'lead', + 'lag', + 'first_value', + 'last_value', + 'abs', + 'round', + 'floor', + 'ceiling', + 'ceil', + 'sqrt', + 'pow', + 'power', + 'mod', + 'log', + 'log10', + 'exp', + 'sign', + 'ln', + 'truncate', + 'sin', + 'cos', + 'tan', + 'asin', + 'acos', + 'atan', + 'atan2', + 'rand', + 'pi', + 'concat', + 'length', + 'char_length', + 'lower', + 'upper', + 'substring', + 'substring_index', + 'left', + 'right', + 'ltrim', + 'rtrim', + 'replace', + 'reverse', + 'lcase', + 'ucase', + 'mid', + 'repeat', + 'position', + 'instr', + 'locate', + 'ascii', + 'ord', + 'char', + 'regexp_substr', + 'str_to_date', + 'date_format', + 'date_add', + 'date_sub', + 'year', + 'month', + 'day', + 'weekday', + 'datediff', + 'unix_timestamp', + 'from_unixtime', + 'extract', + 'hour', + 'minute', + 'second', + 'dayname', + 'monthname', + 'dayofweek', + 'dayofmonth', + 'dayofyear', + 'week', + 'quarter', + 'time_to_sec', + 'sec_to_time', + 'timestampdiff', + 'timestampadd', + 'cast', + 'convert', + 'json_extract', + 'json_object', + 'json_array', + 'json_merge_patch', + 'json_valid', + 'json_contains', + 'json_length', + 'json_type', + 'json_keys', + 'json_search', + 'json_quote', + 'json_unquote', + 'json_set', + 'json_insert', + 'json_replace', + 'json_remove', +];