SQL Expressions: Enable Auto-complete (#106511)

* First draft autocomplete

* Better naming

* Change suggestion to table/column population

* Remove all suggestions, just use table/column population but trigger on space

* Gate feature flag behind a feature flag

* Reorganize and add function list

* Add blurb about autocomplete to docs

* Update public/app/features/expressions/utils/metaSqlExpr.ts

Co-authored-by: Alex Spencer <52186778+alexjonspencer1@users.noreply.github.com>

* Add try catch, remove promise resolve

---------

Co-authored-by: Alex Spencer <52186778+alexjonspencer1@users.noreply.github.com>

docs/sources/panels-visualizations/query-transform-data/sql-expressions/index.md

@@ -150,6 +150,7 @@ The SQL conversion path:
 
 - Currently, only one SQL expression is supported per panel or alert.
 - Grafana supports certain data sources. Refer to [compatible data sources](#compatible-data-sources) for a current list.
+- Autocomplete is available, but column/field autocomplete is only available after enabling the `sqlExpressionsColumnAutoComplete` feature toggle, which is provided on an experimental basis.
 
 ## Supported data source formats
 

packages/grafana-data/src/types/featureToggles.gen.ts

@@ -477,6 +477,10 @@ export interface FeatureToggles {
   */
   sqlExpressions?: boolean;
   /**
+  * Enables column autocomplete for SQL Expressions
+  */
+  sqlExpressionsColumnAutoComplete?: boolean;
+  /**
   * Enables the group to nested table transformation
   * @default true
   */

pkg/services/featuremgmt/registry.go

@@ -803,6 +803,13 @@ var (
 			FrontendOnly: false,
 			Owner:        grafanaDatasourcesCoreServicesSquad,
 		},
+		{
+			Name:         "sqlExpressionsColumnAutoComplete",
+			Description:  "Enables column autocomplete for SQL Expressions",
+			Stage:        FeatureStageExperimental,
+			FrontendOnly: true,
+			Owner:        grafanaDataProSquad,
+		},
 		{
 			Name:         "groupToNestedTableTransformation",
 			Description:  "Enables the group to nested table transformation",

pkg/services/featuremgmt/toggles_gen.csv

@@ -105,6 +105,7 @@ scopeApi,experimental,@grafana/grafana-app-platform-squad,false,false,false
 promQLScope,GA,@grafana/oss-big-tent,false,false,false
 logQLScope,privatePreview,@grafana/observability-logs,false,false,false
 sqlExpressions,privatePreview,@grafana/grafana-datasources-core-services,false,false,false
+sqlExpressionsColumnAutoComplete,experimental,@grafana/datapro,false,false,true
 groupToNestedTableTransformation,GA,@grafana/dataviz-squad,false,false,true
 newPDFRendering,GA,@grafana/grafana-operator-experience-squad,false,false,false
 tlsMemcached,GA,@grafana/grafana-operator-experience-squad,false,false,false

pkg/services/featuremgmt/toggles_gen.go

@@ -431,6 +431,10 @@ const (
 	// Enables SQL Expressions, which can execute SQL queries against data source results.
 	FlagSqlExpressions = "sqlExpressions"
 
+	// FlagSqlExpressionsColumnAutoComplete
+	// Enables column autocomplete for SQL Expressions
+	FlagSqlExpressionsColumnAutoComplete = "sqlExpressionsColumnAutoComplete"
+
 	// FlagGroupToNestedTableTransformation
 	// Enables the group to nested table transformation
 	FlagGroupToNestedTableTransformation = "groupToNestedTableTransformation"

pkg/services/featuremgmt/toggles_gen.json

@@ -3111,6 +3111,19 @@
         "codeowner": "@grafana/grafana-datasources-core-services"
       }
     },
+    {
+      "metadata": {
+        "name": "sqlExpressionsColumnAutoComplete",
+        "resourceVersion": "1751471729972",
+        "creationTimestamp": "2025-07-02T15:55:29Z"
+      },
+      "spec": {
+        "description": "Enables column autocomplete for SQL Expressions",
+        "stage": "experimental",
+        "codeowner": "@grafana/datapro",
+        "frontend": true
+      }
+    },
     {
       "metadata": {
         "name": "sseGroupByDatasource",

public/app/features/alerting/unified/components/expressions/Expression.tsx

@@ -133,7 +133,15 @@ export const Expression: FC<ExpressionProps> = ({
           );
 
         case ExpressionQueryType.sql:
-          return <SqlExpr onChange={(query) => onChangeQuery(query)} query={query} refIds={availableRefIds} alerting />;
+          return (
+            <SqlExpr
+              onChange={(query) => onChangeQuery(query)}
+              query={query}
+              refIds={availableRefIds}
+              alerting
+              queries={[]}
+            />
+          );
 
         default:
           return (

public/app/features/expressions/ExpressionDatasource.ts

@@ -1,15 +1,27 @@
-import { from, mergeMap, Observable } from 'rxjs';
+import { from, lastValueFrom, map, mergeMap, Observable } from 'rxjs';
 
 import {
+  DataFrame,
   DataQueryRequest,
   DataQueryResponse,
   DataSourceInstanceSettings,
   DataSourcePluginMeta,
   PluginType,
   ScopedVars,
+  TimeRange,
 } from '@grafana/data';
-import { DataSourceWithBackend, getDataSourceSrv, getTemplateSrv } from '@grafana/runtime';
+import { SQLQuery } from '@grafana/plugin-ui';
+import {
+  BackendDataSourceResponse,
+  DataSourceWithBackend,
+  FetchResponse,
+  getBackendSrv,
+  getDataSourceSrv,
+  getTemplateSrv,
+  toDataQueryResponse,
+} from '@grafana/runtime';
 import { ExpressionDatasourceRef } from '@grafana/runtime/internal';
+import { DataQuery } from '@grafana/schema/dist/esm/index';
 import icnDatasourceSvg from 'img/icn-datasource.svg';
 
 import { ExpressionQueryEditor } from './ExpressionQueryEditor';
@@ -59,6 +71,38 @@ export class ExpressionDatasourceApi extends DataSourceWithBackend<ExpressionQue
       ...query,
     };
   }
+
+  runMetaSQLExprQuery(request: Partial<SQLQuery>, range: TimeRange, queries: DataQuery[]): Promise<DataFrame> {
+    const refId = request.refId || 'meta';
+    const metaSqlExpressionQuery: ExpressionQuery = {
+      window: '',
+      hide: false,
+      expression: request.rawSql,
+      datasource: ExpressionDatasourceRef,
+      refId,
+      type: ExpressionQueryType.sql,
+    };
+    return lastValueFrom(
+      getBackendSrv()
+        .fetch<BackendDataSourceResponse>({
+          url: '/api/ds/query',
+          method: 'POST',
+          headers: this.getRequestHeaders(),
+          data: {
+            from: range.from.valueOf().toString(),
+            to: range.to.valueOf().toString(),
+            queries: [...queries, metaSqlExpressionQuery],
+          },
+          requestId: refId,
+        })
+        .pipe(
+          map((res: FetchResponse<BackendDataSourceResponse>) => {
+            const rsp = toDataQueryResponse(res, queries);
+            return rsp.data[0] ?? { fields: [] };
+          })
+        )
+    );
+  }
 }
 
 export const instanceSettings: DataSourceInstanceSettings = {

public/app/features/expressions/ExpressionQueryEditor.tsx

@@ -118,7 +118,7 @@ export function ExpressionQueryEditor(props: Props) {
         return <Threshold onChange={onChange} query={query} labelWidth={labelWidth} refIds={refIds} />;
 
       case ExpressionQueryType.sql:
-        return <SqlExpr onChange={onChange} query={query} refIds={refIds} />;
+        return <SqlExpr onChange={onChange} query={query} refIds={refIds} queries={queries} />;
     }
   };
 

public/app/features/expressions/components/SqlExpr.test.tsx

@@ -19,7 +19,7 @@ describe('SqlExpr', () => {
     const refIds = [{ value: 'A' }];
     const query = { refId: 'expr1', type: 'sql', expression: '' } as ExpressionQuery;
 
-    render(<SqlExpr onChange={onChange} refIds={refIds} query={query} />);
+    render(<SqlExpr onChange={onChange} refIds={refIds} query={query} queries={[]} />);
 
     // Verify onChange was called
     expect(onChange).toHaveBeenCalled();
@@ -35,7 +35,7 @@ describe('SqlExpr', () => {
     const existingExpression = 'SELECT 1 AS foo';
     const query = { refId: 'expr1', type: 'sql', expression: existingExpression } as ExpressionQuery;
 
-    render(<SqlExpr onChange={onChange} refIds={refIds} query={query} />);
+    render(<SqlExpr onChange={onChange} refIds={refIds} query={query} queries={[]} />);
 
     // Check if onChange was called
     if (onChange.mock.calls.length > 0) {
@@ -53,7 +53,7 @@ describe('SqlExpr', () => {
     const refIds = [{ value: 'A' }];
     const query = { refId: 'expr1', type: 'sql' } as ExpressionQuery;
 
-    render(<SqlExpr onChange={onChange} refIds={refIds} query={query} alerting />);
+    render(<SqlExpr onChange={onChange} refIds={refIds} query={query} alerting queries={[]} />);
 
     const updatedQuery = onChange.mock.calls[0][0];
     expect(updatedQuery.format).toBe('alerting');

public/app/features/expressions/components/SqlExpr.tsx

@@ -2,34 +2,44 @@ import { css } from '@emotion/css';
 import { useMemo, useRef, useEffect, useState } from 'react';
 
 import { SelectableValue } from '@grafana/data';
-import { SQLEditor, LanguageDefinition } from '@grafana/plugin-ui';
+import { SQLEditor, CompletionItemKind, LanguageDefinition, TableIdentifier } from '@grafana/plugin-ui';
+import { DataQuery } from '@grafana/schema/dist/esm/index';
 import { useStyles2 } from '@grafana/ui';
 
 import { SqlExpressionQuery } from '../types';
+import { fetchSQLFields } from '../utils/metaSqlExpr';
+
+import { getSqlCompletionProvider } from './sqlCompletionProvider';
 
 // Account for Monaco editor's border to prevent clipping
 const EDITOR_BORDER_ADJUSTMENT = 2; // 1px border on top and bottom
 
-// Define the language definition for MySQL syntax highlighting and autocomplete
-const EDITOR_LANGUAGE_DEFINITION: LanguageDefinition = {
-  id: 'mysql',
-  // Additional properties could be added here in the future if needed
-  // eg:
-  // completionProvider: to autocomplete field (ie column) names when given
-  // a table name (dataframe reference)
-  // formatter: to format the SQL query and dashboard variables
-};
-
 interface Props {
   refIds: Array<SelectableValue<string>>;
   query: SqlExpressionQuery;
+  queries: DataQuery[] | undefined;
   onChange: (query: SqlExpressionQuery) => void;
   /** Should the `format` property be set to `alerting`? */
   alerting?: boolean;
 }
 
-export const SqlExpr = ({ onChange, refIds, query, alerting = false }: Props) => {
+export const SqlExpr = ({ onChange, refIds, query, alerting = false, queries }: Props) => {
   const vars = useMemo(() => refIds.map((v) => v.value!), [refIds]);
+  const completionProvider = useMemo(
+    () =>
+      getSqlCompletionProvider({
+        getFields: (identifier: TableIdentifier) => fetchFields(identifier, queries || []),
+        refIds,
+      }),
+    [queries, refIds]
+  );
+
+  // Define the language definition for MySQL syntax highlighting and autocomplete
+  const EDITOR_LANGUAGE_DEFINITION: LanguageDefinition = {
+    id: 'mysql',
+    completionProvider,
+  };
+
   const initialQuery = `SELECT *
   FROM ${vars[0]}
   LIMIT 10`;
@@ -90,3 +100,8 @@ const getStyles = () => ({
     minHeight: '100px',
   }),
 });
+
+async function fetchFields(identifier: TableIdentifier, queries: DataQuery[]) {
+  const fields = await fetchSQLFields({ table: identifier.table }, queries);
+  return fields.map((t) => ({ name: t.name, completion: t.value, kind: CompletionItemKind.Field }));
+}

public/app/features/expressions/components/sqlCompletionProvider.ts

@@ -0,0 +1,46 @@
+import { SelectableValue } from '@grafana/data';
+import { ColumnDefinition, LanguageCompletionProvider, TableDefinition, TableIdentifier } from '@grafana/plugin-ui';
+import { config } from '@grafana/runtime';
+
+import { ALLOWED_FUNCTIONS } from '../utils/metaSqlExpr';
+
+interface CompletionProviderGetterArgs {
+  getFields: (t: TableIdentifier) => Promise<ColumnDefinition[]>;
+  refIds: Array<SelectableValue<string>>;
+}
+
+export const getSqlCompletionProvider: (args: CompletionProviderGetterArgs) => LanguageCompletionProvider =
+  (args) => (monaco, language) => ({
+    ...language,
+    triggerCharacters: [' '],
+    tables: {
+      resolve: async () => {
+        const refIdsToTableDefs = args.refIds.map((refId) => {
+          const tableDef: TableDefinition = {
+            name: refId.label || refId.value || '',
+            completion: refId.label || refId.value || '',
+          };
+          return tableDef;
+        });
+        return refIdsToTableDefs;
+      },
+    },
+    columns: {
+      resolve: async (t?: TableIdentifier) => {
+        if (config.featureToggles.sqlExpressionsColumnAutoComplete) {
+          try {
+            return await args.getFields({ table: t?.table });
+          } catch {
+            return [];
+          }
+        } else {
+          return [];
+        }
+      },
+    },
+    supportedFunctions: () => {
+      return ALLOWED_FUNCTIONS.map((func) => {
+        return { id: func, name: func };
+      });
+    },
+  });

public/app/features/expressions/utils/metaSqlExpr.ts

@@ -0,0 +1,155 @@
+import { v4 as uuidv4 } from 'uuid';
+
+import { getDefaultTimeRange, DataFrameView } from '@grafana/data';
+import { QueryFormat, SQLQuery, SQLSelectableValue } from '@grafana/plugin-ui';
+import { DataQuery } from '@grafana/schema';
+import { mapFieldsToTypes } from 'app/plugins/datasource/mysql/fields';
+import { quoteIdentifierIfNecessary } from 'app/plugins/datasource/mysql/sqlUtil';
+
+import { dataSource } from '../ExpressionDatasource';
+
+export async function fetchSQLFields(query: Partial<SQLQuery>, queries: DataQuery[]): Promise<SQLSelectableValue[]> {
+  const datasource = dataSource;
+  if (!query.table) {
+    return [];
+  }
+
+  const queryString = `SELECT * FROM ${query.table} LIMIT 1`;
+
+  const queryResponse = await datasource.runMetaSQLExprQuery(
+    { rawSql: queryString, format: QueryFormat.Table, refId: `fields-${uuidv4()}` },
+    getDefaultTimeRange(),
+    queries.filter((q) => q.refId === query.table)
+  );
+  const frame = new DataFrameView<string[]>(queryResponse);
+
+  const fields = Object.values(frame.fields).map(({ name, type }) => {
+    return {
+      name,
+      text: name,
+      label: name,
+      value: quoteIdentifierIfNecessary(name),
+      type,
+    };
+  });
+
+  return mapFieldsToTypes(fields);
+}
+
+// based off https://github.com/grafana/grafana/blob/main/pkg/expr/sql/parser_allow.go
+export const ALLOWED_FUNCTIONS = [
+  'if',
+  'coalesce',
+  'ifnull',
+  'nullif',
+  'sum',
+  'avg',
+  'count',
+  'min',
+  'max',
+  'stddev',
+  'std',
+  'stddev_pop',
+  'variance',
+  'var_pop',
+  'group_concat',
+  'row_number',
+  'rank',
+  'dense_rank',
+  'lead',
+  'lag',
+  'first_value',
+  'last_value',
+  'abs',
+  'round',
+  'floor',
+  'ceiling',
+  'ceil',
+  'sqrt',
+  'pow',
+  'power',
+  'mod',
+  'log',
+  'log10',
+  'exp',
+  'sign',
+  'ln',
+  'truncate',
+  'sin',
+  'cos',
+  'tan',
+  'asin',
+  'acos',
+  'atan',
+  'atan2',
+  'rand',
+  'pi',
+  'concat',
+  'length',
+  'char_length',
+  'lower',
+  'upper',
+  'substring',
+  'substring_index',
+  'left',
+  'right',
+  'ltrim',
+  'rtrim',
+  'replace',
+  'reverse',
+  'lcase',
+  'ucase',
+  'mid',
+  'repeat',
+  'position',
+  'instr',
+  'locate',
+  'ascii',
+  'ord',
+  'char',
+  'regexp_substr',
+  'str_to_date',
+  'date_format',
+  'date_add',
+  'date_sub',
+  'year',
+  'month',
+  'day',
+  'weekday',
+  'datediff',
+  'unix_timestamp',
+  'from_unixtime',
+  'extract',
+  'hour',
+  'minute',
+  'second',
+  'dayname',
+  'monthname',
+  'dayofweek',
+  'dayofmonth',
+  'dayofyear',
+  'week',
+  'quarter',
+  'time_to_sec',
+  'sec_to_time',
+  'timestampdiff',
+  'timestampadd',
+  'cast',
+  'convert',
+  'json_extract',
+  'json_object',
+  'json_array',
+  'json_merge_patch',
+  'json_valid',
+  'json_contains',
+  'json_length',
+  'json_type',
+  'json_keys',
+  'json_search',
+  'json_quote',
+  'json_unquote',
+  'json_set',
+  'json_insert',
+  'json_replace',
+  'json_remove',
+];