From e6f359f90bfa8956f6aa29c05f34b45a57c3ebf1 Mon Sep 17 00:00:00 2001 From: Drew Slobodnjak <60050885+drew08t@users.noreply.github.com> Date: Thu, 12 Sep 2024 08:50:23 -0700 Subject: [PATCH] Transformations: Binary operation on all numbers (#92172) * Transformations: Binary operation on all numbers * Handle replaceFields option * Change left clear out to string * Handle time field * Fix filtering * Update new field names to remove double space * Add tests * Add BinaryValue interface and update editor * Fix initial behavior * Rollback rendering standards * Add ctx interpolate * Fix fixed value variable * Add function to convert old binary value type * Update tests for new structures * Add bullet for all number field option * baldm0mma/run content build script * Disable alias control for type matching --- .../transform-data/index.md | 1 + .../transformers/calculateField.test.ts | 67 ++++++++++ .../transformers/calculateField.ts | 120 ++++++++++++++---- .../app/features/transformers/docs/content.ts | 1 + .../BinaryOperationOptionsEditor.tsx | 100 ++++++++++++--- .../CalculateFieldTransformerEditor.tsx | 6 +- 6 files changed, 254 insertions(+), 41 deletions(-) diff --git a/docs/sources/panels-visualizations/query-transform-data/transform-data/index.md b/docs/sources/panels-visualizations/query-transform-data/transform-data/index.md index 437e2f1ad2a..e6a01262253 100644 --- a/docs/sources/panels-visualizations/query-transform-data/transform-data/index.md +++ b/docs/sources/panels-visualizations/query-transform-data/transform-data/index.md @@ -188,6 +188,7 @@ Use this transformation to add a new field calculated from two other fields. Eac - **Field name** - Select the names of fields you want to use in the calculation for the new field. - **Calculation** - If you select **Reduce row** mode, then the **Calculation** field appears. Click in the field to see a list of calculation choices you can use to create the new field. For information about available calculations, refer to [Calculation types][]. - **Operation** - If you select **Binary operation** or **Unary operation** mode, then the **Operation** fields appear. These fields allow you to apply basic math operations on values in a single row from selected fields. You can also use numerical values for binary operations. + - **All number fields** - Set the left side of a **Binary operation** to apply the calculation to all number fields. - **As percentile** - If you select **Row index** mode, then the **As percentile** switch appears. This switch allows you to transform the row index as a percentage of the total number of rows. - **Alias** - (Optional) Enter the name of your new field. If you leave this blank, then the field will be named to match the calculation. - **Replace all fields** - (Optional) Select this option if you want to hide all other fields and display only your calculated field in the visualization. diff --git a/packages/grafana-data/src/transformations/transformers/calculateField.test.ts b/packages/grafana-data/src/transformations/transformers/calculateField.test.ts index 154083e3169..3091be651bd 100644 --- a/packages/grafana-data/src/transformations/transformers/calculateField.test.ts +++ b/packages/grafana-data/src/transformations/transformers/calculateField.test.ts @@ -7,6 +7,7 @@ import { BinaryOperationID } from '../../utils/binaryOperators'; import { mockTransformationsRegistry } from '../../utils/tests/mockTransformationsRegistry'; import { UnaryOperationID } from '../../utils/unaryOperators'; import { ReducerID } from '../fieldReducer'; +import { FieldMatcherID } from '../matchers/ids'; import { transformDataFrame } from '../transformDataFrame'; import { @@ -197,6 +198,72 @@ describe('calculateField transformer w/ timeseries', () => { }); }); + it('all numbers + static number', async () => { + const cfg = { + id: DataTransformerID.calculateField, + options: { + mode: CalculateFieldMode.BinaryOperation, + binary: { + left: { matcher: { id: FieldMatcherID.byType, options: FieldType.number } }, + operator: BinaryOperationID.Add, + right: '2', + }, + replaceFields: true, + }, + }; + + await expect(transformDataFrame([cfg], [seriesBC])).toEmitValuesWith((received) => { + const data = received[0]; + const filtered = data[0]; + const rows = new DataFrameView(filtered).toArray(); + expect(rows).toEqual([ + { + 'B + 2': 4, + 'C + 2': 5, + TheTime: 1000, + }, + { + 'B + 2': 202, + 'C + 2': 302, + TheTime: 2000, + }, + ]); + }); + }); + + it('all numbers + field number', async () => { + const cfg = { + id: DataTransformerID.calculateField, + options: { + mode: CalculateFieldMode.BinaryOperation, + binary: { + left: { matcher: { id: FieldMatcherID.byType, options: FieldType.number } }, + operator: BinaryOperationID.Add, + right: 'C', + }, + replaceFields: true, + }, + }; + + await expect(transformDataFrame([cfg], [seriesBC])).toEmitValuesWith((received) => { + const data = received[0]; + const filtered = data[0]; + const rows = new DataFrameView(filtered).toArray(); + expect(rows).toEqual([ + { + 'B + C': 5, + 'C + C': 6, + TheTime: 1000, + }, + { + 'B + C': 500, + 'C + C': 600, + TheTime: 2000, + }, + ]); + }); + }); + it('unary math', async () => { const unarySeries = toDataFrame({ fields: [ diff --git a/packages/grafana-data/src/transformations/transformers/calculateField.ts b/packages/grafana-data/src/transformations/transformers/calculateField.ts index 820ecdec7cf..72a136c3980 100644 --- a/packages/grafana-data/src/transformations/transformers/calculateField.ts +++ b/packages/grafana-data/src/transformations/transformers/calculateField.ts @@ -5,7 +5,7 @@ import { getTimeField } from '../../dataframe/processDataFrame'; import { getFieldDisplayName } from '../../field/fieldState'; import { NullValueMode } from '../../types/data'; import { DataFrame, FieldType, Field } from '../../types/dataFrame'; -import { DataTransformerInfo } from '../../types/transformations'; +import { DataTransformContext, DataTransformerInfo } from '../../types/transformations'; import { BinaryOperationID, binaryOperators } from '../../utils/binaryOperators'; import { UnaryOperationID, unaryOperators } from '../../utils/unaryOperators'; import { doStandardCalcs, fieldReducers, ReducerID } from '../fieldReducer'; @@ -58,9 +58,14 @@ export interface UnaryOptions { } export interface BinaryOptions { - left: string; + left: BinaryValue; operator: BinaryOperationID; - right: string; + right: BinaryValue; +} + +export interface BinaryValue { + fixed?: string; + matcher?: { id?: FieldMatcherID; options?: string }; } interface IndexOptions { @@ -79,9 +84,9 @@ export const defaultWindowOptions: WindowOptions = { }; const defaultBinaryOptions: BinaryOptions = { - left: '', + left: { fixed: '' }, operator: BinaryOperationID.Add, - right: '', + right: { fixed: '' }, }; const defaultUnaryOptions: UnaryOptions = { @@ -152,13 +157,62 @@ export const calculateFieldTransformer: DataTransformerInfo { + frame.fields.map((field) => { + fieldNames.push(field.name); + }); + }); const binaryOptions = { - ...options.binary, - left: ctx.interpolate(options.binary?.left!), - right: ctx.interpolate(options.binary?.right!), + left: checkBinaryValueType(options.binary?.left ?? '', fieldNames), + operator: options.binary?.operator ?? defaultBinaryOptions.operator, + right: checkBinaryValueType(options.binary?.right ?? '', fieldNames), }; + options.binary = binaryOptions; + if (binaryOptions.left?.matcher?.id && binaryOptions.left?.matcher.id === FieldMatcherID.byType) { + const fieldType = binaryOptions.left.matcher.options; + const operator = binaryOperators.getIfExists(binaryOptions.operator); + return data.map((frame) => { + const { timeField } = getTimeField(frame); + const newFields: Field[] = []; + if (timeField && options.timeSeries !== false) { + newFields.push(timeField); + } + // For each field of type match, apply operator + frame.fields.map((field, index) => { + if (!options.replaceFields) { + newFields.push(field); + } + if (field.type === fieldType) { + const left = field.values; + // TODO consolidate common creator logic + const right = findFieldValuesWithNameOrConstant( + frame, + binaryOptions.right ?? defaultBinaryOptions.right, + data, + ctx + ); + if (!left || !right || !operator) { + return undefined; + } - creator = getBinaryCreator(defaults(binaryOptions, defaultBinaryOptions), data); + const arr = new Array(left.length); + for (let i = 0; i < arr.length; i++) { + arr[i] = operator.operation(left[i], right[i]); + } + const newField = { + ...field, + name: `${field.name} ${options.binary?.operator ?? ''} ${options.binary?.right.matcher?.options ?? options.binary?.right.fixed}`, + values: arr, + }; + newFields.push(newField); + } + }); + return { ...frame, fields: newFields }; + }); + } else { + creator = getBinaryCreator(defaults(binaryOptions, defaultBinaryOptions), data, ctx); + } break; case CalculateFieldMode.Index: return data.map((frame) => { @@ -488,23 +542,27 @@ function getReduceRowCreator(options: ReduceOptions, allFrames: DataFrame[]): Va function findFieldValuesWithNameOrConstant( frame: DataFrame, - name: string, - allFrames: DataFrame[] + value: BinaryValue, + allFrames: DataFrame[], + ctx: DataTransformContext ): number[] | undefined { - if (!name) { + if (!value) { return undefined; } - for (const f of frame.fields) { - if (name === getFieldDisplayName(f, frame, allFrames)) { - if (f.type === FieldType.boolean) { - return f.values.map((v) => (v ? 1 : 0)); + if (value.matcher && value.matcher.id === FieldMatcherID.byName) { + const name = ctx.interpolate(value.matcher.options ?? ''); + for (const f of frame.fields) { + if (name === getFieldDisplayName(f, frame, allFrames)) { + if (f.type === FieldType.boolean) { + return f.values.map((v) => (v ? 1 : 0)); + } + return f.values; } - return f.values; } } - const v = parseFloat(name); + const v = parseFloat(value.fixed ?? ctx.interpolate(value.matcher?.options ?? '')); if (!isNaN(v)) { return new Array(frame.length).fill(v); } @@ -512,12 +570,12 @@ function findFieldValuesWithNameOrConstant( return undefined; } -function getBinaryCreator(options: BinaryOptions, allFrames: DataFrame[]): ValuesCreator { +function getBinaryCreator(options: BinaryOptions, allFrames: DataFrame[], ctx: DataTransformContext): ValuesCreator { const operator = binaryOperators.getIfExists(options.operator); return (frame: DataFrame) => { - const left = findFieldValuesWithNameOrConstant(frame, options.left, allFrames); - const right = findFieldValuesWithNameOrConstant(frame, options.right, allFrames); + const left = findFieldValuesWithNameOrConstant(frame, options.left, allFrames, ctx); + const right = findFieldValuesWithNameOrConstant(frame, options.right, allFrames, ctx); if (!left || !right || !operator) { return undefined; } @@ -530,6 +588,24 @@ function getBinaryCreator(options: BinaryOptions, allFrames: DataFrame[]): Value }; } +export function checkBinaryValueType(value: BinaryValue | string, names: string[]): BinaryValue { + // Support old binary value structure + if (typeof value === 'string') { + if (isNaN(Number(value))) { + return { matcher: { id: FieldMatcherID.byName, options: value } }; + } else { + // If it's a number, check if matches name, otherwise store as fixed number value + if (names.includes(value)) { + return { matcher: { id: FieldMatcherID.byName, options: value } }; + } else { + return { fixed: value }; + } + } + } + // Pass through new BinaryValue structure + return value; +} + function getUnaryCreator(options: UnaryOptions, allFrames: DataFrame[]): ValuesCreator { const operator = unaryOperators.getIfExists(options.operator); @@ -577,7 +653,7 @@ export function getNameFromOptions(options: CalculateFieldTransformerOptions) { } case CalculateFieldMode.BinaryOperation: { const { binary } = options; - const alias = `${binary?.left ?? ''} ${binary?.operator ?? ''} ${binary?.right ?? ''}`; + const alias = `${binary?.left?.matcher?.options ?? binary?.left?.fixed ?? ''} ${binary?.operator ?? ''} ${binary?.right?.matcher?.options ?? binary?.right?.fixed ?? ''}`; //Remove $ signs as they will be interpolated and cause issues. Variables can still be used //in alias but shouldn't in the autogenerated name diff --git a/public/app/features/transformers/docs/content.ts b/public/app/features/transformers/docs/content.ts index d03f6aa4efc..4c1024e867c 100644 --- a/public/app/features/transformers/docs/content.ts +++ b/public/app/features/transformers/docs/content.ts @@ -64,6 +64,7 @@ Use this transformation to add a new field calculated from two other fields. Eac - **Field name** - Select the names of fields you want to use in the calculation for the new field. - **Calculation** - If you select **Reduce row** mode, then the **Calculation** field appears. Click in the field to see a list of calculation choices you can use to create the new field. For information about available calculations, refer to [Calculation types][]. - **Operation** - If you select **Binary operation** or **Unary operation** mode, then the **Operation** fields appear. These fields allow you to apply basic math operations on values in a single row from selected fields. You can also use numerical values for binary operations. + - **All number fields** - Set the left side of a **Binary operation** to apply the calculation to all number fields. - **As percentile** - If you select **Row index** mode, then the **As percentile** switch appears. This switch allows you to transform the row index as a percentage of the total number of rows. - **Alias** - (Optional) Enter the name of your new field. If you leave this blank, then the field will be named to match the calculation. - **Replace all fields** - (Optional) Select this option if you want to hide all other fields and display only your calculated field in the visualization. diff --git a/public/app/features/transformers/editors/CalculateFieldTransformerEditor/BinaryOperationOptionsEditor.tsx b/public/app/features/transformers/editors/CalculateFieldTransformerEditor/BinaryOperationOptionsEditor.tsx index 4371dd82949..0dd7cc033cb 100644 --- a/public/app/features/transformers/editors/CalculateFieldTransformerEditor/BinaryOperationOptionsEditor.tsx +++ b/public/app/features/transformers/editors/CalculateFieldTransformerEditor/BinaryOperationOptionsEditor.tsx @@ -1,10 +1,12 @@ -import { BinaryOperationID, binaryOperators, SelectableValue } from '@grafana/data'; +import { BinaryOperationID, binaryOperators, FieldMatcherID, FieldType, SelectableValue } from '@grafana/data'; import { + BinaryValue, BinaryOptions, CalculateFieldMode, CalculateFieldTransformerOptions, + checkBinaryValueType, } from '@grafana/data/src/transformations/transformers/calculateField'; -import { InlineField, InlineFieldRow, Select } from '@grafana/ui'; +import { getFieldTypeIconName, InlineField, InlineFieldRow, Select } from '@grafana/ui'; import { LABEL_WIDTH } from './constants'; @@ -14,21 +16,65 @@ export const BinaryOperationOptionsEditor = (props: { names: string[]; }) => { const { options, onChange } = props; + const newLeft = checkBinaryValueType(props.options.binary?.left ?? '', props.names); + const newRight = checkBinaryValueType(props.options.binary?.right ?? '', props.names); + // If there is a change due to migration, update save model + if (newLeft !== props.options.binary?.left || newRight !== props.options.binary?.right) { + onChange({ + ...options, + mode: CalculateFieldMode.BinaryOperation, + binary: { operator: options.binary?.operator!, left: newLeft, right: newRight }, + }); + } + const { binary } = options; let foundLeft = !binary?.left; let foundRight = !binary?.right; + + const fixedValueLeft = !binary?.left?.matcher; + const fixedValueRight = !binary?.right?.matcher; + const matcherOptionsLeft = binary?.left?.matcher?.options; + const matcherOptionsRight = binary?.right?.matcher?.options; + + const byNameLeft = binary?.left?.matcher?.id === FieldMatcherID.byName; + const byNameRight = binary?.right?.matcher?.id === FieldMatcherID.byName; const names = props.names.map((v) => { - if (v === binary?.left) { + if (byNameLeft && v === matcherOptionsLeft) { foundLeft = true; } - if (v === binary?.right) { + if (byNameRight && v === matcherOptionsRight) { foundRight = true; } - return { label: v, value: v }; + return { label: v, value: JSON.stringify({ matcher: { id: FieldMatcherID.byName, options: v } }) }; }); - const leftNames = foundLeft ? names : [...names, { label: binary?.left, value: binary?.left }]; - const rightNames = foundRight ? names : [...names, { label: binary?.right, value: binary?.right }]; + + // Populate left and right names with missing name only for byName + const leftNames = foundLeft + ? [...names] + : byNameLeft + ? [...names, { label: matcherOptionsLeft, value: JSON.stringify(binary.left), icon: '' }] + : [...names]; + const rightNames = foundRight + ? [...names] + : byNameRight + ? [...names, { label: matcherOptionsRight, value: JSON.stringify(binary.right), icon: '' }] + : [...names]; + + // Add byTypes to left names ONLY - avoid all number fields operated by all number fields + leftNames.push({ + label: `All ${FieldType.number} fields`, + value: JSON.stringify({ matcher: { id: FieldMatcherID.byType, options: FieldType.number } }), + icon: getFieldTypeIconName(FieldType.number), + }); + + // Add fixed values to left and right names + if (fixedValueLeft && binary?.left?.fixed) { + leftNames.push({ label: binary.left.fixed, value: JSON.stringify(binary.left) ?? '', icon: '' }); + } + if (fixedValueRight && binary?.right?.fixed) { + rightNames.push({ label: binary.right.fixed, value: JSON.stringify(binary.right) ?? '', icon: '' }); + } const ops = binaryOperators.list().map((v) => { return { label: v.binaryOperationID, value: v.binaryOperationID }; @@ -43,17 +89,35 @@ export const BinaryOperationOptionsEditor = (props: { }; const onBinaryLeftChanged = (v: SelectableValue) => { - updateBinaryOptions({ - ...binary!, - left: v.value!, - }); + const vObject: BinaryValue = JSON.parse(v.value ?? ''); + // If no matcher, treat as fixed value + if (!vObject.matcher) { + updateBinaryOptions({ + ...binary!, + left: { fixed: vObject.fixed ?? v.value?.toString() }, + }); + } else { + updateBinaryOptions({ + ...binary!, + left: vObject, + }); + } }; const onBinaryRightChanged = (v: SelectableValue) => { - updateBinaryOptions({ - ...binary!, - right: v.value!, - }); + const vObject: BinaryValue = JSON.parse(v.value ?? ''); + // If no matcher, treat as fixed value + if (!vObject.matcher) { + updateBinaryOptions({ + ...binary!, + right: { fixed: vObject.fixed ?? v.value?.toString() }, + }); + } else { + updateBinaryOptions({ + ...binary!, + right: vObject, + }); + } }; const onBinaryOperationChanged = (v: SelectableValue) => { @@ -69,10 +133,10 @@ export const BinaryOperationOptionsEditor = (props: {