diff --git a/packages/grafana-e2e/cli.js b/packages/grafana-e2e/cli.js index f75cf4ab2fc..2eb4d6f05fa 100644 --- a/packages/grafana-e2e/cli.js +++ b/packages/grafana-e2e/cli.js @@ -3,12 +3,18 @@ const program = require('commander'); const resolveBin = require('resolve-as-bin'); const { resolve, sep } = require('path'); -const cypress = commandName => { +const cypress = (commandName, { updateScreenshots }) => { // Support running an unpublished dev build const dirname = __dirname.split(sep).pop(); const projectPath = resolve(`${__dirname}${dirname === 'dist' ? '/..' : ''}`); - const cypressOptions = [commandName, '--env', `CWD=${process.cwd()}`, `--project=${projectPath}`]; + // For plugins/extendConfig + const CWD = `CWD=${process.cwd()}`; + + // For plugins/compareSnapshots + const UPDATE_SCREENSHOTS = `UPDATE_SCREENSHOTS=${updateScreenshots ? 1 : 0}`; + + const cypressOptions = [commandName, '--env', `${CWD},${UPDATE_SCREENSHOTS}`, `--project=${projectPath}`]; const execaOptions = { cwd: __dirname, @@ -24,20 +30,20 @@ const cypress = commandName => { }; module.exports = () => { - const configOption = '-c, --config '; - const configDescription = 'path to JSON file where configuration values are set; defaults to "cypress.json"'; + const updateOption = '-u, --update-screenshots'; + const updateDescription = 'update expected screenshots'; program .command('open') .description('runs tests within the interactive GUI') - .option(configOption, configDescription) - .action(() => cypress('open')); + .option(updateOption, updateDescription) + .action(options => cypress('open', options)); program .command('run') .description('runs tests from the CLI without the GUI') - .option(configOption, configDescription) - .action(() => cypress('run')); + .option(updateOption, updateDescription) + .action(options => cypress('run', options)); program.parse(process.argv); }; diff --git a/packages/grafana-e2e/cypress/fixtures/example.json b/packages/grafana-e2e/cypress/fixtures/example.json deleted file mode 100644 index 02e4254378e..00000000000 --- a/packages/grafana-e2e/cypress/fixtures/example.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "name": "Using fixtures to represent data", - "email": "hello@cypress.io", - "body": "Fixtures are a great way to mock data for responses to routes" -} diff --git a/packages/grafana-e2e/cypress/plugins/compareScreenshots.js b/packages/grafana-e2e/cypress/plugins/compareScreenshots.js new file mode 100644 index 00000000000..3ca9bf397cb --- /dev/null +++ b/packages/grafana-e2e/cypress/plugins/compareScreenshots.js @@ -0,0 +1,49 @@ +'use strict'; +const BlinkDiff = require('blink-diff'); +const { resolve } = require('path'); + +// @todo use npmjs.com/pixelmatch or an available cypress plugin +const compareSceenshots = async ({ config, screenshotsFolder, specName }) => { + const name = config.name || config; // @todo use `??` + const threshold = config.threshold || 0.001; // @todo use `??` + + const imageAPath = `${screenshotsFolder}/${specName}/${name}.png`; + const imageBPath = resolve(`${screenshotsFolder}/../expected/${specName}/${name}.png`); + + const imageOutputPath = screenshotsFolder.endsWith('actual') ? imageAPath.replace('.png', '.diff.png') : undefined; + + const { code } = await new Promise((resolve, reject) => { + new BlinkDiff({ + imageAPath, + imageBPath, + imageOutputPath, + threshold, + thresholdType: BlinkDiff.THRESHOLD_PERCENT, + }).run((error, result) => { + if (error) { + reject(error); + } else { + resolve(result); + } + }); + }); + + if (code <= 1) { + let msg = `\nThe screenshot [${imageAPath}] differs from [${imageBPath}]`; + msg += '\n'; + msg += '\nCheck the Artifacts tab in the CircleCi build output for the actual screenshots.'; + msg += '\n'; + msg += '\n If the difference between expected and outcome is NOT acceptable then do the following:'; + msg += '\n - Check the code for changes that causes this difference, fix that and retry.'; + msg += '\n'; + msg += '\n If the difference between expected and outcome is acceptable then do the following:'; + msg += '\n - Replace the expected image with the outcome and retry.'; + msg += '\n'; + throw new Error(msg); + } else { + // Must return a value + return true; + } +}; + +module.exports = compareSceenshots; diff --git a/packages/grafana-e2e/cypress/plugins/compareSnapshots.js b/packages/grafana-e2e/cypress/plugins/compareSnapshots.js deleted file mode 100644 index 65702f52f11..00000000000 --- a/packages/grafana-e2e/cypress/plugins/compareSnapshots.js +++ /dev/null @@ -1,25 +0,0 @@ -const BlinkDiff = require('blink-diff'); - -function compareSnapshotsPlugin(args) { - args.threshold = args.threshold || 0.001; - - return new Promise((resolve, reject) => { - const diff = new BlinkDiff({ - imageAPath: args.pathToFileA, - imageBPath: args.pathToFileB, - thresholdType: BlinkDiff.THRESHOLD_PERCENT, - threshold: args.threshold, - imageOutputPath: args.pathToFileA.replace('.png', '.diff.png'), - }); - - diff.run((error, result) => { - if (error) { - reject(error); - } else { - resolve(result); - } - }); - }); -} - -module.exports = compareSnapshotsPlugin; diff --git a/packages/grafana-e2e/cypress/plugins/extendConfig.js b/packages/grafana-e2e/cypress/plugins/extendConfig.js index 56561f108c9..fe37bb9083c 100644 --- a/packages/grafana-e2e/cypress/plugins/extendConfig.js +++ b/packages/grafana-e2e/cypress/plugins/extendConfig.js @@ -4,10 +4,11 @@ const { } = require('fs'); const { resolve } = require('path'); +// @todo use https://github.com/bahmutov/cypress-extends when possible module.exports = async baseConfig => { // From CLI const { - env: { CWD }, + env: { CWD, UPDATE_SCREENSHOTS }, } = baseConfig; if (CWD) { @@ -21,7 +22,7 @@ module.exports = async baseConfig => { reporterOptions: { output: `${CWD}/cypress/report.json`, }, - screenshotsFolder: `${CWD}/cypress/screenshots`, + screenshotsFolder: `${CWD}/cypress/screenshots/${UPDATE_SCREENSHOTS ? 'expected' : 'actual'}`, videosFolder: `${CWD}/cypress/videos`, }; diff --git a/packages/grafana-e2e/cypress/plugins/index.js b/packages/grafana-e2e/cypress/plugins/index.js index f5afa717b24..7c2120b99b0 100644 --- a/packages/grafana-e2e/cypress/plugins/index.js +++ b/packages/grafana-e2e/cypress/plugins/index.js @@ -1,24 +1,22 @@ -const compareSnapshotsPlugin = require('./compareSnapshots'); +const compareScreenshots = require('./compareScreenshots'); const extendConfig = require('./extendConfig'); const readProvisions = require('./readProvisions'); const typescriptPreprocessor = require('./typescriptPreprocessor'); +const { install: installConsoleLogger } = require('cypress-log-to-output'); module.exports = (on, config) => { - // yarn build fails with: - // >> /Users/hugo/go/src/github.com/grafana/grafana/node_modules/stringmap/stringmap.js:99 - // >> throw new Error("StringMap expected string key"); - // on('task', { - // failed: require('cypress-failed-log/src/failed')(), - // }); on('file:preprocessor', typescriptPreprocessor); - on('task', { compareSnapshotsPlugin, readProvisions }); + on('task', { compareScreenshots, readProvisions }); on('task', { + // @todo remove log({ message, optional }) { optional ? console.log(message, optional) : console.log(message); return null; }, }); + installConsoleLogger(on); + // Always extend with this library's config and return for diffing // @todo remove this when possible: https://github.com/cypress-io/cypress/issues/5674 return extendConfig(config); diff --git a/packages/grafana-e2e/cypress/support/commands.ts b/packages/grafana-e2e/cypress/support/commands.ts index b237699e96b..d281d26229f 100644 --- a/packages/grafana-e2e/cypress/support/commands.ts +++ b/packages/grafana-e2e/cypress/support/commands.ts @@ -1,27 +1,17 @@ -interface CompareSnapshotArgs { - pathToFileA: string; - pathToFileB: string; +interface CompareSceenshotsConfig { + name: string; threshold?: number; } -Cypress.Commands.add('compareSnapshot', (args: CompareSnapshotArgs) => { - cy.task('compareSnapshotsPlugin', args).then((results: any) => { - if (results.code <= 1) { - let msg = `\nThe screenshot:[${args.pathToFileA}] differs from :[${args.pathToFileB}]`; - msg += '\n'; - msg += '\nCheck the Artifacts tab in the CircleCi build output for the actual screenshots.'; - msg += '\n'; - msg += '\n If the difference between expected and outcome is NOT acceptable then do the following:'; - msg += '\n - Check the code for changes that causes this difference, fix that and retry.'; - msg += '\n'; - msg += '\n If the difference between expected and outcome is acceptable then do the following:'; - msg += '\n - Replace the expected image with the outcome and retry.'; - msg += '\n'; - throw new Error(msg); - } +Cypress.Commands.add('compareSceenshots', (config: CompareSceenshotsConfig | string) => { + cy.task('compareSceenshots', { + config, + screenshotsFolder: Cypress.config('screenshotsFolder'), + specName: Cypress.spec.name, }); }); +// @todo remove Cypress.Commands.add('logToConsole', (message: string, optional?: any) => { cy.task('log', { message, optional }); }); diff --git a/packages/grafana-e2e/package.json b/packages/grafana-e2e/package.json index 1289e9830b8..5277f8e578b 100644 --- a/packages/grafana-e2e/package.json +++ b/packages/grafana-e2e/package.json @@ -50,6 +50,7 @@ "blink-diff": "1.0.13", "commander": "5.0.0", "cypress": "4.5.0", + "cypress-log-to-output": "^1.0.8", "execa": "4.0.0", "resolve-as-bin": "2.1.0", "ts-loader": "6.2.1", diff --git a/packages/grafana-e2e/src/flows/addPanel.ts b/packages/grafana-e2e/src/flows/addPanel.ts index 45e1c83cccf..540be4aadd6 100644 --- a/packages/grafana-e2e/src/flows/addPanel.ts +++ b/packages/grafana-e2e/src/flows/addPanel.ts @@ -2,27 +2,57 @@ import { e2e } from '../index'; import { getScenarioContext } from '../support/scenarioContext'; export interface AddPanelConfig { + dashboardUid?: string; dataSourceName: string; queriesForm: Function; + visualizationName: string; } const DEFAULT_ADD_PANEL_CONFIG: AddPanelConfig = { dataSourceName: 'TestData DB', queriesForm: () => {}, + visualizationName: 'Graph', }; -export const addPanel = (config?: Partial) => { - const { dataSourceName, queriesForm } = { ...DEFAULT_ADD_PANEL_CONFIG, ...config }; +// @todo this actually returns type `Cypress.Chainable` +export const addPanel = (config?: Partial): any => { + const { dashboardUid, dataSourceName, queriesForm, visualizationName } = { ...DEFAULT_ADD_PANEL_CONFIG, ...config }; + const panelTitle = `e2e-${Date.now()}`; - getScenarioContext().then(({ lastAddedDashboardUid }: any) => { - e2e.flows.openDashboard(lastAddedDashboardUid); - e2e.pages.Dashboard.Toolbar.toolbarItems('Add panel').click(); - e2e.pages.AddDashboard.addNewPanel().click(); - e2e() - .get('.ds-picker') - .click() - .contains('[id^="react-select-"][id*="-option-"]', dataSourceName) - .click(); - queriesForm(); - }); + return getScenarioContext() + .then(({ lastAddedDashboardUid }: any) => { + e2e.flows.openDashboard(dashboardUid ?? lastAddedDashboardUid); + e2e.pages.Dashboard.Toolbar.toolbarItems('Add panel').click(); + e2e.pages.AddDashboard.addNewPanel().click(); + + e2e() + .get('.ds-picker') + .click() + .contains('[id^="react-select-"][id*="-option-"]', dataSourceName) + .click(); + + getOptionsGroup('settings') + .find('[value="Panel Title"]') + .clear() + .type(panelTitle); + toggleOptionsGroup('settings'); + + toggleOptionsGroup('type'); + e2e() + .get(`[aria-label="Plugin visualization item ${visualizationName}"]`) + .scrollIntoView() + .click(); + toggleOptionsGroup('type'); + + queriesForm(); + }) + .then(() => panelTitle); }; + +const getOptionsGroup = (name: string) => e2e().get(`.options-group:has([aria-label="Options group Panel ${name}"])`); + +const toggleOptionsGroup = (name: string) => + getOptionsGroup(name) + .find('.editor-options-group-toggle') + .scrollIntoView() + .click(); diff --git a/yarn.lock b/yarn.lock index 8e6110b77bb..11388c37126 100644 --- a/yarn.lock +++ b/yarn.lock @@ -8912,6 +8912,14 @@ chownr@^1.1.1, chownr@^1.1.2: resolved "https://registry.yarnpkg.com/chownr/-/chownr-1.1.3.tgz#42d837d5239688d55f303003a508230fa6727142" integrity sha512-i70fVHhmV3DtTl6nqvZOnIjbY0Pe4kAUjwHj8z0zAdgBtYrJyYwLKCCuRBQ5ppkyL0AkN7HKRnETdmdp1zqNXw== +chrome-remote-interface@^0.27.1: + version "0.27.2" + resolved "https://registry.yarnpkg.com/chrome-remote-interface/-/chrome-remote-interface-0.27.2.tgz#e5605605f092b7ef8575d95304e004039c9d0ab9" + integrity sha512-pVLljQ29SAx8KIv5tSa9sIf8GrEsAZdPJoeWOmY3/nrIzFmE+EryNNHvDkddGod0cmAFTv+GmPG0uvzxi2NWsA== + dependencies: + commander "2.11.x" + ws "^6.1.0" + chrome-trace-event@^1.0.2: version "1.0.2" resolved "https://registry.yarnpkg.com/chrome-trace-event/-/chrome-trace-event-1.0.2.tgz#234090ee97c7d4ad1a2c4beae27505deffc608a4" @@ -9301,6 +9309,11 @@ commander@2, commander@^2.18.0, commander@^2.19.0, commander@^2.20.0, commander@ resolved "https://registry.yarnpkg.com/commander/-/commander-2.20.3.tgz#fd485e84c03eb4881c20722ba48035e8531aeb33" integrity sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ== +commander@2.11.x: + version "2.11.0" + resolved "https://registry.yarnpkg.com/commander/-/commander-2.11.0.tgz#157152fd1e7a6c8d98a5b715cf376df928004563" + integrity sha512-b0553uYA5YAEGgyYIGYROzKQ7X5RAqedkfjiZxwi0kL1g3bOaBNNZfYkzt/CL0umgD5wc9Jec2FbB98CjkMRvQ== + commander@2.17.x: version "2.17.1" resolved "https://registry.yarnpkg.com/commander/-/commander-2.17.1.tgz#bd77ab7de6de94205ceacc72f1716d29f20a77bf" @@ -10201,6 +10214,14 @@ cyclist@^1.0.1: resolved "https://registry.yarnpkg.com/cyclist/-/cyclist-1.0.1.tgz#596e9698fd0c80e12038c2b82d6eb1b35b6224d9" integrity sha1-WW6WmP0MgOEgOMK4LW6xs1tiJNk= +cypress-log-to-output@^1.0.8: + version "1.0.8" + resolved "https://registry.yarnpkg.com/cypress-log-to-output/-/cypress-log-to-output-1.0.8.tgz#8698db2cd68b88fd62e7f9ea8d1eece20bb210cf" + integrity sha512-o0PwNSXSZho2QLTOa4I/KgyPfZwgqNBqNcz+jBMcKJHPsRBZDUElaosigqSXI28uuSlprUlvcYjpcb/791u/lg== + dependencies: + chalk "^2.4.2" + chrome-remote-interface "^0.27.1" + cypress@4.5.0: version "4.5.0" resolved "https://registry.yarnpkg.com/cypress/-/cypress-4.5.0.tgz#01940d085f6429cec3c87d290daa47bb976a7c7b"