diff --git a/.github/workflows/actions/test-core-screenshot/action.yml b/.github/workflows/actions/test-core-screenshot/action.yml index 8686caf579..afb213fcb1 100644 --- a/.github/workflows/actions/test-core-screenshot/action.yml +++ b/.github/workflows/actions/test-core-screenshot/action.yml @@ -7,8 +7,6 @@ inputs: description: 'Playwright total number of test shards (ex: 4)' update: description: 'Whether or not to update the reference snapshots' - commandModifier: - description: 'Command modifier to run. This is should only be used to temporarily run legacy E2E tests as we migrate to the generator infrastructure.' runs: using: 'composite' @@ -27,7 +25,7 @@ runs: working-directory: ./core - name: Test if: inputs.update != 'true' - run: npm run test.e2e${{ inputs.commandModifier }} -- --shard=${{ inputs.shard }}/${{ inputs.totalShards }} + run: npm run test.e2e -- --shard=${{ inputs.shard }}/${{ inputs.totalShards }} shell: bash working-directory: ./core - name: Test and Update @@ -49,14 +47,14 @@ runs: # which is why we not using the upload-archive # composite step here. run: | - npm run test.e2e${{ inputs.commandModifier }} -- --shard=${{ inputs.shard }}/${{ inputs.totalShards }} --update-snapshots + npm run test.e2e -- --shard=${{ inputs.shard }}/${{ inputs.totalShards }} --update-snapshots git add src/\*.png --force mkdir updated-screenshots cd ../ && rsync -R --progress $(git diff --name-only --cached) core/updated-screenshots if [ -d core/updated-screenshots/core ]; then echo "hasUpdatedScreenshots=$(echo 'true')" >> $GITHUB_OUTPUT cd core/updated-screenshots - zip -q -r ../../UpdatedScreenshots${{ inputs.commandModifier }}-${{ inputs.shard }}-${{ inputs.totalShards }}.zip core + zip -q -r ../../UpdatedScreenshots-${{ inputs.shard }}-${{ inputs.totalShards }}.zip core fi shell: bash working-directory: ./core @@ -64,8 +62,8 @@ runs: if: inputs.update == 'true' && steps.test-and-update.outputs.hasUpdatedScreenshots == 'true' uses: actions/upload-artifact@v3 with: - name: updated-screenshots${{ inputs.commandModifier }}-${{ inputs.shard }}-${{ inputs.totalShards }} - path: UpdatedScreenshots${{ inputs.commandModifier }}-${{ inputs.shard }}-${{ inputs.totalShards }}.zip + name: updated-screenshots-${{ inputs.shard }}-${{ inputs.totalShards }} + path: UpdatedScreenshots-${{ inputs.shard }}-${{ inputs.totalShards }}.zip - name: Archive Test Results # The always() ensures that this step # runs even if the previous step fails. @@ -76,6 +74,6 @@ runs: if: always() uses: ./.github/workflows/actions/upload-archive with: - name: test-results${{ inputs.commandModifier }}-${{ inputs.shard }}-${{ inputs.totalShards }} - output: core/TestResults${{ inputs.commandModifier }}-${{ inputs.shard }}-${{ inputs.totalShards }}.zip + name: test-results-${{ inputs.shard }}-${{ inputs.totalShards }} + output: core/TestResults-${{ inputs.shard }}-${{ inputs.totalShards }}.zip paths: core/playwright-report diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 268eee3d81..3ca6263334 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -81,45 +81,6 @@ jobs: if: ${{ needs.test-core-screenshot.result != 'success' }} run: exit 1 - test-core-screenshot-legacy: - strategy: - matrix: - # Divide the tests into n buckets - # and run those buckets in parallel. - # To increase the number of shards, - # add new items to the shard array - # and change the value of totalShards - # to be the length of the shard array. - shard: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20] - totalShards: [20] - needs: [build-core] - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: ./.github/workflows/actions/test-core-screenshot - with: - shard: ${{ matrix.shard }} - totalShards: ${{ matrix.totalShards }} - commandModifier: '.legacy' - - # Screenshots are required to pass - # in order for the branch to be merge - # eligible. However, the screenshot tests - # are run on n runners where n can change - # over time. The verify-screenshots step allows - # us to have a required status check for screenshot - # results without having to manually add each - # matrix run in the branch protection rules - # Source: https://github.community/t/status-check-for-a-matrix-jobs/127354 - verify-screenshots-legacy: - if: ${{ always() }} - needs: test-core-screenshot-legacy - runs-on: ubuntu-latest - steps: - - name: Check build matrix status - if: ${{ needs.test-core-screenshot-legacy.result != 'success' }} - run: exit 1 - build-vue: needs: [build-core] runs-on: ubuntu-latest diff --git a/.github/workflows/stencil-nightly.yml b/.github/workflows/stencil-nightly.yml index 46b2f0732e..b81d20a04e 100644 --- a/.github/workflows/stencil-nightly.yml +++ b/.github/workflows/stencil-nightly.yml @@ -87,45 +87,6 @@ jobs: if: ${{ needs.test-core-screenshot.result != 'success' }} run: exit 1 - test-core-screenshot-legacy: - strategy: - matrix: - # Divide the tests into n buckets - # and run those buckets in parallel. - # To increase the number of shards, - # add new items to the shard array - # and change the value of totalShards - # to be the length of the shard array. - shard: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20] - totalShards: [20] - needs: [build-core-with-stencil-nightly] - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: ./.github/workflows/actions/test-core-screenshot - with: - shard: ${{ matrix.shard }} - totalShards: ${{ matrix.totalShards }} - commandModifier: '.legacy' - - # Screenshots are required to pass - # in order for the branch to be merge - # eligible. However, the screenshot tests - # are run on n runners where n can change - # over time. The verify-screenshots step allows - # us to have a required status check for screenshot - # results without having to manually add each - # matrix run in the branch protection rules - # Source: https://github.community/t/status-check-for-a-matrix-jobs/127354 - verify-screenshots-legacy: - if: ${{ always() }} - needs: test-core-screenshot-legacy - runs-on: ubuntu-latest - steps: - - name: Check build matrix status - if: ${{ needs.test-core-screenshot-legacy.result != 'success' }} - run: exit 1 - build-vue: needs: [build-core-with-stencil-nightly] runs-on: ubuntu-latest diff --git a/core/package.json b/core/package.json index 76b2809fdb..6bef03ccc9 100644 --- a/core/package.json +++ b/core/package.json @@ -90,8 +90,7 @@ "test": "npm run test.spec && npm run test.e2e", "test.spec": "stencil test --spec --max-workers=2", "test.spec.debug": "npx --node-arg=\"--inspect-brk\" stencil test --spec", - "test.e2e": "npx playwright test --config playwright.config.ts", - "test.e2e.legacy": "npx playwright test --config playwright.config-legacy.ts", + "test.e2e": "npx playwright test", "test.watch": "jest --watch --no-cache", "test.treeshake": "node scripts/treeshaking.js dist/index.js", "validate": "npm run lint && npm run test && npm run build && npm run test.treeshake" diff --git a/core/playwright.config-legacy.ts b/core/playwright.config-legacy.ts deleted file mode 100644 index 6f0b4374d2..0000000000 --- a/core/playwright.config-legacy.ts +++ /dev/null @@ -1,119 +0,0 @@ -import type { PlaywrightTestConfig, PlaywrightTestOptions, PlaywrightWorkerOptions, Project } from '@playwright/test'; -import { devices, expect } from '@playwright/test'; - -import { matchers } from './src/utils/test/playwright'; - -expect.extend(matchers); - -const projects: Project[] = [ - { - /** - * This is really just desktop Firefox - * but with a mobile viewport. - */ - name: 'Mobile Firefox', - use: { - browserName: 'firefox', - /** - * This is the Pixel 5 configuration. - * We can't use devices['Pixel 5'] - * because the "isMobile" option is - * not supported on Firefox. - */ - viewport: { - width: 393, - height: 727 - }, - }, - }, - { - name: 'Mobile Chrome', - use: { - browserName: 'chromium', - ...devices['Pixel 5'] - } - }, - { - name: 'Mobile Safari', - use: { - browserName: 'webkit', - ...devices['iPhone 12'] - } - } -]; - -const modes = ['ios', 'md']; - -const generateProjects = () => { - const projectsWithMetadata: Project[] = []; - - modes.forEach(mode => { - projects.forEach(project => { - projectsWithMetadata.push({ - ...project, - metadata: { - mode, - rtl: false, - _testing: true - } - }); - projectsWithMetadata.push({ - ...project, - metadata: { - mode, - rtl: true, - _testing: true - } - }); - }); - }); - - return projectsWithMetadata; -} - -/** - * See https://playwright.dev/docs/test-configuration. - */ -const config: PlaywrightTestConfig = { - testMatch: '*.e2e-legacy.ts', - expect: { - /** - * Maximum time expect() should wait for the condition to be met. - * For example in `await expect(locator).toHaveText();` - */ - timeout: 5000 - }, - /* Fail the build on CI if you accidentally left test.only in the source code. */ - forbidOnly: !!process.env.CI, - /* Fail fast on CI */ - maxFailures: process.env.CI ? 1 : 0, - /* Flaky test should be either addressed or disabled until we can address them */ - retries: 0, - /* Opt out of parallel tests on CI. */ - workers: process.env.CI ? 1 : undefined, - /* Reporter to use. See https://playwright.dev/docs/test-reporters */ - reporter: 'html', - /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */ - use: { - /* Maximum time each action such as `click()` can take. Defaults to 0 (no limit). */ - actionTimeout: 0, - /** - * All failed tests should create - * a trace file for easier debugging. - * - * See https://playwright.dev/docs/trace-viewer - */ - trace: 'retain-on-failure', - baseURL: 'http://localhost:3333', - }, - - /* Configure projects for major browsers */ - projects: generateProjects(), - webServer: { - command: 'serve -p 3333', - port: 3333, - reuseExistingServer: !process.env.CI - } -}; - -export default config; diff --git a/core/src/utils/test/playwright/docs/README.md b/core/src/utils/test/playwright/docs/README.md index abf2d8f68a..3836f331f4 100644 --- a/core/src/utils/test/playwright/docs/README.md +++ b/core/src/utils/test/playwright/docs/README.md @@ -7,5 +7,4 @@ This directory contains information on how to get the most out of Ionic's E2E te | Directory | Description | | - | - | | [Best Practices](./best-practices.md) | Contains information on conventions to follow as well as pitfalls to avoid when writing tests | -| [API](./api.md) | Documents the custom functionality that has been built on top of Playwright | -| [Migrating to Generators](./migrating-to-generators.md) | A step-by-step guide for migrating existing E2E tests to use the generator architecture | +| [API](./api.md) | Documents the custom functionality that has been built on top of Playwright | \ No newline at end of file diff --git a/core/src/utils/test/playwright/docs/migrating-to-generators.md b/core/src/utils/test/playwright/docs/migrating-to-generators.md deleted file mode 100644 index d00558f898..0000000000 --- a/core/src/utils/test/playwright/docs/migrating-to-generators.md +++ /dev/null @@ -1,115 +0,0 @@ -# Generator Migration - -This guide shows how to migrate an E2E test to use the new generator architecture. - -## Steps to Migrate - -1. Import `configs` from `@utils/test/playwright`. - -```typescript -/** - * test should already have been imported - * for the existing E2E test. - */ -import { configs, test } from '@utils/test/playwright'; -``` - -2. Call `configs` outside of the `test.describe` block, and pass in any of your configuration. This should replace any `skip.rtl` or `skip.mode` calls. `skip.browser` calls can remain in place. If you are not using `skip.rtl` or `skip.mode` then you can call `configs` without passing an options (i.e. `configs().forEach(...)`). - -```typescript -import { configs, test } from '@utils/test/playwright'; - -configs({ modes: ['ios'], directions: ['ltr'] }).forEach(({ config, title, screenshot }) => { - test.describe('accordion: basic', () => { - ... - }); -}); -``` - -:warning: Be careful when replacing `skip.rtl` and `skip.mode` with `configs`. For example, `skip.mode('md')` means that the test should _skip_ MD mode and only run in iOS mode **not** that the test should only run in MD mode. Similarly, `skip.rtl()` means that the test should only run in the LTR direction. - -3. Update the title of the describe block to use the `title` function. - -```typescript -import { configs, test } from '@utils/test/playwright'; - -configs({ modes: ['ios'], directions: ['ltr'] }).forEach(({ config, title, screenshot }) => { - test.describe(title('accordion: basic'), () => { - ... - }); -}); -``` - -4. Pass the `config` object to either `page.goto` or `page.setContent`. - -```typescript -import { configs, test } from '@utils/test/playwright'; - -configs({ modes: ['ios'], directions: ['ltr'] }).forEach(({ config, title, screenshot }) => { - test.describe(title('accordion: basic'), () => { - test('should not have visual regressions', async ({ page }) => { - await page.goto('/src/components/accordion/test/basic', config); - - /** - * If using setContent: - * await page.setContent('...', config); - */ - }); - }); -}); -``` - -5. If taking a screenshot, replace `page.getSnapshotSettings()` with the `screenshot` function. - -```typescript -import { configs, test } from '@utils/test/playwright'; - -configs({ modes: ['ios'], directions: ['ltr'] }).forEach(({ config, title, screenshot }) => { - test.describe(title('accordion: basic'), () => { - test('should not have visual regressions', async ({ page }) => { - await page.goto('/src/components/accordion/test/basic', config); - - /** - * The `screenshot` function will automatically add ".png" - * as well as the test config. - */ - await expect(page).toHaveScreenshot(screenshot('accordion-basic')); - }); - }); -}); -``` - -6. Rename your `*.e2e-legacy.ts` file to `*.e2e.ts`. - -**Example:** `accordion.e2e-legacy.ts` becomes `accordion.e2e.ts` - -7. If the test has a screenshots directory, rename the `*.e2e-legacy.ts-snapshots` directory to `*.e2e.ts-snapshots`. - -**Example:** `accordion.e2e-legacy.ts-snapshots` becomes `accordion.e2e.ts-snapshots` - -8. Run `git add .` to stage your changes. - -:warning: It is very important that you also stage the screenshot files when renaming the directory. The screenshot files are gitignored to prevent team members from accidentally overwriting the ground truth screenshots. As a result, you will need to bypass the gitignore to add the screenshots using the `--force` (`-f`) flag: - -`git add -f src/components/[name of component]/test/[test directory]/**/*-linux.png` - -This will forcibly stage only the screenshots generated on CI (due to the use of `-linux.png`). - -**Example:** `git add -f src/components/accordion/test/basic/**/*-linux.png` - -9. Review your changes. **There should be no deleted or added screenshots.** Instead, the screenshots should be listed as "renamed". - -**Example:** - -``` -renamed: src/components/accordion/test/basic/accordion.e2e-legacy.ts -> src/components/accordion/test/basic/accordion.e2e.ts -renamed: src/components/accordion/test/basic/accordion.e2e-legacy.ts-snapshots/accordion-basic-ios-ltr-Mobile-Chrome-linux.png -> src/components/accordion/test/basic/accordion.e2e.ts-snapshots/accordion-basic-ios-ltr-Mobile-Chrome-linux.png -renamed: src/components/accordion/test/basic/accordion.e2e-legacy.ts-snapshots/accordion-basic-ios-ltr-Mobile-Firefox-linux.png -> src/components/accordion/test/basic/accordion.e2e.ts-snapshots/accordion-basic-ios-ltr-Mobile-Firefox-linux.png -renamed: src/components/accordion/test/basic/accordion.e2e-legacy.ts-snapshots/accordion-basic-ios-ltr-Mobile-Safari-linux.png -> src/components/accordion/test/basic/accordion.e2e.ts-snapshots/accordion-basic-ios-ltr-Mobile-Safari-linux.png -``` - -10. Commit your changes using the following format: `test([component]): migrate [test directory] to generators` - -**Example:** `test(accordion): migrate basic to generators` - -11. Push your changes. \ No newline at end of file