test(e2e): add infrastructure for migration to playwright (#25033)

.github/workflows/actions/build-core/action.yml

@@ -28,3 +28,8 @@ runs:
         name: ionic-core
         output: core/CoreBuild.zip
         paths: core/dist core/components core/css core/hydrate core/loader
+    - uses: ./.github/workflows/actions/upload-archive
+      with:
+        name: ionic-core-src
+        output: core/CoreSrc.zip
+        paths: core/src

.github/workflows/actions/test-core-screenshot-main/action.yml

@@ -1,33 +0,0 @@
-name: 'Test Core Screenshot Main'
-description: 'Test Core Screenshot Main'
-inputs:
-  access-key-id:
-    description: 'AWS_ACCESS_KEY_ID'
-  secret-access-key:
-    description: 'AWS_SECRET_ACCESS_KEY'
-runs:
-  using: 'composite'
-  steps:
-    - uses: actions/setup-node@v1
-      with:
-        node-version: 15.x
-
-    - name: Cache Core Node Modules
-      uses: actions/cache@v2
-      env:
-        cache-name: core-node-modules
-      with:
-        path: ./core/node_modules
-        key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('./core/package-lock.json') }}-v2
-    - uses: ./.github/workflows/actions/download-archive
-      with:
-        name: ionic-core
-        path: ./core
-        filename: CoreBuild.zip
-    - name: Test
-      run: npx stencil test --e2e --screenshot --screenshot-connector=scripts/screenshot/ci.js --ci --update-screenshot --no-build || true
-      shell: bash
-      env:
-        AWS_ACCESS_KEY_ID: ${{ inputs.access-key-id }}
-        AWS_SECRET_ACCESS_KEY: ${{ inputs.secret-access-key }}
-      working-directory: ./core

.github/workflows/actions/test-core-screenshot/action.yml

@@ -1,10 +1,12 @@
 name: 'Test Core Screenshot'
 description: 'Test Core Screenshot'
 inputs:
-  access-key-id:
-    description: 'AWS_ACCESS_KEY_ID'
-  secret-access-key:
-    description: 'AWS_SECRET_ACCESS_KEY'
+  shard:
+    description: 'Playwright Test Shard (ex: 2)'
+  totalShards:
+    description: 'Playwright total number of test shards (ex: 4)'
+  update:
+    description: 'Whether or not to update the reference snapshots'
 runs:
   using: 'composite'
   steps:
@@ -24,10 +26,35 @@ runs:
         name: ionic-core
         path: ./core
         filename: CoreBuild.zip
-    - name: Test
-      run: npx stencil test --e2e --screenshot --screenshot-connector=scripts/screenshot/ci.js --ci --no-build || true
+    - uses: ./.github/workflows/actions/download-archive
+      with:
+        name: ionic-core-src
+        path: ./core
+        filename: CoreSrc.zip
+    - name: Install Playwright Dependencies
+      run: npx playwright install && npx playwright install-deps
       shell: bash
-      env:
-        AWS_ACCESS_KEY_ID: ${{ inputs.access-key-id }}
-        AWS_SECRET_ACCESS_KEY: ${{ inputs.secret-access-key }}
       working-directory: ./core
+    - name: Test
+      if: inputs.update != 'true'
+      run: npx playwright test --shard=${{ inputs.shard }}/${{ inputs.totalShards }}
+      shell: bash
+      working-directory: ./core
+    - name: Test and Update
+      if: inputs.update == 'true'
+      run: npx playwright test --shard=${{ inputs.shard }}/${{ inputs.totalShards }} --update-snapshots
+      shell: bash
+      working-directory: ./core
+    - name: Archive Test Results
+      # The always() ensures that this step
+      # runs even if the previous step fails.
+      # We want the test results to be archived
+      # even if the test fails in the previous
+      # step, otherwise there would be no way
+      # to debug these tests.
+      if: always()
+      uses: ./.github/workflows/actions/upload-archive
+      with:
+        name: test-results-${{ inputs.shard }}-${{ inputs.totalShards }}
+        output: core/TestResults-${{ inputs.shard }}-${{ inputs.totalShards }}.zip
+        paths: core/playwright-report core/src

.github/workflows/actions/update-reference-screenshots/action.yml

@@ -0,0 +1,35 @@
+name: 'Update Reference Screenshots'
+description: 'Update Reference Screenshots'
+
+on:
+  workflow_dispatch:
+
+runs:
+  using: 'composite'
+  steps:
+    - uses: actions/setup-node@v1
+      with:
+        node-version: 15.x
+    - uses: actions/download-artifact@v2
+      with:
+        path: ./artifacts
+    - name: Extract Archives
+      # This finds all .zip files in the ./artifacts
+      # directory, including nested directories.
+      # It then unzips every .zip to the root directory
+      run: |
+        find . -type f -name '*.zip' -exec unzip -q -o -d ../ -- '{}'  -x '*.zip' \;
+      shell: bash
+      working-directory: ./artifacts
+    - name: Push Screenshots
+      # Configure user as Ionitron
+      # and push only the changed .png snapshots
+      # to the remote branch.
+      run: |
+        git config user.name ionitron
+        git config user.email hi@ionicframework.com
+        git add src/\*.png
+        git commit -m "chore(): add updated snapshots"
+        git push
+      shell: bash
+      working-directory: ./core

.github/workflows/build.yml

@@ -40,26 +40,28 @@ jobs:
       - uses: ./.github/workflows/actions/test-core-e2e
 
   test-core-screenshot:
+    strategy:
+      # This ensures that all screenshot shard
+      # failures are reported so the dev can
+      # review everything at once.
+      fail-fast: false
+      matrix:
+        # Divide the tests into n buckets
+        # and run those buckets in parallel.
+        # To increase the number of shards,
+        # add new items to the shard array
+        # and change the value of totalShards
+        # to be the length of the shard array.
+        shard: [1, 2, 3, 4, 5]
+        totalShards: [5]
     needs: [build-core]
     runs-on: ubuntu-latest
-    if: github.ref != 'refs/heads/main' && !github.event.pull_request.head.repo.fork
     steps:
       - uses: actions/checkout@v2
       - uses: ./.github/workflows/actions/test-core-screenshot
         with:
-          access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
-          secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-
-  test-core-screenshot-main:
-    needs: [build-core]
-    runs-on: ubuntu-latest
-    if: github.ref == 'refs/heads/main'
-    steps:
-      - uses: actions/checkout@v2
-      - uses: ./.github/workflows/actions/test-core-screenshot-main
-        with:
-          access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
-          secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          shard: ${{ matrix.shard }}
+          totalShards: ${{ matrix.totalShards }}
 
   build-vue:
     needs: [build-core]

.github/workflows/update-screenshots.yml

@@ -1,11 +1,52 @@
-name: 'Update Screenshot References'
+name: 'Update Reference Screenshots'
 
 on:
   workflow_dispatch:
 
 jobs:
-  stub:
+  build-core:
+    runs-on: ubuntu-latest
     steps:
-      - name: Stub
-        run: echo 'This is a stub'
-        shell: bash
+      - uses: actions/checkout@v2
+      - uses: ./.github/workflows/actions/build-core
+
+  test-core-screenshot:
+    strategy:
+      # This ensures that all screenshot shard
+      # failures are reported so the dev can
+      # review everything at once.
+      fail-fast: false
+      matrix:
+        # Divide the tests into n buckets
+        # and run those buckets in parallel.
+        # To increase the number of shards,
+        # add new items to the shard array
+        # and change the value of totalShards
+        # to be the length of the shard array.
+        shard: [1, 2, 3, 4, 5]
+        totalShards: [5]
+    needs: [build-core]
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: ./.github/workflows/actions/test-core-screenshot
+        with:
+          shard: ${{ matrix.shard }}
+          totalShards: ${{ matrix.totalShards }}
+          update: true
+
+  update-reference-screenshots:
+    runs-on: ubuntu-latest
+    needs: [test-core-screenshot]
+    steps:
+      - uses: actions/checkout@v2
+        # Normally, we could just push with the
+        # default GITHUB_TOKEN, but that will
+        # not cause the build workflow
+        # to re-run. We use Ionitron's
+        # Personal Access Token instead
+        # to allow for this build workflow
+        # to run when the screenshots are pushed.
+        with:
+          token: ${{ secrets.IONITRON_TOKEN }}
+      - uses: ./.github/workflows/actions/update-reference-screenshots

.gitignore

@@ -67,4 +67,8 @@ core/www/
 .stencil/
 angular/build/
 
+# playwright
+core/test-results/
+core/playwright-report/
+
 .npmrc

core/package.json

@@ -38,6 +38,7 @@
   "devDependencies": {
     "@axe-core/puppeteer": "^4.3.2",
     "@jest/core": "^26.6.3",
+    "@playwright/test": "^1.20.0",
     "@rollup/plugin-node-resolve": "^8.4.0",
     "@rollup/plugin-virtual": "^2.0.3",
     "@stencil/angular-output-target": "^0.4.0",

core/playwright.config.ts

@@ -0,0 +1,109 @@
+import type { PlaywrightTestConfig } from '@playwright/test';
+import { devices } from '@playwright/test';
+
+const projects = [
+  {
+    name: 'chromium',
+
+    use: {
+      ...devices['Desktop Chrome'],
+    },
+  },
+  {
+    name: 'firefox',
+    use: {
+      ...devices['Desktop Firefox'],
+    },
+  },
+  {
+    name: 'webkit',
+    use: {
+      ...devices['Desktop Safari'],
+    },
+  },
+  {
+    name: 'Mobile Chrome',
+    use: {
+      ...devices['Pixel 5']
+    }
+  },
+  {
+    name: 'Mobile Safari',
+    use: {
+      ...devices['iPhone 12']
+    }
+  }
+];
+
+const modes = ['ios', 'md'];
+
+const generateProjects = () => {
+  const projectsWithMetadata = [];
+
+  modes.forEach(mode => {
+    projects.forEach(project => {
+      projectsWithMetadata.push({
+        ...project,
+        metadata: {
+          mode,
+          rtl: false
+        }
+      });
+      projectsWithMetadata.push({
+        ...project,
+        metadata: {
+          mode,
+          rtl: true
+        }
+      });
+    });
+  });
+
+  return projectsWithMetadata;
+}
+
+/**
+ * See https://playwright.dev/docs/test-configuration.
+ */
+const config: PlaywrightTestConfig = {
+  testMatch: '*.e2e.ts',
+  expect: {
+    /**
+     * Maximum time expect() should wait for the condition to be met.
+     * For example in `await expect(locator).toHaveText();`
+     */
+    timeout: 5000
+  },
+  /* Fail the build on CI if you accidentally left test.only in the source code. */
+  forbidOnly: !!process.env.CI,
+  retries: 0,
+  /* Opt out of parallel tests on CI. */
+  workers: process.env.CI ? 1 : undefined,
+  /* Reporter to use. See https://playwright.dev/docs/test-reporters */
+  reporter: 'html',
+  /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */
+  use: {
+    /* Maximum time each action such as `click()` can take. Defaults to 0 (no limit). */
+    actionTimeout: 0,
+    /* Base URL to use in actions like `await page.goto('/')`. */
+    // baseURL: 'http://localhost:3000',
+
+    /**
+     * All failed tests should create
+     * a trace file for easier debugging.
+     *
+     * See https://playwright.dev/docs/trace-viewer
+     */
+    trace: 'retain-on-failure',
+    baseURL: 'http://localhost:3333',
+  },
+
+  /* Configure projects for major browsers */
+  projects: generateProjects(),
+  webServer: {
+    command: 'python3 -m http.server 3333',
+    port: '3333'
+  }
+};
+
+export default config;

core/scripts/testing/scripts.js

@@ -17,4 +17,59 @@
   window.Ionic = window.Ionic || {};
   window.Ionic.config = window.Ionic.config || {};
 
+  /**
+   * Waits for all child Stencil components
+   * to be ready before resolving.
+   * This logic is pulled from the Stencil
+   * core codebase for testing with Puppeteer:
+   * https://github.com/ionic-team/stencil/blob/16b8ea4dabb22024872a38bc58ba1dcf1c7cc25b/src/testing/puppeteer/puppeteer-events.ts#L158-L183
+   */
+  const allReady = () => {
+    const promises = [];
+    const waitForDidLoad = (promises, elm) => {
+      if (elm != null && elm.nodeType === 1) {
+        for (let i = 0; i < elm.children.length; i++) {
+          const childElm = elm.children[i];
+          if (childElm.tagName.includes('-') && typeof childElm.componentOnReady === 'function') {
+            promises.push(childElm.componentOnReady());
+          }
+          waitForDidLoad(promises, childElm);
+        }
+      }
+    };
+
+    waitForDidLoad(promises, window.document.documentElement);
+
+    return Promise.all(promises).catch((e) => console.error(e));
+  };
+
+  const waitFrame = () => {
+    return new Promise((resolve) => {
+      requestAnimationFrame(resolve);
+    });
+  };
+
+  const stencilReady = () => {
+    return allReady()
+      .then(() => waitFrame())
+      .then(() => allReady())
+      .then(() => {
+        window.stencilAppLoaded = true;
+      });
+  };
+
+  /**
+   * Testing solutions can wait for `window.stencilAppLoaded === true`
+   * to know when to proceed with the test.
+   */
+  if (window.document.readyState === 'complete') {
+    stencilReady();
+  } else {
+    document.addEventListener('readystatechange', function (e) {
+      if (e.target.readyState == 'complete') {
+        stencilReady();
+      }
+    });
+  }
+
 })();

core/src/components/button/test/basic/button.e2e.ts

@@ -0,0 +1,12 @@
+import { expect, describe } from '@playwright/test';
+import { test } from '@utils/test/playwright';
+
+test.describe('button: basic', () => {
+  test('should not have visual regressions', async ({ page }) => {
+    await page.goto(`/src/components/button/test/basic`);
+
+    await page.setIonViewport();
+
+    expect(await page.screenshot({ fullPage: true })).toMatchSnapshot(`button-diff-${page.getSnapshotSettings()}.png`);
+  });
+});

core/src/components/button/test/basic/e2e.ts

@@ -1,10 +0,0 @@
-import { newE2EPage } from '@stencil/core/testing';
-
-test('button: basic', async () => {
-  const page = await newE2EPage({
-    url: '/src/components/button/test/basic?ionic:_testing=true'
-  });
-
-  const compare = await page.compareScreenshot();
-  expect(compare).toMatchScreenshot();
-});

core/src/utils/test/playwright.ts

@@ -0,0 +1,97 @@
+import { test as base } from '@playwright/test';
+
+export const test = base.extend({
+  page: async ({ page }, use, testInfo) => {
+    const oldGoTo = page.goto.bind(page);
+
+
+    /**
+     * This is an extended version of Playwright's
+     * page.goto method. In addition to performing
+     * the normal page.goto work, this code also
+     * automatically waits for the Stencil components
+     * to be hydrated before proceeding with the test.
+     */
+    page.goto = (url: string) => {
+      const { mode, rtl } = testInfo.project.metadata;
+
+      const splitUrl = url.split('?');
+      const paramsString = splitUrl[1];
+
+      /**
+       * This allows developers to force a
+       * certain mode or LTR/RTL config per test.
+       */
+      const urlToParams = new URLSearchParams(paramsString);
+      const formattedMode = urlToParams.get('ionic:mode') ?? mode;
+      const formattedRtl = urlToParams.get('rtl') ?? rtl;
+
+      const formattedUrl = `${splitUrl[0]}?ionic:_testing=true&ionic:mode=${formattedMode}&rtl=${formattedRtl}`;
+
+      return Promise.all([
+        page.waitForFunction(() => window.stencilAppLoaded === true),
+        oldGoTo(formattedUrl)
+      ])
+    }
+
+    /**
+     * This provides metadata that can be used to
+     * create a unique screenshot URL.
+     * For example, we need to be able to differentiate
+     * between iOS in LTR mode and iOS in RTL mode.
+     */
+    page.getSnapshotSettings = () => {
+      const url = page.url();
+      const splitUrl = url.split('?');
+      const paramsString = splitUrl[1];
+
+      const { mode, rtl } = testInfo.project.metadata;
+
+      /**
+       * Account for custom settings when overriding
+       * the mode/rtl setting. Fall back to the
+       * project metadata if nothing was found. This
+       * will happen if you call page.getSnapshotSettings
+       * before page.goto.
+       */
+      const urlToParams = new URLSearchParams(paramsString);
+      const formattedMode = urlToParams.get('ionic:mode') ?? mode;
+      const formattedRtl = urlToParams.get('rtl') ?? rtl;
+
+      /**
+       * If encoded in the search params, the rtl value
+       * can be `'true'` instead of `true`.
+       */
+      const rtlString = formattedRtl === true || formattedRtl === 'true' ? 'rtl' : 'ltr';
+      return `${formattedMode}-${rtlString}`;
+    }
+
+    /**
+     * Taking fullpage screenshots in Playwright
+     * does not work with ion-content by default.
+     * The reason is that full page screenshots do not
+     * expand any scrollable container on the page. Instead,
+     * they render the full scrollable content of the document itself.
+     * To work around this, we increase the size of the document
+     * so the full scrollable content inside of ion-content
+     * can be captured in a screenshot.
+     */
+    page.setIonViewport = async () => {
+      const currentViewport = await page.viewportSize();
+
+      const pixelAmountRenderedOffscreen = await page.evaluate(() => {
+        const content = document.querySelector('ion-content');
+        const innerScroll = content.shadowRoot.querySelector('.inner-scroll');
+
+        return innerScroll.scrollHeight - content.clientHeight;
+      });
+
+      await page.setViewportSize({
+        width: currentViewport.width,
+        height: currentViewport.height + pixelAmountRenderedOffscreen
+      })
+    }
+
+    await use(page);
+  },
+});

core/stencil.config.ts

@@ -258,6 +258,7 @@ export const config: Config = {
     scriptDataOpts: true
   },
   testing: {
+    testRegex: '(/__tests__/.*|(\\.|/)(test|spec)|[//](e2e))\\.[jt]sx?$',
     allowableMismatchedPixels: 200,
     pixelmatchThreshold: 0.05,
     waitBeforeScreenshot: 20,

core/tsconfig.json

@@ -26,6 +26,7 @@
     "target": "es2017",
     "baseUrl": ".",
     "paths": {
+      "@utils/*": ["src/utils/*"],
       "@utils/test": ["src/utils/test/utils"]
     }
   },