Add Playwright end to end testing (#11912)

* Install playwright Signed-off-by: Michael Telatynski <7t3chguy@gmail.com> * Add foundations for writing tests under Playwright Signed-off-by: Michael Telatynski <7t3chguy@gmail.com> * .gitignore juggling Signed-off-by: Michael Telatynski <7t3chguy@gmail.com> * Add tsconfig and fix eslint rules Signed-off-by: Michael Telatynski <7t3chguy@gmail.com> --------- Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>
2023-11-21 17:33:32 +00:00 · 2023-11-21 17:33:32 +00:00 · 0f1f056503
parent a26c2d3c78
commit 0f1f056503
8 changed files with 268 additions and 2 deletions
--- a/.eslintrc.js
+++ b/.eslintrc.js
@ -169,7 +169,7 @@ module.exports = {
    },
    overrides: [
        {
-            files: ["src/**/*.{ts,tsx}", "test/**/*.{ts,tsx}", "cypress/**/*.ts"],
+            files: ["src/**/*.{ts,tsx}", "test/**/*.{ts,tsx}", "cypress/**/*.ts", "playwright/**/*.ts"],
            extends: ["plugin:matrix-org/typescript", "plugin:matrix-org/react"],
            rules: {
                "@typescript-eslint/explicit-function-return-type": [
@ -233,7 +233,7 @@ module.exports = {
            },
        },
        {
-            files: ["test/**/*.{ts,tsx}", "cypress/**/*.ts"],
+            files: ["test/**/*.{ts,tsx}", "cypress/**/*.ts", "playwright/**/*.ts"],
            extends: ["plugin:matrix-org/jest"],
            rules: {
                // We don't need super strict typing in test utilities
@ -267,6 +267,18 @@ module.exports = {
                "jest/no-done-callback": "off",
            },
        },
+        {
+            files: ["playwright/**/*.ts"],
+            parserOptions: {
+                project: ["./playwright/tsconfig.json"],
+            },
+            rules: {
+                // Cypress "promises" work differently - disable some related rules
+                "jest/valid-expect": "off",
+                "jest/valid-expect-in-promise": "off",
+                "jest/no-done-callback": "off",
+            },
+        },
    ],
    settings: {
        react: {
--- a/.github/workflows/end-to-end-tests.yaml
+++ b/.github/workflows/end-to-end-tests.yaml
@ -0,0 +1,142 @@
+# Triggers after the layered build has finished, taking the artifact and running Playwright on it
+name: End to End Tests
+on:
+    workflow_run:
+        workflows: ["Element Web - Build"]
+        types:
+            - completed
+
+concurrency:
+    group: ${{ github.workflow }}-${{ github.event.workflow_run.head_branch || github.run_id }}
+    cancel-in-progress: ${{ github.event.workflow_run.event == 'pull_request' }}
+
+jobs:
+    prepare:
+        name: Prepare
+        if: github.event.workflow_run.conclusion == 'success'
+        runs-on: ubuntu-latest
+        permissions:
+            statuses: write
+        steps:
+            # We create the status here and then update it to success/failure in the `report` stage
+            # This provides an easy link to this workflow_run from the PR before the tests are done.
+            - uses: Sibz/github-status-action@faaa4d96fecf273bd762985e0e7f9f933c774918 # v1
+              with:
+                  authToken: ${{ secrets.GITHUB_TOKEN }}
+                  state: pending
+                  context: ${{ github.workflow }} / end-to-end-tests
+                  sha: ${{ github.event.workflow_run.head_sha }}
+                  target_url: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}
+
+    tests:
+        name: "Run Tests"
+        needs: prepare
+        runs-on: ubuntu-latest
+        permissions:
+            actions: read
+            issues: read
+            pull-requests: read
+        environment: EndToEndTests
+        strategy:
+            fail-fast: false
+            matrix:
+                # Run 2 instances in Parallel
+                ci_node_total: [2]
+                ci_node_index: [0, 1]
+        steps:
+            - uses: browser-actions/setup-chrome@803ef6dfb4fdf22089c9563225d95e4a515820a0 # v1
+            - run: echo "BROWSER_PATH=$(which chrome)" >> $GITHUB_ENV
+
+            # There's a 'download artifact' action, but it hasn't been updated for the workflow_run action
+            # (https://github.com/actions/download-artifact/issues/60) so instead we get this mess:
+            - name: 📥 Download artifact
+              uses: dawidd6/action-download-artifact@246dbf436b23d7c49e21a7ab8204ca9ecd1fe615 # v2
+              with:
+                  run_id: ${{ github.event.workflow_run.id }}
+                  name: previewbuild
+                  path: webapp
+
+            # The workflow_run.head_sha is the sha of the head commit but the element-web was built using a simulated
+            # merge commit - https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#pull_request
+            # so use the sha from the tarball for the checkout of the tests
+            # to make sure we get a matching set of code and tests.
+            - name: Grab sha from webapp
+              id: sha
+              run: |
+                  echo "sha=$(cat webapp/sha)" >> $GITHUB_OUTPUT
+
+            - uses: actions/checkout@v4
+              with:
+                  # XXX: We're checking out untrusted code in a secure context
+                  # We need to be careful to not trust anything this code outputs/may do
+                  #
+                  # Note that (in the absence of a `react-sdk-repository` input),
+                  # we check out from the default repository, which is (for this workflow) the
+                  # *target* repository for the pull request.
+                  #
+                  ref: ${{ steps.sha.outputs.sha }}
+                  persist-credentials: false
+                  path: matrix-react-sdk
+                  repository: ${{ inputs.react-sdk-repository || github.repository }}
+
+            - uses: actions/setup-node@v3
+              with:
+                  cache: "yarn"
+
+            - name: Install dependencies
+              run: yarn install --frozen-lockfile
+
+            - name: Install Playwright browsers
+              run: yarn playwright install --with-deps
+
+            - name: Run Playwright tests
+              run: yarn playwright test --shard ${{ matrix.ci_node_index }}/${{ matrix.ci_node_total }}
+
+            - name: Upload blob report to GitHub Actions Artifacts
+              if: always()
+              uses: actions/upload-artifact@v3
+              with:
+                  name: all-blob-reports
+                  path: blob-report
+                  retention-days: 1
+
+    report:
+        name: Report results
+        needs: tests
+        runs-on: ubuntu-latest
+        if: always()
+        permissions:
+            statuses: write
+        steps:
+            - uses: actions/checkout@v3
+
+            - uses: actions/setup-node@v3
+              with:
+                  cache: "yarn"
+
+            - name: Install dependencies
+              run: yarn install --frozen-lockfile
+
+            - name: Download blob reports from GitHub Actions Artifacts
+              uses: actions/download-artifact@v3
+              with:
+                  name: all-blob-reports
+                  path: all-blob-reports
+
+            - name: Merge into HTML Report
+              run: yarn playwright merge-reports --reporter=html,github ./all-blob-reports
+
+            - name: Upload HTML report
+              uses: actions/upload-artifact@v3
+              with:
+                  name: html-report--attempt-${{ github.run_attempt }}
+                  path: playwright-report
+                  retention-days: 14
+
+            - uses: Sibz/github-status-action@faaa4d96fecf273bd762985e0e7f9f933c774918 # v1
+              with:
+                  authToken: ${{ secrets.GITHUB_TOKEN }}
+                  state: ${{ needs.tests.result == 'success' && 'success' || 'failure' }}
+                  context: ${{ github.workflow }} / end-to-end-tests
+                  sha: ${{ github.event.workflow_run.head_sha }}
+                  target_url: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}
--- a/package.json
+++ b/package.json
@ -53,6 +53,8 @@
        "test": "jest",
        "test:cypress": "cypress run",
        "test:cypress:open": "cypress open",
+        "test:playwright": "playwright test",
+        "test:playwright:open": "yarn test:playwright --ui",
        "coverage": "yarn test --coverage"
    },
    "resolutions": {
@ -146,6 +148,7 @@
        "@peculiar/webcrypto": "^1.4.3",
        "@percy/cli": "^1.11.0",
        "@percy/cypress": "^3.1.2",
+        "@playwright/test": "^1.40.0",
        "@testing-library/cypress": "^9.0.0",
        "@testing-library/dom": "^9.0.0",
        "@testing-library/jest-dom": "^6.0.0",
--- a/playwright.config.ts
+++ b/playwright.config.ts
@ -0,0 +1,39 @@
+/*
+Copyright 2023 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import type { PlaywrightTestConfig } from "@playwright/test";
+
+const baseURL = process.env["BASE_URL"] ?? "http://localhost:8080";
+
+const config: PlaywrightTestConfig = {
+    use: {
+        headless: false,
+        viewport: { width: 1280, height: 720 },
+        ignoreHTTPSErrors: true,
+        video: "on-first-retry",
+        baseURL,
+    },
+    webServer: {
+        command: process.env.CI ? "npx serve -p 8080 -L ../webapp" : "yarn --cwd ../element-web start",
+        url: `${baseURL}/config.json`,
+        reuseExistingServer: true,
+    },
+    testDir: "playwright/e2e",
+    outputDir: "playwright/test-results",
+    workers: 1,
+    reporter: process.env.CI ? "blob" : [["html", { outputFolder: "playwright/html-report" }]],
+};
+export default config;
--- a/playwright/.gitignore
+++ b/playwright/.gitignore
@ -0,0 +1,2 @@
+/test-results/
+/html-report/
--- a/playwright/e2e/launch.spec.ts
+++ b/playwright/e2e/launch.spec.ts
@ -0,0 +1,29 @@
+/*
+Copyright 2023 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import { test, expect } from "@playwright/test";
+
+test.describe("App launch", () => {
+    test.beforeEach(async ({ page }) => {
+        await page.goto("/");
+    });
+
+    test("should launch and render the welcome view successfully", async ({ page }) => {
+        await page.locator("#matrixchat").waitFor();
+        await page.locator(".mx_Welcome").waitFor();
+        await expect(page).toHaveURL("http://localhost:8080/#/welcome");
+    });
+});
--- a/playwright/tsconfig.json
+++ b/playwright/tsconfig.json
@ -0,0 +1,13 @@
+{
+    "compilerOptions": {
+        "target": "es2016",
+        "jsx": "react",
+        "lib": ["es2021", "dom", "dom.iterable"],
+        "types": ["axe-playwright"],
+        "resolveJsonModule": true,
+        "esModuleInterop": true,
+        "moduleResolution": "node",
+        "module": "es2022"
+    },
+    "include": ["**/*.ts"]
+}
--- a/yarn.lock
+++ b/yarn.lock
@ -2181,6 +2181,13 @@
  resolved "https://registry.yarnpkg.com/@pkgjs/parseargs/-/parseargs-0.11.0.tgz#a77ea742fab25775145434eb1d2328cf5013ac33"
  integrity sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==

+"@playwright/test@^1.40.0":
+  version "1.40.0"
+  resolved "https://registry.yarnpkg.com/@playwright/test/-/test-1.40.0.tgz#d06c506977dd7863aa16e07f2136351ecc1be6ed"
+  integrity sha512-PdW+kn4eV99iP5gxWNSDQCbhMaDVej+RXL5xr6t04nbKLCBwYtA046t7ofoczHOm8u6c+45hpDKQVZqtqwkeQg==
+  dependencies:
+    playwright "1.40.0"
+
 "@radix-ui/primitive@1.0.1":
  version "1.0.1"
  resolved "https://registry.yarnpkg.com/@radix-ui/primitive/-/primitive-1.0.1.tgz#e46f9958b35d10e9f6dc71c497305c22e3e55dbd"
@ -5696,6 +5703,11 @@ fs.realpath@^1.0.0:
  resolved "https://registry.yarnpkg.com/fs.realpath/-/fs.realpath-1.0.0.tgz#1504ad2523158caa40db4a2787cb01411994ea4f"
  integrity sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==

+fsevents@2.3.2:
+  version "2.3.2"
+  resolved "https://registry.yarnpkg.com/fsevents/-/fsevents-2.3.2.tgz#8a526f78b8fdf4623b709e0b975c52c24c02fd1a"
+  integrity sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==
+
 fsevents@^2.3.2, fsevents@~2.3.2:
  version "2.3.3"
  resolved "https://registry.yarnpkg.com/fsevents/-/fsevents-2.3.3.tgz#cac6407785d03675a2a5e1a5305c697b347d90d6"
@ -8191,6 +8203,20 @@ pkg-dir@^4.2.0:
  dependencies:
    find-up "^4.0.0"

+playwright-core@1.40.0:
+  version "1.40.0"
+  resolved "https://registry.yarnpkg.com/playwright-core/-/playwright-core-1.40.0.tgz#82f61e5504cb3097803b6f8bbd98190dd34bdf14"
+  integrity sha512-fvKewVJpGeca8t0ipM56jkVSU6Eo0RmFvQ/MaCQNDYm+sdvKkMBBWTE1FdeMqIdumRaXXjZChWHvIzCGM/tA/Q==
+
+playwright@1.40.0:
+  version "1.40.0"
+  resolved "https://registry.yarnpkg.com/playwright/-/playwright-1.40.0.tgz#2a1824b9fe5c4fe52ed53db9ea68003543a99df0"
+  integrity sha512-gyHAgQjiDf1m34Xpwzaqb76KgfzYrhK7iih+2IzcOCoZWr/8ZqmdBw+t0RU85ZmfJMgtgAiNtBQ/KS2325INXw==
+  dependencies:
+    playwright-core "1.40.0"
+  optionalDependencies:
+    fsevents "2.3.2"
+
 pluralize@^8.0.0:
  version "8.0.0"
  resolved "https://registry.yarnpkg.com/pluralize/-/pluralize-8.0.0.tgz#1a6fa16a38d12a1901e0320fa017051c539ce3b1"