From 0f1f056503db4a1e09be9a0885d1aeab0b119827 Mon Sep 17 00:00:00 2001
From: Michael Telatynski <7t3chguy@gmail.com>
Date: Tue, 21 Nov 2023 17:33:32 +0000
Subject: [PATCH] Add Playwright end to end testing (#11912)
* Install playwright
Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>
* Add foundations for writing tests under Playwright
Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>
* .gitignore juggling
Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>
* Add tsconfig and fix eslint rules
Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>
---------
Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>
---
.eslintrc.js | 16 ++-
.github/workflows/end-to-end-tests.yaml | 142 ++++++++++++++++++++++++
package.json | 3 +
playwright.config.ts | 39 +++++++
playwright/.gitignore | 2 +
playwright/e2e/launch.spec.ts | 29 +++++
playwright/tsconfig.json | 13 +++
yarn.lock | 26 +++++
8 files changed, 268 insertions(+), 2 deletions(-)
create mode 100644 .github/workflows/end-to-end-tests.yaml
create mode 100644 playwright.config.ts
create mode 100644 playwright/.gitignore
create mode 100644 playwright/e2e/launch.spec.ts
create mode 100644 playwright/tsconfig.json
diff --git a/.eslintrc.js b/.eslintrc.js
index 4434aecfdf..f0720a0252 100644
--- a/.eslintrc.js
+++ b/.eslintrc.js
@@ -169,7 +169,7 @@ module.exports = {
},
overrides: [
{
- files: ["src/**/*.{ts,tsx}", "test/**/*.{ts,tsx}", "cypress/**/*.ts"],
+ files: ["src/**/*.{ts,tsx}", "test/**/*.{ts,tsx}", "cypress/**/*.ts", "playwright/**/*.ts"],
extends: ["plugin:matrix-org/typescript", "plugin:matrix-org/react"],
rules: {
"@typescript-eslint/explicit-function-return-type": [
@@ -233,7 +233,7 @@ module.exports = {
},
},
{
- files: ["test/**/*.{ts,tsx}", "cypress/**/*.ts"],
+ files: ["test/**/*.{ts,tsx}", "cypress/**/*.ts", "playwright/**/*.ts"],
extends: ["plugin:matrix-org/jest"],
rules: {
// We don't need super strict typing in test utilities
@@ -267,6 +267,18 @@ module.exports = {
"jest/no-done-callback": "off",
},
},
+ {
+ files: ["playwright/**/*.ts"],
+ parserOptions: {
+ project: ["./playwright/tsconfig.json"],
+ },
+ rules: {
+ // Cypress "promises" work differently - disable some related rules
+ "jest/valid-expect": "off",
+ "jest/valid-expect-in-promise": "off",
+ "jest/no-done-callback": "off",
+ },
+ },
],
settings: {
react: {
diff --git a/.github/workflows/end-to-end-tests.yaml b/.github/workflows/end-to-end-tests.yaml
new file mode 100644
index 0000000000..8a0801e715
--- /dev/null
+++ b/.github/workflows/end-to-end-tests.yaml
@@ -0,0 +1,142 @@
+# Triggers after the layered build has finished, taking the artifact and running Playwright on it
+name: End to End Tests
+on:
+ workflow_run:
+ workflows: ["Element Web - Build"]
+ types:
+ - completed
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.event.workflow_run.head_branch || github.run_id }}
+ cancel-in-progress: ${{ github.event.workflow_run.event == 'pull_request' }}
+
+jobs:
+ prepare:
+ name: Prepare
+ if: github.event.workflow_run.conclusion == 'success'
+ runs-on: ubuntu-latest
+ permissions:
+ statuses: write
+ steps:
+ # We create the status here and then update it to success/failure in the `report` stage
+ # This provides an easy link to this workflow_run from the PR before the tests are done.
+ - uses: Sibz/github-status-action@faaa4d96fecf273bd762985e0e7f9f933c774918 # v1
+ with:
+ authToken: ${{ secrets.GITHUB_TOKEN }}
+ state: pending
+ context: ${{ github.workflow }} / end-to-end-tests
+ sha: ${{ github.event.workflow_run.head_sha }}
+ target_url: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}
+
+ tests:
+ name: "Run Tests"
+ needs: prepare
+ runs-on: ubuntu-latest
+ permissions:
+ actions: read
+ issues: read
+ pull-requests: read
+ environment: EndToEndTests
+ strategy:
+ fail-fast: false
+ matrix:
+ # Run 2 instances in Parallel
+ ci_node_total: [2]
+ ci_node_index: [0, 1]
+ steps:
+ - uses: browser-actions/setup-chrome@803ef6dfb4fdf22089c9563225d95e4a515820a0 # v1
+ - run: echo "BROWSER_PATH=$(which chrome)" >> $GITHUB_ENV
+
+ # There's a 'download artifact' action, but it hasn't been updated for the workflow_run action
+ # (https://github.com/actions/download-artifact/issues/60) so instead we get this mess:
+ - name: 📥 Download artifact
+ uses: dawidd6/action-download-artifact@246dbf436b23d7c49e21a7ab8204ca9ecd1fe615 # v2
+ with:
+ run_id: ${{ github.event.workflow_run.id }}
+ name: previewbuild
+ path: webapp
+
+ # The workflow_run.head_sha is the sha of the head commit but the element-web was built using a simulated
+ # merge commit - https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#pull_request
+ # so use the sha from the tarball for the checkout of the tests
+ # to make sure we get a matching set of code and tests.
+ - name: Grab sha from webapp
+ id: sha
+ run: |
+ echo "sha=$(cat webapp/sha)" >> $GITHUB_OUTPUT
+
+ - uses: actions/checkout@v4
+ with:
+ # XXX: We're checking out untrusted code in a secure context
+ # We need to be careful to not trust anything this code outputs/may do
+ #
+ # Note that (in the absence of a `react-sdk-repository` input),
+ # we check out from the default repository, which is (for this workflow) the
+ # *target* repository for the pull request.
+ #
+ ref: ${{ steps.sha.outputs.sha }}
+ persist-credentials: false
+ path: matrix-react-sdk
+ repository: ${{ inputs.react-sdk-repository || github.repository }}
+
+ - uses: actions/setup-node@v3
+ with:
+ cache: "yarn"
+
+ - name: Install dependencies
+ run: yarn install --frozen-lockfile
+
+ - name: Install Playwright browsers
+ run: yarn playwright install --with-deps
+
+ - name: Run Playwright tests
+ run: yarn playwright test --shard ${{ matrix.ci_node_index }}/${{ matrix.ci_node_total }}
+
+ - name: Upload blob report to GitHub Actions Artifacts
+ if: always()
+ uses: actions/upload-artifact@v3
+ with:
+ name: all-blob-reports
+ path: blob-report
+ retention-days: 1
+
+ report:
+ name: Report results
+ needs: tests
+ runs-on: ubuntu-latest
+ if: always()
+ permissions:
+ statuses: write
+ steps:
+ - uses: actions/checkout@v3
+
+ - uses: actions/setup-node@v3
+ with:
+ cache: "yarn"
+
+ - name: Install dependencies
+ run: yarn install --frozen-lockfile
+
+ - name: Download blob reports from GitHub Actions Artifacts
+ uses: actions/download-artifact@v3
+ with:
+ name: all-blob-reports
+ path: all-blob-reports
+
+ - name: Merge into HTML Report
+ run: yarn playwright merge-reports --reporter=html,github ./all-blob-reports
+
+ - name: Upload HTML report
+ uses: actions/upload-artifact@v3
+ with:
+ name: html-report--attempt-${{ github.run_attempt }}
+ path: playwright-report
+ retention-days: 14
+
+ - uses: Sibz/github-status-action@faaa4d96fecf273bd762985e0e7f9f933c774918 # v1
+ with:
+ authToken: ${{ secrets.GITHUB_TOKEN }}
+ state: ${{ needs.tests.result == 'success' && 'success' || 'failure' }}
+ context: ${{ github.workflow }} / end-to-end-tests
+ sha: ${{ github.event.workflow_run.head_sha }}
+ target_url: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}
diff --git a/package.json b/package.json
index d6e1fbc91e..01abd04fa9 100644
--- a/package.json
+++ b/package.json
@@ -53,6 +53,8 @@
"test": "jest",
"test:cypress": "cypress run",
"test:cypress:open": "cypress open",
+ "test:playwright": "playwright test",
+ "test:playwright:open": "yarn test:playwright --ui",
"coverage": "yarn test --coverage"
},
"resolutions": {
@@ -146,6 +148,7 @@
"@peculiar/webcrypto": "^1.4.3",
"@percy/cli": "^1.11.0",
"@percy/cypress": "^3.1.2",
+ "@playwright/test": "^1.40.0",
"@testing-library/cypress": "^9.0.0",
"@testing-library/dom": "^9.0.0",
"@testing-library/jest-dom": "^6.0.0",
diff --git a/playwright.config.ts b/playwright.config.ts
new file mode 100644
index 0000000000..5844443427
--- /dev/null
+++ b/playwright.config.ts
@@ -0,0 +1,39 @@
+/*
+Copyright 2023 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import type { PlaywrightTestConfig } from "@playwright/test";
+
+const baseURL = process.env["BASE_URL"] ?? "http://localhost:8080";
+
+const config: PlaywrightTestConfig = {
+ use: {
+ headless: false,
+ viewport: { width: 1280, height: 720 },
+ ignoreHTTPSErrors: true,
+ video: "on-first-retry",
+ baseURL,
+ },
+ webServer: {
+ command: process.env.CI ? "npx serve -p 8080 -L ../webapp" : "yarn --cwd ../element-web start",
+ url: `${baseURL}/config.json`,
+ reuseExistingServer: true,
+ },
+ testDir: "playwright/e2e",
+ outputDir: "playwright/test-results",
+ workers: 1,
+ reporter: process.env.CI ? "blob" : [["html", { outputFolder: "playwright/html-report" }]],
+};
+export default config;
diff --git a/playwright/.gitignore b/playwright/.gitignore
new file mode 100644
index 0000000000..ab8ff9582a
--- /dev/null
+++ b/playwright/.gitignore
@@ -0,0 +1,2 @@
+/test-results/
+/html-report/
diff --git a/playwright/e2e/launch.spec.ts b/playwright/e2e/launch.spec.ts
new file mode 100644
index 0000000000..b14fac143d
--- /dev/null
+++ b/playwright/e2e/launch.spec.ts
@@ -0,0 +1,29 @@
+/*
+Copyright 2023 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import { test, expect } from "@playwright/test";
+
+test.describe("App launch", () => {
+ test.beforeEach(async ({ page }) => {
+ await page.goto("/");
+ });
+
+ test("should launch and render the welcome view successfully", async ({ page }) => {
+ await page.locator("#matrixchat").waitFor();
+ await page.locator(".mx_Welcome").waitFor();
+ await expect(page).toHaveURL("http://localhost:8080/#/welcome");
+ });
+});
diff --git a/playwright/tsconfig.json b/playwright/tsconfig.json
new file mode 100644
index 0000000000..6eb5bac738
--- /dev/null
+++ b/playwright/tsconfig.json
@@ -0,0 +1,13 @@
+{
+ "compilerOptions": {
+ "target": "es2016",
+ "jsx": "react",
+ "lib": ["es2021", "dom", "dom.iterable"],
+ "types": ["axe-playwright"],
+ "resolveJsonModule": true,
+ "esModuleInterop": true,
+ "moduleResolution": "node",
+ "module": "es2022"
+ },
+ "include": ["**/*.ts"]
+}
diff --git a/yarn.lock b/yarn.lock
index 18057a5aec..a727768770 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -2181,6 +2181,13 @@
resolved "https://registry.yarnpkg.com/@pkgjs/parseargs/-/parseargs-0.11.0.tgz#a77ea742fab25775145434eb1d2328cf5013ac33"
integrity sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==
+"@playwright/test@^1.40.0":
+ version "1.40.0"
+ resolved "https://registry.yarnpkg.com/@playwright/test/-/test-1.40.0.tgz#d06c506977dd7863aa16e07f2136351ecc1be6ed"
+ integrity sha512-PdW+kn4eV99iP5gxWNSDQCbhMaDVej+RXL5xr6t04nbKLCBwYtA046t7ofoczHOm8u6c+45hpDKQVZqtqwkeQg==
+ dependencies:
+ playwright "1.40.0"
+
"@radix-ui/primitive@1.0.1":
version "1.0.1"
resolved "https://registry.yarnpkg.com/@radix-ui/primitive/-/primitive-1.0.1.tgz#e46f9958b35d10e9f6dc71c497305c22e3e55dbd"
@@ -5696,6 +5703,11 @@ fs.realpath@^1.0.0:
resolved "https://registry.yarnpkg.com/fs.realpath/-/fs.realpath-1.0.0.tgz#1504ad2523158caa40db4a2787cb01411994ea4f"
integrity sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==
+fsevents@2.3.2:
+ version "2.3.2"
+ resolved "https://registry.yarnpkg.com/fsevents/-/fsevents-2.3.2.tgz#8a526f78b8fdf4623b709e0b975c52c24c02fd1a"
+ integrity sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==
+
fsevents@^2.3.2, fsevents@~2.3.2:
version "2.3.3"
resolved "https://registry.yarnpkg.com/fsevents/-/fsevents-2.3.3.tgz#cac6407785d03675a2a5e1a5305c697b347d90d6"
@@ -8191,6 +8203,20 @@ pkg-dir@^4.2.0:
dependencies:
find-up "^4.0.0"
+playwright-core@1.40.0:
+ version "1.40.0"
+ resolved "https://registry.yarnpkg.com/playwright-core/-/playwright-core-1.40.0.tgz#82f61e5504cb3097803b6f8bbd98190dd34bdf14"
+ integrity sha512-fvKewVJpGeca8t0ipM56jkVSU6Eo0RmFvQ/MaCQNDYm+sdvKkMBBWTE1FdeMqIdumRaXXjZChWHvIzCGM/tA/Q==
+
+playwright@1.40.0:
+ version "1.40.0"
+ resolved "https://registry.yarnpkg.com/playwright/-/playwright-1.40.0.tgz#2a1824b9fe5c4fe52ed53db9ea68003543a99df0"
+ integrity sha512-gyHAgQjiDf1m34Xpwzaqb76KgfzYrhK7iih+2IzcOCoZWr/8ZqmdBw+t0RU85ZmfJMgtgAiNtBQ/KS2325INXw==
+ dependencies:
+ playwright-core "1.40.0"
+ optionalDependencies:
+ fsevents "2.3.2"
+
pluralize@^8.0.0:
version "8.0.0"
resolved "https://registry.yarnpkg.com/pluralize/-/pluralize-8.0.0.tgz#1a6fa16a38d12a1901e0320fa017051c539ce3b1"