diff --git a/.github/workflows/end-to-end-tests.yaml b/.github/workflows/end-to-end-tests.yaml index 5a75040866..6afabdb1fe 100644 --- a/.github/workflows/end-to-end-tests.yaml +++ b/.github/workflows/end-to-end-tests.yaml @@ -3,6 +3,9 @@ # as an artifact and run end-to-end tests. name: End to End Tests on: + # CRON to run all Projects at 6am UTC + schedule: + - cron: "0 6 * * *" pull_request: {} merge_group: types: [checks_requested] @@ -32,6 +35,8 @@ concurrency: env: # fetchdep.sh needs to know our PR number PR_NUMBER: ${{ github.event.pull_request.number }} + # Use 6 runners in the default case, but 4 when running on a schedule where we run all 5 projects (20 runners total) + NUM_RUNNERS: ${{ github.event_name == 'schedule' && 4 || 6 }} permissions: {} # No permissions required @@ -40,6 +45,9 @@ jobs: name: "Build Element-Web" runs-on: ubuntu-24.04 if: inputs.skip != true + outputs: + num-runners: ${{ env.NUM_RUNNERS }} + runners-matrix: ${{ steps.runner-vars.outputs.matrix }} steps: - name: Checkout code uses: actions/checkout@v4 @@ -79,8 +87,17 @@ jobs: path: webapp retention-days: 1 + - name: Calculate runner variables + id: runner-vars + uses: actions/github-script@v7 + with: + script: | + const numRunners = parseInt(process.env.NUM_RUNNERS, 10); + const matrix = Array.from({ length: numRunners }, (_, i) => i + 1); + core.setOutput("matrix", JSON.stringify(matrix)); + playwright: - name: "Run Tests ${{ matrix.runner }}/${{ strategy.job-total }}" + name: "Run Tests [${{ matrix.project }}] ${{ matrix.runner }}/${{ needs.build.outputs.num-runners }}" needs: build if: inputs.skip != true runs-on: ubuntu-24.04 @@ -92,7 +109,19 @@ jobs: fail-fast: false matrix: # Run multiple instances in parallel to speed up the tests - runner: [1, 2, 3, 4, 5, 6] + runner: ${{ fromJSON(needs.build.outputs.runners-matrix) }} + project: + - Chrome + - Firefox + - WebKit + isCron: + - ${{ github.event_name == 'schedule' }} + # Skip the Firefox & Safari runs unless this was a cron trigger + exclude: + - isCron: false + project: Firefox + - isCron: false + project: WebKit steps: - uses: actions/checkout@v4 with: @@ -124,24 +153,30 @@ jobs: with: path: | ~/.cache/ms-playwright - key: ${{ runner.os }}-playwright-${{ steps.playwright.outputs.version }}-chromium + key: ${{ runner.os }}-playwright-${{ steps.playwright.outputs.version }} - - name: Install Playwright browser + - name: Install Playwright browsers if: steps.playwright-cache.outputs.cache-hit != 'true' - run: yarn playwright install --with-deps --no-shell chromium + run: yarn playwright install --with-deps --no-shell + + - name: Install system dependencies for WebKit + # Some WebKit dependencies seem to lay outside the cache and will need to be installed separately + if: matrix.project == 'WebKit' && steps.playwright-cache.outputs.cache-hit == 'true' + run: yarn playwright install-deps webkit # We skip tests tagged with @mergequeue when running on PRs, but run them in MQ and everywhere else - name: Run Playwright tests run: | yarn playwright test \ - --shard "${{ matrix.runner }}/${{ strategy.job-total }}" \ + --shard "${{ matrix.runner }}/${{ needs.build.outputs.num-runners }}" \ + --project="${{ matrix.project }}" \ ${{ github.event_name == 'pull_request' && '--grep-invert @mergequeue' || '' }} - name: Upload blob report to GitHub Actions Artifacts if: always() uses: actions/upload-artifact@v4 with: - name: all-blob-reports-${{ matrix.runner }} + name: all-blob-reports-${{ matrix.project }}-${{ matrix.runner }} path: blob-report retention-days: 1 diff --git a/docs/playwright.md b/docs/playwright.md index 4af3194220..73ee77228b 100644 --- a/docs/playwright.md +++ b/docs/playwright.md @@ -53,15 +53,11 @@ yarn run test:playwright:open --headed --debug See more command line options at . -### Running with Rust cryptography +## Projects -`matrix-js-sdk` is currently in the -[process](https://github.com/vector-im/element-web/issues/21972) of being -updated to replace its end-to-end encryption implementation to use the [Matrix -Rust SDK](https://github.com/matrix-org/matrix-rust-sdk). This is not currently -enabled by default, but it is possible to have Playwright configure Element to use -the Rust crypto implementation by passing `--project="Rust Crypto"` or using -the top left options in open mode. +By default, Playwright will run all "Projects", this means tests will run against Chrome, Firefox and "Safari" (Webkit). +We only run tests against Chrome in pull request CI, but all projects in the merge queue. +Some tests are excluded from running on certain browsers due to incompatibilities in the test harness. ## How the Tests Work @@ -224,3 +220,14 @@ We use test tags to categorise tests for running subsets more efficiently. - `@mergequeue`: Tests that are slow or flaky and cover areas of the app we update seldom, should not be run on every PR commit but will be run in the Merge Queue. - `@screenshot`: Tests that use `toMatchScreenshot` to speed up a run of `test:playwright:screenshots`. A test with this tag must not also have the `@mergequeue` tag as this would cause false positives in the stale screenshot detection. +- `@no-$project`: Tests which are unsupported in $Project. These tests will be skipped when running in $Project. + +Anything testing Matrix media will need to have `@no-firefox` and `@no-webkit` as those rely on the service worker which +has to be disabled in Playwright on Firefox & Webkit to retain routing functionality. +Anything testing VoIP/microphone will need to have `@no-webkit` as fake microphone functionality is not available +there at this time. + +## Colima + +If you are running under Colima, you may need to set the environment variable `TMPDIR` to `/tmp/colima` or a path +within `$HOME` to allow bind mounting temporary directories into the Docker containers. diff --git a/package.json b/package.json index 999d1237f9..f41993a687 100644 --- a/package.json +++ b/package.json @@ -64,7 +64,7 @@ "test:playwright:open": "yarn test:playwright --ui", "test:playwright:screenshots": "yarn test:playwright:screenshots:build && yarn test:playwright:screenshots:run", "test:playwright:screenshots:build": "docker build playwright -t element-web-playwright", - "test:playwright:screenshots:run": "docker run --rm --network host -e BASE_URL -e CI -v $(pwd):/work/ -v $(node -e 'console.log(require(`path`).dirname(require.resolve(`matrix-js-sdk/package.json`)))'):/work/node_modules/matrix-js-sdk -v /var/run/docker.sock:/var/run/docker.sock -v /tmp/:/tmp/ -it element-web-playwright --grep @screenshot", + "test:playwright:screenshots:run": "docker run --rm --network host -e BASE_URL -e CI -v $(pwd):/work/ -v $(node -e 'console.log(require(`path`).dirname(require.resolve(`matrix-js-sdk/package.json`)))'):/work/node_modules/matrix-js-sdk -v /var/run/docker.sock:/var/run/docker.sock -v /tmp/:/tmp/ -it element-web-playwright --grep @screenshot --project=Chrome", "coverage": "yarn test --coverage", "analyse:unused-exports": "ts-node ./scripts/analyse_unused_exports.ts", "analyse:webpack-bundles": "webpack-bundle-analyzer webpack-stats.json webapp", diff --git a/playwright.config.ts b/playwright.config.ts index 06c1b05322..0b2bd1bd02 100644 --- a/playwright.config.ts +++ b/playwright.config.ts @@ -11,16 +11,49 @@ import { defineConfig, devices } from "@playwright/test"; const baseURL = process.env["BASE_URL"] ?? "http://localhost:8080"; export default defineConfig({ - projects: [{ name: "Chrome", use: { ...devices["Desktop Chrome"], channel: "chromium" } }], + projects: [ + { + name: "Chrome", + use: { + ...devices["Desktop Chrome"], + channel: "chromium", + permissions: ["clipboard-write", "clipboard-read", "microphone"], + launchOptions: { + args: ["--use-fake-ui-for-media-stream", "--use-fake-device-for-media-stream", "--mute-audio"], + }, + }, + }, + { + name: "Firefox", + use: { + ...devices["Desktop Firefox"], + launchOptions: { + firefoxUserPrefs: { + "permissions.default.microphone": 1, + }, + }, + // This is needed to work around an issue between Playwright routes, Firefox, and Service workers + // https://github.com/microsoft/playwright/issues/33561#issuecomment-2471642120 + serviceWorkers: "block", + }, + ignoreSnapshots: true, + }, + { + name: "WebKit", + use: { + ...devices["Desktop Safari"], + // Seemingly WebKit has the same issue as Firefox in Playwright routes not working + // https://playwright.dev/docs/network#missing-network-events-and-service-workers + serviceWorkers: "block", + }, + ignoreSnapshots: true, + }, + ], use: { viewport: { width: 1280, height: 720 }, ignoreHTTPSErrors: true, video: "retain-on-failure", baseURL, - permissions: ["clipboard-write", "clipboard-read", "microphone"], - launchOptions: { - args: ["--use-fake-ui-for-media-stream", "--use-fake-device-for-media-stream", "--mute-audio"], - }, trace: "on-first-retry", }, webServer: { diff --git a/playwright/e2e/audio-player/audio-player.spec.ts b/playwright/e2e/audio-player/audio-player.spec.ts index 2bb9ab0be4..2749d7eb1d 100644 --- a/playwright/e2e/audio-player/audio-player.spec.ts +++ b/playwright/e2e/audio-player/audio-player.spec.ts @@ -13,7 +13,7 @@ import { SettingLevel } from "../../../src/settings/SettingLevel"; import { Layout } from "../../../src/settings/enums/Layout"; import { ElementAppPage } from "../../pages/ElementAppPage"; -test.describe("Audio player", () => { +test.describe("Audio player", { tag: ["@no-firefox", "@no-webkit"] }, () => { test.use({ displayName: "Hanako", }); diff --git a/playwright/e2e/crypto/backups.spec.ts b/playwright/e2e/crypto/backups.spec.ts index d174cc89e5..40c7dc0ac6 100644 --- a/playwright/e2e/crypto/backups.spec.ts +++ b/playwright/e2e/crypto/backups.spec.ts @@ -51,90 +51,94 @@ test.describe("Backups", () => { displayName: "Hanako", }); - test("Create, delete and recreate a keys backup", async ({ page, user, app }, workerInfo) => { - // Create a backup - const securityTab = await app.settings.openUserSettings("Security & Privacy"); + test( + "Create, delete and recreate a keys backup", + { tag: "@no-webkit" }, + async ({ page, user, app }, workerInfo) => { + // Create a backup + const securityTab = await app.settings.openUserSettings("Security & Privacy"); - await expect(securityTab.getByRole("heading", { name: "Secure Backup" })).toBeVisible(); - await securityTab.getByRole("button", { name: "Set up", exact: true }).click(); + await expect(securityTab.getByRole("heading", { name: "Secure Backup" })).toBeVisible(); + await securityTab.getByRole("button", { name: "Set up", exact: true }).click(); - const currentDialogLocator = page.locator(".mx_Dialog"); + const currentDialogLocator = page.locator(".mx_Dialog"); - // It's the first time and secure storage is not set up, so it will create one - await expect(currentDialogLocator.getByRole("heading", { name: "Set up Secure Backup" })).toBeVisible(); - await currentDialogLocator.getByRole("button", { name: "Continue", exact: true }).click(); - await expect(currentDialogLocator.getByRole("heading", { name: "Save your Security Key" })).toBeVisible(); - await currentDialogLocator.getByRole("button", { name: "Copy", exact: true }).click(); - // copy the recovery key to use it later - const securityKey = await app.getClipboard(); - await currentDialogLocator.getByRole("button", { name: "Continue", exact: true }).click(); + // It's the first time and secure storage is not set up, so it will create one + await expect(currentDialogLocator.getByRole("heading", { name: "Set up Secure Backup" })).toBeVisible(); + await currentDialogLocator.getByRole("button", { name: "Continue", exact: true }).click(); + await expect(currentDialogLocator.getByRole("heading", { name: "Save your Security Key" })).toBeVisible(); + await currentDialogLocator.getByRole("button", { name: "Copy", exact: true }).click(); + // copy the recovery key to use it later + const securityKey = await app.getClipboard(); + await currentDialogLocator.getByRole("button", { name: "Continue", exact: true }).click(); - await expect(currentDialogLocator.getByRole("heading", { name: "Secure Backup successful" })).toBeVisible(); - await currentDialogLocator.getByRole("button", { name: "Done", exact: true }).click(); + await expect(currentDialogLocator.getByRole("heading", { name: "Secure Backup successful" })).toBeVisible(); + await currentDialogLocator.getByRole("button", { name: "Done", exact: true }).click(); - // Open the settings again - await app.settings.openUserSettings("Security & Privacy"); - await expect(securityTab.getByRole("heading", { name: "Secure Backup" })).toBeVisible(); + // Open the settings again + await app.settings.openUserSettings("Security & Privacy"); + await expect(securityTab.getByRole("heading", { name: "Secure Backup" })).toBeVisible(); - // expand the advanced section to see the active version in the reports - await page - .locator(".mx_Dialog .mx_SettingsSubsection_content details .mx_SecureBackupPanel_advanced") - .locator("..") - .click(); + // expand the advanced section to see the active version in the reports + await page + .locator(".mx_Dialog .mx_SettingsSubsection_content details .mx_SecureBackupPanel_advanced") + .locator("..") + .click(); - await expectBackupVersionToBe(page, "1"); + await expectBackupVersionToBe(page, "1"); - await securityTab.getByRole("button", { name: "Delete Backup", exact: true }).click(); - await expect(currentDialogLocator.getByRole("heading", { name: "Delete Backup" })).toBeVisible(); - // Delete it - await currentDialogLocator.getByTestId("dialog-primary-button").click(); // Click "Delete Backup" + await securityTab.getByRole("button", { name: "Delete Backup", exact: true }).click(); + await expect(currentDialogLocator.getByRole("heading", { name: "Delete Backup" })).toBeVisible(); + // Delete it + await currentDialogLocator.getByTestId("dialog-primary-button").click(); // Click "Delete Backup" - // Create another - await securityTab.getByRole("button", { name: "Set up", exact: true }).click(); - await expect(currentDialogLocator.getByRole("heading", { name: "Security Key" })).toBeVisible(); - await currentDialogLocator.getByLabel("Security Key").fill(securityKey); - await currentDialogLocator.getByRole("button", { name: "Continue", exact: true }).click(); + // Create another + await securityTab.getByRole("button", { name: "Set up", exact: true }).click(); + await expect(currentDialogLocator.getByRole("heading", { name: "Security Key" })).toBeVisible(); + await currentDialogLocator.getByLabel("Security Key").fill(securityKey); + await currentDialogLocator.getByRole("button", { name: "Continue", exact: true }).click(); - // Should be successful - await expect(currentDialogLocator.getByRole("heading", { name: "Success!" })).toBeVisible(); - await currentDialogLocator.getByRole("button", { name: "OK", exact: true }).click(); + // Should be successful + await expect(currentDialogLocator.getByRole("heading", { name: "Success!" })).toBeVisible(); + await currentDialogLocator.getByRole("button", { name: "OK", exact: true }).click(); - // Open the settings again - await app.settings.openUserSettings("Security & Privacy"); - await expect(securityTab.getByRole("heading", { name: "Secure Backup" })).toBeVisible(); + // Open the settings again + await app.settings.openUserSettings("Security & Privacy"); + await expect(securityTab.getByRole("heading", { name: "Secure Backup" })).toBeVisible(); - // expand the advanced section to see the active version in the reports - await page - .locator(".mx_Dialog .mx_SettingsSubsection_content details .mx_SecureBackupPanel_advanced") - .locator("..") - .click(); + // expand the advanced section to see the active version in the reports + await page + .locator(".mx_Dialog .mx_SettingsSubsection_content details .mx_SecureBackupPanel_advanced") + .locator("..") + .click(); - await expectBackupVersionToBe(page, "2"); + await expectBackupVersionToBe(page, "2"); - // == - // Ensure that if you don't have the secret storage passphrase the backup won't be created - // == + // == + // Ensure that if you don't have the secret storage passphrase the backup won't be created + // == - // First delete version 2 - await securityTab.getByRole("button", { name: "Delete Backup", exact: true }).click(); - await expect(currentDialogLocator.getByRole("heading", { name: "Delete Backup" })).toBeVisible(); - // Click "Delete Backup" - await currentDialogLocator.getByTestId("dialog-primary-button").click(); + // First delete version 2 + await securityTab.getByRole("button", { name: "Delete Backup", exact: true }).click(); + await expect(currentDialogLocator.getByRole("heading", { name: "Delete Backup" })).toBeVisible(); + // Click "Delete Backup" + await currentDialogLocator.getByTestId("dialog-primary-button").click(); - // Try to create another - await securityTab.getByRole("button", { name: "Set up", exact: true }).click(); - await expect(currentDialogLocator.getByRole("heading", { name: "Security Key" })).toBeVisible(); - // But cancel the security key dialog, to simulate not having the secret storage passphrase - await currentDialogLocator.getByTestId("dialog-cancel-button").click(); + // Try to create another + await securityTab.getByRole("button", { name: "Set up", exact: true }).click(); + await expect(currentDialogLocator.getByRole("heading", { name: "Security Key" })).toBeVisible(); + // But cancel the security key dialog, to simulate not having the secret storage passphrase + await currentDialogLocator.getByTestId("dialog-cancel-button").click(); - await expect(currentDialogLocator.getByRole("heading", { name: "Starting backup…" })).toBeVisible(); - // check that it failed - await expect(currentDialogLocator.getByText("Unable to create key backup")).toBeVisible(); - // cancel - await currentDialogLocator.getByTestId("dialog-cancel-button").click(); + await expect(currentDialogLocator.getByRole("heading", { name: "Starting backup…" })).toBeVisible(); + // check that it failed + await expect(currentDialogLocator.getByText("Unable to create key backup")).toBeVisible(); + // cancel + await currentDialogLocator.getByTestId("dialog-cancel-button").click(); - // go back to the settings to check that no backup was created (the setup button should still be there) - await app.settings.openUserSettings("Security & Privacy"); - await expect(securityTab.getByRole("button", { name: "Set up", exact: true })).toBeVisible(); - }); + // go back to the settings to check that no backup was created (the setup button should still be there) + await app.settings.openUserSettings("Security & Privacy"); + await expect(securityTab.getByRole("button", { name: "Set up", exact: true })).toBeVisible(); + }, + ); }); diff --git a/playwright/e2e/crypto/device-verification.spec.ts b/playwright/e2e/crypto/device-verification.spec.ts index 83a81c260c..032b649b8d 100644 --- a/playwright/e2e/crypto/device-verification.spec.ts +++ b/playwright/e2e/crypto/device-verification.spec.ts @@ -21,7 +21,7 @@ import { } from "./utils"; import { Bot } from "../../pages/bot"; -test.describe("Device verification", () => { +test.describe("Device verification", { tag: "@no-webkit" }, () => { let aliceBotClient: Bot; /** The backup version that was set up by the bot client. */ diff --git a/playwright/e2e/crypto/migration.spec.ts b/playwright/e2e/crypto/migration.spec.ts index a9530a288b..52df22688b 100644 --- a/playwright/e2e/crypto/migration.spec.ts +++ b/playwright/e2e/crypto/migration.spec.ts @@ -25,7 +25,7 @@ const test = base.extend({ }, }); -test.describe("migration", function () { +test.describe("migration", { tag: "@no-webkit" }, function () { test.use({ displayName: "Alice" }); test("Should support migration from legacy crypto", async ({ context, user, page }, workerInfo) => { diff --git a/playwright/e2e/location/location.spec.ts b/playwright/e2e/location/location.spec.ts index e0c23d6c22..2277c16d4f 100644 --- a/playwright/e2e/location/location.spec.ts +++ b/playwright/e2e/location/location.spec.ts @@ -10,7 +10,8 @@ import { Locator, Page } from "@playwright/test"; import { test, expect } from "../../element-web-test"; -test.describe("Location sharing", () => { +// Firefox headless lacks WebGL support https://bugzilla.mozilla.org/show_bug.cgi?id=1375585 +test.describe("Location sharing", { tag: "@no-firefox" }, () => { const selectLocationShareTypeOption = (page: Page, shareType: string): Locator => { return page.getByTestId(`share-location-option-${shareType}`); }; diff --git a/playwright/e2e/oidc/oidc-aware.spec.ts b/playwright/e2e/oidc/oidc-aware.spec.ts index a2f1e62714..7b155f27a4 100644 --- a/playwright/e2e/oidc/oidc-aware.spec.ts +++ b/playwright/e2e/oidc/oidc-aware.spec.ts @@ -9,7 +9,7 @@ Please see LICENSE files in the repository root for full details. import { test, expect, registerAccountMas } from "."; import { isDendrite } from "../../plugins/homeserver/dendrite"; -test.describe("OIDC Aware", () => { +test.describe("OIDC Aware", { tag: ["@no-firefox", "@no-webkit"] }, () => { test.skip(isDendrite, "does not yet support MAS"); test.slow(); // trace recording takes a while here diff --git a/playwright/e2e/oidc/oidc-native.spec.ts b/playwright/e2e/oidc/oidc-native.spec.ts index 3309826b63..2ae5cf83e6 100644 --- a/playwright/e2e/oidc/oidc-native.spec.ts +++ b/playwright/e2e/oidc/oidc-native.spec.ts @@ -10,7 +10,7 @@ import { test, expect, registerAccountMas } from "."; import { isDendrite } from "../../plugins/homeserver/dendrite"; import { ElementAppPage } from "../../pages/ElementAppPage.ts"; -test.describe("OIDC Native", () => { +test.describe("OIDC Native", { tag: ["@no-firefox", "@no-webkit"] }, () => { test.skip(isDendrite, "does not yet support MAS"); test.slow(); // trace recording takes a while here diff --git a/playwright/e2e/register/register.spec.ts b/playwright/e2e/register/register.spec.ts index 19608ee174..c127436266 100644 --- a/playwright/e2e/register/register.spec.ts +++ b/playwright/e2e/register/register.spec.ts @@ -9,7 +9,9 @@ Please see LICENSE files in the repository root for full details. import { test, expect } from "../../element-web-test"; test.describe("Registration", () => { - test.use({ startHomeserverOpts: "consent" }); + test.use({ + startHomeserverOpts: "consent", + }); test.beforeEach(async ({ page }) => { await page.goto("/#/register"); diff --git a/playwright/e2e/right-panel/file-panel.spec.ts b/playwright/e2e/right-panel/file-panel.spec.ts index c535bcdfbb..c5a106d841 100644 --- a/playwright/e2e/right-panel/file-panel.spec.ts +++ b/playwright/e2e/right-panel/file-panel.spec.ts @@ -39,7 +39,7 @@ test.describe("FilePanel", () => { await expect(page.locator(".mx_FilePanel")).toBeVisible(); }); - test.describe("render", () => { + test.describe("render", { tag: ["@no-firefox", "@no-webkit"] }, () => { test("should render empty state", { tag: "@screenshot" }, async ({ page }) => { // Wait until the information about the empty state is rendered await expect(page.locator(".mx_EmptyState")).toBeVisible(); diff --git a/playwright/e2e/room-directory/room-directory.spec.ts b/playwright/e2e/room-directory/room-directory.spec.ts index f299a929bb..b3d2cf0ee9 100644 --- a/playwright/e2e/room-directory/room-directory.spec.ts +++ b/playwright/e2e/room-directory/room-directory.spec.ts @@ -15,37 +15,43 @@ test.describe("Room Directory", () => { botCreateOpts: { displayName: "Paul" }, }); - test("should allow admin to add alias & publish room to directory", async ({ page, app, user, bot }) => { - const roomId = await app.client.createRoom({ - name: "Gaming", - preset: "public_chat" as Preset, - }); + test( + "should allow admin to add alias & publish room to directory", + { tag: "@no-webkit" }, + async ({ page, app, user, bot }) => { + const roomId = await app.client.createRoom({ + name: "Gaming", + preset: "public_chat" as Preset, + }); - await app.viewRoomByName("Gaming"); - await app.settings.openRoomSettings(); + await app.viewRoomByName("Gaming"); + await app.settings.openRoomSettings(); - // First add a local address `gaming` - const localAddresses = page.locator(".mx_SettingsFieldset", { hasText: "Local Addresses" }); - await localAddresses.getByRole("textbox").fill("gaming"); - await localAddresses.getByRole("button", { name: "Add" }).click(); - await expect(localAddresses.getByText("#gaming:localhost")).toHaveClass("mx_EditableItem_item"); + // First add a local address `gaming` + const localAddresses = page.locator(".mx_SettingsFieldset", { hasText: "Local Addresses" }); + await localAddresses.getByRole("textbox").fill("gaming"); + await localAddresses.getByRole("button", { name: "Add" }).click(); + await expect(localAddresses.getByText("#gaming:localhost")).toHaveClass("mx_EditableItem_item"); - // Publish into the public rooms directory - const publishedAddresses = page.locator(".mx_SettingsFieldset", { hasText: "Published Addresses" }); - await expect(publishedAddresses.locator("#canonicalAlias")).toHaveValue("#gaming:localhost"); - const checkbox = publishedAddresses - .locator(".mx_SettingsFlag", { hasText: "Publish this room to the public in localhost's room directory?" }) - .getByRole("switch"); - await checkbox.check(); - await expect(checkbox).toBeChecked(); + // Publish into the public rooms directory + const publishedAddresses = page.locator(".mx_SettingsFieldset", { hasText: "Published Addresses" }); + await expect(publishedAddresses.locator("#canonicalAlias")).toHaveValue("#gaming:localhost"); + const checkbox = publishedAddresses + .locator(".mx_SettingsFlag", { + hasText: "Publish this room to the public in localhost's room directory?", + }) + .getByRole("switch"); + await checkbox.check(); + await expect(checkbox).toBeChecked(); - await app.closeDialog(); + await app.closeDialog(); - const resp = await bot.publicRooms({}); - expect(resp.total_room_count_estimate).toEqual(1); - expect(resp.chunk).toHaveLength(1); - expect(resp.chunk[0].room_id).toEqual(roomId); - }); + const resp = await bot.publicRooms({}); + expect(resp.total_room_count_estimate).toEqual(1); + expect(resp.chunk).toHaveLength(1); + expect(resp.chunk[0].room_id).toEqual(roomId); + }, + ); test( "should allow finding published rooms in directory", diff --git a/playwright/e2e/settings/general-room-settings-tab.spec.ts b/playwright/e2e/settings/general-room-settings-tab.spec.ts index 828ba5285b..7102a258bc 100644 --- a/playwright/e2e/settings/general-room-settings-tab.spec.ts +++ b/playwright/e2e/settings/general-room-settings-tab.spec.ts @@ -36,7 +36,7 @@ test.describe("General room settings tab", () => { await expect(settings.getByText("Show more")).toBeVisible(); }); - test("long address should not cause dialog to overflow", async ({ page, app }) => { + test("long address should not cause dialog to overflow", { tag: "@no-webkit" }, async ({ page, app }) => { const settings = await app.settings.openRoomSettings("General"); // 1. Set the room-address to be a really long string const longString = "abcasdhjasjhdaj1jh1asdhasjdhajsdhjavhjksd".repeat(4); diff --git a/playwright/e2e/settings/preferences-user-settings-tab.spec.ts b/playwright/e2e/settings/preferences-user-settings-tab.spec.ts index 8dc2570b42..fb2dae4eb0 100644 --- a/playwright/e2e/settings/preferences-user-settings-tab.spec.ts +++ b/playwright/e2e/settings/preferences-user-settings-tab.spec.ts @@ -31,7 +31,7 @@ test.describe("Preferences user settings tab", () => { await expect(tab).toMatchScreenshot("Preferences-user-settings-tab-should-be-rendered-properly-1.png"); }); - test("should be able to change the app language", async ({ uut, user }) => { + test("should be able to change the app language", { tag: ["@no-firefox", "@no-webkit"] }, async ({ uut, user }) => { // Check language and region setting dropdown const languageInput = uut.getByRole("button", { name: "Language Dropdown" }); await languageInput.scrollIntoViewIfNeeded(); diff --git a/playwright/e2e/spaces/spaces.spec.ts b/playwright/e2e/spaces/spaces.spec.ts index 233cdee3b4..374fc6b068 100644 --- a/playwright/e2e/spaces/spaces.spec.ts +++ b/playwright/e2e/spaces/spaces.spec.ts @@ -55,38 +55,44 @@ test.describe("Spaces", () => { botCreateOpts: { displayName: "BotBob" }, }); - test("should allow user to create public space", { tag: "@screenshot" }, async ({ page, app, user }) => { - const contextMenu = await openSpaceCreateMenu(page); - await expect(contextMenu).toMatchScreenshot("space-create-menu.png"); + test( + "should allow user to create public space", + { tag: ["@screenshot", "@no-webkit"] }, + async ({ page, app, user }) => { + const contextMenu = await openSpaceCreateMenu(page); + await expect(contextMenu).toMatchScreenshot("space-create-menu.png"); - await contextMenu.getByRole("button", { name: /Public/ }).click(); + await contextMenu.getByRole("button", { name: /Public/ }).click(); - await contextMenu - .locator('.mx_SpaceBasicSettings_avatarContainer input[type="file"]') - .setInputFiles("playwright/sample-files/riot.png"); - await contextMenu.getByRole("textbox", { name: "Name" }).fill("Let's have a Riot"); - await expect(contextMenu.getByRole("textbox", { name: "Address" })).toHaveValue("lets-have-a-riot"); - await contextMenu.getByRole("textbox", { name: "Description" }).fill("This is a space to reminisce Riot.im!"); - await contextMenu.getByRole("button", { name: "Create" }).click(); + await contextMenu + .locator('.mx_SpaceBasicSettings_avatarContainer input[type="file"]') + .setInputFiles("playwright/sample-files/riot.png"); + await contextMenu.getByRole("textbox", { name: "Name" }).fill("Let's have a Riot"); + await expect(contextMenu.getByRole("textbox", { name: "Address" })).toHaveValue("lets-have-a-riot"); + await contextMenu + .getByRole("textbox", { name: "Description" }) + .fill("This is a space to reminisce Riot.im!"); + await contextMenu.getByRole("button", { name: "Create" }).click(); - // Create the default General & Random rooms, as well as a custom "Jokes" room - await expect(page.getByPlaceholder("General")).toBeVisible(); - await expect(page.getByPlaceholder("Random")).toBeVisible(); - await page.getByPlaceholder("Support").fill("Jokes"); - await page.getByRole("button", { name: "Continue" }).click(); + // Create the default General & Random rooms, as well as a custom "Jokes" room + await expect(page.getByPlaceholder("General")).toBeVisible(); + await expect(page.getByPlaceholder("Random")).toBeVisible(); + await page.getByPlaceholder("Support").fill("Jokes"); + await page.getByRole("button", { name: "Continue" }).click(); - // Copy matrix.to link - await page.getByRole("button", { name: "Share invite link" }).click(); - expect(await app.getClipboardText()).toEqual("https://matrix.to/#/#lets-have-a-riot:localhost"); + // Copy matrix.to link + await page.getByRole("button", { name: "Share invite link" }).click(); + expect(await app.getClipboardText()).toEqual("https://matrix.to/#/#lets-have-a-riot:localhost"); - // Go to space home - await page.getByRole("button", { name: "Go to my first room" }).click(); + // Go to space home + await page.getByRole("button", { name: "Go to my first room" }).click(); - // Assert rooms exist in the room list - await expect(page.getByRole("treeitem", { name: "General" })).toBeVisible(); - await expect(page.getByRole("treeitem", { name: "Random" })).toBeVisible(); - await expect(page.getByRole("treeitem", { name: "Jokes" })).toBeVisible(); - }); + // Assert rooms exist in the room list + await expect(page.getByRole("treeitem", { name: "General" })).toBeVisible(); + await expect(page.getByRole("treeitem", { name: "Random" })).toBeVisible(); + await expect(page.getByRole("treeitem", { name: "Jokes" })).toBeVisible(); + }, + ); test("should allow user to create private space", { tag: "@screenshot" }, async ({ page, app, user }) => { const menu = await openSpaceCreateMenu(page); @@ -157,7 +163,7 @@ test.describe("Spaces", () => { ).toBeVisible(); }); - test("should allow user to invite another to a space", async ({ page, app, user, bot }) => { + test("should allow user to invite another to a space", { tag: "@no-webkit" }, async ({ page, app, user, bot }) => { await app.client.createSpace({ visibility: "public" as any, room_alias_name: "space", diff --git a/playwright/e2e/spaces/threads-activity-centre/threadsActivityCentre.spec.ts b/playwright/e2e/spaces/threads-activity-centre/threadsActivityCentre.spec.ts index ecf458c060..965047e75e 100644 --- a/playwright/e2e/spaces/threads-activity-centre/threadsActivityCentre.spec.ts +++ b/playwright/e2e/spaces/threads-activity-centre/threadsActivityCentre.spec.ts @@ -9,7 +9,7 @@ import { expect, test } from "."; import { CommandOrControl } from "../../utils"; -test.describe("Threads Activity Centre", () => { +test.describe("Threads Activity Centre", { tag: "@no-firefox" }, () => { test.use({ displayName: "Alice", botCreateOpts: { displayName: "Other User" }, diff --git a/playwright/e2e/threads/threads.spec.ts b/playwright/e2e/threads/threads.spec.ts index 06ec57653c..b6d72da358 100644 --- a/playwright/e2e/threads/threads.spec.ts +++ b/playwright/e2e/threads/threads.spec.ts @@ -324,7 +324,7 @@ test.describe("Threads", () => { }); }); - test("can send voice messages", async ({ page, app, user }) => { + test("can send voice messages", { tag: ["@no-firefox", "@no-webkit"] }, async ({ page, app, user }) => { // Increase right-panel size, so that voice messages fit await page.evaluate(() => { window.localStorage.setItem("mx_rhs_size", "600"); @@ -353,7 +353,7 @@ test.describe("Threads", () => { test( "should send location and reply to the location on ThreadView", - { tag: "@screenshot" }, + { tag: ["@screenshot", "@no-firefox"] }, async ({ page, app, bot }) => { const roomId = await app.client.createRoom({}); await app.client.inviteUser(roomId, bot.credentials.userId); diff --git a/playwright/e2e/timeline/timeline.spec.ts b/playwright/e2e/timeline/timeline.spec.ts index 7aaabb9759..4761876de4 100644 --- a/playwright/e2e/timeline/timeline.spec.ts +++ b/playwright/e2e/timeline/timeline.spec.ts @@ -90,7 +90,7 @@ test.describe("Timeline", () => { let oldAvatarUrl: string; let newAvatarUrl: string; - test.describe("useOnlyCurrentProfiles", () => { + test.describe("useOnlyCurrentProfiles", { tag: ["@no-firefox", "@no-webkit"] }, () => { test.beforeEach(async ({ app, user }) => { ({ content_uri: oldAvatarUrl } = await app.client.uploadContent(OLD_AVATAR, { type: "image/png" })); await app.client.setAvatarUrl(oldAvatarUrl); @@ -876,7 +876,7 @@ test.describe("Timeline", () => { }); }); - test.describe("message sending", () => { + test.describe("message sending", { tag: ["@no-firefox", "@no-webkit"] }, () => { const MESSAGE = "Hello world"; const reply = "Reply"; const viewRoomSendMessageAndSetupReply = async (page: Page, app: ElementAppPage, roomId: string) => { @@ -914,7 +914,6 @@ test.describe("Timeline", () => { }); test("can reply with a voice message", async ({ page, app, room, context }) => { - await context.grantPermissions(["microphone"]); await viewRoomSendMessageAndSetupReply(page, app, room.roomId); const composerOptions = await app.openMessageComposerOptions(); diff --git a/playwright/e2e/widgets/stickers.spec.ts b/playwright/e2e/widgets/stickers.spec.ts index 418e104037..bd65100baa 100644 --- a/playwright/e2e/widgets/stickers.spec.ts +++ b/playwright/e2e/widgets/stickers.spec.ts @@ -128,7 +128,7 @@ async function setWidgetAccountData( }); } -test.describe("Stickers", () => { +test.describe("Stickers", { tag: ["@no-firefox", "@no-webkit"] }, () => { test.use({ displayName: "Sally", room: async ({ app }, use) => { diff --git a/playwright/element-web-test.ts b/playwright/element-web-test.ts index 6ac0b7226a..4fe51d0a8a 100644 --- a/playwright/element-web-test.ts +++ b/playwright/element-web-test.ts @@ -127,6 +127,14 @@ export interface Fixtures { } export const test = base.extend({ + context: async ({ context }, use, testInfo) => { + // We skip tests instead of using grep-invert to still surface the counts in the html report + test.skip( + testInfo.tags.includes(`@no-${testInfo.project.name.toLowerCase()}`), + `Test does not work on ${testInfo.project.name}`, + ); + await use(context); + }, config: CONFIG_JSON, page: async ({ context, page, config, labsFlags }, use) => { await context.route(`http://localhost:8080/config.json*`, async (route) => { diff --git a/playwright/plugins/docker/index.ts b/playwright/plugins/docker/index.ts index 895a7d0f12..6cc13860be 100644 --- a/playwright/plugins/docker/index.ts +++ b/playwright/plugins/docker/index.ts @@ -140,8 +140,12 @@ export class Docker { * Detects whether the docker command is actually podman. * To do this, it looks for "podman" in the output of "docker --help". */ + static _isPodman?: boolean; static async isPodman(): Promise { - const { stdout } = await exec("docker", ["--help"], true); - return stdout.toLowerCase().includes("podman"); + if (Docker._isPodman === undefined) { + const { stdout } = await exec("docker", ["--help"], true); + Docker._isPodman = stdout.toLowerCase().includes("podman"); + } + return Docker._isPodman; } }