Run Playwright tests on Firefox & "Safari" nightly (#28757)

* Run Playwright tests on Firefox

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

* Update playwright.config.ts

* Iterate

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

* Iterate

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

* Iterate

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

* Iterate

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

* Iterate

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

* Iterate

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

* Iterate

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

* Iterate

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

* Iterate

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

* Iterate

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

* Iterate

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

* Iterate

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

* Iterate

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

* Update end-to-end-tests.yaml

* Iterate

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

* Finalise

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

* Documentation

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

* typo

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

---------

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

.github/workflows/end-to-end-tests.yaml

@@ -3,6 +3,9 @@
 # as an artifact and run end-to-end tests.
 name: End to End Tests
 on:
+    # CRON to run all Projects at 6am UTC
+    schedule:
+        - cron: "0 6 * * *"
     pull_request: {}
     merge_group:
         types: [checks_requested]
@@ -32,6 +35,8 @@ concurrency:
 env:
     # fetchdep.sh needs to know our PR number
     PR_NUMBER: ${{ github.event.pull_request.number }}
+    # Use 6 runners in the default case, but 4 when running on a schedule where we run all 5 projects (20 runners total)
+    NUM_RUNNERS: ${{ github.event_name == 'schedule' && 4 || 6 }}
 
 permissions: {} # No permissions required
 
@@ -40,6 +45,9 @@ jobs:
         name: "Build Element-Web"
         runs-on: ubuntu-24.04
         if: inputs.skip != true
+        outputs:
+            num-runners: ${{ env.NUM_RUNNERS }}
+            runners-matrix: ${{ steps.runner-vars.outputs.matrix }}
         steps:
             - name: Checkout code
               uses: actions/checkout@v4
@@ -79,8 +87,17 @@ jobs:
                   path: webapp
                   retention-days: 1
 
+            - name: Calculate runner variables
+              id: runner-vars
+              uses: actions/github-script@v7
+              with:
+                  script: |
+                      const numRunners = parseInt(process.env.NUM_RUNNERS, 10);
+                      const matrix = Array.from({ length: numRunners }, (_, i) => i + 1);
+                      core.setOutput("matrix", JSON.stringify(matrix));
+
     playwright:
-        name: "Run Tests ${{ matrix.runner }}/${{ strategy.job-total }}"
+        name: "Run Tests [${{ matrix.project }}] ${{ matrix.runner }}/${{ needs.build.outputs.num-runners }}"
         needs: build
         if: inputs.skip != true
         runs-on: ubuntu-24.04
@@ -92,7 +109,19 @@ jobs:
             fail-fast: false
             matrix:
                 # Run multiple instances in parallel to speed up the tests
-                runner: [1, 2, 3, 4, 5, 6]
+                runner: ${{ fromJSON(needs.build.outputs.runners-matrix) }}
+                project:
+                    - Chrome
+                    - Firefox
+                    - WebKit
+                isCron:
+                    - ${{ github.event_name == 'schedule' }}
+                # Skip the Firefox & Safari runs unless this was a cron trigger
+                exclude:
+                    - isCron: false
+                      project: Firefox
+                    - isCron: false
+                      project: WebKit
         steps:
             - uses: actions/checkout@v4
               with:
@@ -124,24 +153,30 @@ jobs:
               with:
                   path: |
                       ~/.cache/ms-playwright
-                  key: ${{ runner.os }}-playwright-${{ steps.playwright.outputs.version }}-chromium
+                  key: ${{ runner.os }}-playwright-${{ steps.playwright.outputs.version }}
 
-            - name: Install Playwright browser
+            - name: Install Playwright browsers
               if: steps.playwright-cache.outputs.cache-hit != 'true'
-              run: yarn playwright install --with-deps --no-shell chromium
+              run: yarn playwright install --with-deps --no-shell
+
+            - name: Install system dependencies for WebKit
+              # Some WebKit dependencies seem to lay outside the cache and will need to be installed separately
+              if: matrix.project == 'WebKit' && steps.playwright-cache.outputs.cache-hit == 'true'
+              run: yarn playwright install-deps webkit
 
             # We skip tests tagged with @mergequeue when running on PRs, but run them in MQ and everywhere else
             - name: Run Playwright tests
               run: |
                   yarn playwright test \
-                      --shard "${{ matrix.runner }}/${{ strategy.job-total }}" \
+                      --shard "${{ matrix.runner }}/${{ needs.build.outputs.num-runners }}" \
+                      --project="${{ matrix.project }}" \
                       ${{ github.event_name == 'pull_request' && '--grep-invert @mergequeue' || '' }}
 
             - name: Upload blob report to GitHub Actions Artifacts
               if: always()
               uses: actions/upload-artifact@v4
               with:
-                  name: all-blob-reports-${{ matrix.runner }}
+                  name: all-blob-reports-${{ matrix.project }}-${{ matrix.runner }}
                   path: blob-report
                   retention-days: 1
 

docs/playwright.md

@@ -53,15 +53,11 @@ yarn run test:playwright:open --headed --debug
 
 See more command line options at <https://playwright.dev/docs/test-cli>.
 
-### Running with Rust cryptography
+## Projects
 
-`matrix-js-sdk` is currently in the
-[process](https://github.com/vector-im/element-web/issues/21972) of being
-updated to replace its end-to-end encryption implementation to use the [Matrix
-Rust SDK](https://github.com/matrix-org/matrix-rust-sdk). This is not currently
-enabled by default, but it is possible to have Playwright configure Element to use
-the Rust crypto implementation by passing `--project="Rust Crypto"` or using
-the top left options in open mode.
+By default, Playwright will run all "Projects", this means tests will run against Chrome, Firefox and "Safari" (Webkit).
+We only run tests against Chrome in pull request CI, but all projects in the merge queue.
+Some tests are excluded from running on certain browsers due to incompatibilities in the test harness.
 
 ## How the Tests Work
 
@@ -224,3 +220,14 @@ We use test tags to categorise tests for running subsets more efficiently.
 
 - `@mergequeue`: Tests that are slow or flaky and cover areas of the app we update seldom, should not be run on every PR commit but will be run in the Merge Queue.
 - `@screenshot`: Tests that use `toMatchScreenshot` to speed up a run of `test:playwright:screenshots`. A test with this tag must not also have the `@mergequeue` tag as this would cause false positives in the stale screenshot detection.
+- `@no-$project`: Tests which are unsupported in $Project. These tests will be skipped when running in $Project.
+
+Anything testing Matrix media will need to have `@no-firefox` and `@no-webkit` as those rely on the service worker which
+has to be disabled in Playwright on Firefox & Webkit to retain routing functionality.
+Anything testing VoIP/microphone will need to have `@no-webkit` as fake microphone functionality is not available
+there at this time.
+
+## Colima
+
+If you are running under Colima, you may need to set the environment variable `TMPDIR` to `/tmp/colima` or a path
+within `$HOME` to allow bind mounting temporary directories into the Docker containers.

package.json

@@ -64,7 +64,7 @@
         "test:playwright:open": "yarn test:playwright --ui",
         "test:playwright:screenshots": "yarn test:playwright:screenshots:build && yarn test:playwright:screenshots:run",
         "test:playwright:screenshots:build": "docker build playwright -t element-web-playwright",
-        "test:playwright:screenshots:run": "docker run --rm --network host -e BASE_URL -e CI -v $(pwd):/work/ -v $(node -e 'console.log(require(`path`).dirname(require.resolve(`matrix-js-sdk/package.json`)))'):/work/node_modules/matrix-js-sdk -v /var/run/docker.sock:/var/run/docker.sock -v /tmp/:/tmp/ -it element-web-playwright --grep @screenshot",
+        "test:playwright:screenshots:run": "docker run --rm --network host -e BASE_URL -e CI -v $(pwd):/work/ -v $(node -e 'console.log(require(`path`).dirname(require.resolve(`matrix-js-sdk/package.json`)))'):/work/node_modules/matrix-js-sdk -v /var/run/docker.sock:/var/run/docker.sock -v /tmp/:/tmp/ -it element-web-playwright --grep @screenshot --project=Chrome",
         "coverage": "yarn test --coverage",
         "analyse:unused-exports": "ts-node ./scripts/analyse_unused_exports.ts",
         "analyse:webpack-bundles": "webpack-bundle-analyzer webpack-stats.json webapp",

playwright.config.ts

@@ -11,16 +11,49 @@ import { defineConfig, devices } from "@playwright/test";
 const baseURL = process.env["BASE_URL"] ?? "http://localhost:8080";
 
 export default defineConfig({
-    projects: [{ name: "Chrome", use: { ...devices["Desktop Chrome"], channel: "chromium" } }],
+    projects: [
+        {
+            name: "Chrome",
+            use: {
+                ...devices["Desktop Chrome"],
+                channel: "chromium",
+                permissions: ["clipboard-write", "clipboard-read", "microphone"],
+                launchOptions: {
+                    args: ["--use-fake-ui-for-media-stream", "--use-fake-device-for-media-stream", "--mute-audio"],
+                },
+            },
+        },
+        {
+            name: "Firefox",
+            use: {
+                ...devices["Desktop Firefox"],
+                launchOptions: {
+                    firefoxUserPrefs: {
+                        "permissions.default.microphone": 1,
+                    },
+                },
+                // This is needed to work around an issue between Playwright routes, Firefox, and Service workers
+                // https://github.com/microsoft/playwright/issues/33561#issuecomment-2471642120
+                serviceWorkers: "block",
+            },
+            ignoreSnapshots: true,
+        },
+        {
+            name: "WebKit",
+            use: {
+                ...devices["Desktop Safari"],
+                // Seemingly WebKit has the same issue as Firefox in Playwright routes not working
+                // https://playwright.dev/docs/network#missing-network-events-and-service-workers
+                serviceWorkers: "block",
+            },
+            ignoreSnapshots: true,
+        },
+    ],
     use: {
         viewport: { width: 1280, height: 720 },
         ignoreHTTPSErrors: true,
         video: "retain-on-failure",
         baseURL,
-        permissions: ["clipboard-write", "clipboard-read", "microphone"],
-        launchOptions: {
-            args: ["--use-fake-ui-for-media-stream", "--use-fake-device-for-media-stream", "--mute-audio"],
-        },
         trace: "on-first-retry",
     },
     webServer: {

playwright/e2e/audio-player/audio-player.spec.ts

@@ -13,7 +13,7 @@ import { SettingLevel } from "../../../src/settings/SettingLevel";
 import { Layout } from "../../../src/settings/enums/Layout";
 import { ElementAppPage } from "../../pages/ElementAppPage";
 
-test.describe("Audio player", () => {
+test.describe("Audio player", { tag: ["@no-firefox", "@no-webkit"] }, () => {
     test.use({
         displayName: "Hanako",
     });

playwright/e2e/crypto/backups.spec.ts

@@ -51,90 +51,94 @@ test.describe("Backups", () => {
         displayName: "Hanako",
     });
 
-    test("Create, delete and recreate a keys backup", async ({ page, user, app }, workerInfo) => {
-        // Create a backup
-        const securityTab = await app.settings.openUserSettings("Security & Privacy");
+    test(
+        "Create, delete and recreate a keys backup",
+        { tag: "@no-webkit" },
+        async ({ page, user, app }, workerInfo) => {
+            // Create a backup
+            const securityTab = await app.settings.openUserSettings("Security & Privacy");
 
-        await expect(securityTab.getByRole("heading", { name: "Secure Backup" })).toBeVisible();
-        await securityTab.getByRole("button", { name: "Set up", exact: true }).click();
+            await expect(securityTab.getByRole("heading", { name: "Secure Backup" })).toBeVisible();
+            await securityTab.getByRole("button", { name: "Set up", exact: true }).click();
 
-        const currentDialogLocator = page.locator(".mx_Dialog");
+            const currentDialogLocator = page.locator(".mx_Dialog");
 
-        // It's the first time and secure storage is not set up, so it will create one
-        await expect(currentDialogLocator.getByRole("heading", { name: "Set up Secure Backup" })).toBeVisible();
-        await currentDialogLocator.getByRole("button", { name: "Continue", exact: true }).click();
-        await expect(currentDialogLocator.getByRole("heading", { name: "Save your Security Key" })).toBeVisible();
-        await currentDialogLocator.getByRole("button", { name: "Copy", exact: true }).click();
-        // copy the recovery key to use it later
-        const securityKey = await app.getClipboard();
-        await currentDialogLocator.getByRole("button", { name: "Continue", exact: true }).click();
+            // It's the first time and secure storage is not set up, so it will create one
+            await expect(currentDialogLocator.getByRole("heading", { name: "Set up Secure Backup" })).toBeVisible();
+            await currentDialogLocator.getByRole("button", { name: "Continue", exact: true }).click();
+            await expect(currentDialogLocator.getByRole("heading", { name: "Save your Security Key" })).toBeVisible();
+            await currentDialogLocator.getByRole("button", { name: "Copy", exact: true }).click();
+            // copy the recovery key to use it later
+            const securityKey = await app.getClipboard();
+            await currentDialogLocator.getByRole("button", { name: "Continue", exact: true }).click();
 
-        await expect(currentDialogLocator.getByRole("heading", { name: "Secure Backup successful" })).toBeVisible();
-        await currentDialogLocator.getByRole("button", { name: "Done", exact: true }).click();
+            await expect(currentDialogLocator.getByRole("heading", { name: "Secure Backup successful" })).toBeVisible();
+            await currentDialogLocator.getByRole("button", { name: "Done", exact: true }).click();
 
-        // Open the settings again
-        await app.settings.openUserSettings("Security & Privacy");
-        await expect(securityTab.getByRole("heading", { name: "Secure Backup" })).toBeVisible();
+            // Open the settings again
+            await app.settings.openUserSettings("Security & Privacy");
+            await expect(securityTab.getByRole("heading", { name: "Secure Backup" })).toBeVisible();
 
-        // expand the advanced section to see the active version in the reports
-        await page
-            .locator(".mx_Dialog .mx_SettingsSubsection_content details .mx_SecureBackupPanel_advanced")
-            .locator("..")
-            .click();
+            // expand the advanced section to see the active version in the reports
+            await page
+                .locator(".mx_Dialog .mx_SettingsSubsection_content details .mx_SecureBackupPanel_advanced")
+                .locator("..")
+                .click();
 
-        await expectBackupVersionToBe(page, "1");
+            await expectBackupVersionToBe(page, "1");
 
-        await securityTab.getByRole("button", { name: "Delete Backup", exact: true }).click();
-        await expect(currentDialogLocator.getByRole("heading", { name: "Delete Backup" })).toBeVisible();
-        // Delete it
-        await currentDialogLocator.getByTestId("dialog-primary-button").click(); // Click "Delete Backup"
+            await securityTab.getByRole("button", { name: "Delete Backup", exact: true }).click();
+            await expect(currentDialogLocator.getByRole("heading", { name: "Delete Backup" })).toBeVisible();
+            // Delete it
+            await currentDialogLocator.getByTestId("dialog-primary-button").click(); // Click "Delete Backup"
 
-        // Create another
-        await securityTab.getByRole("button", { name: "Set up", exact: true }).click();
-        await expect(currentDialogLocator.getByRole("heading", { name: "Security Key" })).toBeVisible();
-        await currentDialogLocator.getByLabel("Security Key").fill(securityKey);
-        await currentDialogLocator.getByRole("button", { name: "Continue", exact: true }).click();
+            // Create another
+            await securityTab.getByRole("button", { name: "Set up", exact: true }).click();
+            await expect(currentDialogLocator.getByRole("heading", { name: "Security Key" })).toBeVisible();
+            await currentDialogLocator.getByLabel("Security Key").fill(securityKey);
+            await currentDialogLocator.getByRole("button", { name: "Continue", exact: true }).click();
 
-        // Should be successful
-        await expect(currentDialogLocator.getByRole("heading", { name: "Success!" })).toBeVisible();
-        await currentDialogLocator.getByRole("button", { name: "OK", exact: true }).click();
+            // Should be successful
+            await expect(currentDialogLocator.getByRole("heading", { name: "Success!" })).toBeVisible();
+            await currentDialogLocator.getByRole("button", { name: "OK", exact: true }).click();
 
-        // Open the settings again
-        await app.settings.openUserSettings("Security & Privacy");
-        await expect(securityTab.getByRole("heading", { name: "Secure Backup" })).toBeVisible();
+            // Open the settings again
+            await app.settings.openUserSettings("Security & Privacy");
+            await expect(securityTab.getByRole("heading", { name: "Secure Backup" })).toBeVisible();
 
-        // expand the advanced section to see the active version in the reports
-        await page
-            .locator(".mx_Dialog .mx_SettingsSubsection_content details .mx_SecureBackupPanel_advanced")
-            .locator("..")
-            .click();
+            // expand the advanced section to see the active version in the reports
+            await page
+                .locator(".mx_Dialog .mx_SettingsSubsection_content details .mx_SecureBackupPanel_advanced")
+                .locator("..")
+                .click();
 
-        await expectBackupVersionToBe(page, "2");
+            await expectBackupVersionToBe(page, "2");
 
-        // ==
-        // Ensure that if you don't have the secret storage passphrase the backup won't be created
-        // ==
+            // ==
+            // Ensure that if you don't have the secret storage passphrase the backup won't be created
+            // ==
 
-        // First delete version 2
-        await securityTab.getByRole("button", { name: "Delete Backup", exact: true }).click();
-        await expect(currentDialogLocator.getByRole("heading", { name: "Delete Backup" })).toBeVisible();
-        // Click "Delete Backup"
-        await currentDialogLocator.getByTestId("dialog-primary-button").click();
+            // First delete version 2
+            await securityTab.getByRole("button", { name: "Delete Backup", exact: true }).click();
+            await expect(currentDialogLocator.getByRole("heading", { name: "Delete Backup" })).toBeVisible();
+            // Click "Delete Backup"
+            await currentDialogLocator.getByTestId("dialog-primary-button").click();
 
-        // Try to create another
-        await securityTab.getByRole("button", { name: "Set up", exact: true }).click();
-        await expect(currentDialogLocator.getByRole("heading", { name: "Security Key" })).toBeVisible();
-        // But cancel the security key dialog, to simulate not having the secret storage passphrase
-        await currentDialogLocator.getByTestId("dialog-cancel-button").click();
+            // Try to create another
+            await securityTab.getByRole("button", { name: "Set up", exact: true }).click();
+            await expect(currentDialogLocator.getByRole("heading", { name: "Security Key" })).toBeVisible();
+            // But cancel the security key dialog, to simulate not having the secret storage passphrase
+            await currentDialogLocator.getByTestId("dialog-cancel-button").click();
 
-        await expect(currentDialogLocator.getByRole("heading", { name: "Starting backup…" })).toBeVisible();
-        // check that it failed
-        await expect(currentDialogLocator.getByText("Unable to create key backup")).toBeVisible();
-        // cancel
-        await currentDialogLocator.getByTestId("dialog-cancel-button").click();
+            await expect(currentDialogLocator.getByRole("heading", { name: "Starting backup…" })).toBeVisible();
+            // check that it failed
+            await expect(currentDialogLocator.getByText("Unable to create key backup")).toBeVisible();
+            // cancel
+            await currentDialogLocator.getByTestId("dialog-cancel-button").click();
 
-        // go back to the settings to check that no backup was created (the setup button should still be there)
-        await app.settings.openUserSettings("Security & Privacy");
-        await expect(securityTab.getByRole("button", { name: "Set up", exact: true })).toBeVisible();
-    });
+            // go back to the settings to check that no backup was created (the setup button should still be there)
+            await app.settings.openUserSettings("Security & Privacy");
+            await expect(securityTab.getByRole("button", { name: "Set up", exact: true })).toBeVisible();
+        },
+    );
 });

playwright/e2e/crypto/device-verification.spec.ts

@@ -21,7 +21,7 @@ import {
 } from "./utils";
 import { Bot } from "../../pages/bot";
 
-test.describe("Device verification", () => {
+test.describe("Device verification", { tag: "@no-webkit" }, () => {
     let aliceBotClient: Bot;
 
     /** The backup version that was set up by the bot client. */

playwright/e2e/crypto/migration.spec.ts

@@ -25,7 +25,7 @@ const test = base.extend<Fixtures>({
     },
 });
 
-test.describe("migration", function () {
+test.describe("migration", { tag: "@no-webkit" }, function () {
     test.use({ displayName: "Alice" });
 
     test("Should support migration from legacy crypto", async ({ context, user, page }, workerInfo) => {

playwright/e2e/location/location.spec.ts

@@ -10,7 +10,8 @@ import { Locator, Page } from "@playwright/test";
 
 import { test, expect } from "../../element-web-test";
 
-test.describe("Location sharing", () => {
+// Firefox headless lacks WebGL support https://bugzilla.mozilla.org/show_bug.cgi?id=1375585
+test.describe("Location sharing", { tag: "@no-firefox" }, () => {
     const selectLocationShareTypeOption = (page: Page, shareType: string): Locator => {
         return page.getByTestId(`share-location-option-${shareType}`);
     };

playwright/e2e/oidc/oidc-aware.spec.ts

@@ -9,7 +9,7 @@ Please see LICENSE files in the repository root for full details.
 import { test, expect, registerAccountMas } from ".";
 import { isDendrite } from "../../plugins/homeserver/dendrite";
 
-test.describe("OIDC Aware", () => {
+test.describe("OIDC Aware", { tag: ["@no-firefox", "@no-webkit"] }, () => {
     test.skip(isDendrite, "does not yet support MAS");
     test.slow(); // trace recording takes a while here
 

playwright/e2e/oidc/oidc-native.spec.ts

@@ -10,7 +10,7 @@ import { test, expect, registerAccountMas } from ".";
 import { isDendrite } from "../../plugins/homeserver/dendrite";
 import { ElementAppPage } from "../../pages/ElementAppPage.ts";
 
-test.describe("OIDC Native", () => {
+test.describe("OIDC Native", { tag: ["@no-firefox", "@no-webkit"] }, () => {
     test.skip(isDendrite, "does not yet support MAS");
     test.slow(); // trace recording takes a while here
 

playwright/e2e/register/register.spec.ts

@@ -9,7 +9,9 @@ Please see LICENSE files in the repository root for full details.
 import { test, expect } from "../../element-web-test";
 
 test.describe("Registration", () => {
-    test.use({ startHomeserverOpts: "consent" });
+    test.use({
+        startHomeserverOpts: "consent",
+    });
 
     test.beforeEach(async ({ page }) => {
         await page.goto("/#/register");

playwright/e2e/right-panel/file-panel.spec.ts

@@ -39,7 +39,7 @@ test.describe("FilePanel", () => {
         await expect(page.locator(".mx_FilePanel")).toBeVisible();
     });
 
-    test.describe("render", () => {
+    test.describe("render", { tag: ["@no-firefox", "@no-webkit"] }, () => {
         test("should render empty state", { tag: "@screenshot" }, async ({ page }) => {
             // Wait until the information about the empty state is rendered
             await expect(page.locator(".mx_EmptyState")).toBeVisible();

playwright/e2e/room-directory/room-directory.spec.ts

@@ -15,37 +15,43 @@ test.describe("Room Directory", () => {
         botCreateOpts: { displayName: "Paul" },
     });
 
-    test("should allow admin to add alias & publish room to directory", async ({ page, app, user, bot }) => {
-        const roomId = await app.client.createRoom({
-            name: "Gaming",
-            preset: "public_chat" as Preset,
-        });
+    test(
+        "should allow admin to add alias & publish room to directory",
+        { tag: "@no-webkit" },
+        async ({ page, app, user, bot }) => {
+            const roomId = await app.client.createRoom({
+                name: "Gaming",
+                preset: "public_chat" as Preset,
+            });
 
-        await app.viewRoomByName("Gaming");
-        await app.settings.openRoomSettings();
+            await app.viewRoomByName("Gaming");
+            await app.settings.openRoomSettings();
 
-        // First add a local address `gaming`
-        const localAddresses = page.locator(".mx_SettingsFieldset", { hasText: "Local Addresses" });
-        await localAddresses.getByRole("textbox").fill("gaming");
-        await localAddresses.getByRole("button", { name: "Add" }).click();
-        await expect(localAddresses.getByText("#gaming:localhost")).toHaveClass("mx_EditableItem_item");
+            // First add a local address `gaming`
+            const localAddresses = page.locator(".mx_SettingsFieldset", { hasText: "Local Addresses" });
+            await localAddresses.getByRole("textbox").fill("gaming");
+            await localAddresses.getByRole("button", { name: "Add" }).click();
+            await expect(localAddresses.getByText("#gaming:localhost")).toHaveClass("mx_EditableItem_item");
 
-        // Publish into the public rooms directory
-        const publishedAddresses = page.locator(".mx_SettingsFieldset", { hasText: "Published Addresses" });
-        await expect(publishedAddresses.locator("#canonicalAlias")).toHaveValue("#gaming:localhost");
-        const checkbox = publishedAddresses
-            .locator(".mx_SettingsFlag", { hasText: "Publish this room to the public in localhost's room directory?" })
-            .getByRole("switch");
-        await checkbox.check();
-        await expect(checkbox).toBeChecked();
+            // Publish into the public rooms directory
+            const publishedAddresses = page.locator(".mx_SettingsFieldset", { hasText: "Published Addresses" });
+            await expect(publishedAddresses.locator("#canonicalAlias")).toHaveValue("#gaming:localhost");
+            const checkbox = publishedAddresses
+                .locator(".mx_SettingsFlag", {
+                    hasText: "Publish this room to the public in localhost's room directory?",
+                })
+                .getByRole("switch");
+            await checkbox.check();
+            await expect(checkbox).toBeChecked();
 
-        await app.closeDialog();
+            await app.closeDialog();
 
-        const resp = await bot.publicRooms({});
-        expect(resp.total_room_count_estimate).toEqual(1);
-        expect(resp.chunk).toHaveLength(1);
-        expect(resp.chunk[0].room_id).toEqual(roomId);
-    });
+            const resp = await bot.publicRooms({});
+            expect(resp.total_room_count_estimate).toEqual(1);
+            expect(resp.chunk).toHaveLength(1);
+            expect(resp.chunk[0].room_id).toEqual(roomId);
+        },
+    );
 
     test(
         "should allow finding published rooms in directory",

playwright/e2e/settings/general-room-settings-tab.spec.ts

@@ -36,7 +36,7 @@ test.describe("General room settings tab", () => {
         await expect(settings.getByText("Show more")).toBeVisible();
     });
 
-    test("long address should not cause dialog to overflow", async ({ page, app }) => {
+    test("long address should not cause dialog to overflow", { tag: "@no-webkit" }, async ({ page, app }) => {
         const settings = await app.settings.openRoomSettings("General");
         // 1. Set the room-address to be a really long string
         const longString = "abcasdhjasjhdaj1jh1asdhasjdhajsdhjavhjksd".repeat(4);

playwright/e2e/settings/preferences-user-settings-tab.spec.ts

@@ -31,7 +31,7 @@ test.describe("Preferences user settings tab", () => {
         await expect(tab).toMatchScreenshot("Preferences-user-settings-tab-should-be-rendered-properly-1.png");
     });
 
-    test("should be able to change the app language", async ({ uut, user }) => {
+    test("should be able to change the app language", { tag: ["@no-firefox", "@no-webkit"] }, async ({ uut, user }) => {
         // Check language and region setting dropdown
         const languageInput = uut.getByRole("button", { name: "Language Dropdown" });
         await languageInput.scrollIntoViewIfNeeded();

playwright/e2e/spaces/spaces.spec.ts

@@ -55,38 +55,44 @@ test.describe("Spaces", () => {
         botCreateOpts: { displayName: "BotBob" },
     });
 
-    test("should allow user to create public space", { tag: "@screenshot" }, async ({ page, app, user }) => {
-        const contextMenu = await openSpaceCreateMenu(page);
-        await expect(contextMenu).toMatchScreenshot("space-create-menu.png");
+    test(
+        "should allow user to create public space",
+        { tag: ["@screenshot", "@no-webkit"] },
+        async ({ page, app, user }) => {
+            const contextMenu = await openSpaceCreateMenu(page);
+            await expect(contextMenu).toMatchScreenshot("space-create-menu.png");
 
-        await contextMenu.getByRole("button", { name: /Public/ }).click();
+            await contextMenu.getByRole("button", { name: /Public/ }).click();
 
-        await contextMenu
-            .locator('.mx_SpaceBasicSettings_avatarContainer input[type="file"]')
-            .setInputFiles("playwright/sample-files/riot.png");
-        await contextMenu.getByRole("textbox", { name: "Name" }).fill("Let's have a Riot");
-        await expect(contextMenu.getByRole("textbox", { name: "Address" })).toHaveValue("lets-have-a-riot");
-        await contextMenu.getByRole("textbox", { name: "Description" }).fill("This is a space to reminisce Riot.im!");
-        await contextMenu.getByRole("button", { name: "Create" }).click();
+            await contextMenu
+                .locator('.mx_SpaceBasicSettings_avatarContainer input[type="file"]')
+                .setInputFiles("playwright/sample-files/riot.png");
+            await contextMenu.getByRole("textbox", { name: "Name" }).fill("Let's have a Riot");
+            await expect(contextMenu.getByRole("textbox", { name: "Address" })).toHaveValue("lets-have-a-riot");
+            await contextMenu
+                .getByRole("textbox", { name: "Description" })
+                .fill("This is a space to reminisce Riot.im!");
+            await contextMenu.getByRole("button", { name: "Create" }).click();
 
-        // Create the default General & Random rooms, as well as a custom "Jokes" room
-        await expect(page.getByPlaceholder("General")).toBeVisible();
-        await expect(page.getByPlaceholder("Random")).toBeVisible();
-        await page.getByPlaceholder("Support").fill("Jokes");
-        await page.getByRole("button", { name: "Continue" }).click();
+            // Create the default General & Random rooms, as well as a custom "Jokes" room
+            await expect(page.getByPlaceholder("General")).toBeVisible();
+            await expect(page.getByPlaceholder("Random")).toBeVisible();
+            await page.getByPlaceholder("Support").fill("Jokes");
+            await page.getByRole("button", { name: "Continue" }).click();
 
-        // Copy matrix.to link
-        await page.getByRole("button", { name: "Share invite link" }).click();
-        expect(await app.getClipboardText()).toEqual("https://matrix.to/#/#lets-have-a-riot:localhost");
+            // Copy matrix.to link
+            await page.getByRole("button", { name: "Share invite link" }).click();
+            expect(await app.getClipboardText()).toEqual("https://matrix.to/#/#lets-have-a-riot:localhost");
 
-        // Go to space home
-        await page.getByRole("button", { name: "Go to my first room" }).click();
+            // Go to space home
+            await page.getByRole("button", { name: "Go to my first room" }).click();
 
-        // Assert rooms exist in the room list
-        await expect(page.getByRole("treeitem", { name: "General" })).toBeVisible();
-        await expect(page.getByRole("treeitem", { name: "Random" })).toBeVisible();
-        await expect(page.getByRole("treeitem", { name: "Jokes" })).toBeVisible();
-    });
+            // Assert rooms exist in the room list
+            await expect(page.getByRole("treeitem", { name: "General" })).toBeVisible();
+            await expect(page.getByRole("treeitem", { name: "Random" })).toBeVisible();
+            await expect(page.getByRole("treeitem", { name: "Jokes" })).toBeVisible();
+        },
+    );
 
     test("should allow user to create private space", { tag: "@screenshot" }, async ({ page, app, user }) => {
         const menu = await openSpaceCreateMenu(page);
@@ -157,7 +163,7 @@ test.describe("Spaces", () => {
         ).toBeVisible();
     });
 
-    test("should allow user to invite another to a space", async ({ page, app, user, bot }) => {
+    test("should allow user to invite another to a space", { tag: "@no-webkit" }, async ({ page, app, user, bot }) => {
         await app.client.createSpace({
             visibility: "public" as any,
             room_alias_name: "space",

playwright/e2e/spaces/threads-activity-centre/threadsActivityCentre.spec.ts

@@ -9,7 +9,7 @@
 import { expect, test } from ".";
 import { CommandOrControl } from "../../utils";
 
-test.describe("Threads Activity Centre", () => {
+test.describe("Threads Activity Centre", { tag: "@no-firefox" }, () => {
     test.use({
         displayName: "Alice",
         botCreateOpts: { displayName: "Other User" },

playwright/e2e/threads/threads.spec.ts

@@ -324,7 +324,7 @@ test.describe("Threads", () => {
             });
         });
 
-        test("can send voice messages", async ({ page, app, user }) => {
+        test("can send voice messages", { tag: ["@no-firefox", "@no-webkit"] }, async ({ page, app, user }) => {
             // Increase right-panel size, so that voice messages fit
             await page.evaluate(() => {
                 window.localStorage.setItem("mx_rhs_size", "600");
@@ -353,7 +353,7 @@ test.describe("Threads", () => {
 
     test(
         "should send location and reply to the location on ThreadView",
-        { tag: "@screenshot" },
+        { tag: ["@screenshot", "@no-firefox"] },
         async ({ page, app, bot }) => {
             const roomId = await app.client.createRoom({});
             await app.client.inviteUser(roomId, bot.credentials.userId);

playwright/e2e/timeline/timeline.spec.ts

@@ -90,7 +90,7 @@ test.describe("Timeline", () => {
     let oldAvatarUrl: string;
     let newAvatarUrl: string;
 
-    test.describe("useOnlyCurrentProfiles", () => {
+    test.describe("useOnlyCurrentProfiles", { tag: ["@no-firefox", "@no-webkit"] }, () => {
         test.beforeEach(async ({ app, user }) => {
             ({ content_uri: oldAvatarUrl } = await app.client.uploadContent(OLD_AVATAR, { type: "image/png" }));
             await app.client.setAvatarUrl(oldAvatarUrl);
@@ -876,7 +876,7 @@ test.describe("Timeline", () => {
         });
     });
 
-    test.describe("message sending", () => {
+    test.describe("message sending", { tag: ["@no-firefox", "@no-webkit"] }, () => {
         const MESSAGE = "Hello world";
         const reply = "Reply";
         const viewRoomSendMessageAndSetupReply = async (page: Page, app: ElementAppPage, roomId: string) => {
@@ -914,7 +914,6 @@ test.describe("Timeline", () => {
         });
 
         test("can reply with a voice message", async ({ page, app, room, context }) => {
-            await context.grantPermissions(["microphone"]);
             await viewRoomSendMessageAndSetupReply(page, app, room.roomId);
 
             const composerOptions = await app.openMessageComposerOptions();

playwright/e2e/widgets/stickers.spec.ts

@@ -128,7 +128,7 @@ async function setWidgetAccountData(
     });
 }
 
-test.describe("Stickers", () => {
+test.describe("Stickers", { tag: ["@no-firefox", "@no-webkit"] }, () => {
     test.use({
         displayName: "Sally",
         room: async ({ app }, use) => {

playwright/element-web-test.ts

@@ -127,6 +127,14 @@ export interface Fixtures {
 }
 
 export const test = base.extend<Fixtures>({
+    context: async ({ context }, use, testInfo) => {
+        // We skip tests instead of using grep-invert to still surface the counts in the html report
+        test.skip(
+            testInfo.tags.includes(`@no-${testInfo.project.name.toLowerCase()}`),
+            `Test does not work on ${testInfo.project.name}`,
+        );
+        await use(context);
+    },
     config: CONFIG_JSON,
     page: async ({ context, page, config, labsFlags }, use) => {
         await context.route(`http://localhost:8080/config.json*`, async (route) => {

playwright/plugins/docker/index.ts

@@ -140,8 +140,12 @@ export class Docker {
      * Detects whether the docker command is actually podman.
      * To do this, it looks for "podman" in the output of "docker --help".
      */
+    static _isPodman?: boolean;
     static async isPodman(): Promise<boolean> {
-        const { stdout } = await exec("docker", ["--help"], true);
-        return stdout.toLowerCase().includes("podman");
+        if (Docker._isPodman === undefined) {
+            const { stdout } = await exec("docker", ["--help"], true);
+            Docker._isPodman = stdout.toLowerCase().includes("podman");
+        }
+        return Docker._isPodman;
     }
 }