Playwright fixes and improvements (#28925)

* Fix playwright-image-updates.yaml workflow

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

* Add `X-Run-All-Tests` label for running all tests

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

* Ignore failing tests in stale-screenshot-reporter.ts to avoid confusing errors

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

---------

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

.github/CODEOWNERS

@@ -15,5 +15,5 @@
 /src/i18n/strings
 /src/i18n/strings/en_EN.json  @element-hq/element-web-reviewers
 # Ignore the synapse plugin as this is updated by GHA for docker image updating
-/playwright/plugins/homeserver/synapse/index.ts
+/playwright/testcontainers/synapse.ts
 

.github/labels.yml

@@ -210,6 +210,9 @@
 - name: "X-Upcoming-Release-Blocker"
   description: "This does not affect the current release cycle but will affect the next one"
   color: "e99695"
+- name: "X-Run-All-Tests"
+  description: "When applied to PRs, it'll run the full gamut of end-to-end tests on the PR"
+  color: "ff7979"
 - name: "Z-Actions"
   color: "ededed"
 - name: "Z-Cache-Confusion"

.github/workflows/end-to-end-tests.yaml

@@ -114,13 +114,13 @@ jobs:
                     - Chrome
                     - Firefox
                     - WebKit
-                isCron:
-                    - ${{ github.event_name == 'schedule' }}
-                # Skip the Firefox & Safari runs unless this was a cron trigger
+                runAllTests:
+                    - ${{ github.event_name == 'schedule' || contains(github.event.pull_request.labels.*.name, 'X-Run-All-Tests') }}
+                # Skip the Firefox & Safari runs unless this was a cron trigger or PR has X-Run-All-Tests label
                 exclude:
-                    - isCron: false
+                    - runAllTests: false
                       project: Firefox
-                    - isCron: false
+                    - runAllTests: false
                       project: WebKit
         steps:
             - uses: actions/checkout@v4
@@ -170,7 +170,7 @@ jobs:
                   yarn playwright test \
                       --shard "${{ matrix.runner }}/${{ needs.build.outputs.num-runners }}" \
                       --project="${{ matrix.project }}" \
-                      ${{ github.event_name == 'pull_request' && '--grep-invert @mergequeue' || '' }}
+                      ${{ (github.event_name == 'pull_request' && matrix.runAllTests == false ) && '--grep-invert @mergequeue' || '' }}
 
             - name: Upload blob report to GitHub Actions Artifacts
               if: always()

.github/workflows/playwright-image-updates.yaml

@@ -17,7 +17,7 @@ jobs:
                   docker pull "$IMAGE"
                   INSPECT=$(docker inspect --format='{{index .RepoDigests 0}}' "$IMAGE")
                   DIGEST=${INSPECT#*@}
-                  sed -i "s/const DOCKER_TAG.*/const DOCKER_TAG = \"develop@$DIGEST\";/" playwright/plugins/homeserver/synapse/index.ts
+                  sed -i "s/const TAG.*/const TAG = \"develop@$DIGEST\";/" playwright/testcontainers/synapse.ts
               env:
                   IMAGE: ghcr.io/element-hq/synapse:develop
 

docs/playwright.md

@@ -227,6 +227,8 @@ has to be disabled in Playwright on Firefox & Webkit to retain routing functiona
 Anything testing VoIP/microphone will need to have `@no-webkit` as fake microphone functionality is not available
 there at this time.
 
+If you wish to run all tests in a PR, you can give it the label `X-Run-All-Tests`.
+
 ## Supporter container runtimes
 
 We use testcontainers to spin up various instances of Synapse, Matrix Authentication Service, and more.

playwright/stale-screenshot-reporter.ts

@@ -23,6 +23,7 @@ class StaleScreenshotReporter implements Reporter {
     private success = true;
 
     public onTestEnd(test: TestCase): void {
+        if (!test.ok()) return;
         for (const annotation of test.annotations) {
             if (annotation.type === "_screenshot") {
                 this.screenshots.add(annotation.description);