From fa8c346dfa4489d08dde9959ecca0d9e76a9cf57 Mon Sep 17 00:00:00 2001 From: Michael Telatynski <7t3chguy@gmail.com> Date: Tue, 7 Jun 2022 09:03:25 +0100 Subject: [PATCH] Rework the Cypress & Percy CI (#8691) * Split Cypress out into its own workflow * Improve PR Details job to use github-script and output labels * Fix wrongly using github.ref in workflow_run actions which always refer to develop * Update pr-details to be far more generic * Tweak how we fill command-prefix * Tweak cypress job to pass more params & fix if condition * Bring in external changes * Add docs * Use new composite action, and an action to update a status check based on this workflow run * Iterate approach --- .github/workflows/cypress.yaml | 101 ++++++++++++++ .github/workflows/element-build-and-test.yaml | 125 ------------------ .github/workflows/element-web.yaml | 52 ++++++++ .github/workflows/netlify.yaml | 2 +- .github/workflows/tests.yml | 13 ++ docs/cypress.md | 5 +- 6 files changed, 169 insertions(+), 129 deletions(-) create mode 100644 .github/workflows/cypress.yaml delete mode 100644 .github/workflows/element-build-and-test.yaml create mode 100644 .github/workflows/element-web.yaml diff --git a/.github/workflows/cypress.yaml b/.github/workflows/cypress.yaml new file mode 100644 index 0000000000..7ccc0b7996 --- /dev/null +++ b/.github/workflows/cypress.yaml @@ -0,0 +1,101 @@ +# Triggers after the layered build has finished, taking the artifact and running cypress on it +name: Cypress End to End Tests +on: + workflow_run: + workflows: [ "Element Web - Build" ] + types: + - completed +jobs: + # This job cannot have a pretty name due to https://github.com/haya14busa/action-workflow_run-status/issues/158 + cypress: + if: github.event.workflow_run.conclusion == 'success' + runs-on: ubuntu-latest + steps: + # Wire up the status check for this workflow_run action + - uses: haya14busa/action-workflow_run-status@967ed83efa565c257675ed70cfe5231f062ddd94 # v1.0.0 + + - id: prdetails + if: github.event.workflow_run.event == 'pull_request' + uses: matrix-org/pr-details-action@v1.1 + with: + owner: ${{ github.event.workflow_run.head_repository.owner.login }} + branch: ${{ github.event.workflow_run.head_branch }} + + - uses: actions/checkout@v2 + + # There's a 'download artifact' action, but it hasn't been updated for the workflow_run action + # (https://github.com/actions/download-artifact/issues/60) so instead we get this mess: + - name: 📥 Download artifact + uses: dawidd6/action-download-artifact@v2 + with: + workflow: element-build-and-test.yaml + run_id: ${{ github.event.workflow_run.id }} + name: previewbuild + path: webapp + + - name: Get commit details + if: github.event.workflow_run.event == 'pull_request' + run: | + echo "COMMIT_INFO_MESSAGE=\"$(git log --format=%B -n 1 ${{ github.event.workflow_run.head_sha }})\"" >> $GITHUB_ENV + echo "COMMIT_INFO_AUTHOR=$(git log --format='%an' -n 1 ${{ github.event.workflow_run.head_sha }})" >> $GITHUB_ENV + echo "COMMIT_INFO_EMAIL=$(git log --format='%ae' -n 1 ${{ github.event.workflow_run.head_sha }})" >> $GITHUB_ENV + # Only run Percy when it is demanded or on develop + if [[ "${{ (contains(fromJSON(steps.prdetails.outputs.data).labels.*.name, 'X-Needs-Percy') || github.ref == 'refs/heads/develop') }}" == "false" ]]; then + echo "PERCY_ENABLE=0" >> $GITHUB_ENV + fi + + - name: Run Cypress tests + uses: cypress-io/github-action@v2 + with: + # The built-in Electron runner seems to grind to a halt trying + # to run the tests, so use chrome. + browser: chrome + start: npx serve -p 8080 webapp + wait-on: 'http://localhost:8080' + record: true + command-prefix: 'yarn percy exec --' + env: + # pass the Dashboard record key as an environment variable + CYPRESS_RECORD_KEY: ${{ secrets.CYPRESS_RECORD_KEY }} + # tell Cypress more details about the context of this run + COMMIT_INFO_BRANCH: ${{ github.event.workflow_run.head_branch }} + COMMIT_INFO_SHA: ${{ github.event.workflow_run.head_sha }} + COMMIT_INFO_REMOTE: ${{ github.repositoryUrl }} + + # pass the Percy token as an environment variable + PERCY_TOKEN: ${{ secrets.PERCY_TOKEN }} + # Use existing chromium rather than downloading another + PUPPETEER_SKIP_CHROMIUM_DOWNLOAD: true + PERCY_BROWSER_EXECUTABLE: /usr/bin/chromium-browser + # tell Percy more details about the context of this run + PERCY_BRANCH: ${{ github.event.workflow_run.head_branch }} + PERCY_COMMIT: ${{ github.event.workflow_run.head_sha }} + PERCY_PULL_REQUEST: ${{ steps.prdetails.outputs.pr_id }} + # pass GitHub token to allow accurately detecting a build vs a re-run build + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + # make Node's os.tmpdir() return something where we actually have permissions + TMPDIR: ${{ runner.temp }} + + - name: Upload Artifact + if: failure() + uses: actions/upload-artifact@v2 + with: + name: cypress-results + path: | + cypress/screenshots + cypress/videos + cypress/synapselogs + + - name: Store benchmark result + if: github.ref == 'refs/heads/develop' + uses: matrix-org/github-action-benchmark@jsperfentry-1 + with: + name: Cypress measurements + tool: 'jsperformanceentry' + output-file-path: cypress/performance/measurements.json + # The dashboard is available at https://matrix-org.github.io/matrix-react-sdk/cypress/bench/ + benchmark-data-dir-path: cypress/bench + fail-on-alert: false + comment-on-alert: false + github-token: ${{ secrets.DEPLOY_GH_PAGES }} + auto-push: ${{ github.ref == 'refs/heads/develop' }} diff --git a/.github/workflows/element-build-and-test.yaml b/.github/workflows/element-build-and-test.yaml deleted file mode 100644 index 9ed06bd8ad..0000000000 --- a/.github/workflows/element-build-and-test.yaml +++ /dev/null @@ -1,125 +0,0 @@ -# Produce a build of element-web with this version of react-sdk -# and any matching branches of element-web and js-sdk, output it -# as an artifact and run integration tests. -name: Element Web - Build and Test -on: - pull_request: { } - push: - branches: [ develop, master ] - repository_dispatch: - types: [ upstream-sdk-notify ] -env: - # These must be set for fetchdep.sh to get the right branch - REPOSITORY: ${{ github.repository }} - PR_NUMBER: ${{ github.event.pull_request.number }} -jobs: - build: - name: "Build Element-Web" - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - - uses: actions/setup-node@v3 - with: - cache: 'yarn' - - - name: Fetch layered build - id: layered_build - run: | - scripts/ci/layered.sh - JSSDK_SHA=$(git -C matrix-js-sdk rev-parse --short=12 HEAD) - REACT_SHA=$(git rev-parse --short=12 HEAD) - VECTOR_SHA=$(git -C element-web rev-parse --short=12 HEAD) - echo "::set-output name=VERSION::$VECTOR_SHA-react-$REACT_SHA-js-$JSSDK_SHA" - - - name: Copy config - run: cp element.io/develop/config.json config.json - working-directory: ./element-web - - - name: Build - env: - CI_PACKAGE: true - VERSION: "${{ steps.layered_build.outputs.VERSION }}" - run: yarn build - working-directory: ./element-web - - - name: Upload Artifact - uses: actions/upload-artifact@v2 - with: - name: previewbuild - path: element-web/webapp - # We'll only use this in a triggered job, then we're done with it - retention-days: 1 - - cypress: - name: "Cypress End to End Tests" - needs: build - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - - name: Download build - uses: actions/download-artifact@v3 - with: - name: previewbuild - path: webapp - - - name: Run Cypress tests - uses: cypress-io/github-action@v2 - with: - # The built-in Electron runner seems to grind to a halt trying - # to run the tests, so use chrome. - browser: chrome - start: npx serve -p 8080 webapp - wait-on: 'http://localhost:8080' - record: true - command-prefix: 'yarn percy exec --' - env: - # pass the Dashboard record key as an environment variable - CYPRESS_RECORD_KEY: ${{ secrets.CYPRESS_RECORD_KEY }} - # pass the Percy token as an environment variable - PERCY_TOKEN: ${{ secrets.PERCY_TOKEN }} - # Use existing chromium rather than downloading another - PUPPETEER_SKIP_CHROMIUM_DOWNLOAD: true - PERCY_BROWSER_EXECUTABLE: /usr/bin/chromium-browser - # pass GitHub token to allow accurately detecting a build vs a re-run build - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - # make Node's os.tmpdir() return something where we actually have permissions - TMPDIR: ${{ runner.temp }} - - - name: Upload Artifact - if: failure() - uses: actions/upload-artifact@v2 - with: - name: cypress-results - path: | - cypress/screenshots - cypress/videos - cypress/synapselogs - - - name: Store benchmark result - if: github.ref == 'refs/heads/develop' - uses: matrix-org/github-action-benchmark@jsperfentry-1 - with: - name: Cypress measurements - tool: 'jsperformanceentry' - output-file-path: cypress/performance/measurements.json - # The dashboard is available at https://matrix-org.github.io/matrix-react-sdk/cypress/bench/ - benchmark-data-dir-path: cypress/bench - fail-on-alert: false - comment-on-alert: false - github-token: ${{ secrets.DEPLOY_GH_PAGES }} - auto-push: ${{ github.ref == 'refs/heads/develop' }} - - app-tests: - name: Element Web Integration Tests - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - - uses: actions/setup-node@v3 - with: - cache: 'yarn' - - - name: Run tests - run: "./scripts/ci/app-tests.sh" diff --git a/.github/workflows/element-web.yaml b/.github/workflows/element-web.yaml new file mode 100644 index 0000000000..315c8bdcf2 --- /dev/null +++ b/.github/workflows/element-web.yaml @@ -0,0 +1,52 @@ +# Produce a build of element-web with this version of react-sdk +# and any matching branches of element-web and js-sdk, output it +# as an artifact and run integration tests. +name: Element Web - Build +on: + pull_request: { } + push: + branches: [ develop, master ] + repository_dispatch: + types: [ upstream-sdk-notify ] +env: + # These must be set for fetchdep.sh to get the right branch + REPOSITORY: ${{ github.repository }} + PR_NUMBER: ${{ github.event.pull_request.number }} +jobs: + build: + name: "Build Element-Web" + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + + - uses: actions/setup-node@v3 + with: + cache: 'yarn' + + - name: Fetch layered build + id: layered_build + run: | + scripts/ci/layered.sh + JSSDK_SHA=$(git -C matrix-js-sdk rev-parse --short=12 HEAD) + REACT_SHA=$(git rev-parse --short=12 HEAD) + VECTOR_SHA=$(git -C element-web rev-parse --short=12 HEAD) + echo "::set-output name=VERSION::$VECTOR_SHA-react-$REACT_SHA-js-$JSSDK_SHA" + + - name: Copy config + run: cp element.io/develop/config.json config.json + working-directory: ./element-web + + - name: Build + env: + CI_PACKAGE: true + VERSION: "${{ steps.layered_build.outputs.VERSION }}" + run: yarn build + working-directory: ./element-web + + - name: Upload Artifact + uses: actions/upload-artifact@v2 + with: + name: previewbuild + path: element-web/webapp + # We'll only use this in a triggered job, then we're done with it + retention-days: 1 diff --git a/.github/workflows/netlify.yaml b/.github/workflows/netlify.yaml index 3b7b57c227..b577e026f5 100644 --- a/.github/workflows/netlify.yaml +++ b/.github/workflows/netlify.yaml @@ -3,7 +3,7 @@ name: Upload Preview Build to Netlify on: workflow_run: - workflows: [ "Element Web - Build and Test" ] + workflows: [ "Element Web - Build" ] types: - completed jobs: diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 9fa7a6f7cf..f2b97bd189 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -35,3 +35,16 @@ jobs: path: | coverage !coverage/lcov-report + + app-tests: + name: Element Web Integration Tests + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + + - uses: actions/setup-node@v3 + with: + cache: 'yarn' + + - name: Run tests + run: "./scripts/ci/app-tests.sh" diff --git a/docs/cypress.md b/docs/cypress.md index 021f10b215..1f3882a7e7 100644 --- a/docs/cypress.md +++ b/docs/cypress.md @@ -171,9 +171,8 @@ should generally try to adhere to them. ## Percy Visual Testing We also support visual testing via Percy, this extracts the DOM from Cypress and renders it using custom renderers for Safari, Firefox, Chrome & Edge, allowing us to spot visual regressions before they become release regressions. -Right now we run it as part of the standard Pull Request CI automation but due to only having 25k screenshots/month, -and each `cy.percySnapshot()` call results in 8 screenshots (4 browsers, 2 sizes) this could quickly be exhausted and -at that point we would likely run it on a CRON interval or before releases. +Each `cy.percySnapshot()` call results in 8 screenshots (4 browsers, 2 sizes) this can quickly be exhausted and +so we only run Percy testing on `develop` and PRs which are labelled `X-Needs-Percy`. To record a snapshot use `cy.percySnapshot()`, you may have to pass `percyCSS` into the 2nd argument to hide certain elements which contain dynamic/generated data to avoid them cause false positives in the Percy screenshot diffs.