Run CI in Github Actions (#22054)

* Tidy workflows * Update get-version-from-git.sh * Run tests in Actions * Upload coverage to SonarCloud * Run linters in Actions * Run builds in Actions, consolidating with sentry uploader * Update sonar-project.properties * chmod +x * Iterate * Use new env vars * Delete yarn-sub.js * Use re-usable sonarqube action * Add README badges * Match reality
2022-05-16 15:23:44 +01:00 · 2022-05-16 15:23:44 +01:00 · 6c87e294c4
parent f080b1fb27
commit 6c87e294c4
18 changed files with 384 additions and 131 deletions
--- a/.editorconfig
+++ b/.editorconfig
@ -24,3 +24,6 @@ trim_trailing_whitespace = true

 [*.{yml,yaml}]
 indent_size = 2
+
+[package.json]
+indent_size = 2
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@ -0,0 +1,26 @@
+name: Build and Package
+on:
+  pull_request: { }
+  push:
+    branches: [ master ]
+# develop pushes and repository_dispatch handled in build_develop.yaml
+env:
+  # These must be set for fetchdep.sh to get the right branch
+  REPOSITORY: ${{ github.repository }}
+  PR_NUMBER: ${{ github.event.pull_request.number }}
+jobs:
+  build:
+    name: "Build"
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+
+      - uses: actions/setup-node@v3
+        with:
+          cache: 'yarn'
+
+      - name: Install Dependencies
+        run: "./scripts/layered.sh"
+
+      - name: Build & Package
+        run: "./scripts/ci_package.sh"
--- a/.github/workflows/build_develop.yml
+++ b/.github/workflows/build_develop.yml
@ -0,0 +1,31 @@
+# Separate to the main build workflow for access to develop
+# environment secrets, largely similar to build.yaml.
+name: Build and Package develop
+on:
+  push:
+    branches: [ develop ]
+  repository_dispatch:
+    types: [ element-web-notify ]
+jobs:
+  build:
+    name: "Build & Upload source maps to Sentry"
+    runs-on: ubuntu-latest
+    environment: develop
+    steps:
+      - uses: actions/checkout@v2
+
+      - uses: actions/setup-node@v3
+        with:
+          cache: 'yarn'
+
+      - name: Install Dependencies
+        run: "./scripts/layered.sh"
+
+      - name: Build, Package & Upload sourcemaps
+        run: "./scripts/ci_package.sh"
+        env:
+          SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }}
+          SENTRY_DSN: ${{ secrets.SENTRY_DSN }}
+          SENTRY_URL: ${{ secrets.SENTRY_URL }}
+          SENTRY_ORG: sentry
+          SENTRY_PROJECT: riot-web
--- a/.github/workflows/sentry-sourcemaps.yaml
+++ b/.github/workflows/sentry-sourcemaps.yaml
@ -1,26 +0,0 @@
-name: Upload Sentry Sourcemaps
-on:
-    push:
-        branches:
-            - develop
-    repository_dispatch:
-        types: [ element-web-notify ]
-jobs:
-    upload-sentry-sourcemaps:
-        runs-on: ubuntu-latest
-        environment: develop
-        steps:
-            -   uses: actions/checkout@v2
-            -   uses: actions/setup-node@v2
-                with:
-                    node-version: '14'
-                    cache: 'yarn'
-            -   run: ./scripts/fetch-develop.deps.sh --depth 1
-            -   run: yarn install
-            -   run: ./scripts/ci_package.sh
-                env:
-                    SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }}
-                    SENTRY_DSN: ${{ secrets.SENTRY_DSN }}
-                    SENTRY_URL: ${{ secrets.SENTRY_URL }}
-                    SENTRY_ORG: sentry
-                    SENTRY_PROJECT: riot-web
--- a/.github/workflows/sonarqube.yml
+++ b/.github/workflows/sonarqube.yml
@ -0,0 +1,37 @@
+name: SonarQube
+on:
+  workflow_run:
+    workflows: [ "Tests" ]
+    types:
+      - completed
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+jobs:
+  prdetails:
+    name: ℹ️ PR Details
+    if: github.event.workflow_run.conclusion == 'success' && github.event.workflow_run.event == 'pull_request'
+    uses: matrix-org/matrix-js-sdk/.github/workflows/pr_details.yml@develop
+    with:
+      owner: ${{ github.event.workflow_run.head_repository.owner.login }}
+      branch: ${{ github.event.workflow_run.head_branch }}
+
+  sonarqube:
+    name: 🩻 SonarQube
+    needs: prdetails
+    # Only wait for prdetails if it isn't skipped
+    if: |
+      always() &&
+      (needs.prdetails.result == 'success' || needs.prdetails.result == 'skipped') &&
+      github.event.workflow_run.conclusion == 'success'
+    uses: matrix-org/matrix-js-sdk/.github/workflows/sonarcloud.yml@develop
+    with:
+      repo: ${{ github.event.workflow_run.head_repository.full_name }}
+      pr_id: ${{ needs.prdetails.outputs.pr_id }}
+      head_branch: ${{ needs.prdetails.outputs.head_branch || github.event.workflow_run.head_branch }}
+      base_branch: ${{ needs.prdetails.outputs.base_branch }}
+      revision: ${{ github.event.workflow_run.head_sha }}
+      coverage_workflow_name: tests.yml
+      coverage_run_id: ${{ github.event.workflow_run.id }}
+    secrets:
+      SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
--- a/.github/workflows/static_analysis.yaml
+++ b/.github/workflows/static_analysis.yaml
@ -0,0 +1,94 @@
+name: Static Analysis
+on:
+  pull_request: { }
+  push:
+    branches: [ develop, master ]
+  repository_dispatch:
+    types: [ element-web-notify ]
+env:
+  # These must be set for fetchdep.sh to get the right branch
+  REPOSITORY: ${{ github.repository }}
+  PR_NUMBER: ${{ github.event.pull_request.number }}
+jobs:
+  ts_lint:
+    name: "Typescript Syntax Check"
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+
+      - uses: actions/setup-node@v3
+        with:
+          cache: 'yarn'
+
+      - name: Install Dependencies
+        run: "./scripts/layered.sh"
+
+      - name: Typecheck
+        run: "yarn run lint:types"
+
+  i18n_lint:
+    name: "i18n Check"
+    runs-on: ubuntu-latest
+    permissions:
+      pull-requests: read
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: "Get modified files"
+        id: changed_files
+        if: github.event_name == 'pull_request'
+        uses: tj-actions/changed-files@v19
+        with:
+          files: |
+            src/i18n/strings/*
+          files_ignore: |
+            src/i18n/strings/en_EN.json
+      - name: "Assert only en_EN was modified"
+        if: github.event_name == 'pull_request' && steps.changed_files.outputs.any_modified == 'true'
+        run: |
+          echo "You can only modify en_EN.json, do not touch any of the other i18n files as Weblate will be confused"
+          exit 1
+      - uses: actions/setup-node@v3
+        with:
+          cache: 'yarn'
+
+      # Does not need branch matching as only analyses this layer
+      - name: Install Deps
+        run: "yarn install --pure-lockfile"
+
+      - name: i18n Check
+        run: "yarn run diff-i18n"
+
+  js_lint:
+    name: "ESLint"
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+
+      - uses: actions/setup-node@v3
+        with:
+          cache: 'yarn'
+
+      # Does not need branch matching as only analyses this layer
+      - name: Install Deps
+        run: "yarn install --pure-lockfile"
+
+      - name: Run Linter
+        run: "yarn run lint:js"
+
+  style_lint:
+    name: "Style Lint"
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+
+      - uses: actions/setup-node@v3
+        with:
+          cache: 'yarn'
+
+      # Does not need branch matching as only analyses this layer
+      - name: Install Deps
+        run: "yarn install --pure-lockfile"
+
+      - name: Run Linter
+        run: "yarn run lint:style"
--- a/.github/workflows/tests.yaml
+++ b/.github/workflows/tests.yaml
@ -0,0 +1,37 @@
+name: Tests
+on:
+  pull_request: { }
+  push:
+    branches: [ develop, master ]
+  repository_dispatch:
+    types: [ element-web-notify ]
+env:
+  # These must be set for fetchdep.sh to get the right branch
+  REPOSITORY: ${{ github.repository }}
+  PR_NUMBER: ${{ github.event.pull_request.number }}
+jobs:
+  jest:
+    name: Jest
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+
+      - name: Yarn cache
+        uses: actions/setup-node@v3
+        with:
+          cache: 'yarn'
+
+      - name: Install Dependencies
+        run: "./scripts/layered.sh"
+
+      - name: Run tests with coverage
+        run: "yarn coverage --ci"
+
+      - name: Upload Artifact
+        uses: actions/upload-artifact@v2
+        with:
+          name: coverage
+          path: |
+            coverage
+            !coverage/lcov-report
--- a/.github/workflows/triage-incoming.yml
+++ b/.github/workflows/triage-incoming.yml
@ -2,7 +2,7 @@ name: Move new issues into Issue triage board

 on:
  issues:
-    types: [opened]
+    types: [ opened ]

 jobs:
  automate-project-columns:
--- a/.github/workflows/triage-labelled.yml
+++ b/.github/workflows/triage-labelled.yml
@ -2,7 +2,7 @@ name: Move labelled issues to correct projects

 on:
  issues:
-    types: [labeled]
+    types: [ labeled ]

 jobs:
  apply_Z-Labs_label:
--- a/.github/workflows/triage-move-review-requests.yml
+++ b/.github/workflows/triage-move-review-requests.yml
@ -1,7 +1,7 @@
 name: Move pull requests asking for review to the relevant project
 on:
  pull_request_target:
-    types: [review_requested]
+    types: [ review_requested ]

 jobs:
  add_design_pr_to_project:
--- a/.github/workflows/triage-priority-bugs.yml
+++ b/.github/workflows/triage-priority-bugs.yml
@ -2,7 +2,7 @@ name: Move P1 bugs to boards

 on:
  issues:
-    types: [labeled, unlabeled]
+    types: [ labeled, unlabeled ]

 jobs:
  p1_issues_to_team_workboard:
--- a/.github/workflows/triage-unlabelled.yml
+++ b/.github/workflows/triage-unlabelled.yml
@ -2,7 +2,7 @@ name: Move unlabelled from needs info columns to triaged

 on:
  issues:
-    types: [unlabeled]
+    types: [ unlabeled ]

 jobs:
  Move_Unabeled_Issue_On_Project_Board:
--- a/README.md
+++ b/README.md
@ -1,3 +1,12 @@
+[![Chat](https://img.shields.io/matrix/element-web:matrix.org)](https://matrix.to/#/#element-web:matrix.org)
+![Tests](https://github.com/vector-im/element-web/actions/workflows/tests.yaml/badge.svg)
+![Static Analysis](https://github.com/vector-im/element-web/actions/workflows/static_analysis.yaml/badge.svg)
+[![Weblate](https://translate.element.io/widgets/element-web/-/element-web/svg-badge.svg)](https://translate.element.io/engage/element-web/)
+[![Quality Gate Status](https://sonarcloud.io/api/project_badges/measure?project=element-web&metric=alert_status)](https://sonarcloud.io/summary/new_code?id=element-web)
+[![Coverage](https://sonarcloud.io/api/project_badges/measure?project=element-web&metric=coverage)](https://sonarcloud.io/summary/new_code?id=element-web)
+[![Vulnerabilities](https://sonarcloud.io/api/project_badges/measure?project=element-web&metric=vulnerabilities)](https://sonarcloud.io/summary/new_code?id=element-web)
+[![Bugs](https://sonarcloud.io/api/project_badges/measure?project=element-web&metric=bugs)](https://sonarcloud.io/summary/new_code?id=element-web)
+
 Element
 =======

--- a/package.json
+++ b/package.json
@ -49,7 +49,8 @@
    "lint:js-fix": "eslint --fix src",
    "lint:types": "tsc --noEmit --jsx react",
    "lint:style": "stylelint \"res/css/**/*.scss\"",
-    "test": "jest"
+    "test": "jest",
+    "coverage": "yarn test --coverage"
  },
  "dependencies": {
    "@matrix-org/olm": "https://gitlab.matrix.org/api/v4/projects/27/packages/npm/@matrix-org/olm/-/@matrix-org/olm-3.2.8.tgz",
--- a/scripts/get-version-from-git.sh
+++ b/scripts/get-version-from-git.sh
@ -4,7 +4,7 @@
 # these dependencies are git checkouts.

 # Since the deps are fetched from git, we can rev-parse
-REACT_SHA=$(cd node_modules/matrix-react-sdk; git rev-parse --short=12 HEAD)
-JSSDK_SHA=$(cd node_modules/matrix-js-sdk; git rev-parse --short=12 HEAD)
+REACT_SHA=$(git -C node_modules/matrix-react-sdk rev-parse --short=12 HEAD)
+JSSDK_SHA=$(git -C node_modules/matrix-js-sdk rev-parse --short=12 HEAD)
 VECTOR_SHA=$(git rev-parse --short=12 HEAD) # use the ACTUAL SHA rather than assume develop
 echo $VECTOR_SHA-react-$REACT_SHA-js-$JSSDK_SHA
--- a/scripts/layered.sh
+++ b/scripts/layered.sh
@ -0,0 +1,49 @@
+#!/bin/bash
+
+set -x
+
+# Creates a layered environment with the full repo for the app and SDKs cloned
+# and linked. This gives an element-web dev environment ready to build with
+# matching branches of react-sdk's dependencies so that changes can be tested
+# in element-web.
+
+# Note that this style is different from the recommended developer setup: this
+# file nests js-sdk and matrix-react-sdk inside element-web, while the local
+# development setup places them all at the same level. We are nesting them here
+# because some CI systems do not allow moving to a directory above the checkout
+# for the primary repo (element-web in this case).
+
+# Install dependencies, as we'll be using fetchdep.sh from matrix-react-sdk
+yarn install --pure-lockfile
+
+# Pass appropriate repo to fetchdep.sh
+export PR_ORG=vector-im
+export PR_REPO=element-web
+
+# Set up the js-sdk first
+node_modules/matrix-react-sdk/scripts/fetchdep.sh matrix-org matrix-js-sdk
+pushd matrix-js-sdk
+yarn link
+yarn install --pure-lockfile
+popd
+
+# Also set up matrix-analytics-events so we get the latest from
+# the main branch or a branch with matching name
+node_modules/matrix-react-sdk/scripts/fetchdep.sh matrix-org matrix-analytics-events main
+pushd matrix-analytics-events
+yarn link
+yarn install --pure-lockfile
+popd
+
+# Now set up the react-sdk
+node_modules/matrix-react-sdk/scripts/fetchdep.sh matrix-org matrix-react-sdk
+pushd matrix-react-sdk
+yarn link
+yarn link matrix-js-sdk
+yarn link matrix-analytics-events
+yarn install --pure-lockfile
+popd
+
+# Link the layers into element-web
+yarn link matrix-js-sdk
+yarn link matrix-react-sdk
--- a/scripts/yarn-sub.js
+++ b/scripts/yarn-sub.js
@ -1,22 +0,0 @@
-const path = require('path');
-const child_process = require('child_process');
-
-const moduleName = process.argv[2];
-if (!moduleName) {
-    console.error("Expected module name");
-    process.exit(1);
-}
-
-const argString = process.argv.length > 3 ? process.argv.slice(3).join(" ") : "";
-if (!argString) {
-    console.error("Expected an yarn argument string to use");
-    process.exit(1);
-}
-
-const modulePath = path.dirname(require.resolve(`${moduleName}/package.json`));
-
-child_process.execSync("yarn " + argString, {
-    env: process.env,
-    cwd: modulePath,
-    stdio: ['inherit', 'inherit', 'inherit'],
-});
--- a/sonar-project.properties
+++ b/sonar-project.properties
@ -0,0 +1,14 @@
+sonar.projectKey=element-web
+sonar.organization=new_vector_ltd_organization
+
+# Encoding of the source code. Default is default system encoding
+#sonar.sourceEncoding=UTF-8
+
+sonar.sources=src,res
+sonar.tests=test
+sonar.exclusions=__mocks__,docs,element.io,nginx
+
+sonar.typescript.tsconfigPath=./tsconfig.json
+sonar.javascript.lcov.reportPaths=coverage/lcov.info
+sonar.coverage.exclusions=test/**/*
+sonar.testExecutionReportPaths=coverage/test-report.xml