diff --git a/.github/actions/reusable-deploy/action.yml b/.github/actions/reusable-deploy/action.yml new file mode 100644 index 000000000..bc69a2e43 --- /dev/null +++ b/.github/actions/reusable-deploy/action.yml @@ -0,0 +1,46 @@ +name: "Reusable deploy on builds.joinpeertube.org" + +description: "Reusable deploy on builds.joinpeertube.org" + +inputs: + source: + required: true + description: "Source file/files/directory/directories to deploy" + destination: + required: true + description: "Destination directory on builds.joinpeertube.org" + knownHosts: + required: true + description: "Known hosts" + deployKey: + required: true + description: "Deploy key" + deployUser: + required: true + description: "Deploy user" + deployHost: + required: true + description: "Deploy host" + + +runs: + using: "composite" + + steps: + - name: "Deploy" + shell: bash + run: | + mkdir -p ~/.ssh + chmod 700 ~/.ssh + + echo "Adding ssh key to known hosts" + echo -e "${{ inputs.knownHosts }}" > ~/.ssh/known_hosts; + + eval `ssh-agent -s` + + echo "Adding ssh deploy key" + ssh-add <(echo "${{ inputs.deployKey }}"); + + echo "Uploading files" + + scp ${{ inputs.source }} ${{ inputs.deployUser }}@${{ inputs.deployHost }}:../../web/${{ inputs.destination }}; diff --git a/.github/actions/reusable-prepare-peertube-build/action.yml b/.github/actions/reusable-prepare-peertube-build/action.yml new file mode 100644 index 000000000..41ebf71c5 --- /dev/null +++ b/.github/actions/reusable-prepare-peertube-build/action.yml @@ -0,0 +1,31 @@ +name: "Reusable prepare PeerTube build" + +description: "Reusable prepare PeerTube build" + +inputs: + node-version: + required: true + description: 'NodeJS version' + +runs: + using: "composite" + + steps: + - name: Use Node.js + uses: actions/setup-node@v1 + with: + node-version: ${{ inputs.node-version }} + + - name: Cache Node.js modules + uses: actions/cache@v2 + with: + path: | + **/node_modules + key: ${{ runner.OS }}-node-${{ hashFiles('**/yarn.lock') }} + restore-keys: | + ${{ runner.OS }}-node- + ${{ runner.OS }}- + + - name: Install dependencies + shell: bash + run: yarn install --frozen-lockfile diff --git a/.github/actions/reusable-prepare-peertube-run/action.yml b/.github/actions/reusable-prepare-peertube-run/action.yml new file mode 100644 index 000000000..1a6cd2cfd --- /dev/null +++ b/.github/actions/reusable-prepare-peertube-run/action.yml @@ -0,0 +1,16 @@ +name: "Reusable prepare PeerTube run" +description: "Reusable prepare PeerTube run" + +runs: + using: "composite" + + steps: + - name: Setup system dependencies + shell: bash + run: | + sudo apt-get install postgresql-client-common redis-tools parallel + wget --quiet --no-check-certificate "https://download.cpy.re/ffmpeg/ffmpeg-release-4.3.1-64bit-static.tar.xz" + tar xf ffmpeg-release-4.3.1-64bit-static.tar.xz + mkdir -p $HOME/bin + cp ffmpeg-*/{ffmpeg,ffprobe} $HOME/bin + echo "$HOME/bin" >> $GITHUB_PATH diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 86f675432..f02b88a42 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -29,48 +29,15 @@ jobs: env: PGUSER: peertube PGHOST: localhost - NODE_PENDING_JOB_WAIT: 500 steps: - uses: actions/checkout@v2 - - name: Use Node.js - uses: actions/setup-node@v1 + - uses: './.github/actions/reusable-prepare-peertube-build' with: node-version: '12.x' - - name: Setup system dependencies - run: | - sudo apt-get install postgresql-client-common redis-tools parallel - wget --quiet --no-check-certificate "https://download.cpy.re/ffmpeg/ffmpeg-release-4.3.1-64bit-static.tar.xz" - tar xf ffmpeg-release-4.3.1-64bit-static.tar.xz - mkdir -p $HOME/bin - cp ffmpeg-*/{ffmpeg,ffprobe} $HOME/bin - echo "$HOME/bin" >> $GITHUB_PATH - - - name: Cache Node.js modules - uses: actions/cache@v2 - with: - path: | - **/node_modules - key: ${{ runner.OS }}-node-${{ hashFiles('**/yarn.lock') }} - restore-keys: | - ${{ runner.OS }}-node- - ${{ runner.OS }}- - - - name: Cache fixtures - uses: actions/cache@v2 - with: - path: | - fixtures - key: ${{ runner.OS }}-fixtures-${{ matrix.test_suite }}-${{ hashFiles('fixtures/*') }} - restore-keys: | - ${{ runner.OS }}-fixtures-${{ matrix.test_suite }}- - ${{ runner.OS }}-fixtures- - ${{ runner.OS }}- - - - name: Install dependencies - run: yarn install --frozen-lockfile + - uses: './.github/actions/reusable-prepare-peertube-run' - name: Build run: | @@ -111,27 +78,11 @@ jobs: cat benchmark.json build-time.json startup-time.json - name: Upload benchmark result - env: - STATS_DEPLOYEMENT_KNOWN_HOSTS: ${{ secrets.STATS_DEPLOYEMENT_KNOWN_HOSTS }} - STATS_DEPLOYEMENT_KEY: ${{ secrets.STATS_DEPLOYEMENT_KEY }} - STATS_DEPLOYEMENT_USER: ${{ secrets.STATS_DEPLOYEMENT_USER }} - STATS_DEPLOYEMENT_HOST: ${{ secrets.STATS_DEPLOYEMENT_HOST }} - run: | - mkdir -p ~/.ssh - chmod 700 ~/.ssh - if [ ! -z ${STATS_DEPLOYEMENT_KNOWN_HOSTS+x} ]; then - echo "Adding ssh key to known hosts" - echo -e "${STATS_DEPLOYEMENT_KNOWN_HOSTS}" > ~/.ssh/known_hosts; - fi - - eval `ssh-agent -s` - - if [ ! -z ${STATS_DEPLOYEMENT_KEY+x} ]; then - echo "Adding ssh reployement key" - ssh-add <(echo "${STATS_DEPLOYEMENT_KEY}"); - fi - - if [ ! -z ${STATS_DEPLOYEMENT_KEY+x} ]; then - echo "Uploading files" - scp benchmark.json build-time.json startup-time.json ${STATS_DEPLOYEMENT_USER}@${STATS_DEPLOYEMENT_HOST}:../../web/peertube-stats; - fi + uses: './.github/actions/reusable-deploy.yml' + with: + source: benchmark.json build-time.json startup-time.json + destination: peertube-stats + knownHosts: ${{ secrets.STATS_DEPLOYEMENT_KNOWN_HOSTS }} + deployKey: ${{ secrets.STATS_DEPLOYEMENT_KEY }} + deployUser: ${{ secrets.STATS_DEPLOYEMENT_USER }} + deployHost: ${{ secrets.STATS_DEPLOYEMENT_HOST }} diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 20803bd86..5c0c93886 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -13,7 +13,7 @@ name: "CodeQL" on: push: - branches: [ next ] + branches: [ develop, next ] pull_request: # The branches below must be a subset of the branches above branches: [ next ] diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml new file mode 100644 index 000000000..156b7143a --- /dev/null +++ b/.github/workflows/nightly.yml @@ -0,0 +1,36 @@ +name: Nightly + +on: + push: + branches: + - 'next' + schedule: + - cron: '0 3 * * *' + +jobs: + + nightly: + runs-on: ubuntu-latest + + steps: + - + name: Checkout develop + uses: actions/checkout@v2 + with: + ref: next + + - uses: './.github/actions/reusable-prepare-peertube-build' + with: + node-version: '14.x' + + - name: Build + run: npm run nightly + + - uses: './.github/actions/reusable-deploy' + with: + source: ./peertube-nightly-* + destination: nightly-test + knownHosts: ${{ secrets.STATS_DEPLOYEMENT_KNOWN_HOSTS }} + deployKey: ${{ secrets.STATS_DEPLOYEMENT_KEY }} + deployUser: ${{ secrets.STATS_DEPLOYEMENT_USER }} + deployHost: ${{ secrets.STATS_DEPLOYEMENT_HOST }} diff --git a/.github/workflows/stats.yml b/.github/workflows/stats.yml index e50c14720..c87e6fb77 100644 --- a/.github/workflows/stats.yml +++ b/.github/workflows/stats.yml @@ -20,24 +20,10 @@ jobs: steps: - uses: actions/checkout@v2 - - name: Use Node.js - uses: actions/setup-node@v1 + - uses: './.github/actions/reusable-prepare-peertube-build' with: node-version: '14.x' - - name: Cache Node.js modules - uses: actions/cache@v2 - with: - path: | - **/node_modules - key: ${{ runner.OS }}-node-${{ hashFiles('**/yarn.lock') }} - restore-keys: | - ${{ runner.OS }}-node- - ${{ runner.OS }}- - - - name: Install dependencies - run: yarn install --frozen-lockfile - - name: Angular bundlewatch uses: jackyef/bundlewatch-gh-action@master with: @@ -73,27 +59,11 @@ jobs: - name: Upload stats if: github.event_name != 'pull_request' - env: - STATS_DEPLOYEMENT_KNOWN_HOSTS: ${{ secrets.STATS_DEPLOYEMENT_KNOWN_HOSTS }} - STATS_DEPLOYEMENT_KEY: ${{ secrets.STATS_DEPLOYEMENT_KEY }} - STATS_DEPLOYEMENT_USER: ${{ secrets.STATS_DEPLOYEMENT_USER }} - STATS_DEPLOYEMENT_HOST: ${{ secrets.STATS_DEPLOYEMENT_HOST }} - run: | - mkdir -p ~/.ssh - chmod 700 ~/.ssh - if [ ! -z ${STATS_DEPLOYEMENT_KNOWN_HOSTS+x} ]; then - echo "Adding ssh key to known hosts" - echo -e "${STATS_DEPLOYEMENT_KNOWN_HOSTS}" > ~/.ssh/known_hosts; - fi - - eval `ssh-agent -s` - - if [ ! -z ${STATS_DEPLOYEMENT_KEY+x} ]; then - echo "Adding ssh reployement key" - ssh-add <(echo "${STATS_DEPLOYEMENT_KEY}"); - fi - - if [ ! -z ${STATS_DEPLOYEMENT_KEY+x} ]; then - echo "Uploading files" - scp lighthouse.json client-build-stats.json scc.json ${STATS_DEPLOYEMENT_USER}@${STATS_DEPLOYEMENT_HOST}:../../web/peertube-stats; - fi + uses: './.github/actions/reusable-deploy' + with: + source: lighthouse.json client-build-stats.json scc.json + destination: peertube-stats + knownHosts: ${{ secrets.STATS_DEPLOYEMENT_KNOWN_HOSTS }} + deployKey: ${{ secrets.STATS_DEPLOYEMENT_KEY }} + deployUser: ${{ secrets.STATS_DEPLOYEMENT_USER }} + deployHost: ${{ secrets.STATS_DEPLOYEMENT_HOST }} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 0366ac49f..030ec3790 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -50,29 +50,11 @@ jobs: steps: - uses: actions/checkout@v2 - - name: Use Node.js - uses: actions/setup-node@v1 + - uses: './.github/actions/reusable-prepare-peertube-build' with: node-version: '12.x' - - name: Setup system dependencies - run: | - sudo apt-get install postgresql-client-common redis-tools parallel - wget --quiet --no-check-certificate "https://download.cpy.re/ffmpeg/ffmpeg-release-4.3.1-64bit-static.tar.xz" - tar xf ffmpeg-release-4.3.1-64bit-static.tar.xz - mkdir -p $HOME/bin - cp ffmpeg-*/{ffmpeg,ffprobe} $HOME/bin - echo "$HOME/bin" >> $GITHUB_PATH - - - name: Cache Node.js modules - uses: actions/cache@v2 - with: - path: | - **/node_modules - key: ${{ runner.OS }}-node-${{ hashFiles('**/yarn.lock') }} - restore-keys: | - ${{ runner.OS }}-node- - ${{ runner.OS }}- + - uses: './.github/actions/reusable-prepare-peertube-run' - name: Cache fixtures uses: actions/cache@v2 @@ -85,9 +67,6 @@ jobs: ${{ runner.OS }}-fixtures- ${{ runner.OS }}- - - name: Install dependencies - run: yarn install --frozen-lockfile - - name: Set env test variable (schedule) if: github.event_name != 'schedule' run: | diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml deleted file mode 100644 index 66c6df761..000000000 --- a/.gitlab-ci.yml +++ /dev/null @@ -1,40 +0,0 @@ -image: chocobozzz/peertube-ci:14 - -stages: - - clients - - docker-nightly - -cache: - key: yarn - paths: - - .yarn-cache - - cached-fixtures - -# build-openapi-clients: -# stage: clients -# only: -# refs: -# - master -# - schedules -# changes: -# - support/doc/api/openapi.yaml -# script: -# - apt-get update -qq -# - apt-get -yqqq install openjdk-8-jre -# - yarn install --pure-lockfile -# - scripts/openapi-peertube-version.sh -# - scripts/openapi-clients.sh - -build-nightly: - stage: docker-nightly - only: - - schedules - script: - - yarn install --pure-lockfile --cache-folder .yarn-cache - - npm run nightly - - mkdir "${HOME}/.ssh" - - chmod 700 "${HOME}/.ssh" - - if [ ! -z ${DEPLOYEMENT_KNOWN_HOSTS+x} ]; then echo -e "${DEPLOYEMENT_KNOWN_HOSTS}" > ${HOME}/.ssh/known_hosts; fi - - eval `ssh-agent -s` - - if [ ! -z ${DEPLOYEMENT_KEY+x} ]; then ssh-add <(echo "${DEPLOYEMENT_KEY}"); fi - - if [ ! -z ${DEPLOYEMENT_KEY+x} ]; then scp ./peertube-nightly-* ${DEPLOYEMENT_USER}@${DEPLOYEMENT_HOST}:../../web/nightly; fi diff --git a/support/doc/development/ci.md b/support/doc/development/ci.md new file mode 100644 index 000000000..7d6eef197 --- /dev/null +++ b/support/doc/development/ci.md @@ -0,0 +1,40 @@ +# Continuous integration + +PeerTube uses Github Actions as a CI platform. +CI tasks are described in `.github/workflows`. + +## benchmark.yml + +*Scheduled* + +Run various benchmarks (build, API etc) and upload results on https://builds.joinpeertube.org/peertube-stats/ to be publicly consumed. + +## codeql.yml + +*Scheduled, on push on develop and on pull request* + +Run CodeQL task to throw code security issues in Github. https://lgtm.com/projects/g/Chocobozzz/PeerTube can also be used. + +## docker.yml + +*Scheduled and on push on master* + +Build `chocobozzz/peertube-webserver:latest`, `chocobozzz/peertube:production-...`, `chocobozzz/peertube:v-...` (only latest PeerTube tag) and `chocobozzz/peertube:develop-...` Docker images. Scheduled to automatically upgrade image software (Debian security issues etc). + +## nightly.yml + +*Scheduled* + +Build PeerTube nightly build (`develop` branch) and upload the release on https://builds.joinpeertube.org/nightly. + +## stats.yml + +*On push on develop* + +Create various PeerTube stats (line of codes, build size, lighthouse report) and upload results on https://builds.joinpeertube.org/peertube-stats/ to be publicly consumed. + +## test.yml + +*Scheduled, on push and pull request* + +Run PeerTube lint and tests.