continuous-integration.yml

# GitHub Actions workflow for RChain continuous integration.
#
# For information GitHub Actions see:
# https://help.github.com/en/actions/automating-your-workflow-with-github-actions/getting-started-with-github-actions
# https://help.github.com/en/actions/automating-your-workflow-with-github-actions/configuring-workflows

name: CI
on:
  push:
    branches:
      - staging
      - trying
    tags: '**'
  pull_request:
    branches:
      - dev
      - 'feature/**'
      - 'release/**'

env:
  # This is read by every new JVM. Every JVM thinks it can use up to 80% of
  # total memory available to the system (used or unused). This may not be
  # appropriate when sbt is configured to run tests in parallel in forked JVMs.
  # However, setting this value too low or leaving it at default value, 25% on
  # OpenJDK 11, makes some unit tests occasionally fail on OutOfMemoryError on
  # GitHub runners which have only 7GB of RAM.
  _JAVA_OPTIONS: -XX:MaxRAMPercentage=80.0 -XX:MaxDirectMemorySize=128M

jobs:

  # Most jobs run in rchain/buildenv Docker container. This container image has
  # everything needed to build RChain and run unit and integration tests. For
  # more information about the image see
  # https://github.com/rchain/buildenv/blob/master/Dockerfile


  # Build RChain and save it for next jobs.
  build_base:
    name: Build Base
    runs-on: ubuntu-latest
    container: rchain/buildenv
    outputs:
      VERSION: ${{ env.VERSION }}
      BRANCH: ${{ env.BRANCH }}
      DEV_LATEST_TAG: ${{ env.DEV_LATEST_TAG }}
    steps:
      - name: Clone Repository
        uses: actions/checkout@v1

      - name: Initialize Environment
        shell: bash
        run: |
          set -eu -o pipefail

          ./.github/init-environment

          # Version from Git repository (tag-commit)
          version="`git describe --tags --always`"
          echo "VERSION=$version" >> $GITHUB_ENV

          # Find latest tag on dev branch
          dev_latest_tag="`git describe --tags --abbrev=0 origin/dev`"
          echo "DEV_LATEST_TAG=$dev_latest_tag" >> $GITHUB_ENV

          # Find related HEAD branch
          branch=""
          if [[ $GITHUB_REF =~ ^refs/tags/ ]]; then
            # Tag related to multiple branches leaves empty branch variable
            raw_branch=$(git branch -r --contains ${{ github.ref }})
            if [[ $raw_branch =~ ^[\ ]*origin/([^ ]*)$ ]]; then
              branch="${BASH_REMATCH[1]}"
            fi
          elif [[ $GITHUB_REF =~ ^refs/heads/ ]]; then
            branch=${GITHUB_REF#refs/*/}
          else
            branch=$GITHUB_HEAD_REF
          fi
          echo "BRANCH=$branch" >> $GITHUB_ENV

      - name: Restore Cache
        uses: actions/cache@v1
        with:
          path: ${{ env.CACHE_ROOT }}
          key: ${{ env.CACHE_KEY_PREFIX_SCALA }}${{ env.CACHE_KEY_HASH_SCALA }}
          restore-keys: ${{ env.CACHE_KEY_PREFIX_SCALA }}

      - name: Compile
        run: |
          export SBT_OPTS="$SBT_OPTS -Dsbt.task.timings=true -Xmx2500m -Xss2m"
          sbt update scalafmtCheckAll compile

      - name: Pack Working Tree
        run: tar -H posix -czf ../rchain-worktree.tar.gz .

      - name: Save Working Tree
        uses: actions/upload-artifact@v1
        with:
          name: rchain-worktree
          path: ../rchain-worktree.tar.gz


  # Get compiled RChain and run unit tests.
  run_unit_tests:
    name: Unit Tests
    needs: build_base
    runs-on: ubuntu-latest
    timeout-minutes: 45
    container: rchain/buildenv
    strategy:
      fail-fast: false
      matrix:
        # This runs unit tests in parallel.
        #
        # For each entry a runner node is spawned with entry value in
        # matrix.tests workflow variable, which is also put into TESTS
        # environment variable (see below) and used by last step, execution of
        # .github/run-unit-test-selection, which splits it according to shell
        # rules (except for REMAINDER) and passes it as arguments to sbt.
        #
        # To learn about REMAINDER, see .github/run-unit-test-selection.
        tests:
          - "'casper/test:testOnly coop.rchain.casper.addblock.*'"
          - "'casper/test:testOnly coop.rchain.casper.api.*'"
          - "'casper/test:testOnly coop.rchain.casper.batch1.*'"
          - "'casper/test:testOnly coop.rchain.casper.batch2.*'"
          - "'casper/test:testOnly coop.rchain.casper.engine.*'"
          - "'casper/test:testOnly coop.rchain.casper.genesis.*'"
          - "'casper/test:testOnly coop.rchain.casper.merging.*'"
          - "'casper/test:testOnly coop.rchain.casper.rholang.*'"
          - "'casper/test:testOnly coop.rchain.casper.util.*'"
          - "'casper/test:testOnly coop.rchain.casper.sync.*'"
          - REMAINDER # Do not modify/remove!
    env:
      TESTS: ${{ matrix.tests }}
    steps:
      - name: Clone Repository
        uses: actions/checkout@v1

      - name: Load Working Tree
        uses: actions/download-artifact@v1
        with:
          name: rchain-worktree

      - name: Restore Working Tree
        run: tar -H posix -xzf rchain-worktree/rchain-worktree.tar.gz

      - name: Initialize Environment
        run: ./.github/init-environment

      - name: Restore Cache
        uses: actions/cache@v1
        with:
          path: ${{ env.CACHE_ROOT }}
          key: ${{ env.CACHE_KEY_PREFIX_SCALA }}${{ env.CACHE_KEY_HASH_SCALA }}
          restore-keys: ${{ env.CACHE_KEY_PREFIX_SCALA }}

      - name: Run Unit Tests
        run: |
          export SBT_OPTS="$SBT_OPTS"
          ./.github/run-unit-test-selection


  # Get compiled RChain, build Docker image, and save it for next jobs.
  build_docker_image:
    name: Build Docker Image
    needs: build_base
    runs-on: ubuntu-latest
    container: rchain/buildenv
    steps:
      - name: Load Working Tree
        uses: actions/download-artifact@v1
        with:
          name: rchain-worktree

      - name: Restore Working Tree
        run: tar -H posix -xzf rchain-worktree/rchain-worktree.tar.gz

      - name: Initialize Environment
        run: ./.github/init-environment

      - name: Restore Cache
        uses: actions/cache@v1
        with:
          path: ${{ env.CACHE_ROOT }}
          key: ${{ env.CACHE_KEY_PREFIX_SCALA }}${{ env.CACHE_KEY_HASH_SCALA }}
          restore-keys: ${{ env.CACHE_KEY_PREFIX_SCALA }}

      - name: Build Docker Image
        run: sbt node/docker:publishLocal

      - name: Export Docker Image
        run: |
          mkdir ../artifacts
          git describe --tags --always >../artifacts/version.txt
          docker image save coop.rchain/rnode \
              | gzip >../artifacts/rnode-docker.tar.gz

      - name: Save Docker Image
        uses: actions/upload-artifact@v1
        with:
          name: artifacts-docker
          path: ../artifacts/


  # Get compiled RChain, build distro packages, and save them for next jobs.
  build_packages:
    name: Build Packages
    needs: build_base
    if: "github.event_name != 'pull_request'"
    runs-on: ubuntu-latest
    container: rchain/buildenv
    steps:
      - name: Load Working Tree
        uses: actions/download-artifact@v1
        with:
          name: rchain-worktree

      - name: Restore Working Tree
        run: tar -H posix -xzf rchain-worktree/rchain-worktree.tar.gz

      - name: Initialize Environment
        run: ./.github/init-environment

      - name: Restore Cache
        uses: actions/cache@v1
        with:
          path: ${{ env.CACHE_ROOT }}
          key: ${{ env.CACHE_KEY_PREFIX_SCALA }}${{ env.CACHE_KEY_HASH_SCALA }}
          restore-keys: ${{ env.CACHE_KEY_PREFIX_SCALA }}

      - name: Build Packages
        run: sbt
          node/universal:packageZipTarball
          node/debian:packageBin
          node/rpm:packageBin

      - name: Export Packages
        run: |
          mkdir ../artifacts
          git describe --tags --always >../artifacts/version.txt
          cp -av \
              node/target/universal/rnode-*.tgz \
              node/target/rnode_*.deb \
              node/target/rpm/RPMS/noarch/rnode-*.rpm \
              ../artifacts/

      - name: Save Packages
        uses: actions/upload-artifact@v1
        with:
          name: artifacts-packages
          path: ../artifacts/


  # Get RNode Docker image and run integration tests.
  #
  # These steps are run directly on runner's host (note there's no container key
  # in the job configuration). That is because bind mounting in container runner
  # is broken[1] and we need to mount host's /tmp onto container's /tmp (see
  # "Running from Docker" in integration-tests/README.md). The host doesn't have
  # everything we need (pipenv, pyenv), so we're going to run integration tests
  # in rchain/buildenv container started manually as the last step.
  #
  # The problem is that host's runner runs everything under a non-privileged
  # account, whereas the rchain/buildenv container runs as root by default. The
  # container image does not have an account corresponding to the host's
  # unprivileged account UID, so we're going to run it as root and do some
  # workarounds (see below).
  #
  # [1] https://github.community/t5/GitHub-Actions/Container-volumes-key-not-mounting-volume/m-p/34798
  run_integration_tests:
    name: Integration Tests
    needs: build_docker_image
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        # This runs integration tests in parallel.
        #
        # For each entry a runner node is spawned with entry value in
        # matrix.tests workflow variable, which is also put into TESTS
        # environment variable (see below) and used by last step, execution of
        # .github/run-integration-test-selection, which passes it verbatim
        # (except for REMAINDER) to Pytest's -k parameter.
        #
        # To learn about REMAINDER, see github/print-integration-test-selection.
        tests:
          - test_backward_compatible
#          - test_dag_correctness
#          - test_finalization
          - test_genesis_ceremony
          - test_propose
#          - test_slash_invalid_block_number
#          - test_slash_invalid_block_seq
#          - test_slash_justification_not_correct
          - test_slash_GHOST_disobeyed
          - test_storage
          - test_wallets
#          - REMAINDER # Do not modify/remove!
    env:
      TESTS: ${{ matrix.tests }}
      # We don't use $CACHE_ROOT in host because it may not be (it is not)
      # writeable by host runner unprivileged account. See note above.
      HOST_CACHE_ROOT: /tmp/rchain-build-cache
      # In integration tests multiple RNode instances are created so memory
      # limit in lower to prevent sporadic crashes.
      _JAVA_OPTIONS: -XX:MaxRAMPercentage=35.0
    steps:
      - name: Clone Repository
        uses: actions/checkout@v1

      - name: Initialize Environment
        # Only initialize cache keys for "Restore Cache". We're running in the
        # host now so we don't want to prepare filesystem here.
        run: _READ_ONLY=1 ./.github/init-environment

      - name: Restore Cache
        uses: actions/cache@v1
        with:
          path: ${{ env.HOST_CACHE_ROOT }}
          key: ${{ env.CACHE_KEY_PREFIX_PYTHON }}${{ env.CACHE_KEY_HASH_PYTHON }}
          restore-keys: ${{ env.CACHE_KEY_PREFIX_PYTHON }}

      - name: Load Docker Image
        uses: actions/download-artifact@v1
        with:
          name: artifacts-docker

      - name: Import Docker Image
        run: zcat artifacts-docker/rnode-docker.tar.gz | docker image load

      # The cached files are owned by host runner unprivileged user. See comment
      # in "Fix Cache Permissions (Post)" step for details. Some programs
      # running in rchain/buildenv container as root may be unhappy about that
      # (pip issues a warning). Change ownership to root to avoid possible
      # problems. There are not that many cached files for integration tests so
      # performance impact is likely minimal.
      - name: Fix Cache Permissions (Pre)
        run: |
          mkdir -p ${{ env.HOST_CACHE_ROOT }}
          sudo chown -R 0:0 ${{ env.HOST_CACHE_ROOT }}

      - name: Run Integration Test
        run: |
          docker run --network=host \
              -v /var/run/docker.sock:/var/run/docker.sock \
              -v /tmp:/tmp \
              -v $PWD:/work \
              -v ${{ env.HOST_CACHE_ROOT }}:$CACHE_ROOT \
              -e TESTS="$TESTS" \
              -e _JAVA_OPTIONS="$_JAVA_OPTIONS" \
              -w /work \
              rchain/buildenv ./.github/run-integration-test-selection

      # Files created in rchain/buildenv container are owned by root, so we need
      # to change ownership to host runner unprivileged account here, because
      # right after this step, the runner is going to save cache preserving file
      # attributes, and it would fail to restore them in "Restore Cache" above
      # in future workflow runs. See note above.
      - name: Fix Cache Permissions (Post)
        run: sudo chown -R $(id -u):$(id -g) ${{ env.HOST_CACHE_ROOT }}


  # release_* jobs make built artifacts available to public and run only on new
  # tags or pushes to "staging" or "trying" branches used by Bors (bors r+ and bors try).
  # These jobs require secrets! See "env" variables and "Secrets" page in GitHub repository
  # settings. Release destinations differ slightly depending on the event that
  # triggered the job (tag or branch push). See "Publish ..." steps for details.
  # VERSION and BRANCH are used from "build_base" job outputs as a new way to share
  # data between jobs and steps. Legacy "version" via artifact file is still used.


  # Upload built Docker image to Docker Hub.
  release_docker_image:
    name: Release Docker Image
    needs:
      - run_unit_tests
      - run_integration_tests
      - build_docker_image
      - build_base
    if: "github.event_name == 'push' && (startsWith(github.ref, 'refs/tags/') || github.ref == 'refs/heads/trying' || github.ref == 'refs/heads/staging')"
    runs-on: ubuntu-latest
    steps:
      - name: Load Docker Image
        uses: actions/download-artifact@v1
        with:
          name: artifacts-docker

      - name: Import Docker Image
        run: zcat artifacts-docker/rnode-docker.tar.gz | docker image load

      - name: Publish Docker Image
        env:
          DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
          DOCKERHUB_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }}
          DOCKER_IMAGE_NAME: ${{ secrets.DOCKER_IMAGE_NAME }}
          # Output from build_base job
          VERSION: ${{ needs.build_base.outputs.VERSION }}
          BRANCH: ${{ needs.build_base.outputs.BRANCH }}
          DEV_LATEST_TAG: ${{ needs.build_base.outputs.DEV_LATEST_TAG }}
        shell: bash
        run: |
          set -eu -o pipefail

          for var in DOCKERHUB_USERNAME DOCKERHUB_PASSWORD DOCKER_IMAGE_NAME; do
            if [[ ! -v $var || -z ${!var} ]]; then
              echo "Required variable $var is not set." >&2
              exit 1
            fi
          done

          suffix=""
          ci_run=""

          # Add suffix if trying or merging
          if [[ $BRANCH =~ ^(trying|staging)$ ]]; then
            suffix="-$BRANCH"

            # Add CI run number if trying
            if [[ $BRANCH =~ ^trying$ ]]; then
              ci_run="-ci-$GITHUB_RUN_NUMBER"
            fi
          fi

          builtin echo "$DOCKERHUB_PASSWORD" \
              | docker login -u "$DOCKERHUB_USERNAME" --password-stdin

          set -x

          image_name="$DOCKER_IMAGE_NAME$suffix"
          img_version_raw="$image_name:$VERSION$ci_run"
          # Replace unsupported character `+`
          img_version="${img_version_raw//[+]/__}"
          img_latest="$image_name:latest"

          # Release Docker image
          docker tag coop.rchain/rnode:latest $img_version
          docker push $img_version

          # Tag Docker image as latest if tag is the latest on dev branch
          if [[ $GITHUB_REF =~ ^refs/tags/ ]] && [[ $DEV_LATEST_TAG =~ $VERSION ]]; then
            docker tag coop.rchain/rnode:latest $img_latest
            docker push $img_latest
          fi

  # GitHub (create) release and packages
  release_packages:
    name: Release Packages
    needs:
      - run_unit_tests
      - run_integration_tests
      - build_packages
    if: "github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')"
    runs-on: ubuntu-latest
    steps:
      - name: Load Packages
        uses: actions/download-artifact@v1
        with:
          name: artifacts-packages
      - name: Update release and release artifacts
        uses: ncipollo/release-action@v1
        with:
          artifacts: artifacts-packages/*
          prerelease: true
          draft: true
          allowUpdates: true
          omitBodyDuringUpdate: true
          token: ${{ secrets.GITHUB_TOKEN }}

  # Job to notify bors when run is successful. Skipped and cancelled job is considered
  # as failure. More info https://github.com/bors-ng/bors-ng/issues/1115.
  bors_success:
    name: bors build finished
    needs:
      - release_docker_image
    if: "github.event_name == 'push' && (github.ref == 'refs/heads/trying' || github.ref == 'refs/heads/staging') && success()"
    runs-on: ubuntu-latest
    steps:
      - run: true