Merge pull request #46 from dylon/dylon/ci-workflow

Adds CI Pipeline with GitHub Actions

Author: dylon

.github/apt-dependencies.txt

@@ -0,0 +1,10 @@
+autoconf
+cmake
+curl
+git
+jflex
+libtool
+make
+protobuf-compiler
+sbt
+unzip

.github/cabal-dependencies.txt

@@ -0,0 +1,3 @@
+alex
+happy
+BNFC

.github/init-environment

@@ -30,12 +30,10 @@ import shlex
 
 from shared import REMAINDER, get_test_selections
 
-JOB_NAME = 'run_integration_tests'
-
 test_sel = os.environ['TESTS'].strip()
 
 if test_sel == REMAINDER:
-    selections = [*get_test_selections(JOB_NAME)]
+    selections = [*get_test_selections()]
     if selections:
         test_sel = 'not ((' + ') or ('.join(selections) + '))'
     else:

.github/print-integration-test-selection

@@ -1,22 +1,6 @@
 #!/bin/bash
 set -eu -o pipefail
 
-# Run init-environment script to prepare filesystem for GitHub CI caching.
-#
-# Normally the init-environment script prints variables in format that GitHub CI
-# understands[1]. Setting GITHUB_ACTIONS to other value than true turns it into
-# format that can be fed into eval. We don't necessarily need the variables it
-# produces. Those are used only in workflow (cache related variables). But
-# someone may initialize more variables there anyway and expect to see them in
-# integration tests.
-#
-# When [2] is fixed, we can remove this and run whole job in the rchain/buildenv
-# container. See comments in run_integration_tests job for more information.
-#
-# [1] https://help.github.com/en/actions/automating-your-workflow-with-github-actions/development-tools-for-github-actions#set-an-environment-variable-set-env
-# [2] https://github.community/t5/GitHub-Actions/Container-volumes-key-not-mounting-volume/m-p/34798
-eval "$(GITHUB_ACTIONS= ./.github/init-environment || echo false)"
-
 # Store virtualenv in integration-tests/.venv instead of the magic path with
 # hash that Pipenv creates, so that it remains on known path for easy caching.
 export PIPENV_VENV_IN_PROJECT=1
@@ -28,19 +12,20 @@ export PIPENV_VENV_IN_PROJECT=1
 # printed after all tests finish.
 export PYTEST_ADDOPTS="${PYTEST_ADDOPTS-} --log-cli-level=ERROR"
 
+# The virtual environment should have been initialized at this point.
+export _SKIP_VIRTUALENV_INIT=1
+
 if [[ $TESTS != REMAINDER ]]; then
 	# Check code (Pylint & mypy) only once, when running the REMAINDER
 	# selection. Do not remove/modify REMAINDER selection!
 	export _SKIP_CHECK_CODE=1
 fi
 
-test_selection="$(./.github/print-integration-test-selection)"
-
-
-cd integration-tests/
+GITHUB_DIR="$(dirname "$0")"
+TEST_SELECTION="$("$GITHUB_DIR/print-integration-test-selection")"
 
 ret=0
-./run_tests -k "$test_selection" || ret=$?
+./run_tests -k "$TEST_SELECTION" || ret=$?
 
 # Exit code 5:	No tests were collected
 # Source: http://doc.pytest.org/en/latest/usage.html#possible-exit-codes

.github/run-integration-test-selection

@@ -36,13 +36,11 @@ from subprocess import run
 
 from shared import REMAINDER, get_test_selections
 
-JOB_NAME = 'run_unit_tests'
-
 test_sel = os.environ['TESTS'].strip()
 
 if test_sel == REMAINDER:
     sbt_args = []
-    for arg in chain(*map(shlex.split, get_test_selections(JOB_NAME))):
+    for arg in chain(*map(shlex.split, get_test_selections())):
         match = re.match('(?:([-\\w]+)/)?((?:it:)?test)', arg)
         if match:
             project, task = match[1], match[2]

.github/run-unit-test-selection

@@ -4,7 +4,9 @@
 
 import yaml
 
-WORKFLOW_PROJECT_PATH = '.github/workflows/continuous-integration.yml'
+GITHUB_DIR = os.path.dirname(os.path.abspath(__file__))
+REQUIRED_WORKFLOW_PATH = GITHUB_DIR + '/workflows/build-test-and-deploy.yml'
+OPTIONAL_WORKFLOW_PATH = GITHUB_DIR + '/workflows/optional-tests.yml'
 REMAINDER = 'REMAINDER'
 
 
@@ -14,21 +16,21 @@ def get_project_root() -> Path:
     return Path(os.getenv('PROJECT_ROOT', os.getcwd()))
 
 
-def read_workflow() -> dict:
-    """Reads continuous-integration.yml workflow as dict"""
-    workflow_path = get_project_root() / WORKFLOW_PROJECT_PATH
+def read_workflow(workflow_path: Path) -> dict:
+    """Reads YAML workflow as dict"""
     with open(workflow_path) as f:
         return yaml.safe_load(f)
 
 
-def get_test_selections(job_name: str) -> Iterable[str]:
-    workflow = read_workflow()
-    job = workflow['jobs'][job_name]
-
-    try:
-        selections = job['strategy']['matrix']['tests']
-    except KeyError:
-        return []
+def get_test_selections() -> Iterable[str]:
+    project_root = get_project_root()
+    required_workflow_path = project_root / REQUIRED_WORKFLOW_PATH
+    optional_workflow_path = project_root / OPTIONAL_WORKFLOW_PATH
+    required_workflow = read_workflow(required_workflow_path)
+    optional_workflow = read_workflow(optional_workflow_path)
+    selections = \
+        required_workflow['jobs']['run_required_scala_unit_tests']['strategy']['matrix']['tests'] + \
+        optional_workflow['jobs']['run_optional_scala_unit_tests']['strategy']['matrix']['tests']
 
     remainder_found = False