Stronger, clearer test suite with cross-engine verification #415

Workflow file for this run

	# This is the main workflow for testing the code before and after
	# packaging it.
	# The workflow is divided into three jobs:
	# 1. env-prepare:
	# - Prepare the environment for testing
	# 2. source-test:
	# - Test the code base against the latest code in the repository
	# - Create the Python package
	# - Upload the Python package for the next job
	# 3. package-test:
	# - Download the Python package (including extra files) from the previous job
	# - Install the downloaded Python package
	# - Test the code base against the installed package
	# 4. dashboard-build-trigger:
	# - Trigger the dashboard build workflow to update the code quality
	# metrics on the dashboard

	name: Code and package tests

	on:
	# Trigger the workflow on push
	push:
	branches-ignore: [master, main] # Already verified in PR
	# But do not run this workflow on creating a new tag starting with
	# 'v', e.g. 'v1.0.3' (see publish-pypi.yml)
	tags-ignore: ['v*']
	# Trigger the workflow on pull request
	pull_request:
	branches: ['**']
	# Allows you to run this workflow manually from the Actions tab
	workflow_dispatch:

	# Need permissions to trigger the dashboard build workflow
	permissions:
	actions: write
	contents: read

	# Allow only one concurrent workflow, skipping runs queued between the run
	# in-progress and latest queued. And cancel in-progress runs.
	concurrency:
	group: ${{ github.workflow }}-${{ github.event.pull_request.number \|\| github.ref }}
	cancel-in-progress: true

	# Set the environment variables to be used in all jobs defined in this workflow
	env:
	CI_BRANCH: ${{ github.head_ref \|\| github.ref_name }}
	PY_VERSIONS: '3.12 3.14'
	PIXI_ENVS: 'py-312-env py-314-env'

	jobs:
	# Job 1: Set up environment variables
	env-prepare:
	runs-on: [ubuntu-latest]

	outputs:
	pytest-marks: ${{ steps.set-mark.outputs.pytest_marks }}
	run-pr-tier: ${{ steps.set-mark.outputs.run_pr_tier }}

	steps:
	# Select the test cost tier by branch (see ADR test-suite-and-validation):
	# the pr tier runs on develop/master and pull requests; feature-branch
	# pushes run only the default (fast) tier. The nightly tier runs
	# separately in nightly.yml.
	- name: Set test tier marker expression
	id: set-mark
	run: \|
	# Pull requests and develop/master pushes run the pr tier;
	# feature-branch pushes run only the default (fast) tier. On a
	# pull_request event CI_BRANCH is the contributor branch, so the
	# event type must be checked explicitly. run_pr_tier gates the
	# integration steps: the integration layer is entirely pr-marked,
	# so on the fast tier its selection is empty and pytest would exit
	# 5 ("no tests collected"); skip the step in that case instead.
	if [[ "${{ github.event_name }}" == "pull_request" \|\| "${{ env.CI_BRANCH }}" == "master" \|\| "${{ env.CI_BRANCH }}" == "develop" ]]; then
	echo 'pytest_marks=-m "not nightly"' >> "$GITHUB_OUTPUT"
	echo 'run_pr_tier=true' >> "$GITHUB_OUTPUT"
	else
	echo 'pytest_marks=-m "not pr and not nightly"' >> "$GITHUB_OUTPUT"
	echo 'run_pr_tier=false' >> "$GITHUB_OUTPUT"
	fi

	# Job 2: Test code
	source-test:
	needs: env-prepare # depend on previous job

	strategy:
	fail-fast: false
	matrix:
	os: [ubuntu-24.04, macos-15, windows-2022]

	runs-on: ${{ matrix.os }}

	steps:
	- name: Checkout repository
	uses: actions/checkout@v6

	- name: Set up pixi
	uses: ./.github/actions/setup-pixi
	with:
	environments: ${{ env.PIXI_ENVS }}

	- name: Run unit tests
	shell: bash
	run: \|
	set -euo pipefail

	for py_ver in $PY_VERSIONS; do
	echo
	echo "🔹🔸🔹🔸🔹 Python: $py_ver 🔹🔸🔹🔸🔹"

	env="py-$(echo $py_ver \| tr -d .)-env" # Converts 3.XX -> py-3XX-env

	echo "Running tests in environment: $env"
	pixi run --environment $env unit-tests ${{ needs.env-prepare.outputs.pytest-marks }}
	done

	- name: Run functional tests
	shell: bash
	run: \|
	set -euo pipefail

	for py_ver in $PY_VERSIONS; do
	echo
	echo "🔹🔸🔹🔸🔹 Python: $py_ver 🔹🔸🔹🔸🔹"

	env="py-$(echo $py_ver \| tr -d .)-env" # Converts 3.XX -> py-3XX-env

	echo "Running tests in environment: $env"
	pixi run --environment $env functional-tests ${{ needs.env-prepare.outputs.pytest-marks }}
	done

	# Integration tests are entirely pr-marked, so they only run in the pr
	# tier (PRs + develop/master). On feature-branch pushes the selection
	# would be empty; skip the step rather than fail on pytest exit code 5.
	- name: Run integration tests ${{ needs.env-prepare.outputs.pytest-marks }}
	if: needs.env-prepare.outputs.run-pr-tier == 'true'
	shell: bash
	run: \|
	set -euo pipefail

	for py_ver in $PY_VERSIONS; do
	echo
	echo "🔹🔸🔹🔸🔹 Python: $py_ver 🔹🔸🔹🔸🔹"

	env="py-$(echo $py_ver \| tr -d .)-env" # Converts 3.XX -> py-3XX-env

	echo "Running tests in environment: $env"
	pixi run --environment $env integration-tests ${{ needs.env-prepare.outputs.pytest-marks }}
	done

	# Delete all local tags when not on a tagged commit to force versioningit
	# to fall back to the configured default-tag, which is '999.0.0' in our case.
	# This is needed for testing the package in the next job, as its version
	# must be higher than the PyPI version for pip to prefer the local version.
	- name: Force using versioningit default tag (non tagged release)
	if: startsWith(github.ref , 'refs/tags/v') != true
	run: git tag --delete $(git tag)

	- name: Build package wheels for all Python versions
	shell: bash
	run: \|
	set -euo pipefail

	for py_ver in $PY_VERSIONS; do
	echo
	echo "🔹🔸🔹🔸🔹 Python: $py_ver 🔹🔸🔹🔸🔹"

	env="py-$(echo $py_ver \| tr -d .)-env" # Converts 3.11 -> py-311-env

	echo "Building wheel in environment: $env"
	pixi run --environment $env dist-build

	echo "Moving built wheel to dist/py$py_ver/"
	pixi run mkdir -p dist/py$py_ver
	pixi run mv dist/*.whl dist/py$py_ver/
	done

	- name: Remove Python cache files before uploading
	shell: bash
	run: pixi run clean-pycache

	# More than one file/dir need to be specified in 'path', to preserve the
	# structure of the dist/ directory, not only its contents.
	- name: Upload package (incl. extras) for next job
	uses: ./.github/actions/upload-artifact
	with:
	name: easydiffraction_${{ matrix.os }}_${{ runner.arch }}
	path: dist/

	# Job 3: Test the package
	package-test:
	# env-prepare provides the tier marker expression; source-test the wheel
	needs: [env-prepare, source-test]

	strategy:
	fail-fast: false
	matrix:
	os: [ubuntu-24.04, macos-15, windows-2022]

	runs-on: ${{ matrix.os }}

	steps:
	- name: Checkout repository
	uses: actions/checkout@v6

	- name: Download package (incl. extras) from previous job
	uses: ./.github/actions/download-artifact
	with:
	# name and path should be taken from the upload step of the previous job
	name: easydiffraction_${{ matrix.os }}_${{ runner.arch }}
	path: dist/

	- name: Set up pixi
	uses: ./.github/actions/setup-pixi
	with:
	environments: ''
	activate-environment: ''
	run-install: false
	frozen: false

	- name: Install easydiffraction from the built wheel
	shell: bash
	run: \|
	set -euo pipefail

	for py_ver in $PY_VERSIONS; do
	echo
	echo "🔹🔸🔹🔸🔹 Python: $py_ver 🔹🔸🔹🔸🔹"

	echo "Initializing pixi project"
	pixi init easydiffraction_py$py_ver
	cd easydiffraction_py$py_ver

	echo "Setting the minimum system requirements"
	pixi project system-requirements add macos 14.0
	pixi project system-requirements add glibc 2.35

	echo "Adding Python $py_ver"
	pixi add "python=$py_ver"

	echo "Adding GNU Scientific Library (required by diffpy.pdffit2)"
	pixi add gsl

	# diffpy.pdffit2 wheel links @rpath/libc++.1.dylib, which must be
	# present in the conda env lib/ dir on macOS (Python propagates its
	# own @loader_path/../lib/ rpath to loaded extensions). Added as
	# platform-specific deps so this is a no-op on Linux/Windows.
	echo "Adding libc++ for macOS (required by diffpy.pdffit2)"
	pixi add --platform osx-arm64 libcxx

	# Doing this in a hacky way, as pixi does not support adding
	# dependencies from a custom PyPI index with a CLI command without
	# specifying full wheel name.
	#echo "Adding pycrysfml from custom PyPI index"
	#echo '' >> pixi.toml
	#echo '[pypi-dependencies]' >> pixi.toml
	#echo 'pycrysfml = { version = ">=0.2.1", index = "https://easyscience.github.io/pypi/" }' >> pixi.toml

	echo "Looking for wheel in ../dist/py$py_ver/"
	ls -l "../dist/py$py_ver/"

	whl_path=(../dist/"py$py_ver"/*.whl)
	if [[ ! -f "${whl_path[0]}" ]]; then
	echo "❌ No wheel found in ../dist/py$py_ver/"
	exit 1
	fi

	whl_abs_path="$(python -c 'import os,sys; print(os.path.abspath(sys.argv[1]))' "${whl_path[0]}")"

	echo "Adding easydiffraction from: $whl_abs_path"
	pixi add --pypi "easydiffraction[dev] @ ${whl_abs_path}"

	echo "Exiting pixi project directory"
	cd ..
	done

	- name: Run unit tests
	shell: bash
	run: \|
	set -euo pipefail

	for py_ver in $PY_VERSIONS; do
	echo
	echo "🔹🔸🔹🔸🔹 Python: $py_ver 🔹🔸🔹🔸🔹"

	echo "Entering pixi project directory easydiffraction_py$py_ver"
	cd easydiffraction_py$py_ver

	echo "Running tests"
	pixi run python -m pytest ../tests/unit/ --color=yes -v ${{ needs.env-prepare.outputs.pytest-marks }}

	echo "Exiting pixi project directory"
	cd ..
	done

	- name: Run functional tests
	shell: bash
	run: \|
	set -euo pipefail

	for py_ver in $PY_VERSIONS; do
	echo
	echo "🔹🔸🔹🔸🔹 Python: $py_ver 🔹🔸🔹🔸🔹"

	echo "Entering pixi project directory easydiffraction_py$py_ver"
	cd easydiffraction_py$py_ver

	echo "Running tests"
	pixi run python -m pytest ../tests/functional/ --color=yes -v ${{ needs.env-prepare.outputs.pytest-marks }}

	echo "Exiting pixi project directory"
	cd ..
	done

	# See the source-test job: integration tests run only in the pr tier.
	- name: Run integration tests ${{ needs.env-prepare.outputs.pytest-marks }}
	if: needs.env-prepare.outputs.run-pr-tier == 'true'
	shell: bash
	run: \|
	set -euo pipefail

	for py_ver in $PY_VERSIONS; do
	echo
	echo "🔹🔸🔹🔸🔹 Python: $py_ver 🔹🔸🔹🔸🔹"

	echo "Entering pixi project directory easydiffraction_py$py_ver"
	cd easydiffraction_py$py_ver

	echo "Running tests"
	pixi run python -m pytest ../tests/integration/ --color=yes -n auto -v ${{ needs.env-prepare.outputs.pytest-marks }}

	echo "Exiting pixi project directory"
	cd ..
	done

	# Job 4: Build and publish dashboard (reusable workflow)
	run-reusable-workflows:
	needs: package-test # depend on previous job
	uses: ./.github/workflows/dashboard.yml
	secrets: inherit

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Stronger, clearer test suite with cross-engine verification #415

Workflow file

Stronger, clearer test suite with cross-engine verification #415

Uh oh!

Workflow file for this run