Run compiled functions as plain Python in the coverage check, and drop the per-test variants

.github/workflows/continuous-integration.yaml

@@ -1,8 +1,9 @@
-# This workflow checks every push to main and every pull request that is ready for review.  Its
-# jobs lint the code and check its formatting and types, build the documentation, run the fast
-# example notebooks, run the tests with the per-module coverage check, and run the tests marked
-# compiled.  build-changes.yaml runs further checks, such as the installation check, only when
-# pyproject.toml changes.
+# This workflow checks every push to main and every pull request that is ready for review.  Its jobs
+# lint the code and check its formatting and types, build the documentation, run the fast example
+# notebooks, run the tests with the per-module coverage check, and make the compiled runs.  The
+# coverage check runs compiled functions as plain Python, so that coverage can trace them.
+# build-changes.yaml runs further checks, such as the installation check, only when pyproject.toml
+# changes.
 
 name: Continuous integration
 
@@ -144,13 +145,23 @@ jobs:
           --jobs "$(nproc)" --shard "${{ matrix.shard }}/$SHARDS"
 
   compiled:
-    name: Compiled variants of tests
+    name: ${{ matrix.name }}
     if: github.event.pull_request.draft == false
     runs-on: ubuntu-latest
     timeout-minutes: 5
     env:
       # Keep each test worker to one Numba thread, as checks/pytest_.py does.
       NUMBA_NUM_THREADS: 1
+    strategy:
+      # Let each run finish when the other fails, so that each reports its own failures.
+      fail-fast: false
+      matrix:
+        # See the options in src/qldpc/conftest.py.
+        include:
+          - name: "Compiled run: tests of modules with compiled functions"
+            args: src/qldpc --compiled-module-tests
+          - name: "Compiled run: distance tests with compiled set-up code"
+            args: src/qldpc/codes/distance --compile-search-setup
     steps:
       - uses: actions/checkout@v6
       - name: Set up Python
@@ -159,30 +170,30 @@ jobs:
           python-version: 3.14
           enable-cache: true
       - name: Install package and dependencies
-        # No other job runs the tests marked compiled.  Install the same extras as the coverage job,
-        # so that none of these tests is skipped for lack of an optional package.
+        # Install the same extras as the coverage job, so that no test here is skipped for lack of
+        # an optional package.
         run: uv sync --compile-bytecode --extra bliss --extra decoders --extra gap --group ci-test
-      - name: Run the tests marked compiled
+      - name: Run the tests
         # Keep each test file's tests on one worker, so that fewer workers compile the same
         # functions.
         run: >-
-          uv run --extra bliss --extra decoders --extra gap --group ci-test python -m pytest src
-          -m compiled -n "$(nproc)" --dist=loadfile
+          uv run --extra bliss --extra decoders --extra gap --group ci-test python -m pytest
+          ${{ matrix.args }} -n "$(nproc)" --dist=loadfile
 
   coverage-result:
-    # Combine the results of the coverage shards and of the tests marked compiled into one check
-    # for branch protection to require.  Unlike the names of the shards, its name does not depend
-    # on their number.  It runs even when a job fails or the run is cancelled, since GitHub counts a
-    # skipped required check as passing.
+    # Combine the results of the coverage shards and of the compiled runs into one check for branch
+    # protection to require.  Unlike the names of the shards, its name does not depend on their
+    # number.  It runs even when a job fails or the run is cancelled, since GitHub counts a skipped
+    # required check as passing.
     name: Pytest and coverage check
     if: always() && github.event.pull_request.draft == false
     needs: [coverage, compiled]
     runs-on: ubuntu-latest
     timeout-minutes: 1
     steps:
-      - name: Check that the coverage shards and the tests marked compiled passed
+      - name: Check that the coverage shards and the compiled runs passed
         run: |
           echo "Result of the coverage shards: ${{ needs.coverage.result }}"
-          echo "Result of the tests marked compiled: ${{ needs.compiled.result }}"
+          echo "Result of the compiled runs: ${{ needs.compiled.result }}"
           test "${{ needs.coverage.result }}" = success
           test "${{ needs.compiled.result }}" = success

checks/coverage_.py

@@ -3,9 +3,10 @@
 
 """Run the modular coverage check of checks-superstaq with ``--modular --single-run``.
 
-This wrapper also skips the tests marked ``compiled``, sets ``QLDPC_CLEAR_CACHES`` (see
-``src/qldpc/conftest.py``), and fails the check for each selected module under ``src/``, other than
-a ``conftest.py`` file, that has no test file.
+This wrapper also sets ``QLDPC_DISABLE_COMPILATION``, so that qLDPC's compiled functions run as
+plain Python, which coverage can trace (see ``src/qldpc/_compilation.py``).  It sets
+``QLDPC_CLEAR_CACHES`` (see ``src/qldpc/conftest.py``), and fails the check for each selected module
+under ``src/``, other than a ``conftest.py`` file, that has no test file.
 
 Until a release of checks-superstaq includes ``--single-run``, this runs copies of its coverage
 modules in ``checks/_superstaq_coverage``.  To switch to such a release:
@@ -28,6 +29,7 @@
 import pytest_
 from _superstaq_coverage import coverage_
 
+from qldpc._compilation import DISABLE_COMPILATION_VARIABLE
 from qldpc.conftest import CLEAR_CACHES_VARIABLE
 
 check_utils = checks_superstaq.check_utils
@@ -40,11 +42,12 @@
 
 # Options of the check, which come before the caller's arguments, so that the caller's own options
 # replace them.  The check runs only *.py tests, not notebooks, so it does not need nbmake.
-OPTIONS = ("--modular", "--single-run", "--sysmon", "-pno:nbmake", "-mnot(compiled)")
+OPTIONS = ("--modular", "--single-run", "--sysmon", "-pno:nbmake")
 
 
 def run(*args: str) -> int:
     """Run the check with command-line arguments, and return its exit code."""
+    os.environ[DISABLE_COMPILATION_VARIABLE] = "1"
     os.environ[CLEAR_CACHES_VARIABLE] = "1"
     # checks-superstaq parses the arguments first, so that it handles -h and usage errors.
     returncode = coverage_.run(*OPTIONS, *args, exclude=EXCLUDE)

docs/source/contributing/index.rst

@@ -87,9 +87,11 @@ CI's exact environments and commands are in ``.github/workflows/``.
      - mypy with the repository's typing settings, over source and tests
    * - ``python checks/coverage_.py``
      - Per-module tests and coverage, as in CI: each module under ``src/`` needs its own test file, which must pass and alone cover 100% of the module's statements.
-       It skips the tests marked ``compiled``.
-   * - ``python -m pytest src -m compiled``
-     - The tests marked ``compiled``
+       It runs compiled functions as plain Python.
+   * - ``python -m pytest src/qldpc --compiled-module-tests``
+     - Compiled run, as in CI: the test files of the modules that define compiled functions, with the always-compiled functions compiled
+   * - ``python -m pytest src/qldpc/codes/distance --compile-search-setup``
+     - Compiled run, as in CI: the distance tests, with the compiled-on-request functions compiled too
    * - ``python checks/build_docs.py``
      - Strict Sphinx and notebook documentation build
    * - ``python checks/all_.py``
@@ -105,11 +107,11 @@ The third tier is the extended checks.
 * **As needed:** when writing, reviewing, or verifying a change, run targeted tests that cover it.
   Select tests by function, or run the test file of a changed module, as in ``python -m pytest src/qldpc/codes/common_test.py::test_name``.
   Run ``pytest_.py`` only on selected paths, since over the whole repository it largely repeats the test run of ``coverage_.py``.
-  When targeted tests do not settle a question, such as whether a module's own tests fully cover it, run ``coverage_.py`` on that module and its test file.
+  When targeted tests do not settle a question, such as whether a module's own tests fully cover it, or pass and run fast as plain Python, run ``coverage_.py`` on that module and its test file.
   For example, ``python checks/coverage_.py src/qldpc/codes/common.py src/qldpc/codes/common_test.py`` checks the coverage of one module and its test file.
 * **Extended:** the checks that CI runs on every pull request, other than the notebook runs.
   The extended checks are needed only when the maintainer asks for them by name, or has said that they will be away while a change is completed.
-  Run the anytime checks, ``coverage_.py`` over the whole repository, the tests marked ``compiled``, and ``build_docs.py``.
+  Run the anytime checks, ``coverage_.py`` over the whole repository, the compiled runs, and ``build_docs.py``.
   After fixing a failure that the change causes, verify the fix with the drafting checks, and rerun the failed checks and every extended check that the fix could affect.
   Report any other failure, such as one that also occurs on the main branch, as an open decision.
   Report a failure whose fix needs a decision in the same way.
@@ -119,7 +121,7 @@ Refreshing the saved outputs of a changed notebook, as the :doc:`writing standar
 
 CI runs on pull requests that are ready for review, not on drafts.
 It runs the checks that need extra dependencies or platforms, such as the notebook and GAP integration tests, so these need not run locally.
-It also runs the tests marked ``compiled``, which ``coverage_.py`` skips.
+It also makes the compiled runs.
 Run these locally only in the extended checks, or as targeted tests of a change to compiled code.
 
 ``all_.py`` runs most checks in one command.

docs/source/contributing/standards/testing.rst

@@ -9,9 +9,10 @@ This standard covers what tests should guard and the conditions under which they
   The gate runs the selected test files together on parallel pytest workers, and credits a module only with the lines that its own test file runs in the test process, not in subprocesses (see the module ``_coverage_contexts`` of checks-superstaq).
   Between test files on one worker, it resets the repository's shared fixtures and qLDPC's ``functools`` caches, but not other module-level state.
   Reach a hard-to-reach branch with a focused test rather than a broad scenario.
-* Coverage cannot trace Numba-compiled code, so a test of a module's compiled functions needs a variant that runs their Python versions, as the ``binary_kernels`` fixture in ``src/qldpc/_binary_test.py`` does.
-  Mark the variant that runs the compiled code, never a whole test, with ``pytest.mark.compiled``.
-  The coverage gate skips the tests so marked, and a separate CI job runs them.
+* The coverage gate runs qLDPC's compiled functions as plain Python, so that coverage can trace them.
+  So every test must be fast as plain Python.
+  CI also runs some test files with these functions compiled, in the compiled runs of the :ref:`validation commands <validation-commands>`.
+  A test of behavior that only compiled code has must also pass as plain Python, and must be in a test file that a compiled run selects, unless it compiles its own functions even in the coverage gate, as ``src/qldpc/_compilation_test.py`` does.
 * Tests run with network sockets disabled.
   Mock only external boundaries such as web, subprocess, clipboard, prompt, and optional-package paths, or a module's own thin wrappers around them, never the logic under test, and do not add a live-service test.
 * Disk caches are bypassed while pytest is imported, so tests must not depend on cache persistence.

pyproject.toml

@@ -154,11 +154,6 @@ show_error_codes = true
 
 [tool.pytest]
 ini_options.addopts = "--disable-socket --strict-markers"  # forbid network calls and unknown markers
-# checks/coverage_.py skips the tests marked compiled, and a CI job of their own runs them (see the
-# testing standard).
-ini_options.markers = [
-  "compiled: runs Numba-compiled code, which coverage cannot trace, rather than its Python version",
-]
 ini_options.filterwarnings = [
   "error::DeprecationWarning",                                                # tests use the modern API
   "error::FutureWarning",                                                     # tests opt in to future changes

src/qldpc/_binary_test.py

@@ -18,21 +18,6 @@
 # linear algebra of unpacked binary matrices
 
 
-@pytest.fixture(
-    params=[
-        pytest.param(False, marks=pytest.mark.compiled, id="compiled"),
-        pytest.param(True, id="python"),
-    ]
-)
-def binary_kernels(request: pytest.FixtureRequest, monkeypatch: pytest.MonkeyPatch) -> None:
-    """Run the compiled functions, or their Python versions, which coverage can trace."""
-    if request.param:
-        for name in ("pack_bits", "unpack_bits", "row_reduce_packed", "get_rank_packed"):
-            monkeypatch.setattr(binary, name, getattr(binary, name).py_func)
-        monkeypatch.setattr(binary, "get_inverse", binary.get_inverse.py_func)
-
-
-@pytest.mark.usefixtures("binary_kernels")
 @pytest.mark.parametrize("height, width", [(5, 130), (0, 7), (3, 0)])
 def test_row_reduction_and_rank_match_galois(height: int, width: int) -> None:
     """In the default column order, row reduction gives the reduced row echelon form."""
@@ -47,7 +32,6 @@ def test_row_reduction_and_rank_match_galois(height: int, width: int) -> None:
     assert rank == np.count_nonzero(expected.any(axis=1))
 
 
-@pytest.mark.usefixtures("binary_kernels")
 @pytest.mark.parametrize("columns", ["subset", "repeated"])
 def test_row_reduction_over_selected_columns(columns: str) -> None:
     """Pivots come from the selected columns, and labels undergo the same row operations."""
@@ -96,7 +80,6 @@ def test_get_kernel(shape: tuple[int, int]) -> None:
 # compiled linear algebra
 
 
-@pytest.mark.usefixtures("binary_kernels")
 @pytest.mark.parametrize("width", [0, 64, 65, 130])
 def test_pack_bits(width: int) -> None:
     """Column j goes to bit j % 64 of word j // 64, and unpacking inverts packing."""
@@ -109,7 +92,6 @@ def test_pack_bits(width: int) -> None:
     np.testing.assert_array_equal(binary.unpack_bits(packed, width), matrix)
 
 
-@pytest.mark.usefixtures("binary_kernels")
 def test_get_inverse() -> None:
     """The inverse of a product of unitriangular matrices and a permutation inverts it."""
     rng = np.random.default_rng(70)

src/qldpc/_compilation.py

@@ -48,19 +48,21 @@
 functions that other compiled code calls: every search needs them, and the disk cache makes
 compiling them a one-time cost.  Use :func:`compiled_on_request` for the larger functions that
 orchestrate a search, which select and bound its information sets, build its basis families, and
-drive its scan.  Compiling them pays off only in a program that makes very many searches.  A
-function that is neither runs as ordinary Python.  A small always-compiled helper that hot loops
-call can be marked ``inline=True`` (see :func:`always_compiled`).  A compiled-on-request function
-may call always-compiled and compiled-on-request functions.  An always-compiled function may call
-only always-compiled functions, since it is compiled and cached by itself.
+drive its scan.  Compiling them pays off only in a program that makes very many searches.  Define
+them only in :mod:`qldpc.codes.distance`, whose tests CI runs with them compiled.  A function that
+is neither runs as ordinary Python.  A small always-compiled helper that hot loops call can be
+marked ``inline=True`` (see :func:`always_compiled`).  A compiled-on-request function may call
+always-compiled and compiled-on-request functions.  An always-compiled function may call only
+always-compiled functions, since it is compiled and cached by itself.
 
 If the environment variable ``QLDPC_DISABLE_COMPILATION`` is ``1`` when qLDPC is imported, every
 function of both kinds runs as ordinary Python, much more slowly.  :func:`always_compiled` then
 returns the function itself, :func:`~qldpc.codes.distance.compile_search_setup` leaves the
 compiled-on-request functions as they are, and qLDPC neither uses nor cleans up its directories of
 compiled code.  Coverage tools can then trace every line of these functions, and errors in them show
 Python tracebacks.  Numba's own ``NUMBA_DISABLE_JIT`` does the same for every package that uses
-numba, such as galois, which then runs slowly too.
+numba, such as galois, which then runs slowly too.  qLDPC's coverage check sets
+``QLDPC_DISABLE_COMPILATION``.
 """
 
 from __future__ import annotations
@@ -101,8 +103,8 @@ class _NumbaConfig(Protocol):
 # Numba creates the attributes of its settings dynamically, which type checkers cannot see.
 _numba_config = cast(_NumbaConfig, config)
 
-# If this environment variable is "1" when qLDPC is imported, the functions that numba would compile
-# run as ordinary Python (see the module docstring).
+# If this environment variable is "1" when qLDPC is imported, qLDPC's compiled functions run as
+# ordinary Python (see the module docstring).
 DISABLE_COMPILATION_VARIABLE = "QLDPC_DISABLE_COMPILATION"
 _DISABLED_BY_VARIABLE = os.environ.get(DISABLE_COMPILATION_VARIABLE) == "1"
 

src/qldpc/_compilation_test.py

@@ -3,8 +3,7 @@
 """Tests for compiled functions, and for running their tests compiled or as plain Python.
 
 They cover the disk cache, the registry of compiled-on-request functions, the switch to plain
-Python, the options of the test runs with compiled code, and the rules that make compiled functions
-safe.
+Python, the options of the compiled runs, and the rules for compiled functions.
 """
 
 from __future__ import annotations
@@ -448,6 +447,19 @@ def test_compiled_on_request_functions_are_not_imported_by_name() -> None:
                 assert not imported, f"{path.name} imports {imported} by name"
 
 
+def test_compiled_on_request_functions_are_defined_in_the_distance_package() -> None:
+    """CI runs only the distance tests with compiled-on-request functions compiled."""
+    distance_package = Path(qldpc.__file__).parent / "codes" / "distance"
+    paths = [
+        path
+        for path, tree in _get_source_trees()
+        if _get_functions_decorated_by(tree, "compiled_on_request")
+    ]
+    assert paths
+    outside = [path for path in paths if not path.is_relative_to(distance_package)]
+    assert not outside, outside
+
+
 def test_always_compiled_functions_use_nothing_from_outside_qldpc() -> None:
     """Only qLDPC's source is hashed, but numba checks its version and implements NumPy itself."""
     for path, tree in _get_source_trees():