From 26f1d0533106d7fa61c1c63e290ad0af1168433e Mon Sep 17 00:00:00 2001 From: Graffioh Date: Sat, 5 Sep 2026 11:49:08 +0000 Subject: [PATCH] test: make unit-test selection fail closed and expand hosted coverage --- .github/workflows/ci.yml | 4 +- harness/tests/README.md | 31 ++++++++ harness/tests/test_cppunit_runner.py | 96 +++++++++++++++++++++++++ pyproject.toml | 9 +++ server/cmake/DiscoverCppUnitTests.cmake | 4 ++ server/test/CppUnitTestFramework.hpp | 9 ++- 6 files changed, 150 insertions(+), 3 deletions(-) create mode 100644 harness/tests/README.md create mode 100644 harness/tests/test_cppunit_runner.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 679bf5d8f..7aacd5506 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -34,8 +34,8 @@ jobs: - name: Lint Python surfaces touched by lucebox tooling run: uv run --frozen --extra dev ruff check . - - name: Test DS4 benchmark tools - run: uv run --frozen --extra dev pytest -q harness/tests/test_ds4_benchmark_tools.py + - name: Run model-free unit tests + run: uv run --frozen --extra dev pytest -q build: name: Build (cmake + uv sync --extra megakernel) diff --git a/harness/tests/README.md b/harness/tests/README.md new file mode 100644 index 000000000..78354717f --- /dev/null +++ b/harness/tests/README.md @@ -0,0 +1,31 @@ +# Model-free merge checks + +From the repository root, run the same suite as hosted CI: + +```sh +uv run --frozen --extra dev pytest -q +``` + +`pyproject.toml` owns discovery: `harness/tests`, the concurrency benchmark +unit suites, and `server/tests/test_server_parallel_unit.py`. New tests under +the two directories are collected automatically. Tests that need model weights, +a live inference server, or GPU execution must remain explicitly invoked. + +The C++ runner contract tests compile tiny binaries with `c++` (C++17) and use +`cmake`/`ctest` to exercise discovery and exit statuses. Those tools are required; +missing tools fail the suite instead of skipping the gate. No CUDA/HIP toolkit +is needed. Subprocesses and binaries are confined to pytest temporary directories. + +The runner returns 1 for assertion failures, exceptions, and empty selections; +77 is reserved for selections whose tests explicitly skip. `--exact` matches +complete test names only; ordinary selectors still match substrings and tags. +CTest discovery rejects binaries that register no tests. + +Run one suite directly when iterating, for example: + +```sh +uv run --frozen --extra dev pytest -q harness/tests/test_cppunit_runner.py +``` + +This suite complements the CMake server unit tests and hardware qualification; +it does not establish kernel correctness or model quality by itself. diff --git a/harness/tests/test_cppunit_runner.py b/harness/tests/test_cppunit_runner.py new file mode 100644 index 000000000..1a10c0474 --- /dev/null +++ b/harness/tests/test_cppunit_runner.py @@ -0,0 +1,96 @@ +"""Exercise the real C++ runner and CTest adapter without a GPU toolkit.""" + +import subprocess +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[2] + + +@pytest.fixture(scope="module") +def binaries(tmp_path_factory): + directory = tmp_path_factory.mktemp("cppunit") + source = directory / "cases.cpp" + source.write_text('''#include "CppUnitTestFramework.hpp" +#include +struct Runner {}; +TEST_CASE(Runner, pass) { CHECK(true); } +TEST_CASE(Runner, pass_suffix) { CHECK(false); } +TEST_CASE_WITH_TAGS(Runner, tagged, "Runner::pass", "tag") { CHECK(false); } +TEST_CASE(Runner, check_failure) { CHECK(false); } +TEST_CASE(Runner, require_failure) { REQUIRE(false); } +TEST_CASE(Runner, exception) { throw std::runtime_error("intentional"); } +TEST_CASE(Runner, skip) { SKIP("unavailable fixture"); } +''') + result = {} + for name, sources in {"cases": [source], "empty": []}.items(): + binary = directory / name + subprocess.run( + ["c++", "-std=c++17", "-Wall", "-Wextra", "-Werror", "-I", str(ROOT / "server/test"), + str(ROOT / "server/test/test_unit_main.cpp"), *map(str, sources), "-o", str(binary)], + check=True, capture_output=True, text=True, timeout=60, + ) + result[name] = binary + return result + + +@pytest.mark.parametrize(("arguments", "expected"), [ + (["--exact", "Runner::pass"], 0), + (["--exact", "Runner::check_failure"], 1), + (["--exact", "Runner::require_failure"], 1), + (["--exact", "Runner::exception"], 1), + (["--exact", "Runner::skip"], 77), + (["--exact", "Runner::missing"], 1), + (["missing"], 1), + (["--exact", "tag"], 1), + (["tag"], 1), + (["Runner::pass"], 1), + (["--exact", "Runner::pass", "Runner::skip"], 0), + (["--exact", "Runner::check_failure", "Runner::skip"], 1), + (["--unknown-option"], 2), +]) +def test_runner_exit_status(binaries, arguments, expected): + result = subprocess.run( + [str(binaries["cases"]), *arguments], capture_output=True, text=True, timeout=10, + ) + assert result.returncode == expected, result.stdout + result.stderr + if arguments == ["tag"]: + assert "Failed: 1" in result.stdout + if arguments == ["--exact", "tag"]: + assert "No test cases matched" in result.stderr + + +def test_empty_binary_fails(binaries): + result = subprocess.run([str(binaries["empty"])], capture_output=True, text=True, timeout=10) + assert result.returncode == 1 + assert "No test cases matched" in result.stderr + + +def discover(binary, directory): + return subprocess.run( + ["cmake", f"-DTEST_EXECUTABLE={binary}", f"-DTEST_WORKING_DIR={directory}", + f"-DCTEST_FILE={directory / 'discovered.cmake'}", "-DTEST_PREFIX=probe.", + "-P", str(ROOT / "server/cmake/DiscoverCppUnitTests.cmake")], + capture_output=True, text=True, timeout=15, + ) + + +def test_discovery_registers_exact_cases_and_preserves_skip(binaries, tmp_path): + result = discover(binaries["cases"], tmp_path) + assert result.returncode == 0, result.stdout + result.stderr + (tmp_path / "CTestTestfile.cmake").write_text('include("discovered.cmake")\n') + result = subprocess.run( + ["ctest", "--test-dir", str(tmp_path), "--output-on-failure", "--no-tests=error", + "-R", r"^probe\.Runner\.(pass|skip)$"], + capture_output=True, text=True, timeout=15, + ) + assert result.returncode == 0, result.stdout + result.stderr + assert "100% tests passed" in result.stdout + assert "(Skipped)" in result.stdout + + +def test_empty_discovery_fails(binaries, tmp_path): + result = discover(binaries["empty"], tmp_path) + assert result.returncode != 0 + assert "No tests discovered" in result.stderr diff --git a/pyproject.toml b/pyproject.toml index 56ae2bf4f..46f9331dd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,6 +16,15 @@ dependencies = [ megakernel = ["qwen35-megakernel-bf16"] dev = ["pytest>=8", "mypy>=1.10,<2", "ruff>=0.14,<1"] +[tool.pytest.ini_options] +# Model-free merge checks. Keep live-server, model, and vendored suites opt-in. +testpaths = [ + "harness/tests", + "harness/benchmarks/concurrency", + "server/tests/test_server_parallel_unit.py", +] +addopts = "--strict-config --strict-markers" + [tool.ruff] target-version = "py312" line-length = 100 diff --git a/server/cmake/DiscoverCppUnitTests.cmake b/server/cmake/DiscoverCppUnitTests.cmake index 081684e7a..37695f24e 100644 --- a/server/cmake/DiscoverCppUnitTests.cmake +++ b/server/cmake/DiscoverCppUnitTests.cmake @@ -35,6 +35,10 @@ endif() string(REPLACE "\r\n" "\n" _discover_output "${_discover_output}") string(REPLACE "\r" "\n" _discover_output "${_discover_output}") +if(_discover_output STREQUAL "") + message(FATAL_ERROR "No tests discovered from ${TEST_EXECUTABLE}") +endif() + file(WRITE "${CTEST_FILE}" "# Generated by DiscoverCppUnitTests.cmake. Do not edit.\n") diff --git a/server/test/CppUnitTestFramework.hpp b/server/test/CppUnitTestFramework.hpp index 28f8f2a18..d157f8157 100644 --- a/server/test/CppUnitTestFramework.hpp +++ b/server/test/CppUnitTestFramework.hpp @@ -359,6 +359,7 @@ namespace CppUnitTestFramework { logger->BeginRun(all_test_cases.size()); + size_t selected_count = 0; size_t pass_count = 0; size_t fail_count = 0; size_t skip_count = 0; @@ -370,6 +371,7 @@ namespace CppUnitTestFramework { continue; } + ++selected_count; logger->EnterTest(test_case.Name); bool test_failed = true; @@ -402,6 +404,11 @@ namespace CppUnitTestFramework { logger->EndRun(pass_count, fail_count, skip_count); + if (selected_count == 0) { + std::cerr << "No test cases matched the selection" << std::endl; + return 1; + } + if (fail_count != 0) { return 1; } @@ -432,7 +439,7 @@ namespace CppUnitTestFramework { } for (auto& tag : test_tags) { - if (tag == keyword) { + if (!options->ExactMatch && tag == keyword) { return true; } }