test: fail closed on empty unit selections and expand hosted coverage

.github/workflows/ci.yml

@@ -34,8 +34,8 @@ jobs:
       - name: Lint Python surfaces touched by lucebox tooling
         run: uv run --frozen --extra dev ruff check .
 
-      - name: Test DS4 benchmark tools
-        run: uv run --frozen --extra dev pytest -q harness/tests/test_ds4_benchmark_tools.py
+      - name: Run model-free unit tests
+        run: uv run --frozen --extra dev pytest -q
 
   build:
     name: Build (cmake + uv sync --extra megakernel)

harness/tests/README.md

@@ -0,0 +1,31 @@
+# Model-free merge checks
+
+From the repository root, run the same suite as hosted CI:
+
+```sh
+uv run --frozen --extra dev pytest -q
+```
+
+`pyproject.toml` owns discovery: `harness/tests`, the concurrency benchmark
+unit suites, and `server/tests/test_server_parallel_unit.py`. New tests under
+the two directories are collected automatically. Tests that need model weights,
+a live inference server, or GPU execution must remain explicitly invoked.
+
+The C++ runner contract tests compile tiny binaries with `c++` (C++17) and use
+`cmake`/`ctest` to exercise discovery and exit statuses. Those tools are required;
+missing tools fail the suite instead of skipping the gate. No CUDA/HIP toolkit
+is needed. Subprocesses and binaries are confined to pytest temporary directories.
+
+The runner returns 1 for assertion failures, exceptions, and empty selections;
+77 is reserved for selections whose tests explicitly skip. `--exact` matches
+complete test names only; ordinary selectors still match substrings and tags.
+CTest discovery rejects binaries that register no tests.
+
+Run one suite directly when iterating, for example:
+
+```sh
+uv run --frozen --extra dev pytest -q harness/tests/test_cppunit_runner.py
+```
+
+This suite complements the CMake server unit tests and hardware qualification;
+it does not establish kernel correctness or model quality by itself.

harness/tests/test_cppunit_runner.py

@@ -0,0 +1,96 @@
+"""Exercise the real C++ runner and CTest adapter without a GPU toolkit."""
+
+import subprocess
+from pathlib import Path
+
+import pytest
+
+ROOT = Path(__file__).resolve().parents[2]
+
+
+@pytest.fixture(scope="module")
+def binaries(tmp_path_factory):
+    directory = tmp_path_factory.mktemp("cppunit")
+    source = directory / "cases.cpp"
+    source.write_text('''#include "CppUnitTestFramework.hpp"
+#include <stdexcept>
+struct Runner {};
+TEST_CASE(Runner, pass) { CHECK(true); }
+TEST_CASE(Runner, pass_suffix) { CHECK(false); }
+TEST_CASE_WITH_TAGS(Runner, tagged, "Runner::pass", "tag") { CHECK(false); }
+TEST_CASE(Runner, check_failure) { CHECK(false); }
+TEST_CASE(Runner, require_failure) { REQUIRE(false); }
+TEST_CASE(Runner, exception) { throw std::runtime_error("intentional"); }
+TEST_CASE(Runner, skip) { SKIP("unavailable fixture"); }
+''')
+    result = {}
+    for name, sources in {"cases": [source], "empty": []}.items():
+        binary = directory / name
+        subprocess.run(
+            ["c++", "-std=c++17", "-Wall", "-Wextra", "-Werror", "-I", str(ROOT / "server/test"),
+             str(ROOT / "server/test/test_unit_main.cpp"), *map(str, sources), "-o", str(binary)],
+            check=True, capture_output=True, text=True, timeout=60,
+        )
+        result[name] = binary
+    return result
+
+
+@pytest.mark.parametrize(("arguments", "expected"), [
+    (["--exact", "Runner::pass"], 0),
+    (["--exact", "Runner::check_failure"], 1),
+    (["--exact", "Runner::require_failure"], 1),
+    (["--exact", "Runner::exception"], 1),
+    (["--exact", "Runner::skip"], 77),
+    (["--exact", "Runner::missing"], 1),
+    (["missing"], 1),
+    (["--exact", "tag"], 1),
+    (["tag"], 1),
+    (["Runner::pass"], 1),
+    (["--exact", "Runner::pass", "Runner::skip"], 0),
+    (["--exact", "Runner::check_failure", "Runner::skip"], 1),
+    (["--unknown-option"], 2),
+])
+def test_runner_exit_status(binaries, arguments, expected):
+    result = subprocess.run(
+        [str(binaries["cases"]), *arguments], capture_output=True, text=True, timeout=10,
+    )
+    assert result.returncode == expected, result.stdout + result.stderr
+    if arguments == ["tag"]:
+        assert "Failed:  1" in result.stdout
+    if arguments == ["--exact", "tag"]:
+        assert "No test cases matched" in result.stderr
+
+
+def test_empty_binary_fails(binaries):
+    result = subprocess.run([str(binaries["empty"])], capture_output=True, text=True, timeout=10)
+    assert result.returncode == 1
+    assert "No test cases matched" in result.stderr
+
+
+def discover(binary, directory):
+    return subprocess.run(
+        ["cmake", f"-DTEST_EXECUTABLE={binary}", f"-DTEST_WORKING_DIR={directory}",
+         f"-DCTEST_FILE={directory / 'discovered.cmake'}", "-DTEST_PREFIX=probe.",
+         "-P", str(ROOT / "server/cmake/DiscoverCppUnitTests.cmake")],
+        capture_output=True, text=True, timeout=15,
+    )
+
+
+def test_discovery_registers_exact_cases_and_preserves_skip(binaries, tmp_path):
+    result = discover(binaries["cases"], tmp_path)
+    assert result.returncode == 0, result.stdout + result.stderr
+    (tmp_path / "CTestTestfile.cmake").write_text('include("discovered.cmake")\n')
+    result = subprocess.run(
+        ["ctest", "--test-dir", str(tmp_path), "--output-on-failure", "--no-tests=error",
+         "-R", r"^probe\.Runner\.(pass|skip)$"],
+        capture_output=True, text=True, timeout=15,
+    )
+    assert result.returncode == 0, result.stdout + result.stderr
+    assert "100% tests passed" in result.stdout
+    assert "(Skipped)" in result.stdout
+
+
+def test_empty_discovery_fails(binaries, tmp_path):
+    result = discover(binaries["empty"], tmp_path)
+    assert result.returncode != 0
+    assert "No tests discovered" in result.stderr

pyproject.toml

@@ -16,6 +16,15 @@ dependencies = [
 megakernel = ["qwen35-megakernel-bf16"]
 dev = ["pytest>=8", "mypy>=1.10,<2", "ruff>=0.14,<1"]
 
+[tool.pytest.ini_options]
+# Model-free merge checks. Keep live-server, model, and vendored suites opt-in.
+testpaths = [
+    "harness/tests",
+    "harness/benchmarks/concurrency",
+    "server/tests/test_server_parallel_unit.py",
+]
+addopts = "--strict-config --strict-markers"
+
 [tool.ruff]
 target-version = "py312"
 line-length = 100

server/cmake/DiscoverCppUnitTests.cmake

@@ -35,6 +35,10 @@ endif()
 string(REPLACE "\r\n" "\n" _discover_output "${_discover_output}")
 string(REPLACE "\r" "\n" _discover_output "${_discover_output}")
 
+if(_discover_output STREQUAL "")
+    message(FATAL_ERROR "No tests discovered from ${TEST_EXECUTABLE}")
+endif()
+
 file(WRITE "${CTEST_FILE}"
     "# Generated by DiscoverCppUnitTests.cmake. Do not edit.\n")
 

server/test/CppUnitTestFramework.hpp

@@ -359,6 +359,7 @@ namespace CppUnitTestFramework {
 
             logger->BeginRun(all_test_cases.size());
 
+            size_t selected_count = 0;
             size_t pass_count = 0;
             size_t fail_count = 0;
             size_t skip_count = 0;
@@ -370,6 +371,7 @@ namespace CppUnitTestFramework {
                     continue;
                 }
 
+                ++selected_count;
                 logger->EnterTest(test_case.Name);
 
                 bool test_failed = true;
@@ -402,6 +404,11 @@ namespace CppUnitTestFramework {
 
             logger->EndRun(pass_count, fail_count, skip_count);
 
+            if (selected_count == 0) {
+                std::cerr << "No test cases matched the selection" << std::endl;
+                return 1;
+            }
+
             if (fail_count != 0) {
                 return 1;
             }
@@ -432,7 +439,7 @@ namespace CppUnitTestFramework {
                 }
 
                 for (auto& tag : test_tags) {
-                    if (tag == keyword) {
+                    if (!options->ExactMatch && tag == keyword) {
                         return true;
                     }
                 }