From 4e3cf6621ca7e5061e586fff9d9c706ce110b052 Mon Sep 17 00:00:00 2001 From: Michael Tautschnig Date: Wed, 24 Jun 2026 10:18:52 +0000 Subject: [PATCH 1/2] compiletest: start cargo-kani tests from a clean target directory The cargo-kani test runner reuses a persistent --target-dir (`output_base_dir()/target`) across regression runs. compiletest re-runs a cargo-kani test whenever its inputs change (the stamp is derived from `kani-compiler` and the library/compiler/driver source trees), and the test then reuses whatever is left in that target directory. If the directory is in a stale or partially-built state -- e.g. left over from a regression run that was interrupted or OOM-killed mid-build, or built by an older kani-compiler -- cargo can treat a path-dependency crate as up to date while its compiled artifact is missing or incompatible. The dependent crate then fails to build with errors such as `error[E0463]: can't find crate for ...`, which do not match the test's expected output. This deterministically affected the multi-crate path-dependency tests (e.g. stubbing-do-not-resolve, stubbing-double-extern-path) and would recur on every kani rebuild until the build directory was wiped by hand. Remove the per-test target directory before invoking `cargo kani` so each run starts from a clean, reproducible state. This adds no meaningful cost: a test is only re-run when its inputs changed, in which case cargo would rebuild these crates anyway. Co-authored-by: Kiro --- tools/compiletest/src/runtest.rs | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/tools/compiletest/src/runtest.rs b/tools/compiletest/src/runtest.rs index 3467b092bb76..8ffb4b9c1950 100644 --- a/tools/compiletest/src/runtest.rs +++ b/tools/compiletest/src/runtest.rs @@ -269,11 +269,23 @@ impl TestCx<'_> { let parent_dir = self.testpaths.file.parent().unwrap(); // The name of the function to test is the same as the stem of `*.expected` file let function_name = self.testpaths.file.file_stem().unwrap().to_str().unwrap(); - cargo - .arg("kani") - .arg("--target-dir") - .arg(self.output_base_dir().join("target")) - .current_dir(parent_dir); + // Start each run from a clean target directory. The target directory is + // persistent across regression runs and is reused whenever the test is + // re-run (e.g., after `kani-compiler` or the libraries change, which + // invalidates the test stamp). A target directory left in a stale or + // partially-built state -- for instance from a regression run that was + // interrupted or OOM-killed mid-build -- can make cargo treat a + // path-dependency crate as up-to-date while its compiled artifact is + // missing, producing spurious `can't find crate` (E0463) errors that + // don't match the expected output. This mostly affects multi-crate + // tests (those with path dependencies). Removing the directory first + // guarantees a clean, reproducible build regardless of any leftover + // state. This adds no meaningful cost: the test is only ever re-run + // when its inputs changed, in which case cargo would rebuild these + // crates anyway. + let target_dir = self.output_base_dir().join("target"); + let _ = fs::remove_dir_all(&target_dir); + cargo.arg("kani").arg("--target-dir").arg(&target_dir).current_dir(parent_dir); if test { cargo.arg("--tests"); } From 30a49169fbe52aaebd4fc8b1f376f63ee62db1e3 Mon Sep 17 00:00:00 2001 From: Michael Tautschnig Date: Wed, 24 Jun 2026 11:17:27 +0000 Subject: [PATCH 2/2] compiletest: fail loudly on target-dir cleanup errors and cover coverage mode Address review feedback on PR #4614: - Don't silently ignore `fs::remove_dir_all` errors. A missing directory (first run) is fine, but any other failure (permissions, a held lock, ...) would leave the stale state in place and reintroduce the very non-determinism this guards against, so fail the test loudly instead. - Apply the same cleanup to the `cargo-coverage` mode (`run_cargo_coverage_test`), which reuses the same persistent `output_base_dir()/target` and is susceptible to the same stale-build failures. The logic is now shared via a `clean_cargo_target_dir` helper used by both runners. Co-authored-by: Kiro --- tools/compiletest/src/runtest.rs | 57 ++++++++++++++++++++++---------- 1 file changed, 40 insertions(+), 17 deletions(-) diff --git a/tools/compiletest/src/runtest.rs b/tools/compiletest/src/runtest.rs index 8ffb4b9c1950..151222b36985 100644 --- a/tools/compiletest/src/runtest.rs +++ b/tools/compiletest/src/runtest.rs @@ -258,6 +258,41 @@ impl TestCx<'_> { } } + /// Returns the per-test cargo target directory, after removing any leftover + /// contents so the build starts from a clean, reproducible state. + /// + /// The target directory is persistent across regression runs and is reused + /// whenever a test is re-run (the compiletest stamp is derived from + /// `kani-compiler` and the library/compiler/driver source trees, so any + /// kani rebuild forces a re-run). A target directory left in a stale or + /// partially-built state -- for instance from a regression run that was + /// interrupted or OOM-killed mid-build, or built by an older + /// `kani-compiler` -- can make cargo treat a path-dependency crate as up to + /// date while its compiled artifact is missing or incompatible, producing + /// spurious `can't find crate` (E0463) errors that don't match the expected + /// output. This mostly affects multi-crate tests (those with path + /// dependencies). Removing the directory first guarantees a clean build + /// regardless of any leftover state, at no meaningful cost: a test is only + /// re-run when its inputs changed, in which case cargo would rebuild these + /// crates anyway. + /// + /// A missing directory is fine (e.g. on the first run), but any other + /// removal failure (permissions, a held lock, ...) would leave the stale + /// state in place and reintroduce the non-determinism this guards against, + /// so we fail the test loudly instead of ignoring it. + fn clean_cargo_target_dir(&self) -> PathBuf { + let target_dir = self.output_base_dir().join("target"); + match fs::remove_dir_all(&target_dir) { + Ok(()) => {} + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => fatal_error(&format!( + "failed to remove stale target directory `{}`: {error}", + target_dir.display() + )), + } + target_dir + } + /// Runs cargo-kani on the function specified by the stem of `self.testpaths.file`. /// The `test` parameter controls whether to specify `--tests` to `cargo kani`. /// An error message is printed to stdout if verification output does not @@ -269,22 +304,8 @@ impl TestCx<'_> { let parent_dir = self.testpaths.file.parent().unwrap(); // The name of the function to test is the same as the stem of `*.expected` file let function_name = self.testpaths.file.file_stem().unwrap().to_str().unwrap(); - // Start each run from a clean target directory. The target directory is - // persistent across regression runs and is reused whenever the test is - // re-run (e.g., after `kani-compiler` or the libraries change, which - // invalidates the test stamp). A target directory left in a stale or - // partially-built state -- for instance from a regression run that was - // interrupted or OOM-killed mid-build -- can make cargo treat a - // path-dependency crate as up-to-date while its compiled artifact is - // missing, producing spurious `can't find crate` (E0463) errors that - // don't match the expected output. This mostly affects multi-crate - // tests (those with path dependencies). Removing the directory first - // guarantees a clean, reproducible build regardless of any leftover - // state. This adds no meaningful cost: the test is only ever re-run - // when its inputs changed, in which case cargo would rebuild these - // crates anyway. - let target_dir = self.output_base_dir().join("target"); - let _ = fs::remove_dir_all(&target_dir); + // Start each run from a clean target directory (see `clean_cargo_target_dir`). + let target_dir = self.clean_cargo_target_dir(); cargo.arg("kani").arg("--target-dir").arg(&target_dir).current_dir(parent_dir); if test { cargo.arg("--tests"); @@ -350,12 +371,14 @@ impl TestCx<'_> { let parent_dir = self.testpaths.file.parent().unwrap(); // The name of the function to test is the same as the stem of `*.expected` file let function_name = self.testpaths.file.file_stem().unwrap().to_str().unwrap(); + // Start each run from a clean target directory (see `clean_cargo_target_dir`). + let target_dir = self.clean_cargo_target_dir(); cargo .arg("kani") .arg("--coverage") .arg("-Zsource-coverage") .arg("--target-dir") - .arg(self.output_base_dir().join("target")) + .arg(&target_dir) .current_dir(parent_dir); if "expected" != self.testpaths.file.file_name().unwrap() {