From 40b39da13c37c29d58f054bb6ac686d3d7b927a1 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 15 Nov 2026 09:58:02 +0000 Subject: [PATCH 1/2] fix(trust): never award level 5 without a kernel check A coqc and a lean binary on PATH raised the trust level to 5. No code checks the emitted .v and .lean files, so the level was false. The maximum is now 4 and --require-level-5 always reports Q-504 in this version. Suite case 16 puts fake coqc and lean on PATH and requires the doctor to report level 4. It fails on the old source. Checked with Zig 0.17.1: 10 of 10 unit tests pass; the suite gives 14 passed and 2 failed (cases 2 and 15 fail before and after this change, off the solver versions that the README names). Co-Authored-By: Claude Claude-Session: https://claude.ai/code/session_0136P7Zmp1i8PGVHkq3u2iNz --- CHANGELOG.adoc | 5 +++++ src/core/engine.zig | 26 ++++++++++++++++---------- src/core/errors.zig | 2 +- src/interface/ffi/src/cli_main.zig | 13 +++++++------ tests/quintessence/run_suite.sh | 13 +++++++++++-- 5 files changed, 40 insertions(+), 19 deletions(-) diff --git a/CHANGELOG.adoc b/CHANGELOG.adoc index 28a368e..c528f23 100644 --- a/CHANGELOG.adoc +++ b/CHANGELOG.adoc @@ -10,6 +10,11 @@ https://semver.org/spec/v2.0.0.html[Semantic Versioning]. ==== Fixed +* Trust level (2026-11-15): a `coqc` and a `lean` binary on PATH raised the + reported trust level to 5, and no kernel check ran. The maximum is now 4, + and `--require-level-5` always reports `Q-504` in this version. Suite case + 16 checks this. The README now states the measured state and the limits of + the evidence. * `repo-init` minting a repository created with GitHub's "Use this template" (measured on metadatastician/lintel, 2026-10-14): ** provenance recorded the CHILD's own HEAD as the template commit. The diff --git a/src/core/engine.zig b/src/core/engine.zig index 14ff10a..c8f28a1 100644 --- a/src/core/engine.zig +++ b/src/core/engine.zig @@ -64,6 +64,19 @@ pub const RunSummary = struct { output_text: []const u8, }; +/// snip: constant 4; raise only together with a real batch kernel run (M2/M5). +pub const max_local_trust_level: u8 = 4; + +/// Highest trust level that this build can award for a run with `diag_count` +/// diagnostics. Level 5 needs a Coq and a Lean 4 kernel check of the emitted +/// `.v` and `.lean` obligations. This version emits those files and does not +/// check them, so a `coqc` or `lean` binary on PATH must not raise the level. +pub fn localTrustLevel(diag_count: usize) u8 { + return if (diag_count > 0) 0 else max_local_trust_level; +} + +// Checked end to end by case 16 of tests/quintessence/run_suite.sh. + fn collectExFiles(alloc: std.mem.Allocator, io: std.Io, target: []const u8, out: *std.ArrayList([]const u8)) !void { if (std.mem.endsWith(u8, target, ".ex")) { try out.append(alloc, target); @@ -215,14 +228,7 @@ pub fn runVerify(alloc: std.mem.Allocator, io: std.Io, opts: VerifyOptions) !Run ); } - // Determine genuine trust level honestly (Doctrine #10: zero overclaim!) - const has_batch_kernels = tc.coq_version != null and tc.lean_version != null; - const trust_level: u8 = if (diags.items.len > 0) - 0 - else if (has_batch_kernels) - 5 - else - 4; + const trust_level = localTrustLevel(diags.items.len); if (diags.items.len == 0 and opts.require_level_5 and trust_level < 5) { try diags.append(alloc, .{ @@ -230,7 +236,7 @@ pub fn runVerify(alloc: std.mem.Allocator, io: std.Io, opts: VerifyOptions) !Run .file = if (all_mods.items.len > 0) all_mods.items[0].file else ".", .line = 1, .col = 1, - .message = "Local Warrant/Receipt verified at TrustLevel 4 (`z3` + `cvc5` + `alethe-kernel` + `zig`); `--require-level-5` additionally requires `coqc` and `lean` on PATH for batch promotion", + .message = "Local Warrant/Receipt verified at TrustLevel 4 (`z3` + `cvc5` + `alethe-kernel` + `zig`); `--require-level-5` needs Coq and Lean 4 kernel checks of the emitted `.v` and `.lean` obligations, and this version does not run them", }); } @@ -292,7 +298,7 @@ pub fn runVerify(alloc: std.mem.Allocator, io: std.Io, opts: VerifyOptions) !Run .{ trust_level, if (trust_level >= 5) "complete" else "verified_level_4", - if (has_batch_kernels) "[]" else "[\"coq\",\"lean4\"]", + "[\"coq\",\"lean4\"]", }, )); try rraw.appendSlice(alloc, "{\"kernel\":\"z3\",\"status\":\"unsat\",\"version\":"); diff --git a/src/core/errors.zig b/src/core/errors.zig index a251f9e..a949134 100644 --- a/src/core/errors.zig +++ b/src/core/errors.zig @@ -262,7 +262,7 @@ pub const CATALOGUE = [_]ErrorSpec{ .exit_code = .incomplete_evidence, .rule = "Trust-Level Gate", .falsifier = "Verification reached local Warrant/Receipt TrustLevel 4, whereas `--require-level-5` requires external batch Coq + Lean 4 kernel QEDs.", - .remedy = "Run ECHIDNA batch verification with `coqc` and `lean` installed to promote the Warrant + emitted `.v`/`.lean` obligations to Level 5.", + .remedy = "Level 5 is not available in this version. The emitted `.v` and `.lean` obligations must pass a Coq and a Lean 4 kernel check first (planned, milestones M2 and M5).", }, // Q-6xx: Vacuity probes (V1-V4), mutation score, and bounded counterexamples diff --git a/src/interface/ffi/src/cli_main.zig b/src/interface/ffi/src/cli_main.zig index fc9e8e5..4bd78f8 100644 --- a/src/interface/ffi/src/cli_main.zig +++ b/src/interface/ffi/src/cli_main.zig @@ -81,7 +81,7 @@ pub fn main(init: std.process.Init) !u8 { var raw: std.ArrayList(u8) = .empty; try raw.appendSlice(alloc, "{\"schema\":\"quintessence.doctor/v1\",\"ready\":"); try raw.appendSlice(alloc, if (ok) "true" else "false"); - try raw.appendSlice(alloc, try std.fmt.allocPrint(alloc, ",\"max_local_trust_level\":{d},\"tools\":{{\"z3\":", .{if (tc.coq_version != null and tc.lean_version != null) @as(u8, 5) else @as(u8, 4)})); + try raw.appendSlice(alloc, try std.fmt.allocPrint(alloc, ",\"max_local_trust_level\":{d},\"tools\":{{\"z3\":", .{core.max_local_trust_level})); if (tc.z3_version) |v| try jcs.writeString(alloc, &raw, v) else try raw.appendSlice(alloc, "null"); try raw.appendSlice(alloc, ",\"cvc5\":"); if (tc.cvc5_version) |v| try jcs.writeString(alloc, &raw, v) else try raw.appendSlice(alloc, "null"); @@ -103,24 +103,25 @@ pub fn main(init: std.process.Init) !u8 { \\ [REQUIRED] cvc5 (Alethe) : {s} \\ [REQUIRED] zig : {s} \\ [REQUIRED] alethe-kernel : built-in 0.1.0 - \\ [OPTIONAL] coqc (Level5) : {s} - \\ [OPTIONAL] lean (Level5) : {s} + \\ [UNUSED] coqc : {s} + \\ [UNUSED] lean : {s} \\ [OPTIONAL] idris2 : {s} \\ [OPTIONAL] apalache-mc : {s} \\------------------------------------------------------------ \\Status: {s} (maximum achievable local trust level: Level {d}) + \\Level 5 needs kernel checks of the emitted `.v` and `.lean` files; this version does not run them. \\ , .{ tc.z3_version orelse "MISSING", tc.cvc5_version orelse "MISSING", tc.zig_version orelse "MISSING", - tc.coq_version orelse "not installed (batch Level-5 check deferred; `.v` emitted)", - tc.lean_version orelse "not installed (batch Level-5 check deferred; `.lean` emitted)", + tc.coq_version orelse "not installed", + tc.lean_version orelse "not installed", tc.idris2_version orelse "not installed (`qir/v1` serialized natively)", tc.apalache_version orelse "not installed (`k = 12` BMC discharged via Z3+CVC5; `.tla` emitted)", if (ok) "READY" else "INCOMPLETE (missing required tool)", - if (tc.coq_version != null and tc.lean_version != null) @as(u8, 5) else @as(u8, 4), + core.max_local_trust_level, }, ); try stdout.writeStreamingAll(io, msg); diff --git a/tests/quintessence/run_suite.sh b/tests/quintessence/run_suite.sh index c703d32..9366177 100755 --- a/tests/quintessence/run_suite.sh +++ b/tests/quintessence/run_suite.sh @@ -4,7 +4,7 @@ # # End-to-End Formal Verification & Falsifier Regression Suite for `quintessence`. # -# Executes the compiled native `quintessence` binary across 14 live positive, +# Executes the compiled native `quintessence` binary across 16 live positive, # firewall, K-CUT falsifier, TINI information-flow, MaxPlus slice-budget, binary # tamper, vacuity, and ECHIDNA mutation test cases, asserting BOTH the exact # error code (`Q-xxx`) AND the exact OS exit status (`0..5`). @@ -48,7 +48,7 @@ assert_case() { fi } -echo "=== Quintessence 15-Case Live Formal Verification Suite ===" +echo "=== Quintessence 16-Case Live Formal Verification Suite ===" # 1. Toolchain doctor check assert_case "01. Toolchain doctor ready" 0 "Status: READY" \ @@ -217,6 +217,15 @@ assert_case "14. ECHIDNA mutation engine catches weak @post (Q-603, exit 5)" 5 " assert_case "15. Honest Level-5 batch kernel gate without Coq/Lean (Q-504, exit 4)" 4 "[Q-504]" \ verify examples/quintessence --no-mutation --require-level-5 --out-dir "$WORK/o15" +# 16. Falsifier: `coqc` and `lean` on PATH must not raise the trust level. +# No kernel check of the emitted `.v` and `.lean` files exists yet. +mkdir -p "$WORK/fakebin" +printf '#!/bin/sh\necho "The Coq Proof Assistant, version 9.9.9"\n' > "$WORK/fakebin/coqc" +printf '#!/bin/sh\necho "Lean (version 9.9.9)"\n' > "$WORK/fakebin/lean" +chmod +x "$WORK/fakebin/coqc" "$WORK/fakebin/lean" +PATH="$WORK/fakebin:$PATH" assert_case "16. Tools on PATH do not grant Level 5 (doctor reports Level 4)" 0 "maximum achievable local trust level: Level 4" \ + doctor + echo "------------------------------------------------------------" echo "Suite summary: $PASS passed, $FAIL failed" [ "$FAIL" -eq 0 ] From 03794179e59a08c0c1f839fd4ae89a01e5249505 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 15 Nov 2026 09:58:02 +0000 Subject: [PATCH 2/2] docs: state the measured status and add the v1.0.0 plan The README claimed a proof-DAG kernel checker, a symbolic BMC engine and honest certificates. The status block now gives the measured results and the limits of the evidence. docs/plan/ULTRAPLAN.adoc lists the open holes H1 to H10, the release gates, the order, the model roles and the effort sizes. Co-Authored-By: Claude Claude-Session: https://claude.ai/code/session_0136P7Zmp1i8PGVHkq3u2iNz --- README.adoc | 46 +++++++++++++----- docs/plan/ULTRAPLAN.adoc | 100 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 133 insertions(+), 13 deletions(-) create mode 100644 docs/plan/ULTRAPLAN.adoc diff --git a/README.adoc b/README.adoc index 8e3d6b0..938905f 100644 --- a/README.adoc +++ b/README.adoc @@ -43,15 +43,35 @@ three properties the standard BEAM toolchain does not: [IMPORTANT] ==== -*Current implementation state (`v0.1.0-alpha` vertical slice):* The native Zig -compiler/verifier (`src/core/` + `src/interface/ffi/`), dual Z3 5.3.1 + CVC5 -1.4.2 SMT & Alethe proof-DAG kernel checker, `k = 12` symbolic BMC engine, -Layer C `MaxPlus` / `dlopen` / SHA-256 receipt verifier, 6-kind ECHIDNA mutation -engine, Elixir Mix frontend (`mix.exs`, `lib/`), and 15-case live regression -suite (`tests/quintessence/run_suite.sh`) *compile and execute today*, producing -honest `TrustLevel 4` local certificates. Batch `coqc` + `lean` `TrustLevel 5` -promotion and `beam_lib` bytecode tracing are scheduled in Milestones M1--M5 of -link:docs/architecture/QUINTESSENCE-IMPLEMENTATION-PLAN.adoc[the Implementation Plan]. +*Status: alpha prototype (`0.1.0`). Its output is not a proof certificate.* + +What runs today, checked on 2026-11-15 with Zig 0.17.1, Z3 4.8.12 and cvc5 1.1.2: + +* The Zig engine and the CLI build. The 10 Zig unit tests pass. +* The end-to-end suite (`tests/quintessence/run_suite.sh`, 16 cases) gives 14 + passed and 2 failed with those solver versions. Cases 2 and 15 fail. Two cvc5 + certificates come back empty, and the committed NIF receipt hash does not match + a fresh build. The suite was written for Z3 5.3.1 and cvc5 1.4.2. `mise.toml` + does not pin them, and no CI workflow runs the suite yet. + +Limits of the evidence. Read them before you rely on a result: + +* The Alethe check is structural (`src/core/smt_alethe.zig`). It checks proof + shape, premise order and the final empty clause, and it counts rule names. It + does not validate each rule application. +* The `k = 12` bounded model check unrolls one generic protocol model + (`src/core/layer_b.zig`). It does not model the GenServer code under test. +* Layer C checks one nullable-pointer rule and a list of banned patterns. The + `MaxPlus` bound is a pattern heuristic and not a worst-case execution time. + The NIF receipt records all of C1 to C12 as `proved`. Treat those entries as + not checked. +* The mutation engine is local code that uses Z3 only. It does not call ECHIDNA. +* The maximum trust level is 4. Level 5 needs Coq and Lean 4 kernel checks of the + emitted obligations, and this version does not run them. +* No Agda, Rocq or Lean proof of the design exists in this repository. + link:docs/status/PROOF-STATUS.adoc[The proof ledger] has no project row. + +The route from here to `1.0.0` is in link:docs/plan/ULTRAPLAN.adoc[the plan]. ==== == The pipeline @@ -73,7 +93,7 @@ QF_UFLIA / QF_BV K-CUT + k=12 BMC Idris2 ABI / Zig C1-C12 / MaxPlus v Z3 + CVC5 + Native Alethe Proof-DAG Kernel -> 6-Kind Mutation (>= 95%) v - TrustLevel 4 Warrant/Receipt -> batch Coq + Lean -> Level 5 Receipt + TrustLevel 4 Warrant/Receipt -> batch Coq + Lean (planned) -> Level 5 Receipt ---- == What exists in this repository @@ -119,8 +139,8 @@ QF_UFLIA / QF_BV K-CUT + k=12 BMC Idris2 ABI / Zig C1-C12 / MaxPlus | link:examples/quintessence/README.adoc[Worked example] | `Verisim.Spec`, `Verisim.Entity`, `Verisim.Nif`, `verisim_nif.zig`, JCS NIF - receipt, Idris2 model, and TLA+ protocol model — verified end to end (12 QEDs, - 25,910 Alethe steps, 100.00% mutation score). + receipt, Idris2 model, and TLA+ protocol model. See the status block above + for what the end-to-end run gives today. |=== Architectural rationale is recorded in @@ -133,7 +153,7 @@ link:docs/decisions/0004-three-layer-architecture.adoc[ADR-0004]. just build # build libquintessence.so + CLI ./src/interface/ffi/zig-out/bin/quintessence doctor # check Z3, CVC5, Zig, Alethe kernel ./src/interface/ffi/zig-out/bin/quintessence verify examples/quintessence -just test # 10 Zig unit tests + 15 E2E cases +just test # 10 Zig unit tests + 16 E2E cases just validate # RSR structure, deed, AI guide ---- diff --git a/docs/plan/ULTRAPLAN.adoc b/docs/plan/ULTRAPLAN.adoc new file mode 100644 index 0000000..05d259a --- /dev/null +++ b/docs/plan/ULTRAPLAN.adoc @@ -0,0 +1,100 @@ +// SPDX-License-Identifier: CC-BY-SA-4.0 +// SPDX-FileCopyrightText: 2026 Jonathan D.A. Jewell += Quintessence: plan to v1.0.0 +:revdate: 2026-11-15 +:toc: + +Quintessence is an alpha prototype of a three-layer verifier for Elixir/OTP. +This plan is the execution layer. It does not replace the design documents. +The work packages WP-01 to WP-27 stay in +link:../status/QUINTESSENCE-ROADMAP.adoc[the roadmap]. The milestones M1 to M5 +stay in link:../architecture/QUINTESSENCE-IMPLEMENTATION-PLAN.adoc[the +implementation plan]. Where those two documents disagree, this plan decides +the order. + +The work that is shared with the sibling verifiers (`atoma`, `Apodeixis.jl`, +`bactrian`, `canonry`, `chancel`) is in the family plan, +`docs/plan/FAMILY-ULTRAPLAN.adoc` in `hyperpolymath/atoma`. IDs F1 to F12 and +waves W0 to W5 come from that plan. + +== Measured state on 2026-11-15 + +Tools: Zig 0.17.1, Z3 4.8.12, cvc5 1.1.2. + +* `cd src/interface/ffi && zig build test`: 10 of 10 unit tests passed. +* `bash tests/quintessence/run_suite.sh`: 14 passed, 2 failed (cases 2 and 15). +* The same 2 cases fail with Zig 0.17.0. +* Not run: the Elixir front end (`mix`), Idris2, Apalache, Coq, Lean. + +== Open holes + +These are soundness holes or false claims. Doctrine rule 1 puts them before +all feature work. `H0` is closed. The others are open. + +[cols="1,4,2",options="header"] +|=== +| ID | Hole | Evidence +| H0 (closed) | A `coqc` and a `lean` binary on PATH gave TrustLevel 5 with no kernel check. | Fixed. Suite case 16 fails on the old code. +| H1 | The Alethe check is structural. It accepts a certificate with invalid rule applications. `:rule hole` is counted and not rejected. | `src/core/smt_alethe.zig` +| H2 | The NIF receipt emitter records C1 to C12 as `proved`. The code checks one nullable-pointer rule and a list of banned patterns. | `src/core/layer_c.zig` +| H3 | The committed NIF receipt hash does not match a fresh build on a second host, with Zig 0.17.0 or 0.17.1. The receipt also hard-codes tool versions. | Suite case 2 +| H4 | The `k = 12` bounded model check unrolls a generic model, not the GenServer under test. The Y-6 falsifier discards the Z3 result and prints a fixed trace. | `src/core/layer_b.zig` +| H5 | The mutation engine uses Z3 only and carries the ECHIDNA name. It does not call ECHIDNA. The `BudgetInflate` result comes from a constant comparison, not from a pipeline run. | `src/core/vacuity_mutation.zig` +| H6 | Layer A has special cases for the functions of the worked example. | `src/core/layer_a.zig` +| H7 | `Qir.idr` is not in `abi.ipkg`. No tool type-checks it. | `src/interface/abi.ipkg` +| H8 | Code and README need Zig 0.17. `mise.lock` pins 0.16.0. Z3 and cvc5 are not pinned. | `mise.toml`, `mise.lock` +| H9 | No CI workflow builds or tests the engine. | `.github/workflows/` +| H10 | `examples/quintessence/mix/tasks/verify.ex` calls modules that do not exist. `examples/quintessence/README.adoc` prints output that no tool produced. | read +|=== + +== Definition of v1.0.0 + +The seven family gates apply. The text below is the Quintessence reading. + +[cols="1,4,3",options="header"] +|=== +| Gate | Acceptance | Falsifier +| Language boundary | Closed Elixir subset for each layer. The firewall works on expanded code, not on names. | An aliased or imported banned call passes. +| Source connection | Obligations come from the function body and the handler code. No special case for an example. | A body-only mutant keeps the old evidence. +| Semantics | Layer A contracts and termination. Layer B protocol rules for the real handler. Layer C obligations checked, not pattern-matched. | A NIF with an unchecked obligation gets `proved`. +| Replay | Each certificate passes a real checker through ECHIDNA. Level 5 needs both kernels to check the emitted files. | A certificate with a `hole` step is accepted. +| Practical coverage | Three pilots beyond the worked example. | A second project needs engine changes. +| Assurance | Mutants are killed by the full pipeline. The suite runs in CI on pinned tools. | A mutant survives because the gate used a fast mode. +| Release and estate | README, deed and changelog are true. The NIF receipt reproduces on a clean runner. | The receipt differs between two hosts. +|=== + +== Sequence + +[cols="1,4,1,1,3",options="header"] +|=== +| Stage | Work | Lead | Size | Exit evidence +| Q0 Truth (W0) | Close H2, H3, H8, H9, H10. Sync the template (F1) to get the lock tooling, then add `quintessence.yml` with the unit tests. Pin Zig, Z3 and cvc5 with digests. Make the receipt record what was checked. | Opus for H2 and H3; Sonnet for H8 to H10 | M | The suite passes on a clean runner. The receipt reproduces. No `proved` entry without a check. +| Q1 Evidence spine (W1) | Replace the private JCS code, SMT driver and mutation name with the family kit (F3) and the ECHIDNA client (F7). Close H1 and H5 by use of the ECHIDNA certificate path. Maps to WP-03, WP-08, WP-11, WP-12. | Opus | L | One obligation replays through `echidna prove`. A certificate with a `hole` step fails. +| Q2 Layer A, source-bound (W2) | Remove H6. One pure function, body-derived obligations, body-only mutant rejected. Type-check `Qir.idr` (H7). Maps to WP-02, WP-05 to WP-07, WP-13, and M1. | Opus | L | The W2 family exit evidence, with the reuse measures. +| Q3 Layer C through `atoma` (W3, F8) | Send the Zig NIF to `atoma` for C1 to C12. Keep only the Rustler rules here. Maps to WP-14, WP-15, WP-17, and M4. | Opus | L | A NIF with an alias defect is rejected by a checked result. +| Q4 Layer B, real protocol (W3, F9) | Model the handler under test. Run Apalache. Close H4. Share the protocol layer with `bactrian`. Maps to WP-18 to WP-22, and M3. | Opus, advisor review | XL (split first) | A handler that breaks the protocol is rejected with a real trace. +| Q5 Kernel chain (W4) | Batch Coq and Lean checks of the emitted obligations. Signed receipts. Level 5 becomes possible. Maps to WP-09, WP-26, WP-27, M2 and M5. | Opus | L | `--require-level-5` passes only when both kernels checked the files. +| Q6 Theory and release (W4, W5) | Composition theorem and reply chain (WP-24, WP-25) in Agda, then the Rocq and Lean legs (F11). Three pilots. Release audit (F12). | Opus, advisor review | XL (split first) | All seven gates pass on the release commit. +|=== + +Sonnet does the implementation tasks inside each stage from an Opus brief. +Haiku runs the scout pass before each step and does the mechanical edits. +The advisor reviews the close of each stage and each change to a status word. + +== What Quintessence takes from the family, and gives + +* Takes: the evidence kit (F3), the pin table (F6), the ECHIDNA client (F7), + Zig checks from `atoma` (F8). +* Gives: the error-code and exit-code contract (`src/core/errors.zig`), the + suite harness (`assert_case`), the SARIF output. These are candidates for + the kit. +* Shares with `bactrian`: the actor protocol layer (F9). + +== Stop points + +* If Q0 cannot make the receipt reproduce, remove the claim of a reproducible + receipt from the README and the design, then continue. +* If Q1 shows that ECHIDNA cannot check the certificates, stop and review the + evidence design with the advisor. Do not improve the local Alethe check as a + substitute. +* Do not start Q4 or Q6 before they are split into parts of size L or smaller.