From 406a43f898a4da21849b15da0fe4e0c0b6e5c995 Mon Sep 17 00:00:00 2001 From: Anton Standrik Date: Sun, 13 Sep 2026 13:52:16 +0300 Subject: [PATCH 1/5] fix(test): isolate benchmark harness daemon runtimes scripts/benchmark-index.sh and scripts/benchmark-search-graph.sh ran the product with no runtime or cache of their own: the benchmark repository was indexed into the operator's live store and every one-shot joined the operator's account daemon. Source scripts/test-runtime.sh in both, start the private daemon before timing, record setup-time.txt and total-time.txt beside index-time.txt, and index from a repository path instead of querying a project in the live store. Add tests/test_benchmark_runtime_isolation_contract.sh, which fails before this change, and wire it into scripts/test.sh. Part of #1696. Signed-off-by: Anton Standrik --- scripts/benchmark-index.sh | 20 ++++ scripts/benchmark-search-graph.sh | 38 +++++++- scripts/test.sh | 3 + ...st_benchmark_runtime_isolation_contract.sh | 96 +++++++++++++++++++ 4 files changed, 152 insertions(+), 5 deletions(-) create mode 100755 tests/test_benchmark_runtime_isolation_contract.sh diff --git a/scripts/benchmark-index.sh b/scripts/benchmark-index.sh index 756bda06ed..71e694c171 100755 --- a/scripts/benchmark-index.sh +++ b/scripts/benchmark-index.sh @@ -9,6 +9,14 @@ LANG="${2:?}" REPO="${3:?}" RESULTS_DIR="${4:?}" +# The index must run against a daemon rendezvous and cache this run owns: only +# CBM_RUNTIME_DIR moves the rendezvous, and without a private cache the +# benchmark repository was indexed into the operator's live store (#1696). +# shellcheck source=test-runtime.sh +source "$(dirname "${BASH_SOURCE[0]}")/test-runtime.sh" +cbm_test_runtime_init +trap 'cbm_test_runtime_cleanup "$BINARY"' EXIT + # Resolve symlinks REPO=$(cd "$REPO" && pwd -P) @@ -33,6 +41,16 @@ LOC=$(find "$REPO" -type f \ echo "$FILE_COUNT" > "$OUT/file-count.txt" echo "$LOC" > "$OUT/loc.txt" +# Start the private daemon before timing so index-time.txt measures the index +# alone. setup-time.txt keeps the activation cost attributable and +# total-time.txt is their sum — the figure comparable with earlier runs, which +# paid activation inside the index timing whenever no daemon was already warm. +SETUP_START_MS=$(python3 -c "import time; print(int(time.time()*1000))") +if ! "$BINARY" daemon start >/dev/null 2>&1; then + echo " $LANG: private daemon did not start" >&2 + exit 1 +fi + # Index via CLI and capture timing START_MS=$(python3 -c "import time; print(int(time.time()*1000))") @@ -43,6 +61,8 @@ ELAPSED=$((END_MS - START_MS)) echo "$INDEX_JSON" > "$OUT/00-index.json" echo "$ELAPSED" > "$OUT/index-time.txt" +echo "$((START_MS - SETUP_START_MS))" > "$OUT/setup-time.txt" +echo "$((END_MS - SETUP_START_MS))" > "$OUT/total-time.txt" # Extract node/edge counts (CLI wraps in MCP content envelope) NODES=$(echo "$INDEX_JSON" | python3 -c " diff --git a/scripts/benchmark-search-graph.sh b/scripts/benchmark-search-graph.sh index cc94147ec6..1d67b95818 100755 --- a/scripts/benchmark-search-graph.sh +++ b/scripts/benchmark-search-graph.sh @@ -3,18 +3,46 @@ # codebase-memory-mcp binary to measure the regex / LIKE pre-filter performance. # # Usage: -# scripts/benchmark-search-graph.sh +# scripts/benchmark-search-graph.sh # # Example: -# scripts/benchmark-search-graph.sh ./build/c/codebase-memory-mcp my-project +# scripts/benchmark-search-graph.sh ./build/c/codebase-memory-mcp ~/src/my-project +# +# The repository is indexed (untimed) into a private runtime and cache first; +# the queries then run against that index through a daemon this run keeps warm, +# so a timing never includes daemon activation and never touches the operator's +# live store (#1696). set -euo pipefail -BINARY="${1:?Usage: $0 }" -PROJECT="${2:?Usage: $0 }" +BINARY="${1:?Usage: $0 }" +REPO="${2:?Usage: $0 }" +REPO=$(cd "$REPO" && pwd -P) + +# shellcheck source=test-runtime.sh +source "$(dirname "${BASH_SOURCE[0]}")/test-runtime.sh" +cbm_test_runtime_init +trap 'cbm_test_runtime_cleanup "$BINARY"' EXIT + +if ! "$BINARY" daemon start >/dev/null 2>&1; then + echo "private daemon did not start" >&2 + exit 1 +fi +INDEX_JSON=$("$BINARY" cli index_repository "{\"repo_path\":\"$REPO\",\"mode\":\"full\"}" 2>/dev/null || echo '{}') +PROJECT=$(echo "$INDEX_JSON" | python3 -c " +import json, sys +d = json.load(sys.stdin) +if 'content' in d: + d = json.loads(d['content'][0]['text']) +print(d.get('project', '')) +" 2>/dev/null || echo "") +if [ -z "$PROJECT" ]; then + echo "index of $REPO did not report a project" >&2 + exit 1 +fi echo "Binary: $BINARY" -echo "Project: $PROJECT" +echo "Project: $PROJECT (indexed from $REPO)" echo "" run_case() { diff --git a/scripts/test.sh b/scripts/test.sh index 50d65643f9..77cbf42d40 100755 --- a/scripts/test.sh +++ b/scripts/test.sh @@ -239,6 +239,9 @@ bash "$ROOT/tests/test_smoke_fixture_contract.sh" echo "=== Step 0i: parallel suite scheduler contract ===" bash "$ROOT/tests/test_parallel_harness_contract.sh" +echo "=== Step 0i2: benchmark harness runtime isolation contract (#1696) ===" +bash "$ROOT/tests/test_benchmark_runtime_isolation_contract.sh" + echo "=== Step 0j: venue parity contract (one harness, every venue) ===" bash "$ROOT/tests/test_venue_parity_contract.sh" diff --git a/tests/test_benchmark_runtime_isolation_contract.sh b/tests/test_benchmark_runtime_isolation_contract.sh new file mode 100755 index 0000000000..b944df7683 --- /dev/null +++ b/tests/test_benchmark_runtime_isolation_contract.sh @@ -0,0 +1,96 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Runtime-isolation contract for the benchmark harnesses (#1696, follow-up to +# #1691). +# +# scripts/benchmark-index.sh and scripts/benchmark-search-graph.sh ran the +# product with no runtime or cache of their own: the index landed in the +# operator's live store, every one-shot joined the operator's account daemon, +# and the timings depended on whatever that daemon was doing. Drive both with +# an environment-probe fixture and require that no product process ever +# receives the caller's runtime or cache, and that the index benchmark records +# the setup cost it now pays explicitly. + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +WORKDIR="$(mktemp -d)" +trap 'rm -rf "$WORKDIR"' EXIT + +fail() { + echo "FAIL: $*" >&2 + exit 1 +} + +normalize_path() { + local path=${1%$'\r'} + if command -v cygpath >/dev/null 2>&1; then + cygpath -u "$path" 2>/dev/null && return 0 + fi + printf '%s\n' "${path//\\//}" +} + +ENV_PROBE="$WORKDIR/environment-probe" +cat > "$ENV_PROBE" <<'EOF' +#!/usr/bin/env bash +printf '%s\t%s\n' "${CBM_CACHE_DIR-}" "${CBM_RUNTIME_DIR-}" >> "$CBM_BENCH_ENV_PROBE" +[[ "${1-} ${2-}" == "daemon status" ]] && exit 1 +exit 0 +EOF +chmod +x "$ENV_PROBE" + +CALLER_CACHE="$WORKDIR/caller-cache" +CALLER_RUNTIME="$WORKDIR/caller-runtime" +REPO="$WORKDIR/repo" +mkdir -p "$CALLER_CACHE" "$CALLER_RUNTIME" "$REPO" +echo 'def bench(): return 1' > "$REPO/bench.py" +CALLER_CACHE_NORMALIZED=$(normalize_path "$CALLER_CACHE") +CALLER_RUNTIME_NORMALIZED=$(normalize_path "$CALLER_RUNTIME") + +# The fixture answers nothing, so the search benchmark stops once the index +# reports no project; only the environment handed to the product is under test. +assert_isolated() { + local harness="$1" env_log="$2" private_root="" + [[ -s "$env_log" ]] || fail "$harness did not execute the environment-probe fixture" + while IFS=$'\t' read -r child_cache_raw child_runtime_raw; do + local child_cache child_runtime + child_cache=$(normalize_path "$child_cache_raw") + child_runtime=$(normalize_path "$child_runtime_raw") + if [[ -z "$child_runtime" || "$child_runtime" == "$CALLER_RUNTIME_NORMALIZED" ]]; then + fail "$harness exposed the caller CBM_RUNTIME_DIR to a product process" + fi + if [[ -z "$child_cache" || "$child_cache" == "$CALLER_CACHE_NORMALIZED" ]]; then + fail "$harness exposed the caller CBM_CACHE_DIR to a product process" + fi + if [[ "${child_runtime%/*}" != "${child_cache%/*}" || + "${child_runtime##*/}" != "runtime" || "${child_cache##*/}" != "cache" ]]; then + fail "$harness runtime/cache were not isolated beneath one private root" + fi + if [[ -n "$private_root" && "$private_root" != "${child_runtime%/*}" ]]; then + fail "$harness switched private roots mid-run" + fi + private_root="${child_runtime%/*}" + done < "$env_log" + [[ ! -e "$private_root" ]] || fail "$harness left its private root behind: $private_root" +} + +INDEX_LOG="$WORKDIR/index-environment.log" +CBM_CACHE_DIR="$CALLER_CACHE" \ +CBM_RUNTIME_DIR="$CALLER_RUNTIME" \ +CBM_BENCH_ENV_PROBE="$INDEX_LOG" \ + "$ROOT/scripts/benchmark-index.sh" "$ENV_PROBE" probe "$REPO" "$WORKDIR/results" \ + > "$WORKDIR/index.out" 2>&1 || true +assert_isolated "benchmark-index" "$INDEX_LOG" +for metric in setup-time total-time index-time; do + [[ -s "$WORKDIR/results/probe/$metric.txt" ]] || + fail "benchmark-index did not record $metric.txt" +done + +SEARCH_LOG="$WORKDIR/search-environment.log" +CBM_CACHE_DIR="$CALLER_CACHE" \ +CBM_RUNTIME_DIR="$CALLER_RUNTIME" \ +CBM_BENCH_ENV_PROBE="$SEARCH_LOG" \ + "$ROOT/scripts/benchmark-search-graph.sh" "$ENV_PROBE" "$REPO" \ + > "$WORKDIR/search.out" 2>&1 || true +assert_isolated "benchmark-search-graph" "$SEARCH_LOG" + +echo "PASS: benchmark harnesses isolate their daemon runtime and cache from the caller" From 8f0ff31e7de014d65cd3c33fc884607a112f2c6b Mon Sep 17 00:00:00 2001 From: Anton Standrik Date: Mon, 21 Sep 2026 12:00:10 +0300 Subject: [PATCH 2/5] fix(test): escape benchmark paths, keep stderr, use monotonic clock Review follow-up for the benchmark harness isolation. - Both benchmark scripts build the index request with a JSON-escaped repository path (python3 json.dumps, the spelling soak-test.sh already uses): a path containing a quote or a backslash produced a payload the server could not parse. - benchmark-index.sh reads its three timestamps from one helper backed by time.monotonic_ns(); an NTP step mid-run no longer skews or negates a figure whose purpose is comparison across runs. - benchmark-search-graph.sh keeps the stderr of the index call and of the response parse in a run-private temp file (removed by the EXIT trap) and prints it, plus the first 500 bytes of the response, under the existing "did not report a project" line. - The contract test now refuses the index from its fixture and requires the cause to reach the output, and (POSIX only) drives a repository path with a quote and a backslash and requires the recorded request to parse as JSON with the resolved path. Both assertions fail against the previous head of this branch. Signed-off-by: Anton Standrik --- scripts/benchmark-index.sh | 19 +++++-- scripts/benchmark-search-graph.sh | 25 ++++++++-- ...st_benchmark_runtime_isolation_contract.sh | 49 +++++++++++++++++++ 3 files changed, 86 insertions(+), 7 deletions(-) diff --git a/scripts/benchmark-index.sh b/scripts/benchmark-index.sh index 71e694c171..0bc24af2f8 100755 --- a/scripts/benchmark-index.sh +++ b/scripts/benchmark-index.sh @@ -19,6 +19,17 @@ trap 'cbm_test_runtime_cleanup "$BINARY"' EXIT # Resolve symlinks REPO=$(cd "$REPO" && pwd -P) +# One pre-escaped spelling of the path for every request below, the way the +# soak harness builds its own: a repository path may legitimately contain a +# quote or a backslash, and hand-built JSON turns that into a parse error. +REPO_JSON=$(python3 -c 'import json,sys; print(json.dumps(sys.argv[1]))' "$REPO") + +# Elapsed time is read from a monotonic clock, never the wall clock: an NTP +# step mid-run would otherwise skew — or negate — a figure whose whole purpose +# is comparison across runs. Its reference point is fixed per boot on every +# platform CPython supports here, so the three readings below are comparable +# even though each comes from its own process. +bench_now_ms() { python3 -c "import time; print(time.monotonic_ns() // 1000000)"; } OUT="$RESULTS_DIR/$LANG" mkdir -p "$OUT" @@ -45,18 +56,18 @@ echo "$LOC" > "$OUT/loc.txt" # alone. setup-time.txt keeps the activation cost attributable and # total-time.txt is their sum — the figure comparable with earlier runs, which # paid activation inside the index timing whenever no daemon was already warm. -SETUP_START_MS=$(python3 -c "import time; print(int(time.time()*1000))") +SETUP_START_MS=$(bench_now_ms) if ! "$BINARY" daemon start >/dev/null 2>&1; then echo " $LANG: private daemon did not start" >&2 exit 1 fi # Index via CLI and capture timing -START_MS=$(python3 -c "import time; print(int(time.time()*1000))") +START_MS=$(bench_now_ms) -INDEX_JSON=$("$BINARY" cli index_repository "{\"repo_path\":\"$REPO\",\"mode\":\"full\"}" 2>/dev/null || echo '{"error":"index failed"}') +INDEX_JSON=$("$BINARY" cli index_repository "{\"repo_path\":$REPO_JSON,\"mode\":\"full\"}" 2>/dev/null || echo '{"error":"index failed"}') -END_MS=$(python3 -c "import time; print(int(time.time()*1000))") +END_MS=$(bench_now_ms) ELAPSED=$((END_MS - START_MS)) echo "$INDEX_JSON" > "$OUT/00-index.json" diff --git a/scripts/benchmark-search-graph.sh b/scripts/benchmark-search-graph.sh index 1d67b95818..6296d31fb2 100755 --- a/scripts/benchmark-search-graph.sh +++ b/scripts/benchmark-search-graph.sh @@ -22,22 +22,41 @@ REPO=$(cd "$REPO" && pwd -P) # shellcheck source=test-runtime.sh source "$(dirname "${BASH_SOURCE[0]}")/test-runtime.sh" cbm_test_runtime_init -trap 'cbm_test_runtime_cleanup "$BINARY"' EXIT +BENCH_TMP="" +trap 'cbm_test_runtime_cleanup "$BINARY"; [ -z "$BENCH_TMP" ] || rm -rf -- "$BENCH_TMP"' EXIT +BENCH_TMP=$(mktemp -d) +INDEX_ERR="$BENCH_TMP/index-stderr.log" if ! "$BINARY" daemon start >/dev/null 2>&1; then echo "private daemon did not start" >&2 exit 1 fi -INDEX_JSON=$("$BINARY" cli index_repository "{\"repo_path\":\"$REPO\",\"mode\":\"full\"}" 2>/dev/null || echo '{}') +# One pre-escaped spelling of the path, the way the soak harness builds its +# own: a repository path may legitimately contain a quote or a backslash, and +# hand-built JSON turns that into a parse error. +REPO_JSON=$(python3 -c 'import json,sys; print(json.dumps(sys.argv[1]))' "$REPO") +# Index and parse keep their stderr instead of discarding it: without it the +# failure below names only its symptom, and the cause — an unreadable +# repository, a refused daemon, a malformed envelope — is unrecoverable. +INDEX_JSON=$("$BINARY" cli index_repository "{\"repo_path\":$REPO_JSON,\"mode\":\"full\"}" \ + 2>"$INDEX_ERR" || echo '{}') PROJECT=$(echo "$INDEX_JSON" | python3 -c " import json, sys d = json.load(sys.stdin) if 'content' in d: d = json.loads(d['content'][0]['text']) print(d.get('project', '')) -" 2>/dev/null || echo "") +" 2>>"$INDEX_ERR" || echo "") if [ -z "$PROJECT" ]; then echo "index of $REPO did not report a project" >&2 + if [ -s "$INDEX_ERR" ]; then + echo "--- index/parse stderr ---" >&2 + cat "$INDEX_ERR" >&2 + fi + if [ -n "$INDEX_JSON" ]; then + echo "--- index response (first 500 bytes) ---" >&2 + printf '%.500s\n' "$INDEX_JSON" >&2 + fi exit 1 fi diff --git a/tests/test_benchmark_runtime_isolation_contract.sh b/tests/test_benchmark_runtime_isolation_contract.sh index b944df7683..1dc618ae69 100755 --- a/tests/test_benchmark_runtime_isolation_contract.sh +++ b/tests/test_benchmark_runtime_isolation_contract.sh @@ -33,6 +33,11 @@ ENV_PROBE="$WORKDIR/environment-probe" cat > "$ENV_PROBE" <<'EOF' #!/usr/bin/env bash printf '%s\t%s\n' "${CBM_CACHE_DIR-}" "${CBM_RUNTIME_DIR-}" >> "$CBM_BENCH_ENV_PROBE" +if [[ "${1-} ${2-}" == "cli index_repository" ]]; then + [[ -z "${CBM_BENCH_REQUEST_LOG-}" ]] || printf '%s\n' "${3-}" >> "$CBM_BENCH_REQUEST_LOG" + echo "probe: index refused" >&2 + exit 1 +fi [[ "${1-} ${2-}" == "daemon status" ]] && exit 1 exit 0 EOF @@ -93,4 +98,48 @@ CBM_BENCH_ENV_PROBE="$SEARCH_LOG" \ > "$WORKDIR/search.out" 2>&1 || true assert_isolated "benchmark-search-graph" "$SEARCH_LOG" +# A refused index must name its cause. The fixture writes one line to stderr and +# exits non-zero; discarding it leaves "did not report a project" as the only +# thing an operator sees. +grep -q -- '--- index/parse stderr ---' "$WORKDIR/search.out" || + fail "benchmark-search-graph hid the index stderr behind its own message" +grep -q 'probe: index refused' "$WORKDIR/search.out" || + fail "benchmark-search-graph did not surface the cause of the index failure" + +# The request carrying the repository path has to be built as JSON. A path may +# legitimately contain a quote or a backslash — hand-built JSON turns that into +# a payload the server cannot parse, or one that means something else. NTFS +# rejects both characters in a path component, so this case is POSIX-only. +case "$(uname -s)" in +MINGW* | MSYS* | CYGWIN*) ;; +*) + QUIRKY_REPO="$WORKDIR/re\"po\\dir" + mkdir -p "$QUIRKY_REPO" + echo 'def bench(): return 1' > "$QUIRKY_REPO/bench.py" + QUIRKY_RESOLVED=$(cd "$QUIRKY_REPO" && pwd -P) + REQUEST_LOG="$WORKDIR/requests.log" + CBM_CACHE_DIR="$CALLER_CACHE" \ + CBM_RUNTIME_DIR="$CALLER_RUNTIME" \ + CBM_BENCH_ENV_PROBE="$WORKDIR/quirky-environment.log" \ + CBM_BENCH_REQUEST_LOG="$REQUEST_LOG" \ + "$ROOT/scripts/benchmark-search-graph.sh" "$ENV_PROBE" "$QUIRKY_REPO" \ + > "$WORKDIR/quirky.out" 2>&1 || true + python3 - "$REQUEST_LOG" "$QUIRKY_RESOLVED" <<'PY' || fail "benchmark-search-graph built an index request that is not valid JSON for a quoted path" +import json +import sys + +request_log, expected = sys.argv[1], sys.argv[2] +try: + lines = [line for line in open(request_log).read().splitlines() if line.strip()] +except OSError: + sys.exit("the search benchmark sent no index request") +if not lines: + sys.exit("the search benchmark sent no index request") +payload = json.loads(lines[0]) +if payload.get("repo_path") != expected: + sys.exit(f"repo_path is {payload.get('repo_path')!r}, expected {expected!r}") +PY + ;; +esac + echo "PASS: benchmark harnesses isolate their daemon runtime and cache from the caller" From 9574c316e43527d6ca90eb007a9c8fc24fcfd88b Mon Sep 17 00:00:00 2001 From: Anton Standrik Date: Mon, 21 Sep 2026 12:25:28 +0300 Subject: [PATCH 3/5] fix(test): hand the benchmark index to the caller on request MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The evaluation plan (docs/EVALUATION_PLAN.md §7) indexes a language with benchmark-index.sh and then answers graph questions against that index from its own MCP session. With a run-private runtime (#1696) the index was gone before that session could start, so the handoff is now explicit and opt-in. With CBM_BENCH_KEEP_RUNTIME set, a SUCCESSFUL run stops its daemon, leaves the private root in place and records the paths that reach it in //runtime-root.txt (sourceable: CBM_BENCH_RUNTIME_ROOT, CBM_RUNTIME_DIR, CBM_CACHE_DIR). Ownership of that root, including its removal, passes to the caller. A failed run cleans up regardless: there is no index worth keeping and nothing may leak. Nothing changes in the default path or in the three timing files. §7's skeleton sets the flag, sources the file before the graph session and removes the root in step 8 instead of the live-store *.db files. The contract test drives a kept run and requires the root to survive, the handoff file to be sourceable, and its paths to be the ones the product processes actually used; the unflagged run still asserts the root is gone. The kept-run assertions fail against the previous head. Signed-off-by: Anton Standrik --- docs/EVALUATION_PLAN.md | 7 +++++- scripts/benchmark-index.sh | 23 ++++++++++++++++- ...st_benchmark_runtime_isolation_contract.sh | 25 +++++++++++++++++++ 3 files changed, 53 insertions(+), 2 deletions(-) diff --git a/docs/EVALUATION_PLAN.md b/docs/EVALUATION_PLAN.md index 5f8bb532cc..0b546d187b 100644 --- a/docs/EVALUATION_PLAN.md +++ b/docs/EVALUATION_PLAN.md @@ -283,8 +283,13 @@ for lang in $ALL_LANGS; do # ALL_LANGS = full 159-name list # --- step 2: cold index in the main channel, TIMED (key metric) --- t0=$(now_ms) + CBM_BENCH_KEEP_RUNTIME=1 \ scripts/benchmark-index.sh ~/.local/bin/codebase-memory-mcp "$lang" /tmp/bench/"$lang" /tmp/eval-results index_ms=$(( $(now_ms) - t0 )) # clone+index wall-clock → manifest + report (§5) + # The harness indexes into a run-private root (#1696). CBM_BENCH_KEEP_RUNTIME leaves that + # root behind on success and records its paths; the graph session in steps 4-7 must be + # started with these CBM_RUNTIME_DIR / CBM_CACHE_DIR, and step 8 removes the root. + set -a; . /tmp/eval-results/"$lang"/runtime-root.txt; set +a # --- step 3: record per-type histograms (zeros back-filled) --- # node-types.json, edge-types.json (every label + all 32 edge types, zeros kept, §7 below) @@ -293,7 +298,7 @@ for lang in $ALL_LANGS; do # ALL_LANGS = full 159-name list # + per-language report + (deferred, blind) judge --- # --- step 8: delete THIS language's index so the next is cold, then mark done --- - rm -f ~/.cache/codebase-memory-mcp/*.db + rm -rf -- "$CBM_BENCH_RUNTIME_ROOT" manifest_mark_done "$lang" "$index_ms" done ``` diff --git a/scripts/benchmark-index.sh b/scripts/benchmark-index.sh index 0bc24af2f8..11d79ee1fa 100755 --- a/scripts/benchmark-index.sh +++ b/scripts/benchmark-index.sh @@ -15,7 +15,28 @@ RESULTS_DIR="${4:?}" # shellcheck source=test-runtime.sh source "$(dirname "${BASH_SOURCE[0]}")/test-runtime.sh" cbm_test_runtime_init -trap 'cbm_test_runtime_cleanup "$BINARY"' EXIT + +# The evaluation plan (docs/EVALUATION_PLAN.md §7) indexes a language here and +# then answers graph questions against that index from its own MCP session. +# A run-private root would be gone before that session starts, so the caller +# may ask for it to be kept: after a SUCCESSFUL run the harness stops its +# daemon, leaves the root in place, records the paths that reach it in +# //runtime-root.txt (sourceable), and ownership of the root — +# including its removal — passes to the caller. A failed run cleans up +# regardless: there is no index worth keeping, and nothing must leak. Never on +# by default, or an unattended run accumulates one root per language. +bench_finish() { + local rc=$? + if [ "$rc" -eq 0 ] && [ -n "${CBM_BENCH_KEEP_RUNTIME:-}" ] && [ -d "${OUT:-}" ]; then + "$BINARY" daemon stop >/dev/null 2>&1 || true + printf 'CBM_BENCH_RUNTIME_ROOT=%q\nCBM_RUNTIME_DIR=%q\nCBM_CACHE_DIR=%q\n' \ + "$CBM_TEST_RUNTIME_ROOT" "$CBM_RUNTIME_DIR" "$CBM_CACHE_DIR" > "$OUT/runtime-root.txt" + echo " $LANG: runtime kept at $CBM_TEST_RUNTIME_ROOT; paths in $OUT/runtime-root.txt" >&2 + return 0 + fi + cbm_test_runtime_cleanup "$BINARY" +} +trap bench_finish EXIT # Resolve symlinks REPO=$(cd "$REPO" && pwd -P) diff --git a/tests/test_benchmark_runtime_isolation_contract.sh b/tests/test_benchmark_runtime_isolation_contract.sh index 1dc618ae69..73c1758f2b 100755 --- a/tests/test_benchmark_runtime_isolation_contract.sh +++ b/tests/test_benchmark_runtime_isolation_contract.sh @@ -90,6 +90,31 @@ for metric in setup-time total-time index-time; do fail "benchmark-index did not record $metric.txt" done +# The evaluation plan indexes a language and then reads that index from its own +# MCP session (docs/EVALUATION_PLAN.md §7). Asked to keep the runtime, a +# successful run must leave its root behind and record, sourceably, the paths +# that reach it — and they must be the paths the product processes actually +# used. Unasked, the root is gone (asserted above). +KEEP_LOG="$WORKDIR/keep-environment.log" +CBM_CACHE_DIR="$CALLER_CACHE" \ +CBM_RUNTIME_DIR="$CALLER_RUNTIME" \ +CBM_BENCH_ENV_PROBE="$KEEP_LOG" \ +CBM_BENCH_KEEP_RUNTIME=1 \ + "$ROOT/scripts/benchmark-index.sh" "$ENV_PROBE" keep "$REPO" "$WORKDIR/results" \ + > "$WORKDIR/keep.out" 2>&1 || true +HANDOFF="$WORKDIR/results/keep/runtime-root.txt" +[[ -s "$HANDOFF" ]] || fail "benchmark-index was asked to keep its runtime but recorded no handoff" +KEPT_ROOT=$(bash -c '. "$1" && printf "%s" "${CBM_BENCH_RUNTIME_ROOT-}"' _ "$HANDOFF") +KEPT_RUNTIME=$(bash -c '. "$1" && printf "%s" "${CBM_RUNTIME_DIR-}"' _ "$HANDOFF") +KEPT_CACHE=$(bash -c '. "$1" && printf "%s" "${CBM_CACHE_DIR-}"' _ "$HANDOFF") +[[ -n "$KEPT_ROOT" && -d "$KEPT_ROOT" && ! -L "$KEPT_ROOT" ]] || + fail "benchmark-index did not keep its runtime root: ${KEPT_ROOT:-}" +[[ -d "$KEPT_ROOT/cache" && -d "$KEPT_ROOT/runtime" ]] || + fail "the kept root $KEPT_ROOT lost its cache or runtime directory" +grep -qF -- "${KEPT_CACHE}"$'\t'"${KEPT_RUNTIME}" "$KEEP_LOG" || + fail "runtime-root.txt does not name the runtime and cache the product processes used" +rm -rf -- "$KEPT_ROOT" + SEARCH_LOG="$WORKDIR/search-environment.log" CBM_CACHE_DIR="$CALLER_CACHE" \ CBM_RUNTIME_DIR="$CALLER_RUNTIME" \ From 07ee617b07c3e2766930859e3e8bc8f1d9fa8a15 Mon Sep 17 00:00:00 2001 From: Anton Standrik Date: Tue, 22 Sep 2026 13:57:30 +0300 Subject: [PATCH 4/5] fix(test): fail the index benchmark when its index is refused MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit bench_finish keeps the private root on a zero exit status, but the index call swallowed its failure (`|| echo '{"error":"index failed"}'`), so a refused index exited 0 and, with CBM_BENCH_KEEP_RUNTIME set, kept an empty root and wrote runtime-root.txt for an index that does not exist. benchmark-index.sh now records the CLI's exit status, keeps 00-index.json valid JSON on failure ({"error":"index failed","exit":N}), and exits 1 once every per-run file — the three timing files included — is written, when the index failed or reported no project (the search-graph twin's own test). bench_finish then cleans up as its comment always promised, and the caller's loop notices. The index call keeps its stderr in a run-private temp file, the twin's shape, and the failure prints it under `--- index stderr ---` with the first 500 bytes of the response: for a one-shot CLI, stderr is the only channel a refusal is reported on. A stale runtime-root.txt from an earlier run in the same results directory is removed at start, so a failed run leaves no handoff at all. The contract test's fixture answers `cli index_repository` with a minimal envelope when CBM_BENCH_PROBE_INDEX_OK is set, so the keep run covers the successful-index handoff and asserts project.txt; the unflagged run asserts a non-zero exit and the surfaced cause after a refused index; a new keep run with a refused index asserts a non-zero exit, the timing files, no surviving root and no handoff, a pre-seeded stale one included. Against the previous head the test fails on "benchmark-index exited 0 after a refused index". Signed-off-by: Anton Standrik --- scripts/benchmark-index.sh | 33 +++++++++++- ...st_benchmark_runtime_isolation_contract.sh | 54 +++++++++++++++++-- 2 files changed, 81 insertions(+), 6 deletions(-) diff --git a/scripts/benchmark-index.sh b/scripts/benchmark-index.sh index 11d79ee1fa..207944a85f 100755 --- a/scripts/benchmark-index.sh +++ b/scripts/benchmark-index.sh @@ -27,6 +27,7 @@ cbm_test_runtime_init # by default, or an unattended run accumulates one root per language. bench_finish() { local rc=$? + [ -z "${BENCH_TMP:-}" ] || rm -rf -- "$BENCH_TMP" || true if [ "$rc" -eq 0 ] && [ -n "${CBM_BENCH_KEEP_RUNTIME:-}" ] && [ -d "${OUT:-}" ]; then "$BINARY" daemon stop >/dev/null 2>&1 || true printf 'CBM_BENCH_RUNTIME_ROOT=%q\nCBM_RUNTIME_DIR=%q\nCBM_CACHE_DIR=%q\n' \ @@ -36,7 +37,13 @@ bench_finish() { fi cbm_test_runtime_cleanup "$BINARY" } +BENCH_TMP="" trap bench_finish EXIT +# The index call keeps its stderr, the way the search-graph twin does: for a +# one-shot CLI that is the only channel a refusal is reported on, and without +# it the failure below could name only its symptom. +BENCH_TMP=$(mktemp -d) +INDEX_ERR="$BENCH_TMP/index-stderr.log" # Resolve symlinks REPO=$(cd "$REPO" && pwd -P) @@ -54,6 +61,10 @@ bench_now_ms() { python3 -c "import time; print(time.monotonic_ns() // 1000000)" OUT="$RESULTS_DIR/$LANG" mkdir -p "$OUT" +# A handoff names this run's index or none: the evaluation loop reuses the +# results directory, and a stale one would point the graph session at a root +# step 8 has already removed. +rm -f -- "$OUT/runtime-root.txt" echo "INDEX: $LANG ($REPO)" @@ -86,7 +97,12 @@ fi # Index via CLI and capture timing START_MS=$(bench_now_ms) -INDEX_JSON=$("$BINARY" cli index_repository "{\"repo_path\":$REPO_JSON,\"mode\":\"full\"}" 2>/dev/null || echo '{"error":"index failed"}') +# The CLI's exit status is the index's verdict. It is recorded here and acted +# on below, once every per-run file is written; an empty response stays valid +# JSON in 00-index.json so the failure is legible there as well. +INDEX_RC=0 +INDEX_JSON=$("$BINARY" cli index_repository "{\"repo_path\":$REPO_JSON,\"mode\":\"full\"}" 2>"$INDEX_ERR") || + { INDEX_RC=$?; INDEX_JSON="{\"error\":\"index failed\",\"exit\":$INDEX_RC}"; } END_MS=$(bench_now_ms) ELAPSED=$((END_MS - START_MS)) @@ -130,5 +146,20 @@ echo "$NODES" > "$OUT/nodes.txt" echo "$EDGES" > "$OUT/edges.txt" echo "$PROJECT" > "$OUT/project.txt" +# An index that failed, or that named no project, fails the run: after the +# timing files, so the caller keeps its figures, and through the exit status, +# so the caller's loop notices and bench_finish never keeps a root with no +# index in it. +if [ "$INDEX_RC" -ne 0 ] || [ -z "$PROJECT" ]; then + echo " $LANG: index failed (cli exit $INDEX_RC, project '$PROJECT'); response in $OUT/00-index.json" >&2 + if [ -s "$INDEX_ERR" ]; then + echo "--- index stderr ---" >&2 + cat "$INDEX_ERR" >&2 + fi + echo "--- index response (first 500 bytes) ---" >&2 + printf '%.500s\n' "$INDEX_JSON" >&2 + exit 1 +fi + printf " %s: %s files, %s LOC, %sms, %s nodes, %s edges\n" \ "$LANG" "$FILE_COUNT" "$LOC" "$ELAPSED" "$NODES" "$EDGES" diff --git a/tests/test_benchmark_runtime_isolation_contract.sh b/tests/test_benchmark_runtime_isolation_contract.sh index 73c1758f2b..f2e862d0dd 100755 --- a/tests/test_benchmark_runtime_isolation_contract.sh +++ b/tests/test_benchmark_runtime_isolation_contract.sh @@ -9,8 +9,9 @@ set -euo pipefail # operator's live store, every one-shot joined the operator's account daemon, # and the timings depended on whatever that daemon was doing. Drive both with # an environment-probe fixture and require that no product process ever -# receives the caller's runtime or cache, and that the index benchmark records -# the setup cost it now pays explicitly. +# receives the caller's runtime or cache, that the index benchmark records +# the setup cost it now pays explicitly, and that a refused index fails the +# run, names its cause, and leaves nothing behind. ROOT="$(cd "$(dirname "$0")/.." && pwd)" WORKDIR="$(mktemp -d)" @@ -35,6 +36,10 @@ cat > "$ENV_PROBE" <<'EOF' printf '%s\t%s\n' "${CBM_CACHE_DIR-}" "${CBM_RUNTIME_DIR-}" >> "$CBM_BENCH_ENV_PROBE" if [[ "${1-} ${2-}" == "cli index_repository" ]]; then [[ -z "${CBM_BENCH_REQUEST_LOG-}" ]] || printf '%s\n' "${3-}" >> "$CBM_BENCH_REQUEST_LOG" + if [[ -n "${CBM_BENCH_PROBE_INDEX_OK-}" ]]; then + printf '%s\n' '{"project":"probe"}' + exit 0 + fi echo "probe: index refused" >&2 exit 1 fi @@ -78,30 +83,46 @@ assert_isolated() { [[ ! -e "$private_root" ]] || fail "$harness left its private root behind: $private_root" } +# The fixture refuses the index. That must fail the run — after the timing +# files are written, so the caller keeps its figures — and name its cause: the +# fixture's one stderr line has to reach the output. INDEX_LOG="$WORKDIR/index-environment.log" +index_rc=0 CBM_CACHE_DIR="$CALLER_CACHE" \ CBM_RUNTIME_DIR="$CALLER_RUNTIME" \ CBM_BENCH_ENV_PROBE="$INDEX_LOG" \ "$ROOT/scripts/benchmark-index.sh" "$ENV_PROBE" probe "$REPO" "$WORKDIR/results" \ - > "$WORKDIR/index.out" 2>&1 || true + > "$WORKDIR/index.out" 2>&1 || index_rc=$? +[[ "$index_rc" -ne 0 ]] || fail "benchmark-index exited 0 after a refused index" assert_isolated "benchmark-index" "$INDEX_LOG" for metric in setup-time total-time index-time; do [[ -s "$WORKDIR/results/probe/$metric.txt" ]] || fail "benchmark-index did not record $metric.txt" done +grep -q -- '--- index stderr ---' "$WORKDIR/index.out" || + fail "benchmark-index hid the index stderr behind its own message" +grep -q 'probe: index refused' "$WORKDIR/index.out" || + fail "benchmark-index did not surface the cause of the index failure" # The evaluation plan indexes a language and then reads that index from its own # MCP session (docs/EVALUATION_PLAN.md §7). Asked to keep the runtime, a # successful run must leave its root behind and record, sourceably, the paths # that reach it — and they must be the paths the product processes actually -# used. Unasked, the root is gone (asserted above). +# used. Unasked, the root is gone (asserted above). The fixture answers this +# index with a minimal envelope, so what is kept is an index that succeeded. KEEP_LOG="$WORKDIR/keep-environment.log" +keep_rc=0 CBM_CACHE_DIR="$CALLER_CACHE" \ CBM_RUNTIME_DIR="$CALLER_RUNTIME" \ CBM_BENCH_ENV_PROBE="$KEEP_LOG" \ +CBM_BENCH_PROBE_INDEX_OK=1 \ CBM_BENCH_KEEP_RUNTIME=1 \ "$ROOT/scripts/benchmark-index.sh" "$ENV_PROBE" keep "$REPO" "$WORKDIR/results" \ - > "$WORKDIR/keep.out" 2>&1 || true + > "$WORKDIR/keep.out" 2>&1 || keep_rc=$? +[[ "$keep_rc" -eq 0 ]] || fail "benchmark-index failed a run whose index succeeded (exit $keep_rc)" +KEPT_PROJECT=$(cat "$WORKDIR/results/keep/project.txt" 2>/dev/null || true) +[[ "${KEPT_PROJECT%$'\r'}" == "probe" ]] || + fail "benchmark-index did not record the project the index reported: '${KEPT_PROJECT:-}'" HANDOFF="$WORKDIR/results/keep/runtime-root.txt" [[ -s "$HANDOFF" ]] || fail "benchmark-index was asked to keep its runtime but recorded no handoff" KEPT_ROOT=$(bash -c '. "$1" && printf "%s" "${CBM_BENCH_RUNTIME_ROOT-}"' _ "$HANDOFF") @@ -115,6 +136,29 @@ grep -qF -- "${KEPT_CACHE}"$'\t'"${KEPT_RUNTIME}" "$KEEP_LOG" || fail "runtime-root.txt does not name the runtime and cache the product processes used" rm -rf -- "$KEPT_ROOT" +# Asked to keep the runtime and refused the index, the harness has nothing worth +# keeping: the run fails, no root survives, and no handoff is written — nor left +# over from an earlier run, since the evaluation loop reuses the results +# directory. +REFUSED_LOG="$WORKDIR/keep-refused-environment.log" +REFUSED_HANDOFF="$WORKDIR/results/keep-refused/runtime-root.txt" +mkdir -p "${REFUSED_HANDOFF%/*}" +echo 'CBM_BENCH_RUNTIME_ROOT=/stale/root/from/an/earlier/run' > "$REFUSED_HANDOFF" +refused_rc=0 +CBM_CACHE_DIR="$CALLER_CACHE" \ +CBM_RUNTIME_DIR="$CALLER_RUNTIME" \ +CBM_BENCH_ENV_PROBE="$REFUSED_LOG" \ +CBM_BENCH_KEEP_RUNTIME=1 \ + "$ROOT/scripts/benchmark-index.sh" "$ENV_PROBE" keep-refused "$REPO" "$WORKDIR/results" \ + > "$WORKDIR/keep-refused.out" 2>&1 || refused_rc=$? +[[ "$refused_rc" -ne 0 ]] || fail "benchmark-index exited 0 after a refused index it was asked to keep" +for metric in setup-time total-time index-time; do + [[ -s "$WORKDIR/results/keep-refused/$metric.txt" ]] || + fail "benchmark-index did not record $metric.txt for a refused index" +done +assert_isolated "benchmark-index (keep, refused index)" "$REFUSED_LOG" +[[ ! -e "$REFUSED_HANDOFF" ]] || fail "benchmark-index left a handoff for an index that failed" + SEARCH_LOG="$WORKDIR/search-environment.log" CBM_CACHE_DIR="$CALLER_CACHE" \ CBM_RUNTIME_DIR="$CALLER_RUNTIME" \ From f913f0eaf5ee308619ef1fdd4f60534d4509fab5 Mon Sep 17 00:00:00 2001 From: Anton Standrik Date: Tue, 22 Sep 2026 13:57:30 +0300 Subject: [PATCH 5/5] docs(eval-plan): stop the kept daemon before its root is removed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit §7 step 8 removed the kept runtime root while the graph session of steps 4-7 may still have a daemon running under it; stop that daemon under the sourced CBM_RUNTIME_DIR first. §13 Reproducibility still cleared ~/.cache/codebase-memory-mcp/*.db and called the harness unflagged, which no longer describes what the harness does: it indexes into a run-private root, and only CBM_BENCH_KEEP_RUNTIME=1 hands that root, via runtime-root.txt, to the graph phase. Signed-off-by: Anton Standrik --- docs/EVALUATION_PLAN.md | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/docs/EVALUATION_PLAN.md b/docs/EVALUATION_PLAN.md index 0b546d187b..1f8f93b273 100644 --- a/docs/EVALUATION_PLAN.md +++ b/docs/EVALUATION_PLAN.md @@ -298,6 +298,9 @@ for lang in $ALL_LANGS; do # ALL_LANGS = full 159-name list # + per-language report + (deferred, blind) judge --- # --- step 8: delete THIS language's index so the next is cold, then mark done --- + # The graph session (steps 4-7) may still have a daemon running under the kept + # CBM_RUNTIME_DIR (exported above); stop it before its root goes away. + ~/.local/bin/codebase-memory-mcp daemon stop || true rm -rf -- "$CBM_BENCH_RUNTIME_ROOT" manifest_mark_done "$lang" "$index_ms" done @@ -864,10 +867,13 @@ Deep-Dive section. # 1. Clone all 159 repos (shallow; skip existing) scripts/clone-bench-repos.sh /tmp/bench -# 2. Cold index all 159 (LSP cohort in full mode) -rm -f ~/.cache/codebase-memory-mcp/*.db +# 2. Cold index all 159 (LSP cohort in full mode). Each language is indexed into a run-private +# runtime and cache (#1696); CBM_BENCH_KEEP_RUNTIME=1 keeps that root after a successful +# index and records its paths in /tmp/eval-results//runtime-root.txt. Source it +# before the graph phase, then stop the daemon and remove the root, as in §7 step 8. mkdir -p /tmp/eval-results for lang in $ALL_LANGS; do + CBM_BENCH_KEEP_RUNTIME=1 \ scripts/benchmark-index.sh ~/.local/bin/codebase-memory-mcp "$lang" /tmp/bench/"$lang" /tmp/eval-results done