fix(test): isolate benchmark harness daemon runtimes

docs/EVALUATION_PLAN.md

@@ -283,8 +283,13 @@ for lang in $ALL_LANGS; do                      # ALL_LANGS = full 159-name list
 
   # --- step 2: cold index in the main channel, TIMED (key metric) ---
   t0=$(now_ms)
+  CBM_BENCH_KEEP_RUNTIME=1 \
   scripts/benchmark-index.sh ~/.local/bin/codebase-memory-mcp "$lang" /tmp/bench/"$lang" /tmp/eval-results
   index_ms=$(( $(now_ms) - t0 ))                 # clone+index wall-clock → manifest + report (§5)
+  # The harness indexes into a run-private root (#1696). CBM_BENCH_KEEP_RUNTIME leaves that
+  # root behind on success and records its paths; the graph session in steps 4-7 must be
+  # started with these CBM_RUNTIME_DIR / CBM_CACHE_DIR, and step 8 removes the root.
+  set -a; . /tmp/eval-results/"$lang"/runtime-root.txt; set +a
 
   # --- step 3: record per-type histograms (zeros back-filled) ---
   #   node-types.json, edge-types.json  (every label + all 32 edge types, zeros kept, §7 below)
@@ -293,7 +298,10 @@ for lang in $ALL_LANGS; do                      # ALL_LANGS = full 159-name list
   #               + per-language report + (deferred, blind) judge ---
 
   # --- step 8: delete THIS language's index so the next is cold, then mark done ---
-  rm -f ~/.cache/codebase-memory-mcp/*.db
+  # The graph session (steps 4-7) may still have a daemon running under the kept
+  # CBM_RUNTIME_DIR (exported above); stop it before its root goes away.
+  ~/.local/bin/codebase-memory-mcp daemon stop || true
+  rm -rf -- "$CBM_BENCH_RUNTIME_ROOT"
   manifest_mark_done "$lang" "$index_ms"
 done
 ```
@@ -859,10 +867,13 @@ Deep-Dive section.
 # 1. Clone all 159 repos (shallow; skip existing)
 scripts/clone-bench-repos.sh /tmp/bench
 
-# 2. Cold index all 159 (LSP cohort in full mode)
-rm -f ~/.cache/codebase-memory-mcp/*.db
+# 2. Cold index all 159 (LSP cohort in full mode). Each language is indexed into a run-private
+#    runtime and cache (#1696); CBM_BENCH_KEEP_RUNTIME=1 keeps that root after a successful
+#    index and records its paths in /tmp/eval-results/<lang>/runtime-root.txt. Source it
+#    before the graph phase, then stop the daemon and remove the root, as in §7 step 8.
 mkdir -p /tmp/eval-results
 for lang in $ALL_LANGS; do
+  CBM_BENCH_KEEP_RUNTIME=1 \
   scripts/benchmark-index.sh ~/.local/bin/codebase-memory-mcp "$lang" /tmp/bench/"$lang" /tmp/eval-results
 done
 

scripts/benchmark-index.sh

@@ -9,11 +9,62 @@ LANG="${2:?}"
 REPO="${3:?}"
 RESULTS_DIR="${4:?}"
 
+# The index must run against a daemon rendezvous and cache this run owns: only
+# CBM_RUNTIME_DIR moves the rendezvous, and without a private cache the
+# benchmark repository was indexed into the operator's live store (#1696).
+# shellcheck source=test-runtime.sh
+source "$(dirname "${BASH_SOURCE[0]}")/test-runtime.sh"
+cbm_test_runtime_init
+
+# The evaluation plan (docs/EVALUATION_PLAN.md §7) indexes a language here and
+# then answers graph questions against that index from its own MCP session.
+# A run-private root would be gone before that session starts, so the caller
+# may ask for it to be kept: after a SUCCESSFUL run the harness stops its
+# daemon, leaves the root in place, records the paths that reach it in
+# <results>/<lang>/runtime-root.txt (sourceable), and ownership of the root —
+# including its removal — passes to the caller. A failed run cleans up
+# regardless: there is no index worth keeping, and nothing must leak. Never on
+# by default, or an unattended run accumulates one root per language.
+bench_finish() {
+  local rc=$?
+  [ -z "${BENCH_TMP:-}" ] || rm -rf -- "$BENCH_TMP" || true
+  if [ "$rc" -eq 0 ] && [ -n "${CBM_BENCH_KEEP_RUNTIME:-}" ] && [ -d "${OUT:-}" ]; then
+    "$BINARY" daemon stop >/dev/null 2>&1 || true
+    printf 'CBM_BENCH_RUNTIME_ROOT=%q\nCBM_RUNTIME_DIR=%q\nCBM_CACHE_DIR=%q\n' \
+      "$CBM_TEST_RUNTIME_ROOT" "$CBM_RUNTIME_DIR" "$CBM_CACHE_DIR" > "$OUT/runtime-root.txt"
+    echo "  $LANG: runtime kept at $CBM_TEST_RUNTIME_ROOT; paths in $OUT/runtime-root.txt" >&2
+    return 0
+  fi
+  cbm_test_runtime_cleanup "$BINARY"
+}
+BENCH_TMP=""
+trap bench_finish EXIT
+# The index call keeps its stderr, the way the search-graph twin does: for a
+# one-shot CLI that is the only channel a refusal is reported on, and without
+# it the failure below could name only its symptom.
+BENCH_TMP=$(mktemp -d)
+INDEX_ERR="$BENCH_TMP/index-stderr.log"
+
 # Resolve symlinks
 REPO=$(cd "$REPO" && pwd -P)
+# One pre-escaped spelling of the path for every request below, the way the
+# soak harness builds its own: a repository path may legitimately contain a
+# quote or a backslash, and hand-built JSON turns that into a parse error.
+REPO_JSON=$(python3 -c 'import json,sys; print(json.dumps(sys.argv[1]))' "$REPO")
+
+# Elapsed time is read from a monotonic clock, never the wall clock: an NTP
+# step mid-run would otherwise skew — or negate — a figure whose whole purpose
+# is comparison across runs. Its reference point is fixed per boot on every
+# platform CPython supports here, so the three readings below are comparable
+# even though each comes from its own process.
+bench_now_ms() { python3 -c "import time; print(time.monotonic_ns() // 1000000)"; }
 
 OUT="$RESULTS_DIR/$LANG"
 mkdir -p "$OUT"
+# A handoff names this run's index or none: the evaluation loop reuses the
+# results directory, and a stale one would point the graph session at a root
+# step 8 has already removed.
+rm -f -- "$OUT/runtime-root.txt"
 
 echo "INDEX: $LANG ($REPO)"
 
@@ -33,16 +84,33 @@ LOC=$(find "$REPO" -type f \
 echo "$FILE_COUNT" > "$OUT/file-count.txt"
 echo "$LOC" > "$OUT/loc.txt"
 
+# Start the private daemon before timing so index-time.txt measures the index
+# alone. setup-time.txt keeps the activation cost attributable and
+# total-time.txt is their sum — the figure comparable with earlier runs, which
+# paid activation inside the index timing whenever no daemon was already warm.
+SETUP_START_MS=$(bench_now_ms)
+if ! "$BINARY" daemon start >/dev/null 2>&1; then
+  echo "  $LANG: private daemon did not start" >&2
+  exit 1
+fi
+
 # Index via CLI and capture timing
-START_MS=$(python3 -c "import time; print(int(time.time()*1000))")
+START_MS=$(bench_now_ms)
 
-INDEX_JSON=$("$BINARY" cli index_repository "{\"repo_path\":\"$REPO\",\"mode\":\"full\"}" 2>/dev/null || echo '{"error":"index failed"}')
+# The CLI's exit status is the index's verdict. It is recorded here and acted
+# on below, once every per-run file is written; an empty response stays valid
+# JSON in 00-index.json so the failure is legible there as well.
+INDEX_RC=0
+INDEX_JSON=$("$BINARY" cli index_repository "{\"repo_path\":$REPO_JSON,\"mode\":\"full\"}" 2>"$INDEX_ERR") ||
+  { INDEX_RC=$?; INDEX_JSON="{\"error\":\"index failed\",\"exit\":$INDEX_RC}"; }
 
-END_MS=$(python3 -c "import time; print(int(time.time()*1000))")
+END_MS=$(bench_now_ms)
 ELAPSED=$((END_MS - START_MS))
 
 echo "$INDEX_JSON" > "$OUT/00-index.json"
 echo "$ELAPSED" > "$OUT/index-time.txt"
+echo "$((START_MS - SETUP_START_MS))" > "$OUT/setup-time.txt"
+echo "$((END_MS - SETUP_START_MS))" > "$OUT/total-time.txt"
 
 # Extract node/edge counts (CLI wraps in MCP content envelope)
 NODES=$(echo "$INDEX_JSON" | python3 -c "
@@ -78,5 +146,20 @@ echo "$NODES" > "$OUT/nodes.txt"
 echo "$EDGES" > "$OUT/edges.txt"
 echo "$PROJECT" > "$OUT/project.txt"
 
+# An index that failed, or that named no project, fails the run: after the
+# timing files, so the caller keeps its figures, and through the exit status,
+# so the caller's loop notices and bench_finish never keeps a root with no
+# index in it.
+if [ "$INDEX_RC" -ne 0 ] || [ -z "$PROJECT" ]; then
+  echo "  $LANG: index failed (cli exit $INDEX_RC, project '$PROJECT'); response in $OUT/00-index.json" >&2
+  if [ -s "$INDEX_ERR" ]; then
+    echo "--- index stderr ---" >&2
+    cat "$INDEX_ERR" >&2
+  fi
+  echo "--- index response (first 500 bytes) ---" >&2
+  printf '%.500s\n' "$INDEX_JSON" >&2
+  exit 1
+fi
+
 printf "  %s: %s files, %s LOC, %sms, %s nodes, %s edges\n" \
   "$LANG" "$FILE_COUNT" "$LOC" "$ELAPSED" "$NODES" "$EDGES"

scripts/benchmark-search-graph.sh

@@ -3,18 +3,65 @@
 # codebase-memory-mcp binary to measure the regex / LIKE pre-filter performance.
 #
 # Usage:
-#   scripts/benchmark-search-graph.sh <binary-path> <project-name>
+#   scripts/benchmark-search-graph.sh <binary-path> <repo-path>
 #
 # Example:
-#   scripts/benchmark-search-graph.sh ./build/c/codebase-memory-mcp my-project
+#   scripts/benchmark-search-graph.sh ./build/c/codebase-memory-mcp ~/src/my-project
+#
+# The repository is indexed (untimed) into a private runtime and cache first;
+# the queries then run against that index through a daemon this run keeps warm,
+# so a timing never includes daemon activation and never touches the operator's
+# live store (#1696).
 
 set -euo pipefail
 
-BINARY="${1:?Usage: $0 <binary-path> <project-name>}"
-PROJECT="${2:?Usage: $0 <binary-path> <project-name>}"
+BINARY="${1:?Usage: $0 <binary-path> <repo-path>}"
+REPO="${2:?Usage: $0 <binary-path> <repo-path>}"
+REPO=$(cd "$REPO" && pwd -P)
+
+# shellcheck source=test-runtime.sh
+source "$(dirname "${BASH_SOURCE[0]}")/test-runtime.sh"
+cbm_test_runtime_init
+BENCH_TMP=""
+trap 'cbm_test_runtime_cleanup "$BINARY"; [ -z "$BENCH_TMP" ] || rm -rf -- "$BENCH_TMP"' EXIT
+BENCH_TMP=$(mktemp -d)
+INDEX_ERR="$BENCH_TMP/index-stderr.log"
+
+if ! "$BINARY" daemon start >/dev/null 2>&1; then
+    echo "private daemon did not start" >&2
+    exit 1
+fi
+# One pre-escaped spelling of the path, the way the soak harness builds its
+# own: a repository path may legitimately contain a quote or a backslash, and
+# hand-built JSON turns that into a parse error.
+REPO_JSON=$(python3 -c 'import json,sys; print(json.dumps(sys.argv[1]))' "$REPO")
+# Index and parse keep their stderr instead of discarding it: without it the
+# failure below names only its symptom, and the cause — an unreadable
+# repository, a refused daemon, a malformed envelope — is unrecoverable.
+INDEX_JSON=$("$BINARY" cli index_repository "{\"repo_path\":$REPO_JSON,\"mode\":\"full\"}" \
+    2>"$INDEX_ERR" || echo '{}')
+PROJECT=$(echo "$INDEX_JSON" | python3 -c "
+import json, sys
+d = json.load(sys.stdin)
+if 'content' in d:
+    d = json.loads(d['content'][0]['text'])
+print(d.get('project', ''))
+" 2>>"$INDEX_ERR" || echo "")
+if [ -z "$PROJECT" ]; then
+    echo "index of $REPO did not report a project" >&2
+    if [ -s "$INDEX_ERR" ]; then
+        echo "--- index/parse stderr ---" >&2
+        cat "$INDEX_ERR" >&2
+    fi
+    if [ -n "$INDEX_JSON" ]; then
+        echo "--- index response (first 500 bytes) ---" >&2
+        printf '%.500s\n' "$INDEX_JSON" >&2
+    fi
+    exit 1
+fi
 
 echo "Binary:  $BINARY"
-echo "Project: $PROJECT"
+echo "Project: $PROJECT (indexed from $REPO)"
 echo ""
 
 run_case() {

scripts/test.sh

@@ -239,6 +239,9 @@ bash "$ROOT/tests/test_smoke_fixture_contract.sh"
 echo "=== Step 0i: parallel suite scheduler contract ==="
 bash "$ROOT/tests/test_parallel_harness_contract.sh"
 
+echo "=== Step 0i2: benchmark harness runtime isolation contract (#1696) ==="
+bash "$ROOT/tests/test_benchmark_runtime_isolation_contract.sh"
+
 echo "=== Step 0j: venue parity contract (one harness, every venue) ==="
 bash "$ROOT/tests/test_venue_parity_contract.sh"