diff --git a/docs/EVALUATION_PLAN.md b/docs/EVALUATION_PLAN.md index 5f8bb532cc..1f8f93b273 100644 --- a/docs/EVALUATION_PLAN.md +++ b/docs/EVALUATION_PLAN.md @@ -283,8 +283,13 @@ for lang in $ALL_LANGS; do # ALL_LANGS = full 159-name list # --- step 2: cold index in the main channel, TIMED (key metric) --- t0=$(now_ms) + CBM_BENCH_KEEP_RUNTIME=1 \ scripts/benchmark-index.sh ~/.local/bin/codebase-memory-mcp "$lang" /tmp/bench/"$lang" /tmp/eval-results index_ms=$(( $(now_ms) - t0 )) # clone+index wall-clock → manifest + report (§5) + # The harness indexes into a run-private root (#1696). CBM_BENCH_KEEP_RUNTIME leaves that + # root behind on success and records its paths; the graph session in steps 4-7 must be + # started with these CBM_RUNTIME_DIR / CBM_CACHE_DIR, and step 8 removes the root. + set -a; . /tmp/eval-results/"$lang"/runtime-root.txt; set +a # --- step 3: record per-type histograms (zeros back-filled) --- # node-types.json, edge-types.json (every label + all 32 edge types, zeros kept, §7 below) @@ -293,7 +298,10 @@ for lang in $ALL_LANGS; do # ALL_LANGS = full 159-name list # + per-language report + (deferred, blind) judge --- # --- step 8: delete THIS language's index so the next is cold, then mark done --- - rm -f ~/.cache/codebase-memory-mcp/*.db + # The graph session (steps 4-7) may still have a daemon running under the kept + # CBM_RUNTIME_DIR (exported above); stop it before its root goes away. + ~/.local/bin/codebase-memory-mcp daemon stop || true + rm -rf -- "$CBM_BENCH_RUNTIME_ROOT" manifest_mark_done "$lang" "$index_ms" done ``` @@ -859,10 +867,13 @@ Deep-Dive section. # 1. Clone all 159 repos (shallow; skip existing) scripts/clone-bench-repos.sh /tmp/bench -# 2. Cold index all 159 (LSP cohort in full mode) -rm -f ~/.cache/codebase-memory-mcp/*.db +# 2. Cold index all 159 (LSP cohort in full mode). Each language is indexed into a run-private +# runtime and cache (#1696); CBM_BENCH_KEEP_RUNTIME=1 keeps that root after a successful +# index and records its paths in /tmp/eval-results//runtime-root.txt. Source it +# before the graph phase, then stop the daemon and remove the root, as in §7 step 8. mkdir -p /tmp/eval-results for lang in $ALL_LANGS; do + CBM_BENCH_KEEP_RUNTIME=1 \ scripts/benchmark-index.sh ~/.local/bin/codebase-memory-mcp "$lang" /tmp/bench/"$lang" /tmp/eval-results done diff --git a/scripts/benchmark-index.sh b/scripts/benchmark-index.sh index 756bda06ed..207944a85f 100755 --- a/scripts/benchmark-index.sh +++ b/scripts/benchmark-index.sh @@ -9,11 +9,62 @@ LANG="${2:?}" REPO="${3:?}" RESULTS_DIR="${4:?}" +# The index must run against a daemon rendezvous and cache this run owns: only +# CBM_RUNTIME_DIR moves the rendezvous, and without a private cache the +# benchmark repository was indexed into the operator's live store (#1696). +# shellcheck source=test-runtime.sh +source "$(dirname "${BASH_SOURCE[0]}")/test-runtime.sh" +cbm_test_runtime_init + +# The evaluation plan (docs/EVALUATION_PLAN.md §7) indexes a language here and +# then answers graph questions against that index from its own MCP session. +# A run-private root would be gone before that session starts, so the caller +# may ask for it to be kept: after a SUCCESSFUL run the harness stops its +# daemon, leaves the root in place, records the paths that reach it in +# //runtime-root.txt (sourceable), and ownership of the root — +# including its removal — passes to the caller. A failed run cleans up +# regardless: there is no index worth keeping, and nothing must leak. Never on +# by default, or an unattended run accumulates one root per language. +bench_finish() { + local rc=$? + [ -z "${BENCH_TMP:-}" ] || rm -rf -- "$BENCH_TMP" || true + if [ "$rc" -eq 0 ] && [ -n "${CBM_BENCH_KEEP_RUNTIME:-}" ] && [ -d "${OUT:-}" ]; then + "$BINARY" daemon stop >/dev/null 2>&1 || true + printf 'CBM_BENCH_RUNTIME_ROOT=%q\nCBM_RUNTIME_DIR=%q\nCBM_CACHE_DIR=%q\n' \ + "$CBM_TEST_RUNTIME_ROOT" "$CBM_RUNTIME_DIR" "$CBM_CACHE_DIR" > "$OUT/runtime-root.txt" + echo " $LANG: runtime kept at $CBM_TEST_RUNTIME_ROOT; paths in $OUT/runtime-root.txt" >&2 + return 0 + fi + cbm_test_runtime_cleanup "$BINARY" +} +BENCH_TMP="" +trap bench_finish EXIT +# The index call keeps its stderr, the way the search-graph twin does: for a +# one-shot CLI that is the only channel a refusal is reported on, and without +# it the failure below could name only its symptom. +BENCH_TMP=$(mktemp -d) +INDEX_ERR="$BENCH_TMP/index-stderr.log" + # Resolve symlinks REPO=$(cd "$REPO" && pwd -P) +# One pre-escaped spelling of the path for every request below, the way the +# soak harness builds its own: a repository path may legitimately contain a +# quote or a backslash, and hand-built JSON turns that into a parse error. +REPO_JSON=$(python3 -c 'import json,sys; print(json.dumps(sys.argv[1]))' "$REPO") + +# Elapsed time is read from a monotonic clock, never the wall clock: an NTP +# step mid-run would otherwise skew — or negate — a figure whose whole purpose +# is comparison across runs. Its reference point is fixed per boot on every +# platform CPython supports here, so the three readings below are comparable +# even though each comes from its own process. +bench_now_ms() { python3 -c "import time; print(time.monotonic_ns() // 1000000)"; } OUT="$RESULTS_DIR/$LANG" mkdir -p "$OUT" +# A handoff names this run's index or none: the evaluation loop reuses the +# results directory, and a stale one would point the graph session at a root +# step 8 has already removed. +rm -f -- "$OUT/runtime-root.txt" echo "INDEX: $LANG ($REPO)" @@ -33,16 +84,33 @@ LOC=$(find "$REPO" -type f \ echo "$FILE_COUNT" > "$OUT/file-count.txt" echo "$LOC" > "$OUT/loc.txt" +# Start the private daemon before timing so index-time.txt measures the index +# alone. setup-time.txt keeps the activation cost attributable and +# total-time.txt is their sum — the figure comparable with earlier runs, which +# paid activation inside the index timing whenever no daemon was already warm. +SETUP_START_MS=$(bench_now_ms) +if ! "$BINARY" daemon start >/dev/null 2>&1; then + echo " $LANG: private daemon did not start" >&2 + exit 1 +fi + # Index via CLI and capture timing -START_MS=$(python3 -c "import time; print(int(time.time()*1000))") +START_MS=$(bench_now_ms) -INDEX_JSON=$("$BINARY" cli index_repository "{\"repo_path\":\"$REPO\",\"mode\":\"full\"}" 2>/dev/null || echo '{"error":"index failed"}') +# The CLI's exit status is the index's verdict. It is recorded here and acted +# on below, once every per-run file is written; an empty response stays valid +# JSON in 00-index.json so the failure is legible there as well. +INDEX_RC=0 +INDEX_JSON=$("$BINARY" cli index_repository "{\"repo_path\":$REPO_JSON,\"mode\":\"full\"}" 2>"$INDEX_ERR") || + { INDEX_RC=$?; INDEX_JSON="{\"error\":\"index failed\",\"exit\":$INDEX_RC}"; } -END_MS=$(python3 -c "import time; print(int(time.time()*1000))") +END_MS=$(bench_now_ms) ELAPSED=$((END_MS - START_MS)) echo "$INDEX_JSON" > "$OUT/00-index.json" echo "$ELAPSED" > "$OUT/index-time.txt" +echo "$((START_MS - SETUP_START_MS))" > "$OUT/setup-time.txt" +echo "$((END_MS - SETUP_START_MS))" > "$OUT/total-time.txt" # Extract node/edge counts (CLI wraps in MCP content envelope) NODES=$(echo "$INDEX_JSON" | python3 -c " @@ -78,5 +146,20 @@ echo "$NODES" > "$OUT/nodes.txt" echo "$EDGES" > "$OUT/edges.txt" echo "$PROJECT" > "$OUT/project.txt" +# An index that failed, or that named no project, fails the run: after the +# timing files, so the caller keeps its figures, and through the exit status, +# so the caller's loop notices and bench_finish never keeps a root with no +# index in it. +if [ "$INDEX_RC" -ne 0 ] || [ -z "$PROJECT" ]; then + echo " $LANG: index failed (cli exit $INDEX_RC, project '$PROJECT'); response in $OUT/00-index.json" >&2 + if [ -s "$INDEX_ERR" ]; then + echo "--- index stderr ---" >&2 + cat "$INDEX_ERR" >&2 + fi + echo "--- index response (first 500 bytes) ---" >&2 + printf '%.500s\n' "$INDEX_JSON" >&2 + exit 1 +fi + printf " %s: %s files, %s LOC, %sms, %s nodes, %s edges\n" \ "$LANG" "$FILE_COUNT" "$LOC" "$ELAPSED" "$NODES" "$EDGES" diff --git a/scripts/benchmark-search-graph.sh b/scripts/benchmark-search-graph.sh index cc94147ec6..6296d31fb2 100755 --- a/scripts/benchmark-search-graph.sh +++ b/scripts/benchmark-search-graph.sh @@ -3,18 +3,65 @@ # codebase-memory-mcp binary to measure the regex / LIKE pre-filter performance. # # Usage: -# scripts/benchmark-search-graph.sh +# scripts/benchmark-search-graph.sh # # Example: -# scripts/benchmark-search-graph.sh ./build/c/codebase-memory-mcp my-project +# scripts/benchmark-search-graph.sh ./build/c/codebase-memory-mcp ~/src/my-project +# +# The repository is indexed (untimed) into a private runtime and cache first; +# the queries then run against that index through a daemon this run keeps warm, +# so a timing never includes daemon activation and never touches the operator's +# live store (#1696). set -euo pipefail -BINARY="${1:?Usage: $0 }" -PROJECT="${2:?Usage: $0 }" +BINARY="${1:?Usage: $0 }" +REPO="${2:?Usage: $0 }" +REPO=$(cd "$REPO" && pwd -P) + +# shellcheck source=test-runtime.sh +source "$(dirname "${BASH_SOURCE[0]}")/test-runtime.sh" +cbm_test_runtime_init +BENCH_TMP="" +trap 'cbm_test_runtime_cleanup "$BINARY"; [ -z "$BENCH_TMP" ] || rm -rf -- "$BENCH_TMP"' EXIT +BENCH_TMP=$(mktemp -d) +INDEX_ERR="$BENCH_TMP/index-stderr.log" + +if ! "$BINARY" daemon start >/dev/null 2>&1; then + echo "private daemon did not start" >&2 + exit 1 +fi +# One pre-escaped spelling of the path, the way the soak harness builds its +# own: a repository path may legitimately contain a quote or a backslash, and +# hand-built JSON turns that into a parse error. +REPO_JSON=$(python3 -c 'import json,sys; print(json.dumps(sys.argv[1]))' "$REPO") +# Index and parse keep their stderr instead of discarding it: without it the +# failure below names only its symptom, and the cause — an unreadable +# repository, a refused daemon, a malformed envelope — is unrecoverable. +INDEX_JSON=$("$BINARY" cli index_repository "{\"repo_path\":$REPO_JSON,\"mode\":\"full\"}" \ + 2>"$INDEX_ERR" || echo '{}') +PROJECT=$(echo "$INDEX_JSON" | python3 -c " +import json, sys +d = json.load(sys.stdin) +if 'content' in d: + d = json.loads(d['content'][0]['text']) +print(d.get('project', '')) +" 2>>"$INDEX_ERR" || echo "") +if [ -z "$PROJECT" ]; then + echo "index of $REPO did not report a project" >&2 + if [ -s "$INDEX_ERR" ]; then + echo "--- index/parse stderr ---" >&2 + cat "$INDEX_ERR" >&2 + fi + if [ -n "$INDEX_JSON" ]; then + echo "--- index response (first 500 bytes) ---" >&2 + printf '%.500s\n' "$INDEX_JSON" >&2 + fi + exit 1 +fi echo "Binary: $BINARY" -echo "Project: $PROJECT" +echo "Project: $PROJECT (indexed from $REPO)" echo "" run_case() { diff --git a/scripts/test.sh b/scripts/test.sh index 50d65643f9..77cbf42d40 100755 --- a/scripts/test.sh +++ b/scripts/test.sh @@ -239,6 +239,9 @@ bash "$ROOT/tests/test_smoke_fixture_contract.sh" echo "=== Step 0i: parallel suite scheduler contract ===" bash "$ROOT/tests/test_parallel_harness_contract.sh" +echo "=== Step 0i2: benchmark harness runtime isolation contract (#1696) ===" +bash "$ROOT/tests/test_benchmark_runtime_isolation_contract.sh" + echo "=== Step 0j: venue parity contract (one harness, every venue) ===" bash "$ROOT/tests/test_venue_parity_contract.sh" diff --git a/tests/test_benchmark_runtime_isolation_contract.sh b/tests/test_benchmark_runtime_isolation_contract.sh new file mode 100755 index 0000000000..f2e862d0dd --- /dev/null +++ b/tests/test_benchmark_runtime_isolation_contract.sh @@ -0,0 +1,214 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Runtime-isolation contract for the benchmark harnesses (#1696, follow-up to +# #1691). +# +# scripts/benchmark-index.sh and scripts/benchmark-search-graph.sh ran the +# product with no runtime or cache of their own: the index landed in the +# operator's live store, every one-shot joined the operator's account daemon, +# and the timings depended on whatever that daemon was doing. Drive both with +# an environment-probe fixture and require that no product process ever +# receives the caller's runtime or cache, that the index benchmark records +# the setup cost it now pays explicitly, and that a refused index fails the +# run, names its cause, and leaves nothing behind. + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +WORKDIR="$(mktemp -d)" +trap 'rm -rf "$WORKDIR"' EXIT + +fail() { + echo "FAIL: $*" >&2 + exit 1 +} + +normalize_path() { + local path=${1%$'\r'} + if command -v cygpath >/dev/null 2>&1; then + cygpath -u "$path" 2>/dev/null && return 0 + fi + printf '%s\n' "${path//\\//}" +} + +ENV_PROBE="$WORKDIR/environment-probe" +cat > "$ENV_PROBE" <<'EOF' +#!/usr/bin/env bash +printf '%s\t%s\n' "${CBM_CACHE_DIR-}" "${CBM_RUNTIME_DIR-}" >> "$CBM_BENCH_ENV_PROBE" +if [[ "${1-} ${2-}" == "cli index_repository" ]]; then + [[ -z "${CBM_BENCH_REQUEST_LOG-}" ]] || printf '%s\n' "${3-}" >> "$CBM_BENCH_REQUEST_LOG" + if [[ -n "${CBM_BENCH_PROBE_INDEX_OK-}" ]]; then + printf '%s\n' '{"project":"probe"}' + exit 0 + fi + echo "probe: index refused" >&2 + exit 1 +fi +[[ "${1-} ${2-}" == "daemon status" ]] && exit 1 +exit 0 +EOF +chmod +x "$ENV_PROBE" + +CALLER_CACHE="$WORKDIR/caller-cache" +CALLER_RUNTIME="$WORKDIR/caller-runtime" +REPO="$WORKDIR/repo" +mkdir -p "$CALLER_CACHE" "$CALLER_RUNTIME" "$REPO" +echo 'def bench(): return 1' > "$REPO/bench.py" +CALLER_CACHE_NORMALIZED=$(normalize_path "$CALLER_CACHE") +CALLER_RUNTIME_NORMALIZED=$(normalize_path "$CALLER_RUNTIME") + +# The fixture answers nothing, so the search benchmark stops once the index +# reports no project; only the environment handed to the product is under test. +assert_isolated() { + local harness="$1" env_log="$2" private_root="" + [[ -s "$env_log" ]] || fail "$harness did not execute the environment-probe fixture" + while IFS=$'\t' read -r child_cache_raw child_runtime_raw; do + local child_cache child_runtime + child_cache=$(normalize_path "$child_cache_raw") + child_runtime=$(normalize_path "$child_runtime_raw") + if [[ -z "$child_runtime" || "$child_runtime" == "$CALLER_RUNTIME_NORMALIZED" ]]; then + fail "$harness exposed the caller CBM_RUNTIME_DIR to a product process" + fi + if [[ -z "$child_cache" || "$child_cache" == "$CALLER_CACHE_NORMALIZED" ]]; then + fail "$harness exposed the caller CBM_CACHE_DIR to a product process" + fi + if [[ "${child_runtime%/*}" != "${child_cache%/*}" || + "${child_runtime##*/}" != "runtime" || "${child_cache##*/}" != "cache" ]]; then + fail "$harness runtime/cache were not isolated beneath one private root" + fi + if [[ -n "$private_root" && "$private_root" != "${child_runtime%/*}" ]]; then + fail "$harness switched private roots mid-run" + fi + private_root="${child_runtime%/*}" + done < "$env_log" + [[ ! -e "$private_root" ]] || fail "$harness left its private root behind: $private_root" +} + +# The fixture refuses the index. That must fail the run — after the timing +# files are written, so the caller keeps its figures — and name its cause: the +# fixture's one stderr line has to reach the output. +INDEX_LOG="$WORKDIR/index-environment.log" +index_rc=0 +CBM_CACHE_DIR="$CALLER_CACHE" \ +CBM_RUNTIME_DIR="$CALLER_RUNTIME" \ +CBM_BENCH_ENV_PROBE="$INDEX_LOG" \ + "$ROOT/scripts/benchmark-index.sh" "$ENV_PROBE" probe "$REPO" "$WORKDIR/results" \ + > "$WORKDIR/index.out" 2>&1 || index_rc=$? +[[ "$index_rc" -ne 0 ]] || fail "benchmark-index exited 0 after a refused index" +assert_isolated "benchmark-index" "$INDEX_LOG" +for metric in setup-time total-time index-time; do + [[ -s "$WORKDIR/results/probe/$metric.txt" ]] || + fail "benchmark-index did not record $metric.txt" +done +grep -q -- '--- index stderr ---' "$WORKDIR/index.out" || + fail "benchmark-index hid the index stderr behind its own message" +grep -q 'probe: index refused' "$WORKDIR/index.out" || + fail "benchmark-index did not surface the cause of the index failure" + +# The evaluation plan indexes a language and then reads that index from its own +# MCP session (docs/EVALUATION_PLAN.md §7). Asked to keep the runtime, a +# successful run must leave its root behind and record, sourceably, the paths +# that reach it — and they must be the paths the product processes actually +# used. Unasked, the root is gone (asserted above). The fixture answers this +# index with a minimal envelope, so what is kept is an index that succeeded. +KEEP_LOG="$WORKDIR/keep-environment.log" +keep_rc=0 +CBM_CACHE_DIR="$CALLER_CACHE" \ +CBM_RUNTIME_DIR="$CALLER_RUNTIME" \ +CBM_BENCH_ENV_PROBE="$KEEP_LOG" \ +CBM_BENCH_PROBE_INDEX_OK=1 \ +CBM_BENCH_KEEP_RUNTIME=1 \ + "$ROOT/scripts/benchmark-index.sh" "$ENV_PROBE" keep "$REPO" "$WORKDIR/results" \ + > "$WORKDIR/keep.out" 2>&1 || keep_rc=$? +[[ "$keep_rc" -eq 0 ]] || fail "benchmark-index failed a run whose index succeeded (exit $keep_rc)" +KEPT_PROJECT=$(cat "$WORKDIR/results/keep/project.txt" 2>/dev/null || true) +[[ "${KEPT_PROJECT%$'\r'}" == "probe" ]] || + fail "benchmark-index did not record the project the index reported: '${KEPT_PROJECT:-}'" +HANDOFF="$WORKDIR/results/keep/runtime-root.txt" +[[ -s "$HANDOFF" ]] || fail "benchmark-index was asked to keep its runtime but recorded no handoff" +KEPT_ROOT=$(bash -c '. "$1" && printf "%s" "${CBM_BENCH_RUNTIME_ROOT-}"' _ "$HANDOFF") +KEPT_RUNTIME=$(bash -c '. "$1" && printf "%s" "${CBM_RUNTIME_DIR-}"' _ "$HANDOFF") +KEPT_CACHE=$(bash -c '. "$1" && printf "%s" "${CBM_CACHE_DIR-}"' _ "$HANDOFF") +[[ -n "$KEPT_ROOT" && -d "$KEPT_ROOT" && ! -L "$KEPT_ROOT" ]] || + fail "benchmark-index did not keep its runtime root: ${KEPT_ROOT:-}" +[[ -d "$KEPT_ROOT/cache" && -d "$KEPT_ROOT/runtime" ]] || + fail "the kept root $KEPT_ROOT lost its cache or runtime directory" +grep -qF -- "${KEPT_CACHE}"$'\t'"${KEPT_RUNTIME}" "$KEEP_LOG" || + fail "runtime-root.txt does not name the runtime and cache the product processes used" +rm -rf -- "$KEPT_ROOT" + +# Asked to keep the runtime and refused the index, the harness has nothing worth +# keeping: the run fails, no root survives, and no handoff is written — nor left +# over from an earlier run, since the evaluation loop reuses the results +# directory. +REFUSED_LOG="$WORKDIR/keep-refused-environment.log" +REFUSED_HANDOFF="$WORKDIR/results/keep-refused/runtime-root.txt" +mkdir -p "${REFUSED_HANDOFF%/*}" +echo 'CBM_BENCH_RUNTIME_ROOT=/stale/root/from/an/earlier/run' > "$REFUSED_HANDOFF" +refused_rc=0 +CBM_CACHE_DIR="$CALLER_CACHE" \ +CBM_RUNTIME_DIR="$CALLER_RUNTIME" \ +CBM_BENCH_ENV_PROBE="$REFUSED_LOG" \ +CBM_BENCH_KEEP_RUNTIME=1 \ + "$ROOT/scripts/benchmark-index.sh" "$ENV_PROBE" keep-refused "$REPO" "$WORKDIR/results" \ + > "$WORKDIR/keep-refused.out" 2>&1 || refused_rc=$? +[[ "$refused_rc" -ne 0 ]] || fail "benchmark-index exited 0 after a refused index it was asked to keep" +for metric in setup-time total-time index-time; do + [[ -s "$WORKDIR/results/keep-refused/$metric.txt" ]] || + fail "benchmark-index did not record $metric.txt for a refused index" +done +assert_isolated "benchmark-index (keep, refused index)" "$REFUSED_LOG" +[[ ! -e "$REFUSED_HANDOFF" ]] || fail "benchmark-index left a handoff for an index that failed" + +SEARCH_LOG="$WORKDIR/search-environment.log" +CBM_CACHE_DIR="$CALLER_CACHE" \ +CBM_RUNTIME_DIR="$CALLER_RUNTIME" \ +CBM_BENCH_ENV_PROBE="$SEARCH_LOG" \ + "$ROOT/scripts/benchmark-search-graph.sh" "$ENV_PROBE" "$REPO" \ + > "$WORKDIR/search.out" 2>&1 || true +assert_isolated "benchmark-search-graph" "$SEARCH_LOG" + +# A refused index must name its cause. The fixture writes one line to stderr and +# exits non-zero; discarding it leaves "did not report a project" as the only +# thing an operator sees. +grep -q -- '--- index/parse stderr ---' "$WORKDIR/search.out" || + fail "benchmark-search-graph hid the index stderr behind its own message" +grep -q 'probe: index refused' "$WORKDIR/search.out" || + fail "benchmark-search-graph did not surface the cause of the index failure" + +# The request carrying the repository path has to be built as JSON. A path may +# legitimately contain a quote or a backslash — hand-built JSON turns that into +# a payload the server cannot parse, or one that means something else. NTFS +# rejects both characters in a path component, so this case is POSIX-only. +case "$(uname -s)" in +MINGW* | MSYS* | CYGWIN*) ;; +*) + QUIRKY_REPO="$WORKDIR/re\"po\\dir" + mkdir -p "$QUIRKY_REPO" + echo 'def bench(): return 1' > "$QUIRKY_REPO/bench.py" + QUIRKY_RESOLVED=$(cd "$QUIRKY_REPO" && pwd -P) + REQUEST_LOG="$WORKDIR/requests.log" + CBM_CACHE_DIR="$CALLER_CACHE" \ + CBM_RUNTIME_DIR="$CALLER_RUNTIME" \ + CBM_BENCH_ENV_PROBE="$WORKDIR/quirky-environment.log" \ + CBM_BENCH_REQUEST_LOG="$REQUEST_LOG" \ + "$ROOT/scripts/benchmark-search-graph.sh" "$ENV_PROBE" "$QUIRKY_REPO" \ + > "$WORKDIR/quirky.out" 2>&1 || true + python3 - "$REQUEST_LOG" "$QUIRKY_RESOLVED" <<'PY' || fail "benchmark-search-graph built an index request that is not valid JSON for a quoted path" +import json +import sys + +request_log, expected = sys.argv[1], sys.argv[2] +try: + lines = [line for line in open(request_log).read().splitlines() if line.strip()] +except OSError: + sys.exit("the search benchmark sent no index request") +if not lines: + sys.exit("the search benchmark sent no index request") +payload = json.loads(lines[0]) +if payload.get("repo_path") != expected: + sys.exit(f"repo_path is {payload.get('repo_path')!r}, expected {expected!r}") +PY + ;; +esac + +echo "PASS: benchmark harnesses isolate their daemon runtime and cache from the caller"