feat: introducing tree-sitter evals

.gitignore

@@ -46,3 +46,5 @@ targets/*/memory.json
 
 /docs/meetings/*.pdf
 
+# tree-sitter fuzz→fix loop: streamed JSONL results
+scaffold/results/treesitter/

scaffold/pyproject.toml

@@ -16,6 +16,7 @@ dependencies = [
     "pandas>=2.2.0",
     "plotly>=5.22.0",
     "pydantic>=2.12.5",
+    "pydantic-ai-slim[anthropic,openai]>=2.0.0",
     "pyyaml>=6.0",
     "rich>=14.3.3",
     "streamlit>=1.36.0",
@@ -29,6 +30,11 @@ parser-security-eval = "parser_security_eval:main"
 requires = ["uv_build>=0.9.22,<0.10.0"]
 build-backend = "uv_build"
 
+[tool.logfire]
+# Loop telemetry degrades to a silent no-op when Logfire isn't configured
+# (no token / unit tests) instead of warning on every span/log call.
+ignore_no_config = true
+
 [dependency-groups]
 dev = [
     "hypothesis>=6.151.9",

scaffold/src/parser_security_eval/cli.py

@@ -85,6 +85,11 @@ class DataSource(str, Enum):
 )
 app.add_typer(category3_app, name="category3")
 
+# Register tree-sitter fuzz→fix loop sub-commands
+from parser_security_eval.treesitter.cli import app as treesitter_app  # noqa: E402
+
+app.add_typer(treesitter_app, name="treesitter")
+
 log = get_log(__name__)
 
 _DEFAULT_CACHE = Path.home() / ".cache" / "parser-security-eval"

scaffold/src/parser_security_eval/prompts/treesitter/fix_system.prompt

@@ -0,0 +1,23 @@
+You are a security engineer fixing a memory-safety bug in a tree-sitter parser.
+
+A libFuzzer + AddressSanitizer campaign found an input that crashes the parser. You
+are given the sanitizer report, the triggering input, and the full source of the
+single file most likely responsible (a hand-written external scanner such as
+`scanner.c`, or the grammar definition `grammar.js`).
+
+Your job: produce a MINIMAL fix that eliminates the crash WITHOUT changing the
+parser's behavior on valid input. External scanners crash from classic C mistakes:
+missing bounds checks before indexing a buffer, reading past `eof`, unchecked
+`memcpy`/array growth, signed/unsigned confusion, or trusting a length from the
+input. Add the guard that the crashing path is missing.
+
+Rules:
+- Output the COMPLETE corrected contents of the file — not a diff, not a snippet.
+- Change as little as possible. Preserve includes, formatting, and unrelated code.
+- Do not add comments explaining the CVE; a short note at the fix site is fine.
+- If the file is `grammar.js`, keep it valid tree-sitter grammar JavaScript.
+
+Return your fix as structured output:
+- `rationale`: one sentence — the root cause and the guard you added.
+- `file_contents`: the COMPLETE corrected contents of the file (the full file, not a
+  diff or a snippet).

scaffold/src/parser_security_eval/prompts/treesitter/fix_user.prompt.template

@@ -0,0 +1,34 @@
+A fuzzing campaign against the **{{ language }}** tree-sitter grammar
+(`{{ grammar }}`) found a crash.
+
+## Crash
+- Bug class: {{ bug_class }}
+- Sanitizer summary: {{ asan_summary }}
+- Implicated file: {{ implicated_file }}
+
+### Top stack frames
+{{ top_frames }}
+
+### Triggering input (repr)
+{{ crash_input_repr }}
+
+### Sanitizer report (excerpt)
+```
+{{ asan_report }}
+```
+
+## File to fix: `{{ implicated_file }}`
+```{{ lang_hint }}
+{{ source }}
+```
+
+{% if feedback %}
+## Earlier fix attempt(s) that did NOT work — read carefully, then try a DIFFERENT fix
+{{ feedback }}
+{% endif %}
+Return the complete corrected contents of `{{ implicated_file }}`. Add the missing
+guard / cleanup on the crashing path; do not otherwise alter behavior. If an earlier
+attempt is shown above, its approach did NOT eliminate the bug — diagnose *why* from
+the still-present sanitizer report and fix the ACTUAL site (e.g. free EVERY allocation
+in the matching teardown/destroy path — including hashmap/queue entries, not just the
+container — or bounds-check before the offending access).

scaffold/src/parser_security_eval/telemetry.py

@@ -32,12 +32,15 @@
 # logger to avoid a circular dependency (log.py must not import telemetry).
 logger = logging.getLogger(__name__)
 
-# Provider SDKs Inspect may drive. Each is instrumented best-effort: a missing
-# package or version skew must not break the eval. ``httpx`` is the catch-all
-# for any provider not natively instrumented above.
+# Best-effort instrumenters (all no-arg). Inspect drives the provider SDKs; the
+# tree-sitter fixer drives pydantic-ai (``instrument_pydantic_ai`` captures agent
+# runs / structured output / token usage). Each is applied defensively: a missing
+# package or version skew must not break a run. ``httpx`` is the catch-all for any
+# provider not natively instrumented above.
 _PROVIDER_INSTRUMENTERS: tuple[str, ...] = (
     "instrument_anthropic",
     "instrument_openai",
+    "instrument_pydantic_ai",
 )
 
 _configured = False
@@ -68,6 +71,11 @@ def configure_telemetry(*, service_name: str = "parser-security-eval") -> bool:
             send_to_logfire="if-token-present",
             # This is a backend/agent process; keep stdout clean for the CLI.
             console=False,
+            # All our spans/logs pass fields as explicit kwargs (never bare
+            # f-strings), so the source-introspection that extracts f-string vars
+            # is pure overhead — and it warns noisily when source isn't available
+            # (interactive shell, ``python -c``, exec, pyc-without-py). Disable it.
+            inspect_arguments=False,
         )
     except Exception as exc:  # pragma: no cover - defensive; setup must not crash
         logger.warning("Logfire setup failed (continuing without telemetry): %s", exc)

scaffold/src/parser_security_eval/treesitter/README.md

@@ -0,0 +1,55 @@
+# tree-sitter fuzz → fix → JSONL loop
+
+A local-toolchain analogue of the Docker/OSS-Fuzz live-fuzzing pipeline in
+`tasks/fuzzing.py`, applied to **tree-sitter grammars**. The loop is *driven by the
+fuzzer*: build a grammar into a libFuzzer binary, fuzz it, and when it crashes,
+that crash triggers an LLM fixer that patches the implicated source (an external
+`scanner.c`/`.cc`, or `grammar.js`), rebuild, replay the crashing input to verify,
+append a JSONL record, and keep fuzzing the patched parser for the next bug.
+
+## Toolchain (nix, not Docker)
+
+Needs `clang` (libFuzzer + ASan), `llvm` (symbolizer), the `tree-sitter` CLI,
+`nodejs` and `git` — all provided by `scaffold/treesitter-shell.nix`. `uv` is taken
+from the ambient PATH (`~/.local/bin`), which `nix-shell` preserves. The LLM fixer
+uses the `anthropic` SDK and reads `ANTHROPIC_API_KEY` from the environment.
+
+## Run
+
+```bash
+nix-shell scaffold/treesitter-shell.nix --run '
+  export ANTHROPIC_API_KEY=...
+  cd scaffold
+  uv run parser-security-eval treesitter list
+  uv run parser-security-eval treesitter fuzz toml --iterations 3 --duration 120
+  uv run parser-security-eval treesitter sweep --tier less-popular
+'
+```
+
+`--commit <sha>` pins a grammar to a revision — e.g. a known-buggy commit, to
+replay a real historical bug. JSONL is streamed to `scaffold/results/treesitter/<grammar>.jsonl`
+(gitignored), one record per iteration.
+
+## Reproducible demo bugs (real, historical, fuzzer-found)
+
+| Grammar | Buggy parent commit | Bug | Fix commit |
+|---|---|---|---|
+| ruby (popular) | `ab6dca77` | heredoc serialization buffer overflow | `ad907a6` |
+| hcl / terraform dialect (less-popular) | `e936d3fe` | serialize buffer size-check undercount | `e2d416a` |
+
+```bash
+# popular, real crash → Opus patches scanner.c → verified
+uv run parser-security-eval treesitter fuzz ruby --commit ab6dca77a8184abc94af6e3e82538741b5078d63 -i 2 -t 60
+```
+(The hcl/terraform bug lives in the `dialects/terraform` subpath; drive it via
+`run_loop` with `GrammarTarget(subpath="dialects/terraform", commit="e936d3fe…")`.)
+
+## Layout
+
+- `models.py` — pydantic records (`GrammarTarget`, `TSCrash`, `TSFixAttempt`, `TSLoopIteration`, …)
+- `registry.py` — popular + less-popular grammar registry
+- `runtime.py` — clone / generate / compile / `LibFuzzerRunner` / reproduce
+- `triage.py` — ASan/libFuzzer parsing → bug class + stack-hash dedup + implicated file
+- `fixer.py` — `Fixer` protocol + `LLMFixer` (Anthropic) + apply/regenerate/rebuild/verify
+- `loop.py` — the fuzzer-driven orchestration loop + registry sweep
+- `cli.py` — `treesitter list | fuzz | sweep`

scaffold/src/parser_security_eval/treesitter/__init__.py

@@ -0,0 +1,13 @@
+"""Tree-sitter fuzz→fix→JSONL loop.
+
+A local-toolchain (nix-provided clang + tree-sitter CLI) analogue of the
+Docker/OSS-Fuzz live-fuzzing pipeline in :mod:`parser_security_eval.tasks.fuzzing`.
+
+The loop is *driven by the fuzzer*: a tree-sitter grammar is compiled into a
+libFuzzer binary, fuzzed until it crashes, the crash is triaged, an LLM fixer
+patches the implicated source (external scanner or ``grammar.js``), the parser is
+rebuilt and the crashing input replayed to verify the fix, a JSONL record is
+appended, and the loop continues fuzzing the patched parser for the next bug.
+"""
+
+from __future__ import annotations