From 042876322ce5eac78152e68ee910b63f5514b69f Mon Sep 17 00:00:00 2001 From: Matthew Doty Date: Wed, 24 Jun 2026 09:35:10 -0400 Subject: [PATCH 1/6] feat: introducing tree-sitter evals --- .gitignore | 2 + scaffold/src/parser_security_eval/cli.py | 5 + .../prompts/treesitter/fix_system.prompt | 25 ++ .../treesitter/fix_user.prompt.template | 27 ++ .../parser_security_eval/treesitter/README.md | 55 +++ .../treesitter/__init__.py | 13 + .../parser_security_eval/treesitter/cli.py | 258 +++++++++++ .../parser_security_eval/treesitter/fixer.py | 315 +++++++++++++ .../parser_security_eval/treesitter/loop.py | 259 +++++++++++ .../parser_security_eval/treesitter/models.py | 158 +++++++ .../treesitter/registry.py | 111 +++++ .../treesitter/runtime.py | 422 ++++++++++++++++++ .../treesitter/survey/__init__.py | 10 + .../treesitter/survey/github.py | 240 ++++++++++ .../treesitter/survey/models.py | 94 ++++ .../treesitter/survey/runner.py | 224 ++++++++++ .../treesitter/survey/scoring.py | 167 +++++++ .../treesitter/survey/sources.py | 270 +++++++++++ .../parser_security_eval/treesitter/triage.py | 174 ++++++++ scaffold/src/tests/test_treesitter_fixer.py | 279 ++++++++++++ scaffold/src/tests/test_treesitter_hunt.py | 116 +++++ scaffold/src/tests/test_treesitter_loop.py | 131 ++++++ .../src/tests/test_treesitter_registry.py | 35 ++ .../src/tests/test_treesitter_runtime_unit.py | 89 ++++ .../tests/test_treesitter_survey_github.py | 101 +++++ .../tests/test_treesitter_survey_scoring.py | 138 ++++++ .../tests/test_treesitter_survey_sources.py | 80 ++++ scaffold/src/tests/test_treesitter_triage.py | 118 +++++ scaffold/treesitter-shell.nix | 22 + 29 files changed, 3938 insertions(+) create mode 100644 scaffold/src/parser_security_eval/prompts/treesitter/fix_system.prompt create mode 100644 scaffold/src/parser_security_eval/prompts/treesitter/fix_user.prompt.template create mode 100644 scaffold/src/parser_security_eval/treesitter/README.md create mode 100644 scaffold/src/parser_security_eval/treesitter/__init__.py create mode 100644 scaffold/src/parser_security_eval/treesitter/cli.py create mode 100644 scaffold/src/parser_security_eval/treesitter/fixer.py create mode 100644 scaffold/src/parser_security_eval/treesitter/loop.py create mode 100644 scaffold/src/parser_security_eval/treesitter/models.py create mode 100644 scaffold/src/parser_security_eval/treesitter/registry.py create mode 100644 scaffold/src/parser_security_eval/treesitter/runtime.py create mode 100644 scaffold/src/parser_security_eval/treesitter/survey/__init__.py create mode 100644 scaffold/src/parser_security_eval/treesitter/survey/github.py create mode 100644 scaffold/src/parser_security_eval/treesitter/survey/models.py create mode 100644 scaffold/src/parser_security_eval/treesitter/survey/runner.py create mode 100644 scaffold/src/parser_security_eval/treesitter/survey/scoring.py create mode 100644 scaffold/src/parser_security_eval/treesitter/survey/sources.py create mode 100644 scaffold/src/parser_security_eval/treesitter/triage.py create mode 100644 scaffold/src/tests/test_treesitter_fixer.py create mode 100644 scaffold/src/tests/test_treesitter_hunt.py create mode 100644 scaffold/src/tests/test_treesitter_loop.py create mode 100644 scaffold/src/tests/test_treesitter_registry.py create mode 100644 scaffold/src/tests/test_treesitter_runtime_unit.py create mode 100644 scaffold/src/tests/test_treesitter_survey_github.py create mode 100644 scaffold/src/tests/test_treesitter_survey_scoring.py create mode 100644 scaffold/src/tests/test_treesitter_survey_sources.py create mode 100644 scaffold/src/tests/test_treesitter_triage.py create mode 100644 scaffold/treesitter-shell.nix diff --git a/.gitignore b/.gitignore index e610b51..a714f8f 100644 --- a/.gitignore +++ b/.gitignore @@ -46,3 +46,5 @@ targets/*/memory.json /docs/meetings/*.pdf +# tree-sitter fuzz→fix loop: streamed JSONL results +scaffold/results/treesitter/ diff --git a/scaffold/src/parser_security_eval/cli.py b/scaffold/src/parser_security_eval/cli.py index b130b5f..2e59399 100644 --- a/scaffold/src/parser_security_eval/cli.py +++ b/scaffold/src/parser_security_eval/cli.py @@ -85,6 +85,11 @@ class DataSource(str, Enum): ) app.add_typer(category3_app, name="category3") +# Register tree-sitter fuzz→fix loop sub-commands +from parser_security_eval.treesitter.cli import app as treesitter_app # noqa: E402 + +app.add_typer(treesitter_app, name="treesitter") + log = get_log(__name__) _DEFAULT_CACHE = Path.home() / ".cache" / "parser-security-eval" diff --git a/scaffold/src/parser_security_eval/prompts/treesitter/fix_system.prompt b/scaffold/src/parser_security_eval/prompts/treesitter/fix_system.prompt new file mode 100644 index 0000000..40c1e73 --- /dev/null +++ b/scaffold/src/parser_security_eval/prompts/treesitter/fix_system.prompt @@ -0,0 +1,25 @@ +You are a security engineer fixing a memory-safety bug in a tree-sitter parser. + +A libFuzzer + AddressSanitizer campaign found an input that crashes the parser. You +are given the sanitizer report, the triggering input, and the full source of the +single file most likely responsible (a hand-written external scanner such as +`scanner.c`, or the grammar definition `grammar.js`). + +Your job: produce a MINIMAL fix that eliminates the crash WITHOUT changing the +parser's behavior on valid input. External scanners crash from classic C mistakes: +missing bounds checks before indexing a buffer, reading past `eof`, unchecked +`memcpy`/array growth, signed/unsigned confusion, or trusting a length from the +input. Add the guard that the crashing path is missing. + +Rules: +- Output the COMPLETE corrected contents of the file — not a diff, not a snippet. +- Change as little as possible. Preserve includes, formatting, and unrelated code. +- Do not add comments explaining the CVE; a short note at the fix site is fine. +- If the file is `grammar.js`, keep it valid tree-sitter grammar JavaScript. + +Respond in EXACTLY this format: + +RATIONALE: +``` + +``` diff --git a/scaffold/src/parser_security_eval/prompts/treesitter/fix_user.prompt.template b/scaffold/src/parser_security_eval/prompts/treesitter/fix_user.prompt.template new file mode 100644 index 0000000..6d97475 --- /dev/null +++ b/scaffold/src/parser_security_eval/prompts/treesitter/fix_user.prompt.template @@ -0,0 +1,27 @@ +A fuzzing campaign against the **{{ language }}** tree-sitter grammar +(`{{ grammar }}`) found a crash. + +## Crash +- Bug class: {{ bug_class }} +- Sanitizer summary: {{ asan_summary }} +- Implicated file: {{ implicated_file }} + +### Top stack frames +{{ top_frames }} + +### Triggering input (repr) +{{ crash_input_repr }} + +### Sanitizer report (excerpt) +``` +{{ asan_report }} +``` + +## File to fix: `{{ implicated_file }}` +```{{ lang_hint }} +{{ source }} +``` + +Return the complete corrected contents of `{{ implicated_file }}` in the required +format. Add the missing safety check on the crashing path; do not otherwise alter +behavior. diff --git a/scaffold/src/parser_security_eval/treesitter/README.md b/scaffold/src/parser_security_eval/treesitter/README.md new file mode 100644 index 0000000..84abb8b --- /dev/null +++ b/scaffold/src/parser_security_eval/treesitter/README.md @@ -0,0 +1,55 @@ +# tree-sitter fuzz → fix → JSONL loop + +A local-toolchain analogue of the Docker/OSS-Fuzz live-fuzzing pipeline in +`tasks/fuzzing.py`, applied to **tree-sitter grammars**. The loop is *driven by the +fuzzer*: build a grammar into a libFuzzer binary, fuzz it, and when it crashes, +that crash triggers an LLM fixer that patches the implicated source (an external +`scanner.c`/`.cc`, or `grammar.js`), rebuild, replay the crashing input to verify, +append a JSONL record, and keep fuzzing the patched parser for the next bug. + +## Toolchain (nix, not Docker) + +Needs `clang` (libFuzzer + ASan), `llvm` (symbolizer), the `tree-sitter` CLI, +`nodejs` and `git` — all provided by `scaffold/treesitter-shell.nix`. `uv` is taken +from the ambient PATH (`~/.local/bin`), which `nix-shell` preserves. The LLM fixer +uses the `anthropic` SDK and reads `ANTHROPIC_API_KEY` from the environment. + +## Run + +```bash +nix-shell scaffold/treesitter-shell.nix --run ' + export ANTHROPIC_API_KEY=... + cd scaffold + uv run parser-security-eval treesitter list + uv run parser-security-eval treesitter fuzz toml --iterations 3 --duration 120 + uv run parser-security-eval treesitter sweep --tier less-popular +' +``` + +`--commit ` pins a grammar to a revision — e.g. a known-buggy commit, to +replay a real historical bug. JSONL is streamed to `scaffold/results/treesitter/.jsonl` +(gitignored), one record per iteration. + +## Reproducible demo bugs (real, historical, fuzzer-found) + +| Grammar | Buggy parent commit | Bug | Fix commit | +|---|---|---|---| +| ruby (popular) | `ab6dca77` | heredoc serialization buffer overflow | `ad907a6` | +| hcl / terraform dialect (less-popular) | `e936d3fe` | serialize buffer size-check undercount | `e2d416a` | + +```bash +# popular, real crash → Opus patches scanner.c → verified +uv run parser-security-eval treesitter fuzz ruby --commit ab6dca77a8184abc94af6e3e82538741b5078d63 -i 2 -t 60 +``` +(The hcl/terraform bug lives in the `dialects/terraform` subpath; drive it via +`run_loop` with `GrammarTarget(subpath="dialects/terraform", commit="e936d3fe…")`.) + +## Layout + +- `models.py` — pydantic records (`GrammarTarget`, `TSCrash`, `TSFixAttempt`, `TSLoopIteration`, …) +- `registry.py` — popular + less-popular grammar registry +- `runtime.py` — clone / generate / compile / `LibFuzzerRunner` / reproduce +- `triage.py` — ASan/libFuzzer parsing → bug class + stack-hash dedup + implicated file +- `fixer.py` — `Fixer` protocol + `LLMFixer` (Anthropic) + apply/regenerate/rebuild/verify +- `loop.py` — the fuzzer-driven orchestration loop + registry sweep +- `cli.py` — `treesitter list | fuzz | sweep` diff --git a/scaffold/src/parser_security_eval/treesitter/__init__.py b/scaffold/src/parser_security_eval/treesitter/__init__.py new file mode 100644 index 0000000..2d36d12 --- /dev/null +++ b/scaffold/src/parser_security_eval/treesitter/__init__.py @@ -0,0 +1,13 @@ +"""Tree-sitter fuzz→fix→JSONL loop. + +A local-toolchain (nix-provided clang + tree-sitter CLI) analogue of the +Docker/OSS-Fuzz live-fuzzing pipeline in :mod:`parser_security_eval.tasks.fuzzing`. + +The loop is *driven by the fuzzer*: a tree-sitter grammar is compiled into a +libFuzzer binary, fuzzed until it crashes, the crash is triaged, an LLM fixer +patches the implicated source (external scanner or ``grammar.js``), the parser is +rebuilt and the crashing input replayed to verify the fix, a JSONL record is +appended, and the loop continues fuzzing the patched parser for the next bug. +""" + +from __future__ import annotations diff --git a/scaffold/src/parser_security_eval/treesitter/cli.py b/scaffold/src/parser_security_eval/treesitter/cli.py new file mode 100644 index 0000000..aa55cbc --- /dev/null +++ b/scaffold/src/parser_security_eval/treesitter/cli.py @@ -0,0 +1,258 @@ +"""CLI commands for the tree-sitter fuzz→fix loop. + +Registered as a sub-application on the main ``parser-security-eval`` CLI. + +Commands: + treesitter list + treesitter fuzz [--iterations N --duration S --model M --out DIR] + treesitter sweep [--tier popular|less-popular] [--iterations N --duration S] + treesitter survey [--token T --limit N --out DIR] # build the priority registry + treesitter hunt [--top N --duration S] # fuzz top un-fuzzed scanner grammars + +Run inside the project's nix-shell so clang/tree-sitter are on PATH: + nix-shell scaffold/treesitter-shell.nix --run \\ + 'cd scaffold && uv run parser-security-eval treesitter fuzz toml' +""" + +from __future__ import annotations + +from pathlib import Path + +import typer + +from parser_security_eval.treesitter import registry +from parser_security_eval.treesitter.fixer import DEFAULT_FIX_MODEL +from parser_security_eval.treesitter.loop import DEFAULT_OUT_DIR, run_loop +from parser_security_eval.treesitter.models import GrammarTarget, Tier, TSLoopResult + +app = typer.Typer( + name="treesitter", + help="Fuzz tree-sitter grammars, fix the crashes, log JSONL, iterate.", +) + + +@app.command("list") +def list_grammars() -> None: + """List the registered tree-sitter grammar targets.""" + for tier in (Tier.popular, Tier.less_popular): + typer.echo(f"\n[{tier.value}]") + for g in registry.by_tier(tier): + pin = f" @{g.commit[:10]}" if g.commit else "" + typer.echo(f" {g.name:<12} {g.language:<12} {g.repo_url}{pin}") + + +def _print_summary(result: TSLoopResult) -> None: + typer.echo(f"\n=== {result.grammar} ({result.tier.value}) ===") + if result.build_error: + typer.echo(f" BUILD FAILED: {result.build_error[:300]}") + return + typer.echo(f" iterations: {len(result.iterations)}") + typer.echo(f" crashes found: {result.crashes_found}") + typer.echo(f" unique crashes: {len(result.unique_stack_hashes)}") + typer.echo(f" fixes verified: {result.fixes_verified}") + for it in result.iterations: + if it.crash is None: + continue + fix = it.fix + status = "no-fix" + if fix and fix.verified: + status = "FIXED ✓" + elif fix and fix.applied: + status = "patched (unverified)" + elif fix and fix.error: + status = f"fix-error: {fix.error[:60]}" + typer.echo( + f" #{it.iteration} {it.crash.bug_class.value} " + f"in {it.crash.implicated_file or '?'} -> {status}" + ) + typer.echo(f" JSONL: {result.jsonl_path}") + + +@app.command("fuzz") +def fuzz( + grammar: str = typer.Argument(help="Grammar name (see `treesitter list`)"), + iterations: int = typer.Option(5, "--iterations", "-i", help="Fuzz→fix iterations"), + duration: int = typer.Option( + 120, "--duration", "-t", help="Seconds per fuzz window" + ), + model: str = typer.Option(DEFAULT_FIX_MODEL, "--model", "-m", help="Fixer model"), + out_dir: Path = typer.Option(DEFAULT_OUT_DIR, "--out", help="JSONL output dir"), + max_len: int = typer.Option(65536, "--max-len", help="libFuzzer -max_len"), + commit: str | None = typer.Option( + None, "--commit", help="Pin the grammar repo to a commit (e.g. a buggy rev)" + ), +) -> None: + """Run the fuzz→fix→JSONL loop against a single grammar.""" + target = registry.get(grammar) + if commit: + target = target.model_copy(update={"commit": commit}) + result = run_loop( + target, + iterations=iterations, + fuzz_seconds=duration, + model=model, + out_dir=out_dir, + max_len=max_len, + ) + _print_summary(result) + + +@app.command("sweep") +def sweep( + tier: str | None = typer.Option( + None, "--tier", help="popular | less-popular (default: all)" + ), + iterations: int = typer.Option(3, "--iterations", "-i", help="Fuzz→fix iterations"), + duration: int = typer.Option( + 120, "--duration", "-t", help="Seconds per fuzz window" + ), + model: str = typer.Option(DEFAULT_FIX_MODEL, "--model", "-m", help="Fixer model"), + out_dir: Path = typer.Option(DEFAULT_OUT_DIR, "--out", help="JSONL output dir"), +) -> None: + """Run the loop across every grammar in a tier (or all).""" + tier_enum = Tier(tier) if tier else None + targets: list[GrammarTarget] = registry.by_tier(tier_enum) + typer.echo( + f"Sweeping {len(targets)} grammars: {', '.join(t.name for t in targets)}" + ) + for target in targets: + result = run_loop( + target, + iterations=iterations, + fuzz_seconds=duration, + model=model, + out_dir=out_dir, + ) + _print_summary(result) + + +@app.command("survey") +def survey( + token: str | None = typer.Option( + None, + "--token", + help="GitHub token (else $GITHUB_TOKEN). Strongly recommended: " + "metrics for ~500 repos exceed the 60 req/hr unauthenticated limit.", + envvar="GITHUB_TOKEN", + ), + limit: int | None = typer.Option( + None, "--limit", "-n", help="Only survey the first N grammars (debug)" + ), + out_dir: Path = typer.Option( + Path("results") / "treesitter", "--out", help="Output dir for the registry" + ), + top_n: int = typer.Option(60, "--top", help="Rows in priority.md"), + refresh_days: float = typer.Option( + 7.0, "--refresh-days", help="Re-fetch GitHub metrics older than this" + ), +) -> None: + """Scrape the tree-sitter wiki + Zed/Emacs + GitHub metrics into a ranked registry.""" + from parser_security_eval.treesitter.survey.runner import run_survey + + def _progress(i: int, total: int, label: str) -> None: + if i == 1 or i % 25 == 0 or i == total: + typer.echo(f" [{i}/{total}] {label}", err=True) + + if not token: + typer.echo( + "No GitHub token — running unauthenticated (60 req/hr). " + "Results will be cached and the run is restartable.", + err=True, + ) + result = run_survey( + token=token, + out_dir=out_dir, + limit=limit, + top_n=top_n, + refresh_days=refresh_days, + progress=_progress, + ) + typer.echo(f"\nGrammars surveyed: {result.total_grammars}") + typer.echo(f"Metrics fetched: {result.metrics_fetched}") + typer.echo(f"Metrics failed: {result.metrics_failed}") + typer.echo("\nTop 15 fuzzing candidates:") + for rank, rec in enumerate(result.records[:15], start=1): + flags = "".join( + [ + "S" if rec.has_external_scanner else "-", + "Z" if rec.used_by_zed else "-", + "E" if rec.used_by_emacs else "-", + ] + ) + stars = rec.metrics.stars if rec.metrics.stars is not None else "?" + typer.echo( + f" {rank:>2}. {rec.fuzz_priority:5.1f} [{flags}] {rec.name:<16} " + f"{stars}★ {rec.repo_url}" + ) + typer.echo( + f"\nWrote:\n {result.jsonl_path}\n {result.csv_path}\n {result.priority_path}" + ) + + +@app.command("hunt") +def hunt( + top: int = typer.Option(10, "--top", "-n", help="How many candidates to fuzz"), + start: int = typer.Option( + 1, "--start", "-s", help="1-based rank to begin at (skip already-hunted ranks)" + ), + duration: int = typer.Option( + 1800, "--duration", "-t", help="Seconds to fuzz each grammar" + ), + model: str = typer.Option(DEFAULT_FIX_MODEL, "--model", "-m", help="Fixer model"), + registry: Path = typer.Option( + Path("results") / "treesitter" / "registry.jsonl", + "--registry", + help="Survey registry.jsonl to pick candidates from", + ), + out_dir: Path = typer.Option( + Path("results") / "treesitter" / "hunt", "--out", help="JSONL output dir" + ), +) -> None: + """Fuzz→fix the top un-fuzzed, scanner-bearing grammars from the survey registry.""" + from parser_security_eval.treesitter.loop import run_hunt, select_hunt_candidates + + if not registry.exists(): + typer.echo( + f"Registry not found: {registry}\n" + "Run `parser-security-eval treesitter survey` first.", + err=True, + ) + raise typer.Exit(1) + + candidates = select_hunt_candidates(registry, top, start=start) + typer.echo( + f"Hunting ranks {start}–{start + len(candidates) - 1} " + f"({len(candidates)} un-fuzzed scanner grammars, {duration}s each): " + f"{', '.join(c.name for c in candidates)}" + ) + + def _progress(i: int, total: int, label: str) -> None: + typer.echo(f"\n[{i}/{total}] fuzzing {label} …", err=True) + + results = run_hunt( + registry, + top_n=top, + start=start, + fuzz_seconds=duration, + model=model, + out_dir=out_dir, + progress=_progress, + ) + + typer.echo("\n===== HUNT SUMMARY =====") + for res in results: + if res.build_error: + typer.echo(f" {res.grammar:<28} build-failed") + continue + hit = next((it for it in res.iterations if it.crash is not None), None) + if hit is None or hit.crash is None: + fz = res.iterations[0].fuzz if res.iterations else None + execs = fz.total_executions if fz else "?" + typer.echo(f" {res.grammar:<28} no crash ({execs} execs)") + else: + fixed = hit.fix is not None and hit.fix.verified + typer.echo( + f" {res.grammar:<28} CRASH {hit.crash.bug_class.value} " + f"-> {'FIXED ✓' if fixed else 'unverified'}" + ) + typer.echo(f"\nJSONL written under: {out_dir}") diff --git a/scaffold/src/parser_security_eval/treesitter/fixer.py b/scaffold/src/parser_security_eval/treesitter/fixer.py new file mode 100644 index 0000000..e6b1c2d --- /dev/null +++ b/scaffold/src/parser_security_eval/treesitter/fixer.py @@ -0,0 +1,315 @@ +"""The "fix the parser error" step. + +Defines a :class:`Fixer` protocol (so a key-less environment can swap in a +heuristic later) and an :class:`LLMFixer` backed by the Anthropic SDK — the same +``anthropic`` dependency the rest of the repo already uses. ``run_fix`` ties the +proposal to the toolchain: write the patch, regenerate if needed, rebuild, and +replay the crashing input to verify the crash is gone. +""" + +from __future__ import annotations + +import difflib +import re +from dataclasses import dataclass +from pathlib import Path +from typing import Protocol + +from parser_security_eval import prompts +from parser_security_eval.treesitter import runtime +from parser_security_eval.treesitter.models import ( + GrammarTarget, + TSBuildResult, + TSCrash, + TSFixAttempt, +) + +DEFAULT_FIX_MODEL = "claude-opus-4-8" +# Replays used to confirm a crash reproduces (pre-fix) and is gone (post-fix). +# >1 because some scanner crashes are flaky (e.g. heap-layout-dependent SEGVs). +VERIFY_ATTEMPTS = 20 + +_RATIONALE_RE = re.compile(r"RATIONALE:\s*(.+)") +_FENCE_RE = re.compile(r"```[^\n]*\n(.*?)```", re.DOTALL) +_OPEN_FENCE_RE = re.compile(r"```[^\n]*\n") +_CRASH_MARKERS = ( + "ERROR:", + "SUMMARY:", + "AddressSanitizer", + "byte(s) leaked", + "runtime error:", +) + + +@dataclass +class FixProposal: + """A fixer's proposed replacement for the implicated source file.""" + + new_content: str + rationale: str + input_tokens: int = 0 + output_tokens: int = 0 + + +class Fixer(Protocol): + """Proposes a corrected version of the crash's implicated source file.""" + + name: str + + def propose( + self, + crash: TSCrash, + target: GrammarTarget, + source: str, + implicated_file: str, + asan_report: str, + lang_hint: str, + ) -> FixProposal: ... + + +def _lang_hint(path: Path) -> str: + return { + ".c": "c", + ".h": "c", + ".cc": "cpp", + ".cpp": "cpp", + ".cxx": "cpp", + ".js": "javascript", + }.get(path.suffix, "") + + +def _crash_input_repr(crash: TSCrash) -> str: + import base64 + + try: + raw = base64.b64decode(crash.crash_input_b64) + except ValueError, TypeError: + return "(unavailable)" + snippet = raw[:512] + return f"{snippet!r}" + (" …(truncated)" if len(raw) > 512 else "") + + +class LLMFixer: + """Anthropic-backed fixer. Reads ANTHROPIC_API_KEY from the environment.""" + + def __init__(self, model: str = DEFAULT_FIX_MODEL, max_tokens: int = 32768) -> None: + self.model = model + self.name = model + self.max_tokens = max_tokens + + def propose( + self, + crash: TSCrash, + target: GrammarTarget, + source: str, + implicated_file: str, + asan_report: str, + lang_hint: str, + ) -> FixProposal: + import anthropic + + system = prompts.load("treesitter.fix_system") + user = prompts.load( + "treesitter.fix_user", + language=target.language, + grammar=target.name, + bug_class=crash.bug_class.value, + asan_summary=crash.asan_summary, + implicated_file=implicated_file, + top_frames="\n".join(crash.top_frames) or "(none)", + crash_input_repr=_crash_input_repr(crash), + asan_report=asan_report[:6000], + source=source, + lang_hint=lang_hint, + ) + client = anthropic.Anthropic() + # Stream: a full-file rewrite at this max_tokens can exceed the SDK's + # non-streaming 10-minute ceiling, which otherwise raises. + with client.messages.stream( + model=self.model, + max_tokens=self.max_tokens, + system=system, + messages=[{"role": "user", "content": user}], + ) as stream: + msg = stream.get_final_message() + text = "".join( + getattr(block, "text", "") + for block in msg.content + if getattr(block, "type", "") == "text" + ) + new_content = _extract_file(text) + rationale_m = _RATIONALE_RE.search(text) + return FixProposal( + new_content=new_content, + rationale=rationale_m.group(1).strip() if rationale_m else "", + input_tokens=msg.usage.input_tokens, + output_tokens=msg.usage.output_tokens, + ) + + +def _extract_file(text: str) -> str: + """Pull the corrected file out of the model's response (largest fenced block). + + Falls back to "from the last opening fence to EOF" when the reply was truncated + mid-block (an unclosed fence), so a large file that hit the token cap still yields + its (partial) content rather than nothing. + """ + blocks: list[str] = _FENCE_RE.findall(text) + if blocks: + return str(max(blocks, key=len)).strip("\n") + "\n" + opens = list(_OPEN_FENCE_RE.finditer(text)) + if opens: + tail = text[opens[-1].end() :].strip("\n") + if tail: + return tail + "\n" + return "" + + +def _is_crash(rc: int, output: str) -> bool: + """Heuristic: did this replay crash (nonzero exit or a sanitizer marker)?""" + return rc != 0 or any(m in output for m in _CRASH_MARKERS) + + +def _crashes_on_replay(binary: Path, crash_file: Path, attempts: int) -> bool: + """True if the binary crashes on at least one of *attempts* replays (flaky-safe).""" + for _ in range(attempts): + rc, out = runtime.reproduce(binary, crash_file) + if _is_crash(rc, out): + return True + return False + + +def _clean_on_replay(binary: Path, crash_file: Path, attempts: int) -> bool: + """True only if the binary never crashes across *attempts* replays.""" + for _ in range(attempts): + rc, out = runtime.reproduce(binary, crash_file) + if _is_crash(rc, out): + return False + return True + + +def select_fix_target( + crash: TSCrash, build_result: TSBuildResult, src_dir: Path +) -> Path | None: + """Choose the source file the fixer should rewrite. + + Prefers the external scanner (hand-written, cleanly patchable) whenever one is + implicated — including serialization-buffer overflows that abort in the runtime + after the scanner frame returns. Falls back to ``grammar.js`` (a + regenerate-then-rebuild fix) for crashes in the generated parser. Returns + ``None`` when the crash is in the tree-sitter runtime itself. + """ + scanner = Path(build_result.scanner_file) if build_result.scanner_file else None + blob = (crash.asan_summary + " " + " ".join(crash.top_frames)).lower() + scanner_implicated = crash.in_scanner or "external_scanner" in blob + if scanner is not None and scanner_implicated: + return scanner + + impl = (crash.implicated_file or "").lower() + grammar_js = src_dir.parent / "grammar.js" + # A crash in the grammar's *generated* parser.c (not the runtime's lib/src) is + # a grammar-definition bug → patch grammar.js and regenerate. + if "parser.c" in impl and "lib/src" not in impl and grammar_js.exists(): + return grammar_js + return None + + +def run_fix( + crash: TSCrash, + crash_file: Path, + target: GrammarTarget, + spec: runtime.BuildSpec, + build_result: TSBuildResult, + fixer: Fixer, + asan_report: str, +) -> tuple[TSFixAttempt, TSBuildResult]: + """Propose+apply a fix, rebuild, and verify the crash no longer reproduces. + + Returns the :class:`TSFixAttempt` and the (possibly rebuilt) build result. + """ + attempt = TSFixAttempt(attempted=True, model=getattr(fixer, "name", "unknown")) + fix_path = select_fix_target(crash, build_result, spec.src_dir) + if fix_path is None or not fix_path.exists(): + attempt.attempted = False + attempt.error = ( + "crash is in the tree-sitter runtime, not a grammar-level source file" + if fix_path is None + else f"implicated file not found: {fix_path}" + ) + return attempt, build_result + + attempt.target_file = str(fix_path) + + # Soundness: confirm the bug actually reproduces on the PRE-fix build before + # claiming any fix "verified" — otherwise a flaky/non-reproducible artifact + # plus a clean post-fix replay is a false positive (the foam SEGV taught us this). + attempt.verify_attempts = VERIFY_ATTEMPTS + orig_binary = Path(build_result.binary_path or "") + attempt.reproduced_original = _crashes_on_replay( + orig_binary, crash_file, VERIFY_ATTEMPTS + ) + if not attempt.reproduced_original: + attempt.error = ( + f"crash did not reproduce on the original build in {VERIFY_ATTEMPTS} " + "replays (flaky/stateful artifact) — skipping fix to avoid a false positive" + ) + return attempt, build_result + + original = fix_path.read_text(encoding="utf-8", errors="replace") + lang_hint = _lang_hint(fix_path) + + try: + proposal = fixer.propose( + crash, target, original, fix_path.name, asan_report, lang_hint + ) + except Exception as exc: # noqa: BLE001 — surface fixer errors into the record + attempt.error = f"fixer.propose failed: {exc}" + return attempt, build_result + + attempt.input_tokens = proposal.input_tokens + attempt.output_tokens = proposal.output_tokens + attempt.rationale = proposal.rationale + if not proposal.new_content.strip(): + attempt.error = "fixer returned no file content" + return attempt, build_result + + attempt.patch = "".join( + difflib.unified_diff( + original.splitlines(keepends=True), + proposal.new_content.splitlines(keepends=True), + fromfile=f"a/{fix_path.name}", + tofile=f"b/{fix_path.name}", + ) + ) + + # Apply (keep a backup so a failed fix can be rolled back). + backup = fix_path.with_suffix(fix_path.suffix + ".orig") + if not backup.exists(): + backup.write_text(original, encoding="utf-8") + fix_path.write_text(proposal.new_content, encoding="utf-8") + attempt.applied = True + + # grammar.js edits require regeneration before rebuild. + if fix_path.name == "grammar.js": + ok, _ = runtime.generate_parser(spec.src_dir) + attempt.regenerated = ok + if not ok: + attempt.error = "tree-sitter generate failed after grammar.js edit" + return attempt, build_result + + new_build = runtime.build(spec) + attempt.rebuilt = new_build.built + if not new_build.built: + attempt.error = f"rebuild failed:\n{new_build.compile_errors[-1500:]}" + return attempt, build_result + + # Verified only if the patched build never crashes across the replay budget + # (reproduced_original is already True here). + attempt.verified = _clean_on_replay( + Path(new_build.binary_path or ""), crash_file, VERIFY_ATTEMPTS + ) + if not attempt.verified: + attempt.error = ( + "patch did not eliminate the crash (still reproduces after rebuild)" + ) + return attempt, new_build diff --git a/scaffold/src/parser_security_eval/treesitter/loop.py b/scaffold/src/parser_security_eval/treesitter/loop.py new file mode 100644 index 0000000..23e726f --- /dev/null +++ b/scaffold/src/parser_security_eval/treesitter/loop.py @@ -0,0 +1,259 @@ +"""The fuzzer-driven fuzz→fix→JSONL loop. + +For a grammar: build it, then repeatedly fuzz. A crash is the event that *triggers* +the fix phase — triage the crash, ask the fixer to patch the implicated source, +rebuild, replay the crashing input to verify, append a JSONL record, and keep +fuzzing the patched parser for the next bug. +""" + +from __future__ import annotations + +from collections.abc import Callable +from pathlib import Path + +from parser_security_eval.treesitter import fixer as fixer_mod +from parser_security_eval.treesitter import runtime, triage +from parser_security_eval.treesitter.fixer import DEFAULT_FIX_MODEL, Fixer, LLMFixer +from parser_security_eval.treesitter.models import ( + GrammarTarget, + Tier, + TSLoopIteration, + TSLoopResult, +) +from parser_security_eval.treesitter.registry import by_tier, get +from parser_security_eval.treesitter.runtime import ( + DEFAULT_CACHE, + BuildSpec, + LibFuzzerRunner, +) + +DEFAULT_OUT_DIR = Path("results") / "treesitter" + + +def _append_jsonl(path: Path, iteration: TSLoopIteration) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("a", encoding="utf-8") as fh: + fh.write(iteration.to_jsonl() + "\n") + + +def run_loop( + target: GrammarTarget, + *, + iterations: int = 5, + fuzz_seconds: int = 120, + model: str = DEFAULT_FIX_MODEL, + cache_dir: Path = DEFAULT_CACHE, + out_dir: Path = DEFAULT_OUT_DIR, + max_len: int = 65536, + fixer: Fixer | None = None, + seed_corpus: bool = True, +) -> TSLoopResult: + """Run the fuzz→fix loop for a single grammar; stream JSONL to ``out_dir``.""" + runtime.check_toolchain() + result = TSLoopResult( + grammar=target.name, tier=target.tier, language=target.language + ) + out_path = out_dir / f"{target.name}.jsonl" + result.jsonl_path = str(out_path) + + runtime_dir = runtime.ensure_runtime(cache_dir) + grammar_dir = runtime.ensure_grammar(target, cache_dir) + work_dir = cache_dir / f"_work_{target.name}" + spec = BuildSpec( + target=target, + runtime_dir=runtime_dir, + grammar_dir=grammar_dir, + work_dir=work_dir, + ) + + build_result = runtime.build(spec) + if not build_result.built: + result.build_error = build_result.compile_errors + it = TSLoopIteration( + iteration=0, + grammar=target.name, + repo_url=target.repo_url, + tier=target.tier, + language=target.language, + built=False, + notes=f"build failed: {build_result.compile_errors[:500]}", + ) + result.iterations.append(it) + _append_jsonl(out_path, it) + return result + + corpus = work_dir / "corpus" + crashes = work_dir / "crashes" + if seed_corpus: + runtime.gather_seeds(grammar_dir, corpus) + runner = LibFuzzerRunner( + binary=Path(build_result.binary_path or ""), + corpus_dir=corpus, + crashes_dir=crashes, + max_len=max_len, + ) + if fixer is None: + fixer = LLMFixer(model=model) + + seen_hashes: set[str] = set() + for i in range(iterations): + fuzz_res, new_crashes = runner.run(fuzz_seconds) + it = TSLoopIteration( + iteration=i, + grammar=target.name, + repo_url=target.repo_url, + tier=target.tier, + language=target.language, + built=True, + fuzz=fuzz_res, + ) + + # Pick the first crash whose stack hash we haven't already handled. + # Hang/OOM artifacts are classified from the libFuzzer filename WITHOUT + # replaying them (replaying a hang would just hang again). + chosen = None + for cf in new_crashes: + if cf.name.startswith("timeout-"): + report = ( + "ERROR: libFuzzer: timeout\n" + "Pathological parse time / hang (libFuzzer -timeout exceeded)." + ) + elif cf.name.startswith("oom-"): + report = "ERROR: libFuzzer: out-of-memory\nrss_limit exceeded." + else: + _, report = runtime.reproduce(Path(build_result.binary_path or ""), cf) + crash = triage.triage(report, cf.read_bytes(), str(cf)) + if crash.stack_hash and crash.stack_hash in seen_hashes: + continue + chosen = (cf, crash, report) + break + + if chosen is None: + it.notes = "no new crash this window — still fuzzing" + result.iterations.append(it) + _append_jsonl(out_path, it) + continue + + crash_file, crash, report = chosen + if crash.stack_hash: + seen_hashes.add(crash.stack_hash) + it.crash = crash + + # The crash triggers the fix phase. + attempt, new_build = fixer_mod.run_fix( + crash, crash_file, target, spec, build_result, fixer, report + ) + it.fix = attempt + if new_build.built and new_build.binary_path: + build_result = new_build + runner.binary = Path(new_build.binary_path) + if attempt.verified and crash_file.exists(): + crash_file.unlink() # don't re-discover a fixed crash + + result.iterations.append(it) + _append_jsonl(out_path, it) + + return result + + +def run_sweep( + tier: Tier | None = None, + *, + names: list[str] | None = None, + **kwargs: object, +) -> list[TSLoopResult]: + """Run :func:`run_loop` over every grammar in a tier (or an explicit list).""" + targets = [get(n) for n in names] if names else by_tier(tier) + results: list[TSLoopResult] = [] + for target in targets: + results.append(run_loop(target, **kwargs)) # type: ignore[arg-type] + return results + + +# --------------------------------------------------------------------------- # +# Registry-driven hunt: fuzz the top un-fuzzed scanner candidates from a survey +# --------------------------------------------------------------------------- # +DEFAULT_REGISTRY = DEFAULT_OUT_DIR / "registry.jsonl" +DEFAULT_HUNT_DIR = DEFAULT_OUT_DIR / "hunt" + + +def select_hunt_candidates( + registry_path: Path = DEFAULT_REGISTRY, + top_n: int = 10, + *, + start: int = 1, + require_scanner: bool = True, + exclude_fuzzed: bool = True, +) -> list[GrammarTarget]: + """Pick a *range* of fuzzing targets from a survey ``registry.jsonl``. + + Defaults select the goldilocks set: scanner-bearing (a real crash surface), + NOT already fuzzed, and maintained (``fuzz_priority > 0`` excludes archived/ + abandoned). Deduplicated by repo and ordered highest-priority first; the slice + ``[start-1 : start-1+top_n]`` is returned (``start`` is 1-based) so a later run + can skip already-hunted ranks. + """ + from parser_security_eval.treesitter.survey.models import GrammarRecord + + records: list[GrammarRecord] = [] + with registry_path.open(encoding="utf-8") as fh: + for line in fh: + line = line.strip() + if line: + records.append(GrammarRecord.model_validate_json(line)) + + seen: set[tuple[str, str]] = set() + eligible: list[GrammarTarget] = [] + for r in sorted(records, key=lambda rec: rec.fuzz_priority, reverse=True): + if require_scanner and not r.has_external_scanner: + continue + if exclude_fuzzed and r.scores.get("already_fuzzed", 0.0) == 1.0: + continue + if r.fuzz_priority <= 0: + continue + key = (r.owner.lower(), r.repo.lower()) + if key in seen: + continue + seen.add(key) + slug = f"{r.owner}-{r.repo.removeprefix('tree-sitter-')}" + eligible.append( + GrammarTarget( + name=slug, + repo_url=r.repo_url, + tier=Tier.less_popular, + language=r.language_group, + ) + ) + + begin = max(0, start - 1) + return eligible[begin : begin + top_n] + + +def run_hunt( + registry_path: Path = DEFAULT_REGISTRY, + *, + top_n: int = 10, + start: int = 1, + fuzz_seconds: int = 1800, + model: str = DEFAULT_FIX_MODEL, + out_dir: Path = DEFAULT_HUNT_DIR, + cache_dir: Path = DEFAULT_CACHE, + progress: Callable[[int, int, str], None] | None = None, +) -> list[TSLoopResult]: + """Fuzz→fix a range of un-fuzzed scanner candidates from the survey registry.""" + targets = select_hunt_candidates(registry_path, top_n, start=start) + results: list[TSLoopResult] = [] + for i, target in enumerate(targets): + if progress is not None: + progress(i + 1, len(targets), target.name) + results.append( + run_loop( + target, + iterations=1, + fuzz_seconds=fuzz_seconds, + model=model, + out_dir=out_dir, + cache_dir=cache_dir, + ) + ) + return results diff --git a/scaffold/src/parser_security_eval/treesitter/models.py b/scaffold/src/parser_security_eval/treesitter/models.py new file mode 100644 index 0000000..f096dd9 --- /dev/null +++ b/scaffold/src/parser_security_eval/treesitter/models.py @@ -0,0 +1,158 @@ +"""Pydantic data models for the tree-sitter fuzz→fix loop. + +Mirrors the conventions in :mod:`parser_security_eval.models.fuzzing` +(``Field(default_factory=...)`` for collections, computed ``@property`` helpers). +""" + +from __future__ import annotations + +from datetime import datetime, timezone +from enum import Enum + +from pydantic import BaseModel, Field + + +def _utcnow() -> datetime: + return datetime.now(tz=timezone.utc) + + +class Tier(str, Enum): + """Popularity tier of the grammar's language.""" + + popular = "popular" + less_popular = "less-popular" + + +class BugClass(str, Enum): + """Sanitizer / libFuzzer crash classification.""" + + heap_buffer_overflow = "heap-buffer-overflow" + stack_buffer_overflow = "stack-buffer-overflow" + global_buffer_overflow = "global-buffer-overflow" + stack_overflow = "stack-overflow" + heap_use_after_free = "heap-use-after-free" + segv = "SEGV" + memory_leak = "memory-leak" + oom = "out-of-memory" + timeout = "timeout" + unknown = "unknown" + + +class GrammarTarget(BaseModel): + """A tree-sitter grammar to fuzz.""" + + name: str # registry key, e.g. "toml" + repo_url: str + tier: Tier + language: str # human-readable language name, e.g. "TOML" + symbol: str | None = None # tree_sitter_; auto-detected from parser.c if None + commit: str | None = None # optional commit pin (e.g. a known-buggy revision) + subpath: str = "" # subdir holding src/parser.c (e.g. "tree-sitter-markdown") + + +class TSBuildResult(BaseModel): + """Outcome of compiling a grammar into a libFuzzer binary.""" + + built: bool = False + binary_path: str | None = None + symbol: str = "" + src_dir: str = "" + had_scanner: bool = False + scanner_file: str | None = None # path to scanner.c / scanner.cc if present + generated: bool = False # whether `tree-sitter generate` was run + compile_errors: str = "" + + +class TSFuzzResult(BaseModel): + """Parsed result of one libFuzzer run (one fuzz window).""" + + duration_seconds: int + execs_per_sec: float | None = None + corpus_size: int | None = None + total_executions: int | None = None + crashes_found: int = 0 + timed_out: bool = False + oom_killed: bool = False + raw_log_tail: str = "" + + +class TSCrash(BaseModel): + """A triaged crash, deduplicated by stack hash.""" + + bug_class: BugClass = BugClass.unknown + stack_hash: str = "" # sha256 of top-N stack frames + asan_summary: str = "" # one-line SUMMARY from the sanitizer + top_frames: list[str] = Field(default_factory=list) + crash_input_b64: str = "" # base64 of the triggering input + crash_file: str | None = None + implicated_file: str | None = None # scanner.c / parser.c / runtime + in_scanner: bool = False # crash is in a hand-written external scanner + + +class TSFixAttempt(BaseModel): + """The fixer's attempt to eliminate a crash.""" + + attempted: bool = False + model: str = "" + target_file: str | None = None # the source file the fixer rewrote + patch: str = "" # unified diff (old → new) for the record + rationale: str = "" + applied: bool = False + regenerated: bool = False # `tree-sitter generate` re-run after grammar.js edit + rebuilt: bool = False + # verified == reproduced_original AND the patched build never crashed on replay. + reproduced_original: bool = False # the bug reproduced on the pre-fix build + verify_attempts: int = 0 # replays used to confirm repro / clearance (flaky-safe) + verified: bool = False # crashing input no longer reproduces after the fix + error: str = "" + input_tokens: int = 0 + output_tokens: int = 0 + + +class TSLoopIteration(BaseModel): + """One fuzz→(triage→fix) iteration; serialized as one JSONL line.""" + + iteration: int + grammar: str + repo_url: str + tier: Tier + language: str + built: bool + fuzz: TSFuzzResult | None = None + crash: TSCrash | None = None + fix: TSFixAttempt | None = None + started_at: datetime = Field(default_factory=_utcnow) + notes: str = "" + + def to_jsonl(self) -> str: + """Render this iteration as a single JSON line (no trailing newline).""" + return self.model_dump_json() + + +class TSLoopResult(BaseModel): + """Aggregate result of a full loop over one grammar.""" + + grammar: str + tier: Tier + language: str + iterations: list[TSLoopIteration] = Field(default_factory=list) + jsonl_path: str | None = None + build_error: str = "" + + @property + def crashes_found(self) -> int: + return sum(1 for it in self.iterations if it.crash is not None) + + @property + def unique_stack_hashes(self) -> list[str]: + seen: list[str] = [] + for it in self.iterations: + if it.crash and it.crash.stack_hash and it.crash.stack_hash not in seen: + seen.append(it.crash.stack_hash) + return seen + + @property + def fixes_verified(self) -> int: + return sum( + 1 for it in self.iterations if it.fix is not None and it.fix.verified + ) diff --git a/scaffold/src/parser_security_eval/treesitter/registry.py b/scaffold/src/parser_security_eval/treesitter/registry.py new file mode 100644 index 0000000..edde389 --- /dev/null +++ b/scaffold/src/parser_security_eval/treesitter/registry.py @@ -0,0 +1,111 @@ +"""Registry of tree-sitter grammar targets (popular + less-popular tiers). + +Symbols are auto-detected from ``src/parser.c`` at build time unless pinned here +(needed for multi-grammar repos like yaml, whose ``grammar.json`` name is wrong). +""" + +from __future__ import annotations + +from parser_security_eval.treesitter.models import GrammarTarget, Tier + +_GRAMMARS: list[GrammarTarget] = [ + # ---- Popular languages --------------------------------------------------- + GrammarTarget( + name="json", + repo_url="https://github.com/tree-sitter/tree-sitter-json", + tier=Tier.popular, + language="JSON", + ), + GrammarTarget( + name="python", + repo_url="https://github.com/tree-sitter/tree-sitter-python", + tier=Tier.popular, + language="Python", + ), + GrammarTarget( + name="javascript", + repo_url="https://github.com/tree-sitter/tree-sitter-javascript", + tier=Tier.popular, + language="JavaScript", + ), + GrammarTarget( + name="c", + repo_url="https://github.com/tree-sitter/tree-sitter-c", + tier=Tier.popular, + language="C", + ), + GrammarTarget( + name="bash", + repo_url="https://github.com/tree-sitter/tree-sitter-bash", + tier=Tier.popular, + language="Bash", + ), + GrammarTarget( + name="ruby", + repo_url="https://github.com/tree-sitter/tree-sitter-ruby", + tier=Tier.popular, + language="Ruby", + ), + # ---- Less-popular languages ---------------------------------------------- + GrammarTarget( + name="toml", + repo_url="https://github.com/tree-sitter-grammars/tree-sitter-toml", + tier=Tier.less_popular, + language="TOML", + ), + GrammarTarget( + name="lua", + repo_url="https://github.com/tree-sitter-grammars/tree-sitter-lua", + tier=Tier.less_popular, + language="Lua", + ), + GrammarTarget( + name="hcl", + repo_url="https://github.com/tree-sitter-grammars/tree-sitter-hcl", + tier=Tier.less_popular, + language="HCL", + ), + GrammarTarget( + name="wgsl", + repo_url="https://github.com/szebniok/tree-sitter-wgsl", + tier=Tier.less_popular, + language="WGSL", + ), + GrammarTarget( + name="markdown", + repo_url="https://github.com/tree-sitter-grammars/tree-sitter-markdown", + tier=Tier.less_popular, + language="Markdown", + subpath="tree-sitter-markdown", + ), + GrammarTarget( + name="yaml", + repo_url="https://github.com/tree-sitter-grammars/tree-sitter-yaml", + tier=Tier.less_popular, + language="YAML", + symbol="tree_sitter_yaml", + ), +] + +REGISTRY: dict[str, GrammarTarget] = {g.name: g for g in _GRAMMARS} + + +def get(name: str) -> GrammarTarget: + """Look up a grammar by registry key; raise ``KeyError`` if unknown.""" + try: + return REGISTRY[name] + except KeyError as exc: + known = ", ".join(sorted(REGISTRY)) + raise KeyError(f"Unknown grammar {name!r}. Known: {known}") from exc + + +def by_tier(tier: Tier | None = None) -> list[GrammarTarget]: + """Return all grammars, optionally filtered to a single tier.""" + if tier is None: + return list(_GRAMMARS) + return [g for g in _GRAMMARS if g.tier == tier] + + +def all_names() -> list[str]: + """Return all registry keys in declaration order.""" + return [g.name for g in _GRAMMARS] diff --git a/scaffold/src/parser_security_eval/treesitter/runtime.py b/scaffold/src/parser_security_eval/treesitter/runtime.py new file mode 100644 index 0000000..cbee36b --- /dev/null +++ b/scaffold/src/parser_security_eval/treesitter/runtime.py @@ -0,0 +1,422 @@ +"""Local toolchain for the tree-sitter loop: clone, build, fuzz, reproduce. + +This is the nix-provided-clang analogue of +:mod:`parser_security_eval.sandbox.campaign`. It expects ``git``, ``clang`` / +``clang++`` and (only when a grammar ships no generated ``src/parser.c``) +``tree-sitter`` on ``PATH`` — i.e. it is meant to run inside the project's +``nix-shell`` (see ``scaffold/treesitter-shell.nix``). +""" + +from __future__ import annotations + +import re +import shutil +import subprocess +from dataclasses import dataclass, field +from pathlib import Path + +from parser_security_eval.treesitter.models import ( + GrammarTarget, + TSBuildResult, + TSFuzzResult, +) + +DEFAULT_CACHE = Path.home() / ".cache" / "parser-security-eval" / "treesitter" +RUNTIME_REPO = "https://github.com/tree-sitter/tree-sitter" + +# Compile flags shared by every translation unit. ``fuzzer-no-link`` instruments +# each TU; the final link uses ``fuzzer`` to pull in libFuzzer's main(). +_COMMON_FLAGS = [ + "-g", + "-O1", + "-fno-omit-frame-pointer", + "-fsanitize=fuzzer-no-link,address", +] + +_LANG_FUNC_DEF_RE = re.compile(r"\btree_sitter_([A-Za-z0-9_]+)\(void\)\s*\{") +_LANG_FUNC_DECL_RE = re.compile(r"\btree_sitter_([A-Za-z0-9_]+)\(void\)") + +_HARNESS_TEMPLATE = """\ +/* Auto-generated libFuzzer harness for tree-sitter grammar `{symbol}`. */ +#include +#include +#include +#include + +const TSLanguage *{symbol}(void); + +int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {{ + TSParser *parser = ts_parser_new(); + if (!ts_parser_set_language(parser, {symbol}())) {{ + ts_parser_delete(parser); + return 0; + }} + TSTree *tree = + ts_parser_parse_string(parser, NULL, (const char *)data, (uint32_t)size); + if (tree != NULL) {{ + /* Walk the tree (recursive s-expression) to exercise more parser code. */ + char *s = ts_node_string(ts_tree_root_node(tree)); + if (s != NULL) {{ + free(s); + }} + ts_tree_delete(tree); + }} + ts_parser_delete(parser); + return 0; +}} +""" + + +class ToolchainError(RuntimeError): + """Raised when a required external tool is missing or a build step fails.""" + + +def render_harness(symbol: str) -> str: + """Render the standard tree-sitter libFuzzer harness for *symbol*.""" + return _HARNESS_TEMPLATE.format(symbol=symbol) + + +def check_toolchain(*, need_generate: bool = False) -> None: + """Verify required executables are on PATH; raise ToolchainError otherwise.""" + required = ["git", "clang", "clang++"] + if need_generate: + required.append("tree-sitter") + missing = [tool for tool in required if shutil.which(tool) is None] + if missing: + raise ToolchainError( + "Missing tools on PATH: " + + ", ".join(missing) + + ". Run inside `nix-shell scaffold/treesitter-shell.nix`." + ) + + +def _run( + cmd: list[str], + *, + cwd: Path | None = None, + timeout: int | None = None, +) -> subprocess.CompletedProcess[str]: + return subprocess.run( # noqa: S603 — args are constructed, not shell-interpolated + cmd, + cwd=str(cwd) if cwd else None, + capture_output=True, + text=True, + timeout=timeout, + check=False, + ) + + +def ensure_runtime(cache_dir: Path = DEFAULT_CACHE) -> Path: + """Clone the tree-sitter runtime (shallow) if absent; return its path.""" + cache_dir.mkdir(parents=True, exist_ok=True) + dest = cache_dir / "tree-sitter" + if not (dest / "lib" / "src" / "lib.c").exists(): + res = _run(["git", "clone", "--depth", "1", RUNTIME_REPO, str(dest)]) + if res.returncode != 0: + raise ToolchainError(f"Failed to clone tree-sitter runtime:\n{res.stderr}") + return dest + + +def ensure_grammar(target: GrammarTarget, cache_dir: Path = DEFAULT_CACHE) -> Path: + """Clone *target*'s grammar repo (and checkout a pinned commit); return path.""" + cache_dir.mkdir(parents=True, exist_ok=True) + dest = cache_dir / f"tree-sitter-{target.name}" + if not dest.exists(): + clone = ["git", "clone"] + if target.commit is None: + clone += ["--depth", "1"] + clone += [target.repo_url, str(dest)] + res = _run(clone) + if res.returncode != 0: + raise ToolchainError(f"Failed to clone {target.repo_url}:\n{res.stderr}") + if target.commit is not None: + res = _run(["git", "checkout", target.commit], cwd=dest) + if res.returncode != 0: + raise ToolchainError(f"Failed to checkout {target.commit}:\n{res.stderr}") + return dest + + +def find_src_dir(grammar_dir: Path, subpath: str = "") -> Path: + """Locate the directory containing ``src/parser.c`` (or where it should go).""" + if subpath: + candidate = grammar_dir / subpath + if (candidate / "src").exists() or (candidate / "grammar.js").exists(): + return candidate / "src" if (candidate / "src").exists() else candidate + # Prefer an existing src/parser.c. + parsers = sorted(grammar_dir.glob("**/src/parser.c")) + if parsers: + return parsers[0].parent + # Otherwise the dir holding grammar.js (generation target). + grammars = sorted(grammar_dir.glob("**/grammar.js")) + if grammars: + return grammars[0].parent / "src" + return grammar_dir / "src" + + +def detect_symbol(src_dir: Path, fallback: str) -> str: + """Read ``parser.c`` for the ``tree_sitter_(void)`` language function.""" + parser_c = src_dir / "parser.c" + if parser_c.exists(): + text = parser_c.read_text(encoding="utf-8", errors="replace") + m = _LANG_FUNC_DEF_RE.search(text) + if m: + return f"tree_sitter_{m.group(1)}" + m = _LANG_FUNC_DECL_RE.search(text) + if m: + return f"tree_sitter_{m.group(1)}" + return fallback + + +def _find_scanner(src_dir: Path) -> Path | None: + for name in ("scanner.c", "scanner.cc", "scanner.cpp", "scanner.cxx"): + p = src_dir / name + if p.exists(): + return p + return None + + +def gather_seeds(grammar_dir: Path, corpus_dir: Path, limit: int = 400) -> int: + """Seed *corpus_dir* from the grammar's own test/example files + literals.""" + corpus_dir.mkdir(parents=True, exist_ok=True) + count = 0 + patterns = ("test", "examples", "grammars", "queries") + for path in grammar_dir.rglob("*"): + if count >= limit: + break + if not path.is_file(): + continue + parts = {p.lower() for p in path.parts} + if not (parts & set(patterns)): + continue + try: + data = path.read_bytes() + except OSError: + continue + if not data or len(data) > 65536: + continue + (corpus_dir / f"seed{count}").write_bytes(data) + count += 1 + # A few adversarial literals so fuzzing always has something to mutate. + for i, lit in enumerate( + (b"{}", b"\xff\xfe\x00bad", b" x", b"[[[[]]]]", b"\x00") + ): + (corpus_dir / f"lit{i}").write_bytes(lit) + count += 1 + return count + + +def generate_parser(src_or_grammar_dir: Path) -> tuple[bool, str]: + """Run ``tree-sitter generate`` in the grammar.js directory; return (ok, log).""" + # tree-sitter generate must run where grammar.js lives. + gdir = src_or_grammar_dir + if gdir.name == "src": + gdir = gdir.parent + if not (gdir / "grammar.js").exists(): + return False, f"no grammar.js in {gdir}" + res = _run(["tree-sitter", "generate"], cwd=gdir, timeout=300) + return res.returncode == 0, (res.stdout + res.stderr)[-4000:] + + +@dataclass +class BuildSpec: + """Inputs needed to (re)compile a grammar's fuzz binary.""" + + target: GrammarTarget + runtime_dir: Path + grammar_dir: Path + work_dir: Path + + @property + def src_dir(self) -> Path: + return find_src_dir(self.grammar_dir, self.target.subpath) + + @property + def binary_path(self) -> Path: + return self.work_dir / f"fuzz_{self.target.name}" + + +def build(spec: BuildSpec) -> TSBuildResult: + """Compile the grammar + runtime + harness into a libFuzzer binary.""" + check_toolchain() + spec.work_dir.mkdir(parents=True, exist_ok=True) + src_dir = spec.src_dir + generated = False + + if not (src_dir / "parser.c").exists(): + check_toolchain(need_generate=True) + ok, log = generate_parser(src_dir) + generated = ok + if not ok or not (src_dir / "parser.c").exists(): + return TSBuildResult( + built=False, generated=ok, compile_errors=f"generate failed:\n{log}" + ) + + symbol = spec.target.symbol or detect_symbol( + src_dir, f"tree_sitter_{spec.target.name}" + ) + scanner = _find_scanner(src_dir) + + harness_c = spec.work_dir / "harness.c" + harness_c.write_text(render_harness(symbol), encoding="utf-8") + + includes = [ + f"-I{spec.runtime_dir / 'lib' / 'include'}", + f"-I{spec.runtime_dir / 'lib' / 'src'}", + # Some grammars vendor their own next to scanner.c + # and include it with angle brackets, so the grammar src dir must be on + # the search path too. + f"-I{src_dir}", + ] + + # (source, compiler) translation units → object files. + units: list[tuple[Path, str]] = [ + (harness_c, "clang"), + (spec.runtime_dir / "lib" / "src" / "lib.c", "clang"), + (src_dir / "parser.c", "clang"), + ] + if scanner is not None: + cc = "clang++" if scanner.suffix in (".cc", ".cpp", ".cxx") else "clang" + units.append((scanner, cc)) + + objects: list[str] = [] + for source, cc in units: + obj = spec.work_dir / (source.stem + "__" + source.suffix.lstrip(".") + ".o") + res = _run([cc, "-c", *_COMMON_FLAGS, *includes, str(source), "-o", str(obj)]) + if res.returncode != 0: + return TSBuildResult( + built=False, + symbol=symbol, + src_dir=str(src_dir), + had_scanner=scanner is not None, + scanner_file=str(scanner) if scanner else None, + generated=generated, + compile_errors=f"compiling {source.name}:\n{res.stderr[-4000:]}", + ) + objects.append(str(obj)) + + link = [ + "clang++", + "-fsanitize=fuzzer,address", + *objects, + "-o", + str(spec.binary_path), + ] + res = _run(link) + if res.returncode != 0: + return TSBuildResult( + built=False, + symbol=symbol, + src_dir=str(src_dir), + had_scanner=scanner is not None, + scanner_file=str(scanner) if scanner else None, + generated=generated, + compile_errors=f"linking:\n{res.stderr[-4000:]}", + ) + + return TSBuildResult( + built=True, + binary_path=str(spec.binary_path), + symbol=symbol, + src_dir=str(src_dir), + had_scanner=scanner is not None, + scanner_file=str(scanner) if scanner else None, + generated=generated, + ) + + +# libFuzzer final-stats lines, e.g. "stat::number_of_executed_units: 249359". +_STAT_RE = { + "executions": re.compile(r"stat::number_of_executed_units:\s*(\d+)"), + "execs_per_sec": re.compile(r"stat::average_exec_per_sec:\s*(\d+)"), +} +_CRASH_PREFIXES = ("crash-", "oom-", "timeout-", "leak-") + + +@dataclass +class LibFuzzerRunner: + """Runs a compiled tree-sitter fuzz binary and collects crashes.""" + + binary: Path + corpus_dir: Path + crashes_dir: Path + max_len: int = 65536 + rss_limit_mb: int = 2048 + per_input_timeout: int = 20 + extra_flags: list[str] = field(default_factory=list) + + def run(self, duration_seconds: int) -> tuple[TSFuzzResult, list[Path]]: + """Fuzz for *duration_seconds*; return (result, new crash files).""" + self.corpus_dir.mkdir(parents=True, exist_ok=True) + self.crashes_dir.mkdir(parents=True, exist_ok=True) + before = self._crash_files() + + cmd = [ + str(self.binary), + str(self.corpus_dir), + f"-artifact_prefix={self.crashes_dir}/", + f"-max_total_time={duration_seconds}", + f"-max_len={self.max_len}", + f"-rss_limit_mb={self.rss_limit_mb}", + f"-timeout={self.per_input_timeout}", + "-print_final_stats=1", + *self.extra_flags, + ] + safety = duration_seconds + 60 + run_timed_out = False + try: + res = _run(cmd, timeout=safety) + rc, log = res.returncode, res.stdout + "\n" + res.stderr + except subprocess.TimeoutExpired as exc: + run_timed_out = True + rc = -1 + log = ( + (exc.stdout or b"").decode("utf-8", "replace") + if isinstance(exc.stdout, bytes) + else (exc.stdout or "") + ) + + after = self._crash_files() + new_crashes = [p for p in after if p not in before] + + result = TSFuzzResult( + duration_seconds=duration_seconds, + crashes_found=len(new_crashes), + raw_log_tail=log[-4000:], + corpus_size=sum(1 for _ in self.corpus_dir.iterdir()), + ) + m = _STAT_RE["executions"].search(log) + if m: + result.total_executions = int(m.group(1)) + m = _STAT_RE["execs_per_sec"].search(log) + if m: + result.execs_per_sec = float(m.group(1)) + + result.oom_killed = rc == 137 or "out-of-memory" in log + result.timed_out = run_timed_out or "libFuzzer: timeout" in log + return result, new_crashes + + def _crash_files(self) -> set[Path]: + if not self.crashes_dir.exists(): + return set() + return { + p + for p in self.crashes_dir.iterdir() + if p.is_file() and p.name.startswith(_CRASH_PREFIXES) + } + + +def reproduce(binary: Path, crash_file: Path, timeout: int = 60) -> tuple[int, str]: + """Replay a single crashing input; return (returncode, combined output). + + A *timeout/hang* input would hang the replay too, so a TimeoutExpired is + caught and reported as a libFuzzer timeout rather than propagating (which would + otherwise abort a whole campaign). + """ + try: + res = _run([str(binary), str(crash_file)], timeout=timeout) + except subprocess.TimeoutExpired: + return -1, ( + f"ERROR: libFuzzer: timeout\nReplay of {crash_file.name} exceeded " + f"{timeout}s — the input hangs the parser (pathological parse time)." + ) + return res.returncode, res.stdout + "\n" + res.stderr diff --git a/scaffold/src/parser_security_eval/treesitter/survey/__init__.py b/scaffold/src/parser_security_eval/treesitter/survey/__init__.py new file mode 100644 index 0000000..68da663 --- /dev/null +++ b/scaffold/src/parser_security_eval/treesitter/survey/__init__.py @@ -0,0 +1,10 @@ +"""Tree-sitter grammar survey: build a prioritized registry for fuzzing. + +Gathers, for every grammar on the tree-sitter wiki, the data needed to pick good +fuzzing-survey candidates — grammars with real-world impact (used by Zed/Emacs, +RedMonk-ranked language, moderate popularity) that are also *likely to have bugs* +(hand-written external scanner, less scrutiny) and *tractable to fix* (actively +maintained, not archived). Outputs JSONL + CSV + a ranked priority list. +""" + +from __future__ import annotations diff --git a/scaffold/src/parser_security_eval/treesitter/survey/github.py b/scaffold/src/parser_security_eval/treesitter/survey/github.py new file mode 100644 index 0000000..c10c294 --- /dev/null +++ b/scaffold/src/parser_security_eval/treesitter/survey/github.py @@ -0,0 +1,240 @@ +"""A small, cached, restartable GitHub REST client (stdlib only). + +Reads a token from the constructor or ``GITHUB_TOKEN``. Unauthenticated runs work +but are capped at 60 req/hr — fine for incremental, cache-backed re-runs. Each +repo's raw API responses are cached on disk so a survey can be resumed and re-run +cheaply. +""" + +from __future__ import annotations + +import json +import os +import re +import time +import urllib.error +import urllib.request +from pathlib import Path +from typing import cast + +from parser_security_eval.treesitter.survey.models import RepoMetrics + +_API = "https://api.github.com" +_UA = "parser-security-eval-treesitter-survey" + +# Last-page number in a GitHub `Link` header. Anchored on `[?&]page=` so it does +# NOT match the `per_page=` query parameter (a substring-match bug otherwise). +_LAST_PAGE_RE = re.compile(r'[?&]page=(\d+)>;\s*rel="last"') + + +def _as_dict(obj: object) -> dict[str, object]: + """Return obj as a str-keyed dict (empty if it isn't a dict).""" + return cast("dict[str, object]", obj) if isinstance(obj, dict) else {} + + +def contributors_from_link(link: str) -> int | None: + """Extract the contributor count (last page, page size 1) from a Link header.""" + m = _LAST_PAGE_RE.search(link) + return int(m.group(1)) if m else None + + +def detect_fuzz_setup(paths: list[str]) -> tuple[bool, str]: + """Detect existing fuzzing infra from a repo's file paths. + + Any path segment containing ``fuzz`` (e.g. ``fuzz/``, ``fuzzer.cc``, + ``test/fuzz/...``, ``.github/workflows/fuzz.yml``, ``oss-fuzz``) counts. + Returns ``(found, first_matching_path)``. + """ + for path in paths: + if any("fuzz" in seg.lower() for seg in path.split("/")): + return True, path + return False, "" + + +class RateLimitError(RuntimeError): + """Raised when the GitHub rate limit is exhausted.""" + + def __init__(self, reset_epoch: int) -> None: + self.reset_epoch = reset_epoch + wait = max(0, reset_epoch - int(time.time())) + super().__init__(f"GitHub rate limit exhausted; resets in ~{wait}s") + + +class GitHubClient: + def __init__( + self, + token: str | None = None, + cache_dir: Path | None = None, + refresh_days: float = 7.0, + ) -> None: + self.token = token or os.environ.get("GITHUB_TOKEN") or None + self.cache_dir = cache_dir or ( + Path.home() / ".cache" / "parser-security-eval" / "treesitter" / "gh_cache" + ) + self.cache_dir.mkdir(parents=True, exist_ok=True) + self.refresh_seconds = refresh_days * 86400.0 + self.rate_remaining: int | None = None + + @property + def authenticated(self) -> bool: + return self.token is not None + + # -- low-level ---------------------------------------------------------- # + def _request(self, url: str) -> tuple[int, object, dict[str, str]]: + headers = { + "User-Agent": _UA, + "Accept": "application/vnd.github+json", + "X-GitHub-Api-Version": "2022-11-28", + } + if self.token: + headers["Authorization"] = f"Bearer {self.token}" + req = urllib.request.Request(url, headers=headers) # noqa: S310 — https only + try: + with urllib.request.urlopen(req, timeout=30) as resp: # noqa: S310 + self._note_rate(dict(resp.headers)) + body = resp.read().decode("utf-8", "replace") + data = json.loads(body) if body else None + return resp.status, data, dict(resp.headers) + except urllib.error.HTTPError as exc: + hdrs = dict(exc.headers or {}) + self._note_rate(hdrs) + if exc.code == 403 and hdrs.get("X-RateLimit-Remaining") == "0": + raise RateLimitError(int(hdrs.get("X-RateLimit-Reset", "0"))) from exc + return exc.code, None, hdrs + + def _note_rate(self, headers: dict[str, str]) -> None: + rem = headers.get("X-RateLimit-Remaining") + if rem is not None and rem.isdigit(): + self.rate_remaining = int(rem) + + # -- caching ------------------------------------------------------------ # + def _cache_path(self, owner: str, repo: str) -> Path: + return self.cache_dir / f"{owner.lower()}__{repo.lower()}.json" + + def _load_cache(self, owner: str, repo: str) -> dict[str, object] | None: + path = self._cache_path(owner, repo) + if not path.exists(): + return None + try: + blob = json.loads(path.read_text(encoding="utf-8")) + except ValueError, OSError: + return None + fetched = float(blob.get("_fetched", 0)) + if time.time() - fetched > self.refresh_seconds: + return None + return blob + + # -- high-level --------------------------------------------------------- # + def fetch_metrics( + self, owner: str, repo: str, *, use_cache: bool = True + ) -> RepoMetrics: + """Fetch (or load cached) metrics for ``owner/repo``.""" + if use_cache: + cached = self._load_cache(owner, repo) + if cached is not None: + return self._to_metrics(cached) + + blob: dict[str, object] = {} + status, repo_data, _ = self._request(f"{_API}/repos/{owner}/{repo}") + if status == 404: + return RepoMetrics(fetched=False, error="repo not found (404)") + if status != 200 or not isinstance(repo_data, dict): + return RepoMetrics(fetched=False, error=f"repo fetch HTTP {status}") + blob["repo"] = repo_data + blob["commit_activity"] = self._commit_activity(owner, repo) + blob["contributors"] = self._contributors_count(owner, repo) + default_branch = _as_dict(repo_data).get("default_branch") + blob["fuzz"] = self.fuzz_signals( + owner, repo, default_branch if isinstance(default_branch, str) else "HEAD" + ) + blob["_fetched"] = time.time() + self._cache_path(owner, repo).write_text(json.dumps(blob), encoding="utf-8") + return self._to_metrics(blob) + + def _commit_activity(self, owner: str, repo: str) -> list[object]: + # Stats endpoints return 202 while GitHub computes; retry briefly. + url = f"{_API}/repos/{owner}/{repo}/stats/commit_activity" + for _ in range(4): + status, data, _ = self._request(url) + if status == 200 and isinstance(data, list): + return cast("list[object]", data) + if status == 202: + time.sleep(1.5) + continue + break + return [] + + def _contributors_count(self, owner: str, repo: str) -> int | None: + url = f"{_API}/repos/{owner}/{repo}/contributors?per_page=1&anon=1" + status, data, headers = self._request(url) + if status != 200: + return None + return contributors_from_link(headers.get("Link", "")) or ( + len(data) if isinstance(data, list) else None + ) + + def fuzz_signals(self, owner: str, repo: str, branch: str) -> dict[str, object]: + """Scan the repo's file tree for existing fuzzing infrastructure.""" + url = f"{_API}/repos/{owner}/{repo}/git/trees/{branch}?recursive=1" + status, data, _ = self._request(url) + tree = _as_dict(data).get("tree") if status == 200 else None + paths = [ + str(_as_dict(t).get("path", "")) + for t in (tree if isinstance(tree, list) else []) + ] + found, evidence = detect_fuzz_setup(paths) + return {"has_fuzz_setup": found, "fuzz_evidence": evidence} + + @staticmethod + def _to_metrics(blob: dict[str, object]) -> RepoMetrics: + repo = _as_dict(blob.get("repo")) + if not repo: + return RepoMetrics(fetched=False, error="no repo data") + activity = blob.get("commit_activity") + weeks = activity if isinstance(activity, list) else [] + totals: list[int] = [] + for w in weeks: + total = _as_dict(w).get("total") + if isinstance(total, int): + totals.append(total) + last_year = sum(totals) if totals else None + last_90d = sum(totals[-13:]) if totals else None + active_weeks = sum(1 for t in totals if t > 0) if totals else None + + license_id = _as_dict(repo.get("license")).get("spdx_id") + parent_name = _as_dict(repo.get("parent")).get("full_name") + contributors = blob.get("contributors") + topics_raw = repo.get("topics") + topics = [str(t) for t in topics_raw] if isinstance(topics_raw, list) else [] + fuzz = _as_dict(blob.get("fuzz")) + + # Build via model_validate so pydantic coerces the dynamically-typed + # (``object``) API values into the typed fields. + return RepoMetrics.model_validate( + { + "fetched": True, + "stars": repo.get("stargazers_count"), + "forks": repo.get("forks_count"), + "is_fork": repo.get("fork"), + "parent": parent_name, + "archived": bool(repo.get("archived", False)), + "disabled": bool(repo.get("disabled", False)), + "pushed_at": repo.get("pushed_at"), + "created_at": repo.get("created_at"), + "updated_at": repo.get("updated_at"), + "open_issues": repo.get("open_issues_count"), + "has_issues": repo.get("has_issues"), + "license": license_id + if license_id and license_id != "NOASSERTION" + else None, + "size_kb": repo.get("size"), + "topics": topics, + "subscribers": repo.get("subscribers_count"), + "contributors": contributors if isinstance(contributors, int) else None, + "commits_last_year": last_year, + "commits_last_90d": last_90d, + "active_weeks_last_year": active_weeks, + "has_fuzz_setup": bool(fuzz.get("has_fuzz_setup", False)), + "fuzz_evidence": str(fuzz.get("fuzz_evidence", "") or ""), + } + ) diff --git a/scaffold/src/parser_security_eval/treesitter/survey/models.py b/scaffold/src/parser_security_eval/treesitter/survey/models.py new file mode 100644 index 0000000..babb2e8 --- /dev/null +++ b/scaffold/src/parser_security_eval/treesitter/survey/models.py @@ -0,0 +1,94 @@ +"""Pydantic models for the tree-sitter grammar survey/registry.""" + +from __future__ import annotations + +from pydantic import BaseModel, Field + + +class WikiGrammar(BaseModel): + """One row of the tree-sitter wiki 'List of parsers' table.""" + + name: str # language name as listed (lower-cased) + repo_url: str # normalized https URL (no .git, no trailing path) + owner: str + repo: str + wiki_last_commit: str | None = None # YYYY-MM-DD from the wiki snapshot + abi: str | None = None + has_grammar_json: bool = False + has_external_scanner: bool = False + + +class RepoMetrics(BaseModel): + """GitHub metrics for a grammar repository.""" + + fetched: bool = False + error: str = "" + stars: int | None = None + forks: int | None = None + is_fork: bool | None = None + parent: str | None = None # "owner/repo" of the upstream if this is a fork + archived: bool = False + disabled: bool = False + pushed_at: str | None = None # ISO timestamp of last push + created_at: str | None = None + updated_at: str | None = None + open_issues: int | None = None + has_issues: bool | None = None + license: str | None = None # SPDX id + size_kb: int | None = None + topics: list[str] = Field(default_factory=list) + subscribers: int | None = None # watchers + contributors: int | None = None + commits_last_year: int | None = None + commits_last_90d: int | None = None + active_weeks_last_year: int | None = None # weeks (of 52) with >=1 commit + # Existing fuzzing infrastructure in the repo (a `fuzz/` dir, fuzzing CI, + # oss-fuzz refs). If present, the easy crashes are likely already found. + has_fuzz_setup: bool = False + fuzz_evidence: str = "" # the path that matched, for auditing + + +class GrammarRecord(BaseModel): + """A fully-enriched registry entry: wiki + adoption + GitHub metrics + score.""" + + # identity + name: str # wiki language name + language_group: str # canonical language label for grouping + repo_url: str + owner: str + repo: str + + # from the wiki + has_external_scanner: bool = False + has_grammar_json: bool = False + abi: str | None = None + wiki_last_commit: str | None = None + competing_impls: int = 1 # # of wiki repos for the same language group + + # adoption / impact + used_by_zed: bool = False + used_by_emacs: bool = False + redmonk_rank: int | None = None # 1 = most popular; None = unranked + + # github + metrics: RepoMetrics = Field(default_factory=RepoMetrics) + + # scoring (all 0..1 sub-scores retained so the list can be re-weighted) + scores: dict[str, float] = Field(default_factory=dict) + fuzz_priority: float = 0.0 # 0..100 composite + priority_reasons: list[str] = Field(default_factory=list) + + def to_jsonl(self) -> str: + return self.model_dump_json() + + +class SurveyResult(BaseModel): + """Aggregate output of a survey run.""" + + total_grammars: int = 0 + metrics_fetched: int = 0 + metrics_failed: int = 0 + records: list[GrammarRecord] = Field(default_factory=list) + jsonl_path: str | None = None + csv_path: str | None = None + priority_path: str | None = None diff --git a/scaffold/src/parser_security_eval/treesitter/survey/runner.py b/scaffold/src/parser_security_eval/treesitter/survey/runner.py new file mode 100644 index 0000000..f13cd46 --- /dev/null +++ b/scaffold/src/parser_security_eval/treesitter/survey/runner.py @@ -0,0 +1,224 @@ +"""Orchestrate the grammar survey: collect sources, fetch metrics, score, write.""" + +from __future__ import annotations + +import csv +from collections import Counter +from collections.abc import Callable +from datetime import datetime, timezone +from pathlib import Path + +from parser_security_eval.treesitter.survey import sources +from parser_security_eval.treesitter.survey.github import GitHubClient, RateLimitError +from parser_security_eval.treesitter.survey.models import GrammarRecord, SurveyResult +from parser_security_eval.treesitter.survey.scoring import compute_priority +from parser_security_eval.treesitter.survey.sources import repo_key + +DEFAULT_OUT_DIR = Path("results") / "treesitter" + +ProgressFn = Callable[[int, int, str], None] + + +def _build_records(token: str | None) -> list[GrammarRecord]: + """Collect wiki + Zed + Emacs + RedMonk into un-scored records.""" + wiki = sources.fetch_wiki_grammars(token=token) + zed_keys, zed_basenames = sources.fetch_zed_grammars() + emacs_keys, emacs_basenames = sources.fetch_emacs_grammars(token=token) + per_lang = Counter(g.name for g in wiki) + + records: list[GrammarRecord] = [] + for g in wiki: + key = repo_key(g.owner, g.repo) + base = g.repo.lower() + records.append( + GrammarRecord( + name=g.name, + language_group=g.name, + repo_url=g.repo_url, + owner=g.owner, + repo=g.repo, + has_external_scanner=g.has_external_scanner, + has_grammar_json=g.has_grammar_json, + abi=g.abi, + wiki_last_commit=g.wiki_last_commit, + competing_impls=per_lang[g.name], + used_by_zed=key in zed_keys or base in zed_basenames, + used_by_emacs=key in emacs_keys or base in emacs_basenames, + redmonk_rank=sources.redmonk_rank(g.name), + ) + ) + return records + + +def run_survey( + *, + token: str | None = None, + out_dir: Path = DEFAULT_OUT_DIR, + cache_dir: Path | None = None, + limit: int | None = None, + refresh_days: float = 7.0, + top_n: int = 60, + progress: ProgressFn | None = None, +) -> SurveyResult: + """Run the full survey and write registry.jsonl, registry.csv, priority.md.""" + records = _build_records(token) + if limit is not None: + records = records[:limit] + + client = GitHubClient(token=token, cache_dir=cache_dir, refresh_days=refresh_days) + now = datetime.now(tz=timezone.utc) + fetched = failed = 0 + rate_limited = False + + for i, rec in enumerate(records): + if progress: + progress(i + 1, len(records), f"{rec.owner}/{rec.repo}") + if not rate_limited: + try: + rec.metrics = client.fetch_metrics(rec.owner, rec.repo) + except RateLimitError: + rate_limited = True # stop hitting the API; score with what we have + else: + if rec.metrics.fetched: + fetched += 1 + elif rec.metrics.error: + failed += 1 + compute_priority(rec, now) + + records.sort(key=lambda r: r.fuzz_priority, reverse=True) + + out_dir.mkdir(parents=True, exist_ok=True) + jsonl_path = out_dir / "registry.jsonl" + csv_path = out_dir / "registry.csv" + priority_path = out_dir / "priority.md" + _write_jsonl(jsonl_path, records) + _write_csv(csv_path, records) + _write_priority_md(priority_path, records, top_n=top_n, rate_limited=rate_limited) + + return SurveyResult( + total_grammars=len(records), + metrics_fetched=fetched, + metrics_failed=failed, + records=records, + jsonl_path=str(jsonl_path), + csv_path=str(csv_path), + priority_path=str(priority_path), + ) + + +def _write_jsonl(path: Path, records: list[GrammarRecord]) -> None: + with path.open("w", encoding="utf-8") as fh: + for rec in records: + fh.write(rec.to_jsonl() + "\n") + + +_CSV_COLUMNS = [ + "rank", + "name", + "language_group", + "repo_url", + "fuzz_priority", + "used_by_zed", + "used_by_emacs", + "redmonk_rank", + "has_external_scanner", + "has_fuzz_setup", + "already_fuzzed", + "stars", + "forks", + "is_fork", + "archived", + "pushed_at", + "months_since_push", + "commits_last_year", + "commits_last_90d", + "contributors", + "open_issues", + "license", + "competing_impls", + "impact", + "bug_likelihood", + "tractability", +] + + +def _write_csv(path: Path, records: list[GrammarRecord]) -> None: + with path.open("w", encoding="utf-8", newline="") as fh: + writer = csv.writer(fh) + writer.writerow(_CSV_COLUMNS) + for rank, rec in enumerate(records, start=1): + m = rec.metrics + writer.writerow( + [ + rank, + rec.name, + rec.language_group, + rec.repo_url, + rec.fuzz_priority, + int(rec.used_by_zed), + int(rec.used_by_emacs), + rec.redmonk_rank if rec.redmonk_rank is not None else "", + int(rec.has_external_scanner), + int(m.has_fuzz_setup), + int(rec.scores.get("already_fuzzed", 0.0)), + m.stars if m.stars is not None else "", + m.forks if m.forks is not None else "", + int(m.is_fork) if m.is_fork is not None else "", + int(m.archived), + m.pushed_at or "", + rec.scores.get("months_since_push", ""), + m.commits_last_year if m.commits_last_year is not None else "", + m.commits_last_90d if m.commits_last_90d is not None else "", + m.contributors if m.contributors is not None else "", + m.open_issues if m.open_issues is not None else "", + m.license or "", + rec.competing_impls, + rec.scores.get("impact", ""), + rec.scores.get("bug_likelihood", ""), + rec.scores.get("tractability", ""), + ] + ) + + +def _write_priority_md( + path: Path, records: list[GrammarRecord], *, top_n: int, rate_limited: bool +) -> None: + lines = [ + "# Tree-sitter fuzzing-survey priority list", + "", + f"Total grammars surveyed: **{len(records)}**. " + "Ranked by `fuzz_priority` = impact × **crash-surface** (external scanner) × " + "**neglect** (no existing fuzz harness / not OSS-Fuzz'd) × maintenance " + "activity. Already-fuzzed and no-scanner grammars are pushed down — the " + "goldilocks is an actively-maintained scanner grammar nobody has fuzzed yet.", + "", + ] + if rate_limited: + lines += [ + "> ⚠️ GitHub rate limit hit mid-run — some rows lack metrics. " + "Set `GITHUB_TOKEN` and re-run (cached rows are reused).", + "", + ] + lines += [ + "| # | grammar | priority | ★ | scanner | fuzzed? | zed | emacs | last push | why |", + "| - | ------- | -------- | - | ------- | ------- | --- | ----- | --------- | --- |", + ] + for rank, rec in enumerate(records[:top_n], start=1): + m = rec.metrics + push = rec.scores.get("months_since_push", -1.0) + push_s = ( + f"{push:.0f}mo" if isinstance(push, (int, float)) and push >= 0 else "?" + ) + why = ", ".join( + r for r in rec.priority_reasons if r not in {"used by Zed", "used by Emacs"} + )[:80] + already = rec.scores.get("already_fuzzed", 0.0) + lines.append( + f"| {rank} | [{rec.name}]({rec.repo_url}) | {rec.fuzz_priority:.1f} " + f"| {m.stars if m.stars is not None else '?'} " + f"| {'✓' if rec.has_external_scanner else ''} " + f"| {'✓' if already else ''} " + f"| {'✓' if rec.used_by_zed else ''} | {'✓' if rec.used_by_emacs else ''} " + f"| {push_s} | {why} |" + ) + path.write_text("\n".join(lines) + "\n", encoding="utf-8") diff --git a/scaffold/src/parser_security_eval/treesitter/survey/scoring.py b/scaffold/src/parser_security_eval/treesitter/survey/scoring.py new file mode 100644 index 0000000..b58bd8a --- /dev/null +++ b/scaffold/src/parser_security_eval/treesitter/survey/scoring.py @@ -0,0 +1,167 @@ +"""Transparent fuzzing-priority scoring for grammar records. + +The composite favors grammars that are **impactful** (used by Zed/Emacs, popular +language, some stars), **likely to have bugs** (hand-written external scanner, less +scrutiny, older), and **tractable** (actively maintained, not archived). All +sub-scores are stored on the record so the priority list can be re-weighted later. +""" + +from __future__ import annotations + +import math +from datetime import datetime + +from parser_security_eval.treesitter.survey.models import GrammarRecord + +# Orgs whose grammars are well-maintained and continuously fuzzed by OSS-Fuzz, so +# latent memory bugs are far less likely — deprioritized as fuzzing targets. +CORE_ORGS = {"tree-sitter", "tree-sitter-grammars"} + + +def _clamp(x: float) -> float: + return max(0.0, min(1.0, x)) + + +def _months_between(iso: str | None, now: datetime) -> float | None: + if not iso: + return None + try: + dt = datetime.fromisoformat(iso.replace("Z", "+00:00")) + except ValueError: + return None + return (now - dt).days / 30.44 + + +def _years_between(iso: str | None, now: datetime) -> float | None: + months = _months_between(iso, now) + return None if months is None else months / 12.0 + + +def _activity_gate(months_since_push: float | None) -> float: + """1.0 if pushed within a year, decaying to 0 by ~30 months (abandoned).""" + if months_since_push is None: + return 0.35 # unknown — neither reward nor exclude + if months_since_push <= 12: + return 1.0 + if months_since_push >= 30: + return 0.0 + return _clamp(1.0 - (months_since_push - 12) / 18.0 * 0.85) + + +def compute_priority(rec: GrammarRecord, now: datetime) -> None: + """Populate ``rec.scores``, ``rec.fuzz_priority`` and ``rec.priority_reasons``.""" + m = rec.metrics + reasons: list[str] = [] + + # --- impact --------------------------------------------------------------- + adoption = _clamp(0.6 * rec.used_by_zed + 0.6 * rec.used_by_emacs) + if rec.used_by_zed: + reasons.append("used by Zed") + if rec.used_by_emacs: + reasons.append("used by Emacs") + + redmonk = 0.0 + if rec.redmonk_rank is not None: + redmonk = _clamp(1.0 - (rec.redmonk_rank - 1) / 30.0) + reasons.append(f"RedMonk #{rec.redmonk_rank}") + + stars = m.stars if m.stars is not None else None + pop = 0.0 + if stars is not None: + pop = _clamp(math.log10(stars + 1) / math.log10(5000)) + + impact = _clamp(0.45 * adoption + 0.30 * redmonk + 0.25 * pop) + + # --- bug likelihood (crash surface × neglect × latent) -------------------- + # Crash surface: a grammar's only hand-written C is its external scanner. + # Generated parser.c is table-driven and memory-safe by construction, so a + # no-scanner grammar offers almost no grammar-specific crash surface. + crash_surface = 1.0 if rec.has_external_scanner else 0.15 + if rec.has_external_scanner: + reasons.append("external scanner (hand-written C)") + else: + reasons.append("no scanner — minimal crash surface") + + # Neglect: if the repo is already fuzzed, the easy crashes are likely found. + already_fuzzed = ( + m.has_fuzz_setup + or rec.owner.lower() in CORE_ORGS + or any("fuzz" in t.lower() for t in m.topics) + ) + neglect = 1.0 + if already_fuzzed: + neglect = 0.2 + if m.has_fuzz_setup: + reasons.append(f"already fuzzed ({m.fuzz_evidence})") + elif rec.owner.lower() in CORE_ORGS: + reasons.append("core tree-sitter org (continuously fuzzed by OSS-Fuzz)") + else: + reasons.append("already fuzzed (oss-fuzz topic)") + else: + reasons.append("no existing fuzz harness") + + low_scrutiny = 1.0 - pop # fewer stars/eyeballs → more latent bugs + years = _years_between(m.created_at, now) + age = _clamp((years or 0.0) / 5.0) + latent = _clamp(0.5 + 0.3 * low_scrutiny + 0.2 * age) # in [0.5, 1.0] + bug = _clamp(crash_surface * neglect * latent) + + if stars is not None: + if stars >= 3000: + reasons.append(f"{stars}★ — very popular, likely well-tested") + elif stars >= 50: + reasons.append(f"{stars}★") + else: + reasons.append(f"{stars}★ — niche") + + # --- tractability gate ---------------------------------------------------- + months_push = _months_between(m.pushed_at, now) + gate = _activity_gate(months_push) + if m.archived or m.disabled: + gate = 0.0 + reasons.append("ARCHIVED/disabled — excluded") + elif months_push is not None: + if months_push <= 12: + reasons.append(f"active ({months_push:.0f}mo ago)") + elif months_push >= 30: + reasons.append(f"abandoned ({months_push:.0f}mo) — excluded") + else: + reasons.append(f"slowing ({months_push:.0f}mo ago)") + + fork_factor = 1.0 + if m.is_fork and not (rec.used_by_zed or rec.used_by_emacs): + fork_factor = 0.85 + reasons.append("fork") + issues_factor = 0.7 if m.has_issues is False else 1.0 + if m.has_issues is False: + reasons.append("issues disabled") + + tractability = _clamp(gate * fork_factor * issues_factor) + + if rec.competing_impls > 1: + reasons.append(f"{rec.competing_impls} competing impls") + + # Multiplicative: impact modulates (floor 0.3 so a great niche target still + # scores), but crash-surface/neglect/activity can each veto. This pushes + # already-fuzzed and no-scanner grammars down hard, as intended. + impact_term = 0.3 + 0.7 * impact + fuzz_priority = round(100.0 * impact_term * bug * tractability, 2) + + rec.scores = { + "impact": round(impact, 3), + "bug_likelihood": round(bug, 3), + "tractability": round(tractability, 3), + "crash_surface": crash_surface, + "neglect": neglect, + "already_fuzzed": float(already_fuzzed), + "latent": round(latent, 3), + "adoption": round(adoption, 3), + "redmonk": round(redmonk, 3), + "popularity": round(pop, 3), + "low_scrutiny": round(low_scrutiny, 3), + "age": round(age, 3), + "activity_gate": round(gate, 3), + "months_since_push": round(months_push, 1) if months_push is not None else -1.0, + } + rec.fuzz_priority = fuzz_priority + rec.priority_reasons = reasons diff --git a/scaffold/src/parser_security_eval/treesitter/survey/sources.py b/scaffold/src/parser_security_eval/treesitter/survey/sources.py new file mode 100644 index 0000000..3a3aa6b --- /dev/null +++ b/scaffold/src/parser_security_eval/treesitter/survey/sources.py @@ -0,0 +1,270 @@ +"""Source collectors: tree-sitter wiki, Zed, Emacs, and RedMonk rankings. + +All network access uses stdlib ``urllib`` (no new dependency) and degrades +gracefully — a failed fetch returns an empty result rather than aborting the run. +""" + +from __future__ import annotations + +import json +import re +import urllib.error +import urllib.parse +import urllib.request + +from parser_security_eval.treesitter.survey.models import WikiGrammar + +WIKI_URL = ( + "https://raw.githubusercontent.com/wiki/tree-sitter/tree-sitter/List-of-parsers.md" +) +ZED_CARGO_LOCK = "https://raw.githubusercontent.com/zed-industries/zed/main/Cargo.lock" +EMACS_TREE = ( + "https://api.github.com/repos/emacs-mirror/emacs/git/trees/master?recursive=1" +) +EMACS_RAW = "https://raw.githubusercontent.com/emacs-mirror/emacs/master/" +CRATES_API = "https://crates.io/api/v1/crates/" + +_UA = "parser-security-eval-treesitter-survey" + + +def _get_text(url: str, *, token: str | None = None, timeout: int = 30) -> str: + headers = {"User-Agent": _UA} + if token and "api.github.com" in url: + headers["Authorization"] = f"Bearer {token}" + req = urllib.request.Request(url, headers=headers) # noqa: S310 — https only + with urllib.request.urlopen(req, timeout=timeout) as resp: # noqa: S310 + return resp.read().decode("utf-8", "replace") + + +def _get_text_safe(url: str, *, token: str | None = None) -> str: + try: + return _get_text(url, token=token) + except urllib.error.URLError, TimeoutError, OSError: + return "" + + +# --------------------------------------------------------------------------- # +# URL normalization +# --------------------------------------------------------------------------- # +def normalize_repo_url(raw: str) -> tuple[str, str, str] | None: + """Return ``(https_url, owner, repo)`` for a GitHub repo URL, or None. + + Tolerates ``github.com/o/r``, trailing ``.git``, and ``/tree/...`` suffixes. + """ + raw = raw.strip() + if raw.startswith("github.com"): + raw = "https://" + raw + if "github.com" not in raw: + return None + parsed = urllib.parse.urlparse(raw) + if "github.com" not in parsed.netloc: + return None + parts = [p for p in parsed.path.split("/") if p] + if len(parts) < 2: + return None + owner, repo = parts[0], parts[1] + if repo.endswith(".git"): + repo = repo[:-4] + return f"https://github.com/{owner}/{repo}", owner, repo + + +def repo_key(owner: str, repo: str) -> str: + """Case-insensitive ``owner/repo`` match key.""" + return f"{owner.lower()}/{repo.lower()}" + + +# --------------------------------------------------------------------------- # +# tree-sitter wiki +# --------------------------------------------------------------------------- # +_LINK_RE = re.compile(r"\((https?://[^)]+)\)") + + +def parse_wiki(markdown: str) -> list[WikiGrammar]: + """Parse the 'List of parsers' markdown table into WikiGrammar rows.""" + grammars: list[WikiGrammar] = [] + for line in markdown.splitlines(): + if not line.startswith("| "): + continue + parts = [c.strip() for c in line.split("|")] + # parts: ['', name, link, date, abi, grammar.json, external scanner, ''] + if len(parts) < 8: + continue + name, link_cell, date, abi, gj, es = ( + parts[1], + parts[2], + parts[3], + parts[4], + parts[5], + parts[6], + ) + if name in {"name", "---"} or "github.com" not in link_cell: + continue + m = _LINK_RE.search(link_cell) + url_raw = m.group(1) if m else link_cell + norm = normalize_repo_url(url_raw) + if norm is None: + continue + https, owner, repo = norm + grammars.append( + WikiGrammar( + name=name.lower(), + repo_url=https, + owner=owner, + repo=repo, + wiki_last_commit=date if date and date != "-" else None, + abi=abi if abi and abi != "-" else None, + has_grammar_json=gj.lower() == "yes", + has_external_scanner=es.lower() == "yes", + ) + ) + return grammars + + +def fetch_wiki_grammars(token: str | None = None) -> list[WikiGrammar]: + return parse_wiki(_get_text_safe(WIKI_URL, token=token)) + + +# --------------------------------------------------------------------------- # +# Zed +# --------------------------------------------------------------------------- # +_ZED_GIT_RE = re.compile(r'source = "git\+(https://github\.com/[^?"#]+)') +_ZED_CRATE_RE = re.compile(r'^name = "(tree-sitter-[^"]+)"', re.MULTILINE) + + +def parse_zed_cargo_lock(text: str) -> tuple[set[str], set[str]]: + """Return (repo_keys, crate_names) of tree-sitter deps in a Zed Cargo.lock.""" + repo_keys: set[str] = set() + for url in _ZED_GIT_RE.findall(text): + norm = normalize_repo_url(url) + if norm: + repo_keys.add(repo_key(norm[1], norm[2])) + crate_names = set(_ZED_CRATE_RE.findall(text)) + return repo_keys, crate_names + + +def resolve_crate_repo(crate: str) -> tuple[str, str] | None: + """Resolve a crates.io crate to its ``(owner, repo)`` via the crates.io API.""" + body = _get_text_safe(f"{CRATES_API}{crate}") + if not body: + return None + try: + data = json.loads(body) + repository = data.get("crate", {}).get("repository") or "" + except ValueError, AttributeError: + return None + norm = normalize_repo_url(repository) + return (norm[1], norm[2]) if norm else None + + +def fetch_zed_grammars() -> tuple[set[str], set[str]]: + """Return (repo_keys, basenames) of grammars Zed depends on. + + Combines git-sourced deps with crates.io crates resolved to their repos. + Basenames (the ``tree-sitter-x`` repo name) catch Zed's forks of wiki grammars. + """ + text = _get_text_safe(ZED_CARGO_LOCK) + if not text: + return set(), set() + repo_keys, crate_names = parse_zed_cargo_lock(text) + for crate in sorted(crate_names): + resolved = resolve_crate_repo(crate) + if resolved: + repo_keys.add(repo_key(*resolved)) + basenames = {key.split("/", 1)[1] for key in repo_keys} + basenames |= {c.lower() for c in crate_names} + return repo_keys, basenames + + +# --------------------------------------------------------------------------- # +# Emacs +# --------------------------------------------------------------------------- # +_GH_TS_URL_RE = re.compile( + r"https://github\.com/[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]*tree[-_]?sitter[A-Za-z0-9_.-]*" +) + + +def fetch_emacs_grammars(token: str | None = None) -> tuple[set[str], set[str]]: + """Return (repo_keys, basenames) of grammars referenced by Emacs *-ts-mode.el.""" + tree_body = _get_text_safe(EMACS_TREE, token=token) + if not tree_body: + return set(), set() + try: + tree = json.loads(tree_body).get("tree", []) + except ValueError: + return set(), set() + paths = [t["path"] for t in tree if str(t.get("path", "")).endswith("-ts-mode.el")] + repo_keys: set[str] = set() + for path in paths: + content = _get_text_safe(EMACS_RAW + path, token=token) + for url in _GH_TS_URL_RE.findall(content): + norm = normalize_repo_url(url) + if norm: + repo_keys.add(repo_key(norm[1], norm[2])) + basenames = {key.split("/", 1)[1] for key in repo_keys} + return repo_keys, basenames + + +# --------------------------------------------------------------------------- # +# RedMonk programming-language rankings (RedMonk, ~2024; lower rank = more popular) +# --------------------------------------------------------------------------- # +REDMONK_RANKS: dict[str, int] = { + "javascript": 1, + "python": 2, + "java": 3, + "php": 4, + "csharp": 5, + "typescript": 6, + "css": 6, + "cpp": 8, + "ruby": 9, + "c": 9, + "swift": 11, + "r": 12, + "objc": 12, + "shell": 14, + "scala": 15, + "go": 15, + "powershell": 17, + "kotlin": 17, + "rust": 19, + "dart": 19, + "lua": 21, + "groovy": 22, + "perl": 23, + "haskell": 24, + "elixir": 25, + "clojure": 26, + "julia": 27, + "elm": 28, + "ocaml": 29, + "erlang": 30, +} + +# Map wiki language names → RedMonk canonical key. +_REDMONK_ALIASES: dict[str, str] = { + "c_sharp": "csharp", + "c-sharp": "csharp", + "cs": "csharp", + "c++": "cpp", + "objective_c": "objc", + "objective-c": "objc", + "objc": "objc", + "bash": "shell", + "sh": "shell", + "zsh": "shell", + "fish": "shell", + "js": "javascript", + "jsx": "javascript", + "ts": "typescript", + "tsx": "typescript", + "golang": "go", + "rs": "rust", + "py": "python", +} + + +def redmonk_rank(name: str) -> int | None: + """Look up a RedMonk rank for a wiki language name (with aliases).""" + key = name.lower().strip() + key = _REDMONK_ALIASES.get(key, key) + return REDMONK_RANKS.get(key) diff --git a/scaffold/src/parser_security_eval/treesitter/triage.py b/scaffold/src/parser_security_eval/treesitter/triage.py new file mode 100644 index 0000000..8a248e8 --- /dev/null +++ b/scaffold/src/parser_security_eval/treesitter/triage.py @@ -0,0 +1,174 @@ +"""Crash triage for the tree-sitter loop. + +Parses AddressSanitizer / libFuzzer output into a :class:`TSCrash`, classifies the +bug, extracts the top stack frames, and derives a SHA-256 stack hash for +deduplication (the same idea as +:func:`parser_security_eval.triage.casr.CASRTriager._stack_hash_dedup`). +""" + +from __future__ import annotations + +import base64 +import hashlib +import re + +from parser_security_eval.treesitter.models import BugClass, TSCrash + +MAX_STACK_FRAMES = 5 # top-N frames used for the stack hash (matches fuzzing task) + +# Symbolized: " #0 0x55ad in ts_lex /src/scanner.c:42:10" +# Unsymbolized: " #0 0x55ad (/path/fuzz+0x12ab)" +_FRAME_RE = re.compile(r"^\s*#\d+\s+0x[0-9a-fA-F]+\s+(.+?)\s*$", re.MULTILINE) +# Drop the trailing ":col" so equivalent crashes hash identically. +_COL_RE = re.compile(r":(\d+):(\d+)$") + +_ASAN_CLASS_RE = re.compile(r"AddressSanitizer:\s+([a-z0-9-]+)") +_UBSAN_RE = re.compile(r"runtime error:|UndefinedBehaviorSanitizer") +_LIBFUZZER_RE = re.compile(r"libFuzzer:\s+(timeout|out-of-memory|deadly signal)") +_SUMMARY_RE = re.compile(r"^SUMMARY:\s*(.+?)\s*$", re.MULTILINE) + +_ASAN_TO_CLASS: dict[str, BugClass] = { + "heap-buffer-overflow": BugClass.heap_buffer_overflow, + "stack-buffer-overflow": BugClass.stack_buffer_overflow, + "global-buffer-overflow": BugClass.global_buffer_overflow, + "stack-overflow": BugClass.stack_overflow, + "heap-use-after-free": BugClass.heap_use_after_free, + "dynamic-stack-buffer-overflow": BugClass.stack_buffer_overflow, + "SEGV": BugClass.segv, + "segv": BugClass.segv, +} + +_SCANNER_NAMES = ("scanner.c", "scanner.cc", "scanner.cpp", "scanner.cxx") + + +def extract_frames(report: str) -> list[str]: + """Return normalized ``func file:line`` strings for every stack frame.""" + frames: list[str] = [] + for raw in _FRAME_RE.findall(report): + frame = raw.strip() + # Normalize symbolized frames ("in func file:line") by dropping "in ". + if frame.startswith("in "): + frame = frame[3:] + frames.append(_COL_RE.sub(r":\1", frame)) + return frames + + +def classify(report: str) -> BugClass: + """Map a sanitizer/libFuzzer report to a :class:`BugClass`.""" + m = _LIBFUZZER_RE.search(report) + if m: + kind = m.group(1) + if kind == "timeout": + return BugClass.timeout + if kind == "out-of-memory": + return BugClass.oom + return BugClass.segv # "deadly signal" + # Leak detection must precede the generic ASan-class regex: ASan's leak summary + # reads "AddressSanitizer: N byte(s) leaked", which the class regex would + # otherwise capture as the (numeric) "class" and drop to unknown. + if ( + "byte(s) leaked" in report + or "LeakSanitizer" in report + or "detected memory leaks" in report + ): + return BugClass.memory_leak + m = _ASAN_CLASS_RE.search(report) + if m: + return _ASAN_TO_CLASS.get(m.group(1), BugClass.unknown) + if "SEGV" in report or "SIGSEGV" in report: + return BugClass.segv + if _UBSAN_RE.search(report): + return BugClass.unknown + return BugClass.unknown + + +def stack_hash(frames: list[str]) -> str: + """SHA-256 over the top-N frames (function + file:line).""" + top = frames[:MAX_STACK_FRAMES] + return hashlib.sha256("\n".join(top).encode("utf-8", "replace")).hexdigest() + + +def summary_line(report: str) -> str: + """Return the sanitizer ``SUMMARY:`` line, or a best-effort first error line.""" + m = _SUMMARY_RE.search(report) + if m: + return m.group(1) + for pat in (_LIBFUZZER_RE, _ASAN_CLASS_RE): + m = pat.search(report) + if m: + # Return the whole line containing the match. + line_start = report.rfind("\n", 0, m.start()) + 1 + line_end = report.find("\n", m.end()) + return report[line_start : line_end if line_end != -1 else None].strip() + return "" + + +def involves_external_scanner(frames: list[str], report: str = "") -> bool: + """True if a hand-written external scanner is implicated. + + Catches both direct scanner frames and the runtime's + ``ts_parser__external_scanner_*`` wrappers — the latter matters for + serialization-buffer overflows, where the scanner's ``serialize`` frame has + already returned by the time the runtime aborts. + """ + blob = (" ".join(frames) + " " + report).lower() + if "external_scanner" in blob: + return True + return any(name in blob for name in _SCANNER_NAMES) + + +def implicate(frames: list[str], report: str = "") -> tuple[str | None, bool]: + """Pick the source file most likely responsible for the crash. + + Prefers a hand-written external scanner (``scanner.c``/``.cc``) frame, then a + generated ``parser.c`` frame, then the first frame's file. The boolean is True + whenever an external scanner is implicated (see :func:`involves_external_scanner`). + Returns ``(file, in_scanner)``. + """ + in_scanner = involves_external_scanner(frames, report) + for frame in frames: + low = frame.lower() + if any(name in low for name in _SCANNER_NAMES): + return _frame_file(frame), True + for frame in frames: + if "parser.c" in frame.lower(): + return _frame_file(frame), in_scanner + if frames: + return _frame_file(frames[0]), in_scanner + return None, in_scanner + + +def _frame_file(frame: str) -> str | None: + """Extract the file path (without ``:line``) from a normalized frame.""" + m = re.search(r"\s(\S+):\d+$", frame) + if m: + return m.group(1) + # Frame may be just "func /path/file" with no line number. + parts = frame.rsplit(" ", 1) + if len(parts) == 2 and "/" in parts[1]: + return parts[1] + return None + + +def triage(report: str, crash_bytes: bytes, crash_file: str | None = None) -> TSCrash: + """Build a :class:`TSCrash` from a sanitizer report + the triggering input.""" + frames = extract_frames(report) + impl_file, in_scanner = implicate(frames, report) + bug_class = classify(report) + # Frameless crashes (hangs, OOM, unsymbolized aborts) would all collapse to the + # empty-string hash; fall back to a per-bug-class hash so they stay distinct. + sh = ( + stack_hash(frames) + if frames + else hashlib.sha256(f"noframes:{bug_class.value}".encode()).hexdigest() + ) + return TSCrash( + bug_class=bug_class, + stack_hash=sh, + asan_summary=summary_line(report), + top_frames=frames[:MAX_STACK_FRAMES], + crash_input_b64=base64.b64encode(crash_bytes).decode("ascii"), + crash_file=crash_file, + implicated_file=impl_file, + in_scanner=in_scanner, + ) diff --git a/scaffold/src/tests/test_treesitter_fixer.py b/scaffold/src/tests/test_treesitter_fixer.py new file mode 100644 index 0000000..3b11574 --- /dev/null +++ b/scaffold/src/tests/test_treesitter_fixer.py @@ -0,0 +1,279 @@ +"""Tests for the tree-sitter fixer (LLM mocked) and the apply/verify flow.""" + +from __future__ import annotations + +import sys +import types +from pathlib import Path + +from parser_security_eval.treesitter import fixer as fixer_mod +from parser_security_eval.treesitter import runtime +from parser_security_eval.treesitter.fixer import ( + FixProposal, + LLMFixer, + run_fix, + select_fix_target, +) +from parser_security_eval.treesitter.models import ( + BugClass, + GrammarTarget, + Tier, + TSBuildResult, + TSCrash, +) + +MODEL_REPLY = """\ +RATIONALE: missing bounds check before indexing the lexeme buffer. +```c +#include "tree_sitter/parser.h" +int scan(void) { return 0; /* fixed */ } +``` +trailing prose that must be ignored +""" + + +def _target() -> GrammarTarget: + return GrammarTarget( + name="toml", + repo_url="https://example/x", + tier=Tier.less_popular, + language="TOML", + ) + + +def test_extract_file_picks_largest_fenced_block() -> None: + content = fixer_mod._extract_file(MODEL_REPLY) + assert "fixed" in content + assert "trailing prose" not in content + + +def test_extract_file_handles_truncated_unclosed_fence() -> None: + # Model hit the token cap mid-file: opening fence, no closing fence. + truncated = "RATIONALE: bounds check\n```c\nint scan(void){ return 0; }\n// cut off" + out = fixer_mod._extract_file(truncated) + assert "int scan" in out + assert "```" not in out + assert "RATIONALE" not in out + + +def test_llm_fixer_parses_reply(monkeypatch) -> None: + block = types.SimpleNamespace(type="text", text=MODEL_REPLY) + usage = types.SimpleNamespace(input_tokens=11, output_tokens=22) + msg = types.SimpleNamespace(content=[block], usage=usage) + + class _Stream: # context manager mimicking client.messages.stream(...) + def __enter__(self): # noqa: ANN204 + return self + + def __exit__(self, *a): # noqa: ANN002, ANN204 + return False + + def get_final_message(self): # noqa: ANN202 + return msg + + client = types.SimpleNamespace( + messages=types.SimpleNamespace(stream=lambda **_: _Stream()) + ) + fake = types.ModuleType("anthropic") + fake.Anthropic = lambda *a, **k: client # type: ignore[attr-defined] + monkeypatch.setitem(sys.modules, "anthropic", fake) + + crash = TSCrash(bug_class=BugClass.heap_buffer_overflow, asan_summary="boom") + proposal = LLMFixer(model="claude-test").propose( + crash, _target(), "old source", "scanner.c", "asan report", "c" + ) + assert isinstance(proposal, FixProposal) + assert "fixed" in proposal.new_content + assert proposal.rationale.startswith("missing bounds check") + assert proposal.input_tokens == 11 + assert proposal.output_tokens == 22 + + +def test_select_fix_target_prefers_scanner(tmp_path: Path) -> None: + scanner = tmp_path / "scanner.c" + scanner.write_text("// scanner") + crash = TSCrash(in_scanner=True) + build = TSBuildResult(built=True, scanner_file=str(scanner)) + assert select_fix_target(crash, build, tmp_path) == scanner + + +def test_select_fix_target_runtime_crash_returns_none(tmp_path: Path) -> None: + crash = TSCrash(in_scanner=False, implicated_file="lib/src/subtree.c") + build = TSBuildResult(built=True) + assert select_fix_target(crash, build, tmp_path) is None + + +class _StubFixer: + name = "stub" + + def propose(self, crash, target, source, implicated_file, asan_report, lang_hint): # noqa: ANN001, ANN201, PLR0913 + return FixProposal( + new_content="int scan(void){return 0;}\n", + rationale="add guard", + input_tokens=3, + output_tokens=4, + ) + + +def test_run_fix_applies_rebuilds_and_verifies(tmp_path: Path, monkeypatch) -> None: + # Lay out a grammar with src/scanner.c so select_fix_target finds it. + grammar = tmp_path / "g" + src = grammar / "src" + src.mkdir(parents=True) + scanner = src / "scanner.c" + scanner.write_text("int scan(void){buf[i];}\n") + (src / "parser.c").write_text("// parser") + + target = _target() + spec = runtime.BuildSpec( + target=target, + runtime_dir=tmp_path / "rt", + grammar_dir=grammar, + work_dir=tmp_path / "work", + ) + build = TSBuildResult( + built=True, binary_path=str(tmp_path / "fuzz_orig"), scanner_file=str(scanner) + ) + crash = TSCrash( + bug_class=BugClass.heap_buffer_overflow, + in_scanner=True, + implicated_file="scanner.c", + ) + crash_file = tmp_path / "crash-1" + crash_file.write_bytes(b"bad") + + # Rebuild produces a *different* binary; original crashes, patched is clean. + monkeypatch.setattr( + runtime, + "build", + lambda _spec: TSBuildResult( + built=True, binary_path=str(tmp_path / "fuzz_patched") + ), + ) + + def fake_reproduce(binary, crash_file, timeout=60): # noqa: ANN001, ANN202 + if "orig" in str(binary): + return 1, "==ERROR: AddressSanitizer: heap-buffer-overflow" + return 0, "clean exit" + + monkeypatch.setattr(runtime, "reproduce", fake_reproduce) + + attempt, new_build = run_fix( + crash, crash_file, target, spec, build, _StubFixer(), "asan report" + ) + assert attempt.reproduced_original is True + assert attempt.applied is True + assert attempt.rebuilt is True + assert attempt.verified is True + assert attempt.target_file == str(scanner) + assert attempt.patch # unified diff captured + assert scanner.with_suffix(".c.orig").exists() # backup made + assert new_build.built is True + + +def test_run_fix_skips_when_original_does_not_reproduce( + tmp_path: Path, monkeypatch +) -> None: + # Foam-style: the saved artifact does not crash the original build → no fix, + # no false-positive "verified". + grammar = tmp_path / "g" + src = grammar / "src" + src.mkdir(parents=True) + scanner = src / "scanner.c" + scanner.write_text("int scan(void){return 0;}\n") + (src / "parser.c").write_text("// parser") + target = _target() + spec = runtime.BuildSpec( + target=target, + runtime_dir=tmp_path / "rt", + grammar_dir=grammar, + work_dir=tmp_path / "work", + ) + build = TSBuildResult( + built=True, binary_path=str(tmp_path / "fuzz"), scanner_file=str(scanner) + ) + crash = TSCrash( + bug_class=BugClass.segv, in_scanner=True, implicated_file="scanner.c" + ) + crash_file = tmp_path / "crash-1" + crash_file.write_bytes(b"bad") + + # Original never crashes on replay. + monkeypatch.setattr(runtime, "reproduce", lambda *a, **k: (0, "clean")) + build_calls = {"n": 0} + + def counting_build(_spec): # noqa: ANN001, ANN202 + build_calls["n"] += 1 + return TSBuildResult(built=True, binary_path=str(tmp_path / "fuzz2")) + + monkeypatch.setattr(runtime, "build", counting_build) + + attempt, _ = run_fix(crash, crash_file, target, spec, build, _StubFixer(), "report") + assert attempt.reproduced_original is False + assert attempt.verified is False + assert attempt.applied is False # never patched + assert build_calls["n"] == 0 # no rebuild / no fixer call wasted + assert "did not reproduce" in attempt.error + + +def test_run_fix_patch_that_does_not_eliminate_is_not_verified( + tmp_path: Path, monkeypatch +) -> None: + grammar = tmp_path / "g" + src = grammar / "src" + src.mkdir(parents=True) + scanner = src / "scanner.c" + scanner.write_text("int scan(void){buf[i];}\n") + (src / "parser.c").write_text("// parser") + target = _target() + spec = runtime.BuildSpec( + target=target, + runtime_dir=tmp_path / "rt", + grammar_dir=grammar, + work_dir=tmp_path / "work", + ) + build = TSBuildResult( + built=True, binary_path=str(tmp_path / "fuzz"), scanner_file=str(scanner) + ) + crash = TSCrash( + bug_class=BugClass.heap_buffer_overflow, + in_scanner=True, + implicated_file="scanner.c", + ) + crash_file = tmp_path / "crash-1" + crash_file.write_bytes(b"bad") + + # Crash reproduces on BOTH original and patched (fix didn't help). + monkeypatch.setattr( + runtime, "reproduce", lambda *a, **k: (1, "==ERROR: AddressSanitizer: SEGV") + ) + monkeypatch.setattr( + runtime, + "build", + lambda _spec: TSBuildResult(built=True, binary_path=str(tmp_path / "f2")), + ) + attempt, _ = run_fix(crash, crash_file, target, spec, build, _StubFixer(), "report") + assert attempt.reproduced_original is True + assert attempt.applied is True + assert attempt.verified is False + assert "did not eliminate" in attempt.error + + +def test_run_fix_skips_runtime_crash(tmp_path: Path) -> None: + target = _target() + grammar = tmp_path / "g" + (grammar / "src").mkdir(parents=True) + (grammar / "src" / "parser.c").write_text("// parser") + spec = runtime.BuildSpec( + target=target, + runtime_dir=tmp_path / "rt", + grammar_dir=grammar, + work_dir=tmp_path / "work", + ) + build = TSBuildResult(built=True) + crash = TSCrash(in_scanner=False, implicated_file="lib/src/subtree.c") + attempt, new_build = run_fix( + crash, tmp_path / "c", target, spec, build, _StubFixer(), "report" + ) + assert attempt.attempted is False + assert "runtime" in attempt.error diff --git a/scaffold/src/tests/test_treesitter_hunt.py b/scaffold/src/tests/test_treesitter_hunt.py new file mode 100644 index 0000000..7cb15df --- /dev/null +++ b/scaffold/src/tests/test_treesitter_hunt.py @@ -0,0 +1,116 @@ +"""Tests for registry-driven hunt candidate selection (run_loop mocked).""" + +from __future__ import annotations + +from pathlib import Path + +from parser_security_eval.treesitter import loop as loop_mod +from parser_security_eval.treesitter.loop import run_hunt, select_hunt_candidates +from parser_security_eval.treesitter.models import Tier, TSLoopResult +from parser_security_eval.treesitter.survey.models import GrammarRecord + + +def _rec(name, owner, repo, *, scanner, fuzzed, prio) -> GrammarRecord: # noqa: ANN001 + return GrammarRecord( + name=name, + language_group=name, + repo_url=f"https://github.com/{owner}/{repo}", + owner=owner, + repo=repo, + has_external_scanner=scanner, + scores={"already_fuzzed": 1.0 if fuzzed else 0.0}, + fuzz_priority=prio, + ) + + +def _write_registry(tmp_path: Path) -> Path: + records = [ + _rec( + "cmake", "uyha", "tree-sitter-cmake", scanner=True, fuzzed=False, prio=50.0 + ), + _rec( + "fish", "ram02z", "tree-sitter-fish", scanner=True, fuzzed=False, prio=40.0 + ), + # excluded: already fuzzed + _rec( + "ruby", + "tree-sitter", + "tree-sitter-ruby", + scanner=True, + fuzzed=True, + prio=60.0, + ), + # excluded: no scanner + _rec( + "vimdoc", + "neovim", + "tree-sitter-vimdoc", + scanner=False, + fuzzed=False, + prio=55.0, + ), + # excluded: archived/abandoned (priority 0) + _rec("dead", "x", "tree-sitter-dead", scanner=True, fuzzed=False, prio=0.0), + # duplicate repo of cmake, lower priority → deduped + _rec( + "cmake", "uyha", "tree-sitter-cmake", scanner=True, fuzzed=False, prio=10.0 + ), + ] + p = tmp_path / "registry.jsonl" + p.write_text("\n".join(r.to_jsonl() for r in records) + "\n", encoding="utf-8") + return p + + +def test_select_filters_and_orders(tmp_path: Path) -> None: + reg = _write_registry(tmp_path) + cands = select_hunt_candidates(reg, top_n=10) + names = [c.name for c in cands] + # only un-fuzzed scanner grammars, priority-ordered, deduped + assert names == ["uyha-cmake", "ram02z-fish"] + assert all(c.tier == Tier.less_popular for c in cands) + assert len(names) == len(set(names)) # unique slugs + + +def test_select_excludes_fuzzed_noscanner_archived(tmp_path: Path) -> None: + reg = _write_registry(tmp_path) + cands = select_hunt_candidates(reg, top_n=10) + repos = {c.repo_url for c in cands} + assert "https://github.com/tree-sitter/tree-sitter-ruby" not in repos # fuzzed + assert "https://github.com/neovim/tree-sitter-vimdoc" not in repos # no scanner + assert "https://github.com/x/tree-sitter-dead" not in repos # archived (prio 0) + + +def test_select_respects_top_n(tmp_path: Path) -> None: + reg = _write_registry(tmp_path) + assert len(select_hunt_candidates(reg, top_n=1)) == 1 + + +def test_select_start_offset_skips_earlier_ranks(tmp_path: Path) -> None: + reg = _write_registry(tmp_path) + # eligible order is [uyha-cmake (50), ram02z-fish (40)] + assert [c.name for c in select_hunt_candidates(reg, top_n=10, start=2)] == [ + "ram02z-fish" + ] + assert [c.name for c in select_hunt_candidates(reg, top_n=1, start=1)] == [ + "uyha-cmake" + ] + # start past the end → empty + assert select_hunt_candidates(reg, top_n=10, start=3) == [] + + +def test_run_hunt_invokes_loop_per_candidate(tmp_path: Path, monkeypatch) -> None: + reg = _write_registry(tmp_path) + calls: list[str] = [] + + def fake_run_loop(target, **kwargs): # noqa: ANN001, ANN202 + calls.append(target.name) + assert kwargs["iterations"] == 1 + assert kwargs["fuzz_seconds"] == 99 + return TSLoopResult( + grammar=target.name, tier=target.tier, language=target.language + ) + + monkeypatch.setattr(loop_mod, "run_loop", fake_run_loop) + results = run_hunt(reg, top_n=10, fuzz_seconds=99, out_dir=tmp_path / "hunt") + assert calls == ["uyha-cmake", "ram02z-fish"] + assert len(results) == 2 diff --git a/scaffold/src/tests/test_treesitter_loop.py b/scaffold/src/tests/test_treesitter_loop.py new file mode 100644 index 0000000..587074b --- /dev/null +++ b/scaffold/src/tests/test_treesitter_loop.py @@ -0,0 +1,131 @@ +"""Tests for the fuzz→fix loop control flow + JSONL emission (toolchain mocked).""" + +from __future__ import annotations + +import json +from pathlib import Path + +from parser_security_eval.treesitter import fixer as fixer_mod +from parser_security_eval.treesitter import loop as loop_mod +from parser_security_eval.treesitter import runtime +from parser_security_eval.treesitter.models import ( + GrammarTarget, + Tier, + TSBuildResult, + TSFixAttempt, + TSFuzzResult, +) + +HEAP_OVERFLOW = """\ +==1==ERROR: AddressSanitizer: heap-buffer-overflow on address 0x60 + #0 0x1 in scan /g/src/scanner.c:10:3 + #1 0x2 in LLVMFuzzerTestOneInput /w/harness.c:9 +SUMMARY: AddressSanitizer: heap-buffer-overflow /g/src/scanner.c:10:3 in scan +""" + + +def _target() -> GrammarTarget: + return GrammarTarget( + name="demo", + repo_url="https://example/demo", + tier=Tier.less_popular, + language="Demo", + ) + + +def test_loop_fuzzer_triggers_fix_and_writes_jsonl(tmp_path: Path, monkeypatch) -> None: + cache = tmp_path / "cache" + out_dir = tmp_path / "out" + + monkeypatch.setattr(runtime, "ensure_runtime", lambda c: tmp_path / "rt") + monkeypatch.setattr(runtime, "ensure_grammar", lambda t, c: tmp_path / "grammar") + monkeypatch.setattr(runtime, "gather_seeds", lambda *a, **k: 0) + monkeypatch.setattr( + runtime, + "build", + lambda spec: TSBuildResult( + built=True, + binary_path=str(tmp_path / "fuzz"), + scanner_file=str(tmp_path / "scanner.c"), + ), + ) + monkeypatch.setattr(runtime, "reproduce", lambda *a, **k: (1, HEAP_OVERFLOW)) + + calls = {"n": 0} + + def fake_run(self: runtime.LibFuzzerRunner, duration: int): # noqa: ANN202 + calls["n"] += 1 + if calls["n"] == 1: + self.crashes_dir.mkdir(parents=True, exist_ok=True) + cf = self.crashes_dir / "crash-abc" + cf.write_bytes(b"boom") + return TSFuzzResult(duration_seconds=duration, crashes_found=1), [cf] + return TSFuzzResult(duration_seconds=duration, crashes_found=0), [] + + monkeypatch.setattr(runtime.LibFuzzerRunner, "run", fake_run) + + fix_calls = {"n": 0} + + def fake_run_fix(crash, crash_file, target, spec, build_result, fixer, report): # noqa: ANN001, ANN202, PLR0913 + fix_calls["n"] += 1 + return ( + TSFixAttempt( + attempted=True, applied=True, rebuilt=True, verified=True, model="stub" + ), + build_result, + ) + + monkeypatch.setattr(fixer_mod, "run_fix", fake_run_fix) + monkeypatch.setattr(runtime, "check_toolchain", lambda **k: None) + + result = loop_mod.run_loop( + _target(), + iterations=2, + fuzz_seconds=1, + cache_dir=cache, + out_dir=out_dir, + fixer=object(), # never used; run_fix is stubbed + seed_corpus=False, + ) + + # The fuzzer found a crash in window 0 → fix triggered exactly once. + assert fix_calls["n"] == 1 + assert result.crashes_found == 1 + assert result.fixes_verified == 1 + assert len(result.iterations) == 2 + assert result.iterations[0].crash is not None + assert result.iterations[0].fix is not None and result.iterations[0].fix.verified + assert result.iterations[1].crash is None + + # JSONL: one line per iteration, each a valid object. + jsonl = Path(result.jsonl_path or "") + lines = jsonl.read_text().strip().splitlines() + assert len(lines) == 2 + rec0 = json.loads(lines[0]) + assert rec0["crash"]["bug_class"] == "heap-buffer-overflow" + assert rec0["fix"]["verified"] is True + + +def test_loop_records_build_failure(tmp_path: Path, monkeypatch) -> None: + monkeypatch.setattr(runtime, "check_toolchain", lambda **k: None) + monkeypatch.setattr(runtime, "ensure_runtime", lambda c: tmp_path / "rt") + monkeypatch.setattr(runtime, "ensure_grammar", lambda t, c: tmp_path / "grammar") + monkeypatch.setattr( + runtime, + "build", + lambda spec: TSBuildResult( + built=False, compile_errors="boom: undefined symbol" + ), + ) + + result = loop_mod.run_loop( + _target(), + iterations=3, + fuzz_seconds=1, + cache_dir=tmp_path, + out_dir=tmp_path / "o", + ) + assert result.build_error + assert result.crashes_found == 0 + assert len(result.iterations) == 1 + assert result.iterations[0].built is False diff --git a/scaffold/src/tests/test_treesitter_registry.py b/scaffold/src/tests/test_treesitter_registry.py new file mode 100644 index 0000000..67c84e2 --- /dev/null +++ b/scaffold/src/tests/test_treesitter_registry.py @@ -0,0 +1,35 @@ +"""Tests for the tree-sitter grammar registry.""" + +from __future__ import annotations + +import pytest + +from parser_security_eval.treesitter import registry +from parser_security_eval.treesitter.models import Tier + + +def test_registry_nonempty_and_has_both_tiers() -> None: + assert registry.by_tier(Tier.popular) + assert registry.by_tier(Tier.less_popular) + + +def test_all_names_unique() -> None: + names = registry.all_names() + assert len(names) == len(set(names)) + + +def test_get_known_and_unknown() -> None: + assert registry.get("json").name == "json" + with pytest.raises(KeyError): + registry.get("does-not-exist") + + +def test_every_target_has_https_repo() -> None: + for g in registry.by_tier(): + assert g.repo_url.startswith("https://") + assert g.language + + +def test_popular_includes_mainstream_languages() -> None: + popular = {g.name for g in registry.by_tier(Tier.popular)} + assert {"json", "python", "javascript", "c"} <= popular diff --git a/scaffold/src/tests/test_treesitter_runtime_unit.py b/scaffold/src/tests/test_treesitter_runtime_unit.py new file mode 100644 index 0000000..05c6792 --- /dev/null +++ b/scaffold/src/tests/test_treesitter_runtime_unit.py @@ -0,0 +1,89 @@ +"""Unit tests for tree-sitter runtime helpers (no compiler/network needed).""" + +from __future__ import annotations + +from pathlib import Path + +from parser_security_eval.treesitter import runtime + +PARSER_C_STUB = """\ +#include "tree_sitter/parser.h" +/* ... generated tables ... */ +extern const TSLanguage *tree_sitter_toml(void) { + static const TSLanguage language = {0}; + return &language; +} +""" + + +def test_render_harness_contains_symbol_and_entrypoint() -> None: + src = runtime.render_harness("tree_sitter_toml") + assert "tree_sitter_toml()" in src + assert "LLVMFuzzerTestOneInput" in src + assert "ts_parser_parse_string" in src + + +def test_detect_symbol_from_parser_definition(tmp_path: Path) -> None: + (tmp_path / "parser.c").write_text(PARSER_C_STUB) + assert runtime.detect_symbol(tmp_path, "tree_sitter_fallback") == "tree_sitter_toml" + + +def test_detect_symbol_falls_back_when_missing(tmp_path: Path) -> None: + assert runtime.detect_symbol(tmp_path, "tree_sitter_x") == "tree_sitter_x" + + +def test_find_src_dir_prefers_existing_parser_c(tmp_path: Path) -> None: + src = tmp_path / "src" + src.mkdir() + (src / "parser.c").write_text("// parser") + assert runtime.find_src_dir(tmp_path) == src + + +def test_gather_seeds_collects_test_files_and_literals(tmp_path: Path) -> None: + grammar = tmp_path / "grammar" + test_dir = grammar / "test" / "corpus" + test_dir.mkdir(parents=True) + (test_dir / "a.txt").write_text("sample = 1\n") + (test_dir / "b.txt").write_text("[table]\nx = true\n") + corpus = tmp_path / "corpus" + count = runtime.gather_seeds(grammar, corpus) + files = list(corpus.iterdir()) + # 2 test files + the adversarial literals + assert count == len(files) + assert count >= 2 + 5 + + +def test_reproduce_handles_hang_input(tmp_path: Path, monkeypatch) -> None: + import subprocess + + def boom(*a, **k): # noqa: ANN002, ANN003, ANN202 + raise subprocess.TimeoutExpired(cmd="fuzz", timeout=1) + + monkeypatch.setattr(runtime, "_run", boom) + rc, out = runtime.reproduce(tmp_path / "bin", tmp_path / "timeout-1", timeout=1) + assert rc == -1 + assert "timeout" in out.lower() + + +def test_libfuzzer_runner_parses_stats(tmp_path: Path, monkeypatch) -> None: + log = "stat::number_of_executed_units: 12345\nstat::average_exec_per_sec: 678\n" + + class _CP: + returncode = 0 + stdout = log + stderr = "" + + monkeypatch.setattr(runtime, "_run", lambda *a, **k: _CP()) + corpus = tmp_path / "corpus" + corpus.mkdir() + (corpus / "seed").write_text("x") + runner = runtime.LibFuzzerRunner( + binary=tmp_path / "fuzz", + corpus_dir=corpus, + crashes_dir=tmp_path / "crashes", + ) + result, crashes = runner.run(5) + assert result.total_executions == 12345 + assert result.execs_per_sec == 678.0 + assert result.crashes_found == 0 + assert crashes == [] diff --git a/scaffold/src/tests/test_treesitter_survey_github.py b/scaffold/src/tests/test_treesitter_survey_github.py new file mode 100644 index 0000000..fd7dae8 --- /dev/null +++ b/scaffold/src/tests/test_treesitter_survey_github.py @@ -0,0 +1,101 @@ +"""Tests for the GitHub client (mocked HTTP, no network).""" + +from __future__ import annotations + +from pathlib import Path + +from parser_security_eval.treesitter.survey.github import ( + GitHubClient, + detect_fuzz_setup, +) + + +def test_detect_fuzz_setup() -> None: + assert detect_fuzz_setup(["src/parser.c", "grammar.js"]) == (False, "") + assert detect_fuzz_setup(["src/parser.c", "fuzz/fuzzer.cc"]) == ( + True, + "fuzz/fuzzer.cc", + ) + assert detect_fuzz_setup([".github/workflows/fuzz.yml"])[0] is True + assert detect_fuzz_setup(["test/fuzzer/run.sh"])[0] is True + + +REPO_JSON = { + "stargazers_count": 321, + "forks_count": 12, + "fork": False, + "archived": False, + "disabled": False, + "pushed_at": "2026-05-01T00:00:00Z", + "created_at": "2021-01-01T00:00:00Z", + "updated_at": "2026-05-02T00:00:00Z", + "open_issues_count": 7, + "has_issues": True, + "license": {"spdx_id": "MIT"}, + "size": 2048, + "topics": ["tree-sitter", "parser"], + "subscribers_count": 9, +} +COMMIT_ACTIVITY = [{"total": t, "week": 0, "days": []} for t in ([1] * 39 + [2] * 13)] + + +def test_to_metrics_maps_fields() -> None: + blob = { + "repo": REPO_JSON, + "commit_activity": COMMIT_ACTIVITY, + "contributors": 5, + "fuzz": {"has_fuzz_setup": True, "fuzz_evidence": "fuzz/fuzzer.cc"}, + } + m = GitHubClient._to_metrics(blob) + assert m.fetched is True + assert m.stars == 321 + assert m.is_fork is False + assert m.license == "MIT" + assert m.contributors == 5 + assert m.has_fuzz_setup is True + assert m.fuzz_evidence == "fuzz/fuzzer.cc" + assert m.commits_last_year == 39 + 26 # 39*1 + 13*2 + assert m.commits_last_90d == 26 # last 13 weeks * 2 + assert m.active_weeks_last_year == 52 + + +def test_fetch_metrics_caches(tmp_path: Path, monkeypatch) -> None: + client = GitHubClient(token="x", cache_dir=tmp_path) + + # A realistic Link header contains `per_page=1` too — the parser must not + # confuse it with the `page=` it wants. + link = ( + "; rel="next", ' + "; rel="last"' + ) + + def fake_request(url: str): # noqa: ANN202 + if url.endswith("/contributors?per_page=1&anon=1"): + return 200, [{"login": "a"}], {"Link": link} + if url.endswith("/stats/commit_activity"): + return 200, COMMIT_ACTIVITY, {} + return 200, REPO_JSON, {} + + monkeypatch.setattr(client, "_request", fake_request) + m = client.fetch_metrics("o", "tree-sitter-x") + assert m.stars == 321 + assert m.contributors == 26 # last page, not per_page=1 + assert (tmp_path / "o__tree-sitter-x.json").exists() + + # Second call must hit the cache, not the network. + def boom(url: str): # noqa: ANN202 + raise AssertionError("network must not be called when cache is fresh") + + monkeypatch.setattr(client, "_request", boom) + m2 = client.fetch_metrics("o", "tree-sitter-x") + assert m2.stars == 321 + + +def test_fetch_metrics_404(tmp_path: Path, monkeypatch) -> None: + client = GitHubClient(token="x", cache_dir=tmp_path) + monkeypatch.setattr(client, "_request", lambda url: (404, None, {})) + m = client.fetch_metrics("o", "missing") + assert m.fetched is False + assert "404" in m.error diff --git a/scaffold/src/tests/test_treesitter_survey_scoring.py b/scaffold/src/tests/test_treesitter_survey_scoring.py new file mode 100644 index 0000000..92d1e14 --- /dev/null +++ b/scaffold/src/tests/test_treesitter_survey_scoring.py @@ -0,0 +1,138 @@ +"""Tests for survey priority scoring (bands + gates).""" + +from __future__ import annotations + +from datetime import datetime, timezone + +from parser_security_eval.treesitter.survey.models import GrammarRecord, RepoMetrics +from parser_security_eval.treesitter.survey.scoring import compute_priority + +NOW = datetime(2026, 6, 23, tzinfo=timezone.utc) + + +def _rec(**kw) -> GrammarRecord: # noqa: ANN003 + base = { + "name": "x", + "language_group": "x", + "repo_url": "https://github.com/o/tree-sitter-x", + "owner": "o", + "repo": "tree-sitter-x", + } + base.update(kw) + return GrammarRecord(**base) + + +def _metrics(**kw) -> RepoMetrics: # noqa: ANN003 + base = { + "fetched": True, + "stars": 200, + "is_fork": False, + "archived": False, + "has_issues": True, + "pushed_at": "2026-05-01T00:00:00Z", + "created_at": "2021-01-01T00:00:00Z", + } + base.update(kw) + return RepoMetrics(**base) + + +def test_archived_is_excluded() -> None: + rec = _rec(has_external_scanner=True, metrics=_metrics(archived=True)) + compute_priority(rec, NOW) + assert rec.fuzz_priority == 0.0 + assert rec.scores["tractability"] == 0.0 + + +def test_abandoned_is_excluded() -> None: + rec = _rec( + has_external_scanner=True, metrics=_metrics(pushed_at="2022-01-01T00:00:00Z") + ) + compute_priority(rec, NOW) + assert rec.scores["activity_gate"] == 0.0 + assert rec.fuzz_priority == 0.0 + + +def test_scanner_beats_no_scanner() -> None: + with_scanner = _rec(has_external_scanner=True, metrics=_metrics()) + without = _rec(has_external_scanner=False, metrics=_metrics()) + compute_priority(with_scanner, NOW) + compute_priority(without, NOW) + assert with_scanner.fuzz_priority > without.fuzz_priority + + +def test_core_org_penalized() -> None: + niche = _rec(owner="someone", has_external_scanner=True, metrics=_metrics()) + core = _rec( + owner="tree-sitter", + has_external_scanner=True, + metrics=_metrics(), + ) + compute_priority(niche, NOW) + compute_priority(core, NOW) + # Core org is treated as already-fuzzed (OSS-Fuzz) → lower neglect → lower bug. + assert core.scores["bug_likelihood"] < niche.scores["bug_likelihood"] + assert core.scores["neglect"] < niche.scores["neglect"] + + +def test_already_fuzzed_grammar_deprioritized() -> None: + fresh = _rec(has_external_scanner=True, metrics=_metrics()) + fuzzed = _rec( + has_external_scanner=True, + metrics=_metrics(has_fuzz_setup=True, fuzz_evidence="fuzz/fuzz_parser.cc"), + ) + compute_priority(fresh, NOW) + compute_priority(fuzzed, NOW) + assert fuzzed.scores["already_fuzzed"] == 1.0 + assert fresh.scores["already_fuzzed"] == 0.0 + assert fuzzed.scores["neglect"] < fresh.scores["neglect"] + assert fuzzed.fuzz_priority < fresh.fuzz_priority + + +def test_unfuzzed_scanner_beats_fuzzed_scanner_even_if_more_popular() -> None: + # The whole point: a neglected scanner grammar should outrank a more popular + # but already-fuzzed one. + neglected = _rec(has_external_scanner=True, metrics=_metrics(stars=60)) + fuzzed_popular = _rec( + has_external_scanner=True, + metrics=_metrics(stars=2000, has_fuzz_setup=True, fuzz_evidence="fuzz/"), + ) + compute_priority(neglected, NOW) + compute_priority(fuzzed_popular, NOW) + assert neglected.fuzz_priority > fuzzed_popular.fuzz_priority + + +def test_no_scanner_has_minimal_crash_surface() -> None: + rec = _rec(has_external_scanner=False, metrics=_metrics()) + compute_priority(rec, NOW) + assert rec.scores["crash_surface"] == 0.15 + + +def test_very_popular_has_lower_bug_likelihood() -> None: + midpop = _rec(has_external_scanner=True, metrics=_metrics(stars=150)) + superpop = _rec(has_external_scanner=True, metrics=_metrics(stars=20000)) + compute_priority(midpop, NOW) + compute_priority(superpop, NOW) + # More stars → more scrutiny → lower latent-bug likelihood. + assert superpop.scores["low_scrutiny"] < midpop.scores["low_scrutiny"] + assert superpop.scores["bug_likelihood"] < midpop.scores["bug_likelihood"] + + +def test_adoption_raises_impact() -> None: + plain = _rec(has_external_scanner=True, metrics=_metrics()) + adopted = _rec( + has_external_scanner=True, + used_by_zed=True, + used_by_emacs=True, + metrics=_metrics(), + ) + compute_priority(plain, NOW) + compute_priority(adopted, NOW) + assert adopted.scores["impact"] > plain.scores["impact"] + assert adopted.fuzz_priority > plain.fuzz_priority + + +def test_missing_metrics_does_not_crash() -> None: + rec = _rec(has_external_scanner=True, metrics=RepoMetrics(fetched=False)) + compute_priority(rec, NOW) + assert 0.0 <= rec.fuzz_priority <= 100.0 + assert rec.scores["months_since_push"] == -1.0 diff --git a/scaffold/src/tests/test_treesitter_survey_sources.py b/scaffold/src/tests/test_treesitter_survey_sources.py new file mode 100644 index 0000000..fb74f2c --- /dev/null +++ b/scaffold/src/tests/test_treesitter_survey_sources.py @@ -0,0 +1,80 @@ +"""Offline tests for survey source parsers (wiki / zed / redmonk / url normalize).""" + +from __future__ import annotations + +from parser_security_eval.treesitter.survey import sources + +WIKI_FIXTURE = """\ +Some preamble text. + +| name | url | last commit date | abi | grammar.json | external scanner | +| --- | --- | --- | --- | --- | --- | +| abl | [github.com/usagi-coffee/tree-sitter-abl](https://github.com/usagi-coffee/tree-sitter-abl) | 2026-04-07 | 15 | yes | yes | +| ada | [github.com/briot/tree-sitter-ada](https://github.com/briot/tree-sitter-ada) | 2024-05-23 | 14 | yes | no | +| json | [github.com/tree-sitter/tree-sitter-json](https://github.com/tree-sitter/tree-sitter-json) | 2025-01-01 | 15 | yes | no | +""" + +ZED_LOCK_FIXTURE = """\ +[[package]] +name = "tree-sitter-bash" +version = "0.23.0" + +[[package]] +name = "tree-sitter-heex" +version = "0.0.1" +source = "git+https://github.com/zed-industries/tree-sitter-heex?rev=1dd45142#1dd45142" +""" + + +def test_normalize_repo_url_variants() -> None: + for raw in ( + "https://github.com/tree-sitter/tree-sitter-json", + "github.com/tree-sitter/tree-sitter-json", + "https://github.com/tree-sitter/tree-sitter-json.git", + "https://github.com/tree-sitter/tree-sitter-json/tree/master", + ): + norm = sources.normalize_repo_url(raw) + assert norm == ( + "https://github.com/tree-sitter/tree-sitter-json", + "tree-sitter", + "tree-sitter-json", + ) + + +def test_normalize_repo_url_rejects_non_github() -> None: + assert sources.normalize_repo_url("https://gitlab.com/x/y") is None + assert sources.normalize_repo_url("not a url") is None + + +def test_parse_wiki_extracts_fields() -> None: + rows = sources.parse_wiki(WIKI_FIXTURE) + assert len(rows) == 3 + abl = next(r for r in rows if r.name == "abl") + assert abl.owner == "usagi-coffee" + assert abl.repo == "tree-sitter-abl" + assert abl.has_external_scanner is True + assert abl.has_grammar_json is True + assert abl.wiki_last_commit == "2026-04-07" + assert abl.abi == "15" + ada = next(r for r in rows if r.name == "ada") + assert ada.has_external_scanner is False + + +def test_parse_zed_cargo_lock() -> None: + repo_keys, crate_names = sources.parse_zed_cargo_lock(ZED_LOCK_FIXTURE) + assert "zed-industries/tree-sitter-heex" in repo_keys + assert "tree-sitter-bash" in crate_names + assert "tree-sitter-heex" in crate_names + + +def test_redmonk_rank_aliases() -> None: + assert sources.redmonk_rank("javascript") == 1 + assert sources.redmonk_rank("c_sharp") == sources.redmonk_rank("csharp") + assert sources.redmonk_rank("bash") == sources.redmonk_rank("shell") + assert sources.redmonk_rank("nonexistent-lang") is None + + +def test_repo_key_is_lowercase() -> None: + assert sources.repo_key("Tree-Sitter", "Tree-Sitter-JSON") == ( + "tree-sitter/tree-sitter-json" + ) diff --git a/scaffold/src/tests/test_treesitter_triage.py b/scaffold/src/tests/test_treesitter_triage.py new file mode 100644 index 0000000..e249fbf --- /dev/null +++ b/scaffold/src/tests/test_treesitter_triage.py @@ -0,0 +1,118 @@ +"""Tests for tree-sitter crash triage (ASAN/libFuzzer parsing + dedup).""" + +from __future__ import annotations + +import base64 + +from parser_security_eval.treesitter import triage +from parser_security_eval.treesitter.models import BugClass + +HEAP_OVERFLOW = """\ +==12345==ERROR: AddressSanitizer: heap-buffer-overflow on address 0x602000000051 +READ of size 1 at 0x602000000051 thread T0 + #0 0x4f1a2b in scan_string /home/u/tree-sitter-toml/src/scanner.c:120:14 + #1 0x4f0010 in tree_sitter_toml_external_scanner_scan /home/u/tree-sitter-toml/src/scanner.c:300:10 + #2 0x4e9999 in ts_lexer__advance /home/u/tree-sitter/lib/src/lexer.c:200 + #3 0x4d0000 in LLVMFuzzerTestOneInput /home/u/work/harness.c:14:18 +SUMMARY: AddressSanitizer: heap-buffer-overflow /home/u/tree-sitter-toml/src/scanner.c:120:14 in scan_string +""" + +STACK_OVERFLOW = """\ +==1==ERROR: AddressSanitizer: stack-overflow on address 0x7ffd00000000 + #0 0x55 in ts_subtree__print_dot_graph /home/u/tree-sitter/lib/src/subtree.c:900 + #1 0x56 in ts_node_string /home/u/tree-sitter/lib/src/node.c:100 + #2 0x57 in LLVMFuzzerTestOneInput /home/u/work/harness.c:14 +""" + +LIBFUZZER_TIMEOUT = """\ +==2== ERROR: libFuzzer: timeout after 20 seconds + #0 0x10 in scan /home/u/tree-sitter-x/src/scanner.c:5 +SUMMARY: libFuzzer: timeout +""" + +PARSER_CRASH = """\ +==3==ERROR: AddressSanitizer: SEGV on unknown address 0x000000000000 + #0 0x20 in ts_parser__lex /home/u/tree-sitter-x/src/parser.c:88:3 + #1 0x21 in LLVMFuzzerTestOneInput /home/u/work/harness.c:14 +SUMMARY: AddressSanitizer: SEGV +""" + + +def test_classify_heap_overflow() -> None: + assert triage.classify(HEAP_OVERFLOW) is BugClass.heap_buffer_overflow + + +def test_classify_stack_overflow() -> None: + assert triage.classify(STACK_OVERFLOW) is BugClass.stack_overflow + + +def test_classify_timeout_takes_priority() -> None: + assert triage.classify(LIBFUZZER_TIMEOUT) is BugClass.timeout + + +def test_classify_segv() -> None: + assert triage.classify(PARSER_CRASH) is BugClass.segv + + +LEAK = """\ +==1==ERROR: AddressSanitizer: 2024 byte(s) leaked in 527 allocation(s). + #0 0x1 in malloc + #1 0x2 in tree_sitter_sql_external_scanner_deserialize /g/src/scanner.c:50 +SUMMARY: AddressSanitizer: 2024 byte(s) leaked +""" + + +def test_classify_memory_leak_from_asan_wording() -> None: + # ASan writes "AddressSanitizer: N byte(s) leaked"; must not fall to unknown. + assert triage.classify(LEAK) is BugClass.memory_leak + crash = triage.triage(LEAK, b"x") + assert crash.bug_class is BugClass.memory_leak + assert crash.in_scanner is True # deserialize frame + + +def test_extract_frames_drops_column() -> None: + frames = triage.extract_frames(HEAP_OVERFLOW) + assert frames[0] == "scan_string /home/u/tree-sitter-toml/src/scanner.c:120" + assert len(frames) == 4 + + +def test_stack_hash_is_stable_and_distinct() -> None: + a = triage.stack_hash(triage.extract_frames(HEAP_OVERFLOW)) + b = triage.stack_hash(triage.extract_frames(HEAP_OVERFLOW)) + c = triage.stack_hash(triage.extract_frames(STACK_OVERFLOW)) + assert a == b + assert a != c + assert len(a) == 64 + + +def test_implicate_prefers_scanner() -> None: + file, in_scanner = triage.implicate(triage.extract_frames(HEAP_OVERFLOW)) + assert file is not None and file.endswith("scanner.c") + assert in_scanner is True + + +def test_implicate_parser_not_scanner() -> None: + file, in_scanner = triage.implicate(triage.extract_frames(PARSER_CRASH)) + assert file is not None and file.endswith("parser.c") + assert in_scanner is False + + +def test_frameless_crashes_get_distinct_stable_hashes() -> None: + t = triage.triage("ERROR: libFuzzer: timeout\n", b"x") + o = triage.triage("ERROR: libFuzzer: out-of-memory\n", b"x") + assert t.bug_class is BugClass.timeout + assert o.bug_class is BugClass.oom + # Frameless crashes must not collapse to the same (empty-string) hash. + assert t.stack_hash and o.stack_hash + assert t.stack_hash != o.stack_hash + # Stable across calls (dedup-friendly). + assert t.stack_hash == triage.triage("ERROR: libFuzzer: timeout\n", b"y").stack_hash + + +def test_triage_roundtrips_input_and_summary() -> None: + crash = triage.triage(HEAP_OVERFLOW, b"\x00bad\xff", crash_file="/tmp/crash-1") + assert crash.bug_class is BugClass.heap_buffer_overflow + assert crash.in_scanner is True + assert "heap-buffer-overflow" in crash.asan_summary + assert base64.b64decode(crash.crash_input_b64) == b"\x00bad\xff" + assert crash.crash_file == "/tmp/crash-1" diff --git a/scaffold/treesitter-shell.nix b/scaffold/treesitter-shell.nix new file mode 100644 index 0000000..993a7b3 --- /dev/null +++ b/scaffold/treesitter-shell.nix @@ -0,0 +1,22 @@ +# Toolchain for the tree-sitter fuzz→fix loop (parser_security_eval.treesitter). +# +# Provides clang (with libFuzzer + ASan), llvm (llvm-symbolizer, so ASan stack +# traces are symbolized → accurate crash dedup + implicated-file detection), the +# tree-sitter CLI, nodejs (for `tree-sitter generate`) and git. `uv` is +# intentionally NOT pinned here — it is picked up from ~/.local/bin on the ambient +# PATH, which nix-shell preserves. +# +# Usage: +# nix-shell scaffold/treesitter-shell.nix --run \ +# 'cd scaffold && ANTHROPIC_API_KEY=... uv run parser-security-eval treesitter fuzz toml' +# +{ pkgs ? import { } }: +pkgs.mkShell { + packages = [ + pkgs.clang + pkgs.llvm + pkgs.tree-sitter + pkgs.nodejs + pkgs.git + ]; +} From a938143fe296c678101ec3f9af4e543b3903e019 Mon Sep 17 00:00:00 2001 From: Matthew Doty Date: Mon, 29 Jun 2026 12:50:17 -0400 Subject: [PATCH 2/6] feat(scaffold): introducing logfire --- scaffold/pyproject.toml | 6 + .../prompts/treesitter/fix_system.prompt | 10 +- .../src/parser_security_eval/telemetry.py | 9 +- .../parser_security_eval/treesitter/fixer.py | 86 ++++---- .../parser_security_eval/treesitter/loop.py | 187 ++++++++++++------ scaffold/src/tests/test_treesitter_fixer.py | 76 +++---- scaffold/src/tests/test_treesitter_hunt.py | 1 + scaffold/src/tests/test_treesitter_loop.py | 2 + scaffold/uv.lock | 118 ++++++++++- 9 files changed, 328 insertions(+), 167 deletions(-) diff --git a/scaffold/pyproject.toml b/scaffold/pyproject.toml index 636d871..ca1fa6e 100644 --- a/scaffold/pyproject.toml +++ b/scaffold/pyproject.toml @@ -16,6 +16,7 @@ dependencies = [ "pandas>=2.2.0", "plotly>=5.22.0", "pydantic>=2.12.5", + "pydantic-ai-slim[anthropic,openai]>=2.0.0", "pyyaml>=6.0", "rich>=14.3.3", "streamlit>=1.36.0", @@ -29,6 +30,11 @@ parser-security-eval = "parser_security_eval:main" requires = ["uv_build>=0.9.22,<0.10.0"] build-backend = "uv_build" +[tool.logfire] +# Loop telemetry degrades to a silent no-op when Logfire isn't configured +# (no token / unit tests) instead of warning on every span/log call. +ignore_no_config = true + [dependency-groups] dev = [ "hypothesis>=6.151.9", diff --git a/scaffold/src/parser_security_eval/prompts/treesitter/fix_system.prompt b/scaffold/src/parser_security_eval/prompts/treesitter/fix_system.prompt index 40c1e73..c918f23 100644 --- a/scaffold/src/parser_security_eval/prompts/treesitter/fix_system.prompt +++ b/scaffold/src/parser_security_eval/prompts/treesitter/fix_system.prompt @@ -17,9 +17,7 @@ Rules: - Do not add comments explaining the CVE; a short note at the fix site is fine. - If the file is `grammar.js`, keep it valid tree-sitter grammar JavaScript. -Respond in EXACTLY this format: - -RATIONALE: -``` - -``` +Return your fix as structured output: +- `rationale`: one sentence — the root cause and the guard you added. +- `file_contents`: the COMPLETE corrected contents of the file (the full file, not a + diff or a snippet). diff --git a/scaffold/src/parser_security_eval/telemetry.py b/scaffold/src/parser_security_eval/telemetry.py index 4bfa9b7..f10e487 100644 --- a/scaffold/src/parser_security_eval/telemetry.py +++ b/scaffold/src/parser_security_eval/telemetry.py @@ -32,12 +32,15 @@ # logger to avoid a circular dependency (log.py must not import telemetry). logger = logging.getLogger(__name__) -# Provider SDKs Inspect may drive. Each is instrumented best-effort: a missing -# package or version skew must not break the eval. ``httpx`` is the catch-all -# for any provider not natively instrumented above. +# Best-effort instrumenters (all no-arg). Inspect drives the provider SDKs; the +# tree-sitter fixer drives pydantic-ai (``instrument_pydantic_ai`` captures agent +# runs / structured output / token usage). Each is applied defensively: a missing +# package or version skew must not break a run. ``httpx`` is the catch-all for any +# provider not natively instrumented above. _PROVIDER_INSTRUMENTERS: tuple[str, ...] = ( "instrument_anthropic", "instrument_openai", + "instrument_pydantic_ai", ) _configured = False diff --git a/scaffold/src/parser_security_eval/treesitter/fixer.py b/scaffold/src/parser_security_eval/treesitter/fixer.py index e6b1c2d..f7bccad 100644 --- a/scaffold/src/parser_security_eval/treesitter/fixer.py +++ b/scaffold/src/parser_security_eval/treesitter/fixer.py @@ -10,10 +10,13 @@ from __future__ import annotations import difflib -import re from dataclasses import dataclass from pathlib import Path -from typing import Protocol +from typing import Protocol, cast + +from pydantic import BaseModel, Field +from pydantic_ai import Agent +from pydantic_ai.settings import ModelSettings from parser_security_eval import prompts from parser_security_eval.treesitter import runtime @@ -24,14 +27,13 @@ TSFixAttempt, ) -DEFAULT_FIX_MODEL = "claude-opus-4-8" +# Model-agnostic: a pydantic-ai "provider:model" string. Swap the provider freely +# (e.g. "openai:gpt-4o", "google-gla:gemini-2.5-pro"); keys are read from the env. +DEFAULT_FIX_MODEL = "anthropic:claude-opus-4-8" # Replays used to confirm a crash reproduces (pre-fix) and is gone (post-fix). # >1 because some scanner crashes are flaky (e.g. heap-layout-dependent SEGVs). VERIFY_ATTEMPTS = 20 -_RATIONALE_RE = re.compile(r"RATIONALE:\s*(.+)") -_FENCE_RE = re.compile(r"```[^\n]*\n(.*?)```", re.DOTALL) -_OPEN_FENCE_RE = re.compile(r"```[^\n]*\n") _CRASH_MARKERS = ( "ERROR:", "SUMMARY:", @@ -41,6 +43,18 @@ ) +class FixOutput(BaseModel): + """Structured fixer result — enforced by pydantic-ai (no text parsing).""" + + rationale: str = Field( + description="One sentence: the root cause and the guard you added." + ) + file_contents: str = Field( + description="The COMPLETE corrected contents of the file — the full file, " + "not a diff or a snippet." + ) + + @dataclass class FixProposal: """A fixer's proposed replacement for the implicated source file.""" @@ -90,7 +104,13 @@ def _crash_input_repr(crash: TSCrash) -> str: class LLMFixer: - """Anthropic-backed fixer. Reads ANTHROPIC_API_KEY from the environment.""" + """Model-agnostic fixer via pydantic-ai. + + ``model`` is a ``provider:model`` string (e.g. ``anthropic:claude-opus-4-8``, + ``openai:gpt-4o``); the provider's API key is read from the environment. The fix + comes back as a validated :class:`FixOutput` (structured output), so there is no + text/regex parsing of the model's reply. + """ def __init__(self, model: str = DEFAULT_FIX_MODEL, max_tokens: int = 32768) -> None: self.model = model @@ -106,8 +126,6 @@ def propose( asan_report: str, lang_hint: str, ) -> FixProposal: - import anthropic - system = prompts.load("treesitter.fix_system") user = prompts.load( "treesitter.fix_user", @@ -122,49 +140,23 @@ def propose( source=source, lang_hint=lang_hint, ) - client = anthropic.Anthropic() - # Stream: a full-file rewrite at this max_tokens can exceed the SDK's - # non-streaming 10-minute ceiling, which otherwise raises. - with client.messages.stream( - model=self.model, - max_tokens=self.max_tokens, - system=system, - messages=[{"role": "user", "content": user}], - ) as stream: - msg = stream.get_final_message() - text = "".join( - getattr(block, "text", "") - for block in msg.content - if getattr(block, "type", "") == "text" + agent = Agent( + self.model, + output_type=FixOutput, + system_prompt=system, + model_settings=ModelSettings(max_tokens=self.max_tokens), ) - new_content = _extract_file(text) - rationale_m = _RATIONALE_RE.search(text) + result = agent.run_sync(user) + out = cast("FixOutput", result.output) + content = out.file_contents.strip("\n") return FixProposal( - new_content=new_content, - rationale=rationale_m.group(1).strip() if rationale_m else "", - input_tokens=msg.usage.input_tokens, - output_tokens=msg.usage.output_tokens, + new_content=(content + "\n") if content.strip() else "", + rationale=out.rationale.strip(), + input_tokens=result.usage.input_tokens or 0, + output_tokens=result.usage.output_tokens or 0, ) -def _extract_file(text: str) -> str: - """Pull the corrected file out of the model's response (largest fenced block). - - Falls back to "from the last opening fence to EOF" when the reply was truncated - mid-block (an unclosed fence), so a large file that hit the token cap still yields - its (partial) content rather than nothing. - """ - blocks: list[str] = _FENCE_RE.findall(text) - if blocks: - return str(max(blocks, key=len)).strip("\n") + "\n" - opens = list(_OPEN_FENCE_RE.finditer(text)) - if opens: - tail = text[opens[-1].end() :].strip("\n") - if tail: - return tail + "\n" - return "" - - def _is_crash(rc: int, output: str) -> bool: """Heuristic: did this replay crash (nonzero exit or a sanitizer marker)?""" return rc != 0 or any(m in output for m in _CRASH_MARKERS) diff --git a/scaffold/src/parser_security_eval/treesitter/loop.py b/scaffold/src/parser_security_eval/treesitter/loop.py index 23e726f..c4f9ff9 100644 --- a/scaffold/src/parser_security_eval/treesitter/loop.py +++ b/scaffold/src/parser_security_eval/treesitter/loop.py @@ -11,6 +11,9 @@ from collections.abc import Callable from pathlib import Path +import logfire + +from parser_security_eval.telemetry import configure_telemetry from parser_security_eval.treesitter import fixer as fixer_mod from parser_security_eval.treesitter import runtime, triage from parser_security_eval.treesitter.fixer import DEFAULT_FIX_MODEL, Fixer, LLMFixer @@ -49,6 +52,7 @@ def run_loop( seed_corpus: bool = True, ) -> TSLoopResult: """Run the fuzz→fix loop for a single grammar; stream JSONL to ``out_dir``.""" + configure_telemetry() # Logfire: instruments pydantic-ai + provider SDKs runtime.check_toolchain() result = TSLoopResult( grammar=target.name, tier=target.tier, language=target.language @@ -67,6 +71,13 @@ def run_loop( ) build_result = runtime.build(spec) + logfire.info( + "build {grammar} built={built}", + grammar=target.name, + built=build_result.built, + has_scanner=build_result.had_scanner, + symbol=build_result.symbol, + ) if not build_result.built: result.build_error = build_result.compile_errors it = TSLoopIteration( @@ -96,62 +107,102 @@ def run_loop( fixer = LLMFixer(model=model) seen_hashes: set[str] = set() - for i in range(iterations): - fuzz_res, new_crashes = runner.run(fuzz_seconds) - it = TSLoopIteration( - iteration=i, - grammar=target.name, - repo_url=target.repo_url, - tier=target.tier, - language=target.language, - built=True, - fuzz=fuzz_res, - ) - - # Pick the first crash whose stack hash we haven't already handled. - # Hang/OOM artifacts are classified from the libFuzzer filename WITHOUT - # replaying them (replaying a hang would just hang again). - chosen = None - for cf in new_crashes: - if cf.name.startswith("timeout-"): - report = ( - "ERROR: libFuzzer: timeout\n" - "Pathological parse time / hang (libFuzzer -timeout exceeded)." + with logfire.span( + "treesitter fuzz→fix {grammar}", + grammar=target.name, + tier=target.tier.value, + repo=target.repo_url, + iterations=iterations, + fuzz_seconds=fuzz_seconds, + ): + for i in range(iterations): + with logfire.span("ts iteration {grammar} #{i}", grammar=target.name, i=i): + fuzz_res, new_crashes = runner.run(fuzz_seconds) + logfire.info( + "fuzz window {grammar} #{i}: {execs} execs, {n} new crash(es)", + grammar=target.name, + i=i, + execs=fuzz_res.total_executions, + n=fuzz_res.crashes_found, + execs_per_sec=fuzz_res.execs_per_sec, + corpus_size=fuzz_res.corpus_size, + oom=fuzz_res.oom_killed, + timed_out=fuzz_res.timed_out, + ) + it = TSLoopIteration( + iteration=i, + grammar=target.name, + repo_url=target.repo_url, + tier=target.tier, + language=target.language, + built=True, + fuzz=fuzz_res, ) - elif cf.name.startswith("oom-"): - report = "ERROR: libFuzzer: out-of-memory\nrss_limit exceeded." - else: - _, report = runtime.reproduce(Path(build_result.binary_path or ""), cf) - crash = triage.triage(report, cf.read_bytes(), str(cf)) - if crash.stack_hash and crash.stack_hash in seen_hashes: - continue - chosen = (cf, crash, report) - break - - if chosen is None: - it.notes = "no new crash this window — still fuzzing" - result.iterations.append(it) - _append_jsonl(out_path, it) - continue - crash_file, crash, report = chosen - if crash.stack_hash: - seen_hashes.add(crash.stack_hash) - it.crash = crash + # Pick the first crash whose stack hash we haven't already handled. + # Hang/OOM artifacts are classified from the libFuzzer filename + # WITHOUT replaying them (replaying a hang would just hang again). + chosen = None + for cf in new_crashes: + if cf.name.startswith("timeout-"): + report = ( + "ERROR: libFuzzer: timeout\n" + "Pathological parse time / hang (libFuzzer -timeout exceeded)." + ) + elif cf.name.startswith("oom-"): + report = "ERROR: libFuzzer: out-of-memory\nrss_limit exceeded." + else: + _, report = runtime.reproduce( + Path(build_result.binary_path or ""), cf + ) + crash = triage.triage(report, cf.read_bytes(), str(cf)) + if crash.stack_hash and crash.stack_hash in seen_hashes: + continue + chosen = (cf, crash, report) + break - # The crash triggers the fix phase. - attempt, new_build = fixer_mod.run_fix( - crash, crash_file, target, spec, build_result, fixer, report - ) - it.fix = attempt - if new_build.built and new_build.binary_path: - build_result = new_build - runner.binary = Path(new_build.binary_path) - if attempt.verified and crash_file.exists(): - crash_file.unlink() # don't re-discover a fixed crash + if chosen is None: + it.notes = "no new crash this window — still fuzzing" + result.iterations.append(it) + _append_jsonl(out_path, it) + continue - result.iterations.append(it) - _append_jsonl(out_path, it) + crash_file, crash, report = chosen + if crash.stack_hash: + seen_hashes.add(crash.stack_hash) + it.crash = crash + logfire.warn( + "crash {grammar}: {bug_class} (in_scanner={in_scanner})", + grammar=target.name, + bug_class=crash.bug_class.value, + in_scanner=crash.in_scanner, + stack_hash=crash.stack_hash[:16], + ) + + # The crash triggers the fix phase. + attempt, new_build = fixer_mod.run_fix( + crash, crash_file, target, spec, build_result, fixer, report + ) + it.fix = attempt + logfire.info( + "fix {grammar}: verified={verified}", + grammar=target.name, + verified=attempt.verified, + reproduced_original=attempt.reproduced_original, + applied=attempt.applied, + rebuilt=attempt.rebuilt, + fixer_model=attempt.model, + out_tokens=attempt.output_tokens, + error=attempt.error or None, + ) + if new_build.built and new_build.binary_path: + build_result = new_build + runner.binary = Path(new_build.binary_path) + if attempt.verified and crash_file.exists(): + crash_file.unlink() # don't re-discover a fixed crash + + result.iterations.append(it) + _append_jsonl(out_path, it) return result @@ -241,19 +292,27 @@ def run_hunt( progress: Callable[[int, int, str], None] | None = None, ) -> list[TSLoopResult]: """Fuzz→fix a range of un-fuzzed scanner candidates from the survey registry.""" + configure_telemetry() targets = select_hunt_candidates(registry_path, top_n, start=start) results: list[TSLoopResult] = [] - for i, target in enumerate(targets): - if progress is not None: - progress(i + 1, len(targets), target.name) - results.append( - run_loop( - target, - iterations=1, - fuzz_seconds=fuzz_seconds, - model=model, - out_dir=out_dir, - cache_dir=cache_dir, + with logfire.span( + "treesitter hunt ranks {start}-{end} ({n} grammars)", + start=start, + end=start + len(targets) - 1, + n=len(targets), + fuzz_seconds=fuzz_seconds, + ): + for i, target in enumerate(targets): + if progress is not None: + progress(i + 1, len(targets), target.name) + results.append( + run_loop( + target, + iterations=1, + fuzz_seconds=fuzz_seconds, + model=model, + out_dir=out_dir, + cache_dir=cache_dir, + ) ) - ) return results diff --git a/scaffold/src/tests/test_treesitter_fixer.py b/scaffold/src/tests/test_treesitter_fixer.py index 3b11574..b433b0d 100644 --- a/scaffold/src/tests/test_treesitter_fixer.py +++ b/scaffold/src/tests/test_treesitter_fixer.py @@ -2,13 +2,13 @@ from __future__ import annotations -import sys import types from pathlib import Path from parser_security_eval.treesitter import fixer as fixer_mod from parser_security_eval.treesitter import runtime from parser_security_eval.treesitter.fixer import ( + FixOutput, FixProposal, LLMFixer, run_fix, @@ -22,15 +22,6 @@ TSCrash, ) -MODEL_REPLY = """\ -RATIONALE: missing bounds check before indexing the lexeme buffer. -```c -#include "tree_sitter/parser.h" -int scan(void) { return 0; /* fixed */ } -``` -trailing prose that must be ignored -""" - def _target() -> GrammarTarget: return GrammarTarget( @@ -41,54 +32,51 @@ def _target() -> GrammarTarget: ) -def test_extract_file_picks_largest_fenced_block() -> None: - content = fixer_mod._extract_file(MODEL_REPLY) - assert "fixed" in content - assert "trailing prose" not in content - - -def test_extract_file_handles_truncated_unclosed_fence() -> None: - # Model hit the token cap mid-file: opening fence, no closing fence. - truncated = "RATIONALE: bounds check\n```c\nint scan(void){ return 0; }\n// cut off" - out = fixer_mod._extract_file(truncated) - assert "int scan" in out - assert "```" not in out - assert "RATIONALE" not in out - +def _patch_agent( + monkeypatch, output: FixOutput, *, in_tok: int = 11, out_tok: int = 22 +): # noqa: ANN001, ANN202 + """Replace pydantic-ai Agent in the fixer with one returning a canned result.""" + result = types.SimpleNamespace( + output=output, + usage=types.SimpleNamespace(input_tokens=in_tok, output_tokens=out_tok), + ) -def test_llm_fixer_parses_reply(monkeypatch) -> None: - block = types.SimpleNamespace(type="text", text=MODEL_REPLY) - usage = types.SimpleNamespace(input_tokens=11, output_tokens=22) - msg = types.SimpleNamespace(content=[block], usage=usage) + class _FakeAgent: + def __init__(self, *a, **k) -> None: # noqa: ANN002, ANN003 + pass - class _Stream: # context manager mimicking client.messages.stream(...) - def __enter__(self): # noqa: ANN204 - return self + def run_sync(self, _user): # noqa: ANN001, ANN202 + return result - def __exit__(self, *a): # noqa: ANN002, ANN204 - return False + monkeypatch.setattr(fixer_mod, "Agent", _FakeAgent) - def get_final_message(self): # noqa: ANN202 - return msg - client = types.SimpleNamespace( - messages=types.SimpleNamespace(stream=lambda **_: _Stream()) +def test_llm_fixer_returns_structured_proposal(monkeypatch) -> None: + _patch_agent( + monkeypatch, + FixOutput( + rationale="missing bounds check", + file_contents="int scan(void) { return 0; }", + ), ) - fake = types.ModuleType("anthropic") - fake.Anthropic = lambda *a, **k: client # type: ignore[attr-defined] - monkeypatch.setitem(sys.modules, "anthropic", fake) - crash = TSCrash(bug_class=BugClass.heap_buffer_overflow, asan_summary="boom") - proposal = LLMFixer(model="claude-test").propose( + proposal = LLMFixer(model="anthropic:claude-test").propose( crash, _target(), "old source", "scanner.c", "asan report", "c" ) assert isinstance(proposal, FixProposal) - assert "fixed" in proposal.new_content - assert proposal.rationale.startswith("missing bounds check") + assert proposal.new_content == "int scan(void) { return 0; }\n" # trailing newline + assert proposal.rationale == "missing bounds check" assert proposal.input_tokens == 11 assert proposal.output_tokens == 22 +def test_llm_fixer_empty_file_contents_yields_no_content(monkeypatch) -> None: + _patch_agent(monkeypatch, FixOutput(rationale="n/a", file_contents=" \n")) + crash = TSCrash(bug_class=BugClass.heap_buffer_overflow) + proposal = LLMFixer().propose(crash, _target(), "src", "scanner.c", "report", "c") + assert proposal.new_content == "" + + def test_select_fix_target_prefers_scanner(tmp_path: Path) -> None: scanner = tmp_path / "scanner.c" scanner.write_text("// scanner") diff --git a/scaffold/src/tests/test_treesitter_hunt.py b/scaffold/src/tests/test_treesitter_hunt.py index 7cb15df..6ba9c6e 100644 --- a/scaffold/src/tests/test_treesitter_hunt.py +++ b/scaffold/src/tests/test_treesitter_hunt.py @@ -100,6 +100,7 @@ def test_select_start_offset_skips_earlier_ranks(tmp_path: Path) -> None: def test_run_hunt_invokes_loop_per_candidate(tmp_path: Path, monkeypatch) -> None: reg = _write_registry(tmp_path) + monkeypatch.setattr(loop_mod, "configure_telemetry", lambda *a, **k: False) calls: list[str] = [] def fake_run_loop(target, **kwargs): # noqa: ANN001, ANN202 diff --git a/scaffold/src/tests/test_treesitter_loop.py b/scaffold/src/tests/test_treesitter_loop.py index 587074b..0bfa7b6 100644 --- a/scaffold/src/tests/test_treesitter_loop.py +++ b/scaffold/src/tests/test_treesitter_loop.py @@ -37,6 +37,7 @@ def test_loop_fuzzer_triggers_fix_and_writes_jsonl(tmp_path: Path, monkeypatch) cache = tmp_path / "cache" out_dir = tmp_path / "out" + monkeypatch.setattr(loop_mod, "configure_telemetry", lambda *a, **k: False) monkeypatch.setattr(runtime, "ensure_runtime", lambda c: tmp_path / "rt") monkeypatch.setattr(runtime, "ensure_grammar", lambda t, c: tmp_path / "grammar") monkeypatch.setattr(runtime, "gather_seeds", lambda *a, **k: 0) @@ -107,6 +108,7 @@ def fake_run_fix(crash, crash_file, target, spec, build_result, fixer, report): def test_loop_records_build_failure(tmp_path: Path, monkeypatch) -> None: + monkeypatch.setattr(loop_mod, "configure_telemetry", lambda *a, **k: False) monkeypatch.setattr(runtime, "check_toolchain", lambda **k: None) monkeypatch.setattr(runtime, "ensure_runtime", lambda c: tmp_path / "rt") monkeypatch.setattr(runtime, "ensure_grammar", lambda t, c: tmp_path / "grammar") diff --git a/scaffold/uv.lock b/scaffold/uv.lock index 7270727..5cd6ac0 100644 --- a/scaffold/uv.lock +++ b/scaffold/uv.lock @@ -169,7 +169,7 @@ wheels = [ [[package]] name = "anthropic" -version = "0.84.0" +version = "0.111.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "anyio" }, @@ -181,9 +181,9 @@ dependencies = [ { name = "sniffio" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/04/ea/0869d6df9ef83dcf393aeefc12dd81677d091c6ffc86f783e51cf44062f2/anthropic-0.84.0.tar.gz", hash = "sha256:72f5f90e5aebe62dca316cb013629cfa24996b0f5a4593b8c3d712bc03c43c37", size = 539457, upload-time = "2026-02-25T05:22:38.54Z" } +sdist = { url = "https://files.pythonhosted.org/packages/b9/8a/9afc7305a2ce4b52b30e137f83cd2a6a90b918b3997073db11bb5a1de55a/anthropic-0.111.0.tar.gz", hash = "sha256:39cbda0ac17a6d423e5bf609811bd69b26eddf6299d7a468126e05bc711ce826", size = 934001, upload-time = "2026-06-18T17:31:44.733Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/64/ca/218fa25002a332c0aa149ba18ffc0543175998b1f65de63f6d106689a345/anthropic-0.84.0-py3-none-any.whl", hash = "sha256:861c4c50f91ca45f942e091d83b60530ad6d4f98733bfe648065364da05d29e7", size = 455156, upload-time = "2026-02-25T05:22:40.468Z" }, + { url = "https://files.pythonhosted.org/packages/f1/bb/09e82a81885d787f350fb55ca9df865b63140dd28b3b5b3104c4ae261657/anthropic-0.111.0-py3-none-any.whl", hash = "sha256:c14edb36ed80da9099acbd26b5cec810d76606c31f32a0d56a4cf9d4fa9e25ae", size = 929774, upload-time = "2026-06-18T17:31:43.116Z" }, ] [[package]] @@ -420,6 +420,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/47/71/70db47e4f6ce3e5c37a607355f80da8860a33226be640226ac52cb05ef2e/fsspec-2025.9.0-py3-none-any.whl", hash = "sha256:530dc2a2af60a414a832059574df4a6e10cce927f6f4a78209390fe38955cfb7", size = 199289, upload-time = "2025-09-02T19:10:47.708Z" }, ] +[[package]] +name = "genai-prices" +version = "0.0.66" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "httpx2" }, + { name = "pydantic" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/0e/7c/c50fc8d18b283e9b56ff625d45dbe9577c676e1d16636c1011967182d813/genai_prices-0.0.66.tar.gz", hash = "sha256:f087dfe56da28a4c3933dcf846cf2b7111ba733cef674c0cbc66de80212bcd6b", size = 71130, upload-time = "2026-06-09T21:51:14.561Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5c/87/1a36166f91906e47f430f6d4150ec6bfc0001032a363e49fc389021c0609/genai_prices-0.0.66-py3-none-any.whl", hash = "sha256:86b83f107c1cf04bb449a120cd8d4439ceb6843660d9128cde560eb511686d7b", size = 73745, upload-time = "2026-06-09T21:51:13.523Z" }, +] + [[package]] name = "gitdb" version = "4.0.12" @@ -456,6 +469,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e7/c8/e2645aa8ed02fd4c7a2f59d68783b65b1f3cbdfe39a6308e156509d1fee8/googleapis_common_protos-1.75.0-py3-none-any.whl", hash = "sha256:961ed60399c457ceb0ee8f285a84c870aabc9c6a832b9d37bb281b5bebde43ed", size = 300631, upload-time = "2026-05-07T08:03:30.345Z" }, ] +[[package]] +name = "griffelib" +version = "2.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/33/e4/8d187ea29c2e30b3a09505c567513077d6117861bde1fbd997a167f262ec/griffelib-2.1.0.tar.gz", hash = "sha256:762a186d2c6fd6794d4ea20d428d597ffb857cb56b66421651cbba15bdd5e813", size = 216234, upload-time = "2026-06-19T12:05:42.278Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e4/d3/5268aeabf2ad82658c4e2ff3a060648d0f02f3926cb53247c0e4d0dab49e/griffelib-2.1.0-py3-none-any.whl", hash = "sha256:cc7b3d2d2865ad0b909fcc38086e3f554b5ea7acbaa7bbb7ecaa3f5dfb7d9f00", size = 142560, upload-time = "2026-06-19T12:05:38.742Z" }, +] + [[package]] name = "h11" version = "0.16.0" @@ -478,6 +500,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/7e/f5/f66802a942d491edb555dd61e3a9961140fd64c90bce1eafd741609d334d/httpcore-1.0.9-py3-none-any.whl", hash = "sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55", size = 78784, upload-time = "2025-04-24T22:06:20.566Z" }, ] +[[package]] +name = "httpcore2" +version = "2.3.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "h11" }, + { name = "truststore" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e6/34/18f1c596e677962f040284246f393b10a1f8ce440b3a7e69c637d0f1c7ad/httpcore2-2.3.0.tar.gz", hash = "sha256:07327e251560960eea8e969d92d4c6a325feb13cca39e25340731336c3baf924", size = 64300, upload-time = "2026-06-01T13:15:02.998Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c2/dd/3357218c69360d1cecc196c230c9a1d5c9afd5dba362056e23e60a5e64e5/httpcore2-2.3.0-py3-none-any.whl", hash = "sha256:477e9e334f74e5240dcac002e890580f36a57d40ff0fb14cc9655731d23b8415", size = 80024, upload-time = "2026-06-01T13:15:00.001Z" }, +] + [[package]] name = "httpx" version = "0.28.1" @@ -493,6 +528,21 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad", size = 73517, upload-time = "2024-12-06T15:37:21.509Z" }, ] +[[package]] +name = "httpx2" +version = "2.3.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "httpcore2" }, + { name = "idna" }, + { name = "truststore" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9f/9a/cca0b9145f13d8ae34b885ae28d403a1469a433abc78e0f94f4ce94e650b/httpx2-2.3.0.tar.gz", hash = "sha256:227e7c41d95a76d4077a52640564132777215fc3394e07b66a3116c33d668fa9", size = 81115, upload-time = "2026-06-01T13:15:04.324Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/87/ce/ae2911859847f9ba1d6b23027e53481cbeb50b93234f355a968d300ca2cb/httpx2-2.3.0-py3-none-any.whl", hash = "sha256:6f393663bdf6dbe7fe90118e3eb5b2bd024a675cae0390ac08cec9198812d8b7", size = 74538, upload-time = "2026-06-01T13:15:01.566Z" }, +] + [[package]] name = "hypothesis" version = "6.151.9" @@ -763,6 +813,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c1/e1/a008ba3fcf1afdce09f3afb85124b25906e9e06b45c853315b5a0bb323e7/logfire-4.36.0-py3-none-any.whl", hash = "sha256:8ccb2f276fe46888d7ef90aa7ba7ade0c053946666b9218c5115688316bf6d04", size = 378474, upload-time = "2026-06-09T19:15:57.96Z" }, ] +[[package]] +name = "logfire-api" +version = "4.37.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/03/04/471b916249fe7e22056818ca734af46418cd3ff9b9b920c1829c3627b4d2/logfire_api-4.37.0.tar.gz", hash = "sha256:0f62debd6ed593d51307277bd6d5636b57bda07935b5604b96db10fe64441af4", size = 88906, upload-time = "2026-06-12T20:47:08.163Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/10/2f/23e5b8fa22f75f73965c72e5c29e6fb8715263457394601e254fe26fbe31/logfire_api-4.37.0-py3-none-any.whl", hash = "sha256:1d756f8ba23aa56d438e0ba2c0f529a00fcac975b8785c561b058267f9465088", size = 138710, upload-time = "2026-06-12T20:47:05.526Z" }, +] + [[package]] name = "markdown-it-py" version = "4.0.0" @@ -1130,6 +1189,7 @@ dependencies = [ { name = "pandas" }, { name = "plotly" }, { name = "pydantic" }, + { name = "pydantic-ai-slim", extra = ["anthropic", "openai"] }, { name = "pyyaml" }, { name = "rich" }, { name = "streamlit" }, @@ -1155,6 +1215,7 @@ requires-dist = [ { name = "pandas", specifier = ">=2.2.0" }, { name = "plotly", specifier = ">=5.22.0" }, { name = "pydantic", specifier = ">=2.12.5" }, + { name = "pydantic-ai-slim", extras = ["anthropic", "openai"], specifier = ">=2.0.0" }, { name = "pyyaml", specifier = ">=6.0" }, { name = "rich", specifier = ">=14.3.3" }, { name = "streamlit", specifier = ">=1.36.0" }, @@ -1356,6 +1417,33 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/5a/87/b70ad306ebb6f9b585f114d0ac2137d792b48be34d732d60e597c2f8465a/pydantic-2.12.5-py3-none-any.whl", hash = "sha256:e561593fccf61e8a20fc46dfc2dfe075b8be7d0188df33f221ad1f0139180f9d", size = 463580, upload-time = "2025-11-26T15:11:44.605Z" }, ] +[[package]] +name = "pydantic-ai-slim" +version = "2.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "genai-prices" }, + { name = "griffelib" }, + { name = "httpx" }, + { name = "opentelemetry-api" }, + { name = "pydantic" }, + { name = "pydantic-graph" }, + { name = "typing-inspection" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/81/5b/c930c2c645677d1069172a71be9a22ea37b2aa170dcf3c6051aa1c105e72/pydantic_ai_slim-2.0.0.tar.gz", hash = "sha256:056ea466d67b47a832736ac0f33172264b09da2110dbcce09d26b82772173218", size = 734369, upload-time = "2026-06-23T15:48:58.379Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5f/e6/814efd5693a97589bfd15c51c9324f77758b82a04f172f3fec9723c7777b/pydantic_ai_slim-2.0.0-py3-none-any.whl", hash = "sha256:39979b459a7bc73ae5294c071a7e474f123858a9f7922e9cbb662018d6431198", size = 905291, upload-time = "2026-06-23T15:48:51.628Z" }, +] + +[package.optional-dependencies] +anthropic = [ + { name = "anthropic" }, +] +openai = [ + { name = "openai" }, + { name = "tiktoken" }, +] + [[package]] name = "pydantic-core" version = "2.41.5" @@ -1395,6 +1483,21 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/9f/ed/068e41660b832bb0b1aa5b58011dea2a3fe0ba7861ff38c4d4904c1c1a99/pydantic_core-2.41.5-cp314-cp314t-win_arm64.whl", hash = "sha256:35b44f37a3199f771c3eaa53051bc8a70cd7b54f333531c59e29fd4db5d15008", size = 1974769, upload-time = "2025-11-04T13:42:01.186Z" }, ] +[[package]] +name = "pydantic-graph" +version = "2.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "httpx" }, + { name = "logfire-api" }, + { name = "pydantic" }, + { name = "typing-inspection" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c7/9d/81b09921d360614f65bb2d4f936d5cae856b4ebe827fcf43ae7abb81383a/pydantic_graph-2.0.0.tar.gz", hash = "sha256:f0bffe84a46a5118bce0824de63d08f3f32ba4dfc1064674f449b07e15128287", size = 43054, upload-time = "2026-06-23T15:49:00.488Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c9/2c/ff64a971e26bb2d0d67a70ef436f2cfc778341a4cf89d5aca35cbc1f071c/pydantic_graph-2.0.0-py3-none-any.whl", hash = "sha256:36d69fa01cd316be8584b90eef58bd21675c11c0a081b500a5c4ebe9b68310a5", size = 50773, upload-time = "2026-06-23T15:48:54.562Z" }, +] + [[package]] name = "pydeck" version = "0.9.1" @@ -1863,6 +1966,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/16/e1/3079a9ff9b8e11b846c6ac5c8b5bfb7ff225eee721825310c91b3b50304f/tqdm-4.67.3-py3-none-any.whl", hash = "sha256:ee1e4c0e59148062281c49d80b25b67771a127c85fc9676d3be5f243206826bf", size = 78374, upload-time = "2026-02-03T17:35:50.982Z" }, ] +[[package]] +name = "truststore" +version = "0.10.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/53/a3/1585216310e344e8102c22482f6060c7a6ea0322b63e026372e6dcefcfd6/truststore-0.10.4.tar.gz", hash = "sha256:9d91bd436463ad5e4ee4aba766628dd6cd7010cf3e2461756b3303710eebc301", size = 26169, upload-time = "2025-08-12T18:49:02.73Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/19/97/56608b2249fe206a67cd573bc93cd9896e1efb9e98bce9c163bcdc704b88/truststore-0.10.4-py3-none-any.whl", hash = "sha256:adaeaecf1cbb5f4de3b1959b42d41f6fab57b2b1666adb59e89cb0b53361d981", size = 18660, upload-time = "2025-08-12T18:49:01.46Z" }, +] + [[package]] name = "ty" version = "0.0.19" From 0fad11f9db85b7242cd5db7cbda38002fc9b8347 Mon Sep 17 00:00:00 2001 From: Matthew Doty Date: Thu, 2 Jul 2026 13:54:11 -0400 Subject: [PATCH 3/6] refactor(treesitter): improving telemetry --- .../src/parser_security_eval/telemetry.py | 5 +++ .../parser_security_eval/treesitter/fixer.py | 35 +++++++++++++++++-- .../parser_security_eval/treesitter/loop.py | 3 ++ 3 files changed, 40 insertions(+), 3 deletions(-) diff --git a/scaffold/src/parser_security_eval/telemetry.py b/scaffold/src/parser_security_eval/telemetry.py index f10e487..a57a18f 100644 --- a/scaffold/src/parser_security_eval/telemetry.py +++ b/scaffold/src/parser_security_eval/telemetry.py @@ -71,6 +71,11 @@ def configure_telemetry(*, service_name: str = "parser-security-eval") -> bool: send_to_logfire="if-token-present", # This is a backend/agent process; keep stdout clean for the CLI. console=False, + # All our spans/logs pass fields as explicit kwargs (never bare + # f-strings), so the source-introspection that extracts f-string vars + # is pure overhead — and it warns noisily when source isn't available + # (interactive shell, ``python -c``, exec, pyc-without-py). Disable it. + inspect_arguments=False, ) except Exception as exc: # pragma: no cover - defensive; setup must not crash logger.warning("Logfire setup failed (continuing without telemetry): %s", exc) diff --git a/scaffold/src/parser_security_eval/treesitter/fixer.py b/scaffold/src/parser_security_eval/treesitter/fixer.py index f7bccad..f85b033 100644 --- a/scaffold/src/parser_security_eval/treesitter/fixer.py +++ b/scaffold/src/parser_security_eval/treesitter/fixer.py @@ -14,6 +14,7 @@ from pathlib import Path from typing import Protocol, cast +import logfire from pydantic import BaseModel, Field from pydantic_ai import Agent from pydantic_ai.settings import ModelSettings @@ -146,9 +147,37 @@ def propose( system_prompt=system, model_settings=ModelSettings(max_tokens=self.max_tokens), ) - result = agent.run_sync(user) - out = cast("FixOutput", result.output) - content = out.file_contents.strip("\n") + # Log the FULL prompt + solution as named attributes (not interpolated into + # the span message, which Logfire caps). pydantic-ai's auto-instrumentation + # also captures the messages but renders a truncated preview; these explicit + # attributes carry the complete challenge and the complete fix. + with logfire.span( + "repair agent {grammar} ({bug_class})", + grammar=target.name, + bug_class=crash.bug_class.value, + model=self.model, + implicated_file=implicated_file, + ): + logfire.info( + "repair prompt {grammar}", + grammar=target.name, + system_prompt=system, + user_prompt=user, # the full challenge: report + source + crash input + source_chars=len(source), + user_prompt_chars=len(user), + ) + result = agent.run_sync(user) + out = cast("FixOutput", result.output) + content = out.file_contents.strip("\n") + logfire.info( + "repair solution {grammar}: {rationale}", + grammar=target.name, + rationale=out.rationale.strip(), + solution=out.file_contents, # the agent's full corrected file + solution_chars=len(out.file_contents), + input_tokens=result.usage.input_tokens or 0, + output_tokens=result.usage.output_tokens or 0, + ) return FixProposal( new_content=(content + "\n") if content.strip() else "", rationale=out.rationale.strip(), diff --git a/scaffold/src/parser_security_eval/treesitter/loop.py b/scaffold/src/parser_security_eval/treesitter/loop.py index c4f9ff9..0631a16 100644 --- a/scaffold/src/parser_security_eval/treesitter/loop.py +++ b/scaffold/src/parser_security_eval/treesitter/loop.py @@ -194,6 +194,9 @@ def run_loop( fixer_model=attempt.model, out_tokens=attempt.output_tokens, error=attempt.error or None, + rationale=attempt.rationale or None, + patch=attempt.patch or None, # the applied unified diff (solution) + target_file=attempt.target_file, ) if new_build.built and new_build.binary_path: build_result = new_build From 1e5a3c87d4f11c28f9e4f4e40a2c5ca4869b0a92 Mon Sep 17 00:00:00 2001 From: Matthew Doty Date: Tue, 7 Jul 2026 11:00:10 -0400 Subject: [PATCH 4/6] feat(tree-sitter): better iteration --- .../treesitter/fix_user.prompt.template | 13 +- .../parser_security_eval/treesitter/fixer.py | 157 ++++++++++++------ .../parser_security_eval/treesitter/loop.py | 40 +++-- .../parser_security_eval/treesitter/models.py | 1 + scaffold/src/tests/test_treesitter_fixer.py | 98 ++++++++++- scaffold/src/tests/test_treesitter_hunt.py | 21 +++ scaffold/src/tests/test_treesitter_loop.py | 2 +- 7 files changed, 268 insertions(+), 64 deletions(-) diff --git a/scaffold/src/parser_security_eval/prompts/treesitter/fix_user.prompt.template b/scaffold/src/parser_security_eval/prompts/treesitter/fix_user.prompt.template index 6d97475..e33d6c4 100644 --- a/scaffold/src/parser_security_eval/prompts/treesitter/fix_user.prompt.template +++ b/scaffold/src/parser_security_eval/prompts/treesitter/fix_user.prompt.template @@ -22,6 +22,13 @@ A fuzzing campaign against the **{{ language }}** tree-sitter grammar {{ source }} ``` -Return the complete corrected contents of `{{ implicated_file }}` in the required -format. Add the missing safety check on the crashing path; do not otherwise alter -behavior. +{% if feedback %} +## Earlier fix attempt(s) that did NOT work — read carefully, then try a DIFFERENT fix +{{ feedback }} +{% endif %} +Return the complete corrected contents of `{{ implicated_file }}`. Add the missing +guard / cleanup on the crashing path; do not otherwise alter behavior. If an earlier +attempt is shown above, its approach did NOT eliminate the bug — diagnose *why* from +the still-present sanitizer report and fix the ACTUAL site (e.g. free EVERY allocation +in the matching teardown/destroy path — including hashmap/queue entries, not just the +container — or bounds-check before the offending access). diff --git a/scaffold/src/parser_security_eval/treesitter/fixer.py b/scaffold/src/parser_security_eval/treesitter/fixer.py index f85b033..8d43e8d 100644 --- a/scaffold/src/parser_security_eval/treesitter/fixer.py +++ b/scaffold/src/parser_security_eval/treesitter/fixer.py @@ -79,6 +79,7 @@ def propose( implicated_file: str, asan_report: str, lang_hint: str, + feedback: str = "", ) -> FixProposal: ... @@ -126,6 +127,7 @@ def propose( implicated_file: str, asan_report: str, lang_hint: str, + feedback: str = "", ) -> FixProposal: system = prompts.load("treesitter.fix_system") user = prompts.load( @@ -140,6 +142,7 @@ def propose( asan_report=asan_report[:6000], source=source, lang_hint=lang_hint, + feedback=feedback, ) agent = Agent( self.model, @@ -235,6 +238,16 @@ def select_fix_target( return None +def _accumulate_feedback(prev: str, i: int, rationale: str, failure: str) -> str: + """Append a failed-attempt block for the next repair iteration's prompt.""" + block = ( + f"### Attempt {i} (FAILED)\n" + f"- What it tried: {rationale or '(no rationale)'}\n" + f"- Why it failed: {failure}\n" + ) + return f"{prev}\n{block}" if prev else block + + def run_fix( crash: TSCrash, crash_file: Path, @@ -243,9 +256,16 @@ def run_fix( build_result: TSBuildResult, fixer: Fixer, asan_report: str, + max_repair_iters: int = 1, ) -> tuple[TSFixAttempt, TSBuildResult]: """Propose+apply a fix, rebuild, and verify the crash no longer reproduces. + With ``max_repair_iters > 1``, a patch that fails to compile or fails to + eliminate the crash is fed back to the fixer (what it tried + why it failed + + the still-present sanitizer report) and it retries, up to that many times. The + source file is reverted to the original between attempts, so each retry is a + fresh rewrite informed by the accumulated failures. + Returns the :class:`TSFixAttempt` and the (possibly rebuilt) build result. """ attempt = TSFixAttempt(attempted=True, model=getattr(fixer, "name", "unknown")) @@ -278,59 +298,98 @@ def run_fix( original = fix_path.read_text(encoding="utf-8", errors="replace") lang_hint = _lang_hint(fix_path) - - try: - proposal = fixer.propose( - crash, target, original, fix_path.name, asan_report, lang_hint - ) - except Exception as exc: # noqa: BLE001 — surface fixer errors into the record - attempt.error = f"fixer.propose failed: {exc}" - return attempt, build_result - - attempt.input_tokens = proposal.input_tokens - attempt.output_tokens = proposal.output_tokens - attempt.rationale = proposal.rationale - if not proposal.new_content.strip(): - attempt.error = "fixer returned no file content" - return attempt, build_result - - attempt.patch = "".join( - difflib.unified_diff( - original.splitlines(keepends=True), - proposal.new_content.splitlines(keepends=True), - fromfile=f"a/{fix_path.name}", - tofile=f"b/{fix_path.name}", - ) - ) - - # Apply (keep a backup so a failed fix can be rolled back). backup = fix_path.with_suffix(fix_path.suffix + ".orig") if not backup.exists(): backup.write_text(original, encoding="utf-8") - fix_path.write_text(proposal.new_content, encoding="utf-8") - attempt.applied = True - - # grammar.js edits require regeneration before rebuild. - if fix_path.name == "grammar.js": - ok, _ = runtime.generate_parser(spec.src_dir) - attempt.regenerated = ok - if not ok: - attempt.error = "tree-sitter generate failed after grammar.js edit" - return attempt, build_result - - new_build = runtime.build(spec) - attempt.rebuilt = new_build.built - if not new_build.built: - attempt.error = f"rebuild failed:\n{new_build.compile_errors[-1500:]}" - return attempt, build_result - # Verified only if the patched build never crashes across the replay budget - # (reproduced_original is already True here). - attempt.verified = _clean_on_replay( - Path(new_build.binary_path or ""), crash_file, VERIFY_ATTEMPTS - ) - if not attempt.verified: + feedback = "" + result_build = build_result + for i in range(1, max_repair_iters + 1): + attempt.iters_used = i + try: + proposal = fixer.propose( + crash, + target, + original, + fix_path.name, + asan_report, + lang_hint, + feedback=feedback, + ) + except Exception as exc: # noqa: BLE001 — surface fixer errors into the record + attempt.error = f"fixer.propose failed: {exc}" + return attempt, result_build + + attempt.input_tokens += proposal.input_tokens + attempt.output_tokens += proposal.output_tokens + attempt.rationale = proposal.rationale + if not proposal.new_content.strip(): + attempt.error = "fixer returned no file content" + return attempt, result_build + + attempt.patch = "".join( + difflib.unified_diff( + original.splitlines(keepends=True), + proposal.new_content.splitlines(keepends=True), + fromfile=f"a/{fix_path.name}", + tofile=f"b/{fix_path.name}", + ) + ) + fix_path.write_text(proposal.new_content, encoding="utf-8") + attempt.applied = True + + # grammar.js edits require regeneration before rebuild. + if fix_path.name == "grammar.js": + ok, gen_log = runtime.generate_parser(spec.src_dir) + attempt.regenerated = ok + if not ok: + attempt.error = "tree-sitter generate failed after grammar.js edit" + feedback = _accumulate_feedback( + feedback, + i, + proposal.rationale, + f"`tree-sitter generate` failed:\n{gen_log[-800:]}", + ) + fix_path.write_text(original, encoding="utf-8") + continue + + new_build = runtime.build(spec) + attempt.rebuilt = new_build.built + if not new_build.built: + attempt.error = f"rebuild failed:\n{new_build.compile_errors[-1500:]}" + feedback = _accumulate_feedback( + feedback, + i, + proposal.rationale, + f"the patch did NOT COMPILE:\n{new_build.compile_errors[-1000:]}", + ) + fix_path.write_text(original, encoding="utf-8") + continue + + result_build = new_build + # Verified only if the patched build never crashes across the replay budget + # (reproduced_original is already True here). + if _clean_on_replay( + Path(new_build.binary_path or ""), crash_file, VERIFY_ATTEMPTS + ): + attempt.verified = True + attempt.error = "" + return attempt, new_build + + # Still crashes — capture the post-patch report as feedback and retry. + _, post_report = runtime.reproduce( + Path(new_build.binary_path or ""), crash_file + ) attempt.error = ( "patch did not eliminate the crash (still reproduces after rebuild)" ) - return attempt, new_build + feedback = _accumulate_feedback( + feedback, + i, + proposal.rationale, + "the bug STILL reproduces after this patch. Current (post-patch) " + f"sanitizer report:\n{post_report[:2000]}", + ) + fix_path.write_text(original, encoding="utf-8") # clean base for next attempt + + return attempt, result_build diff --git a/scaffold/src/parser_security_eval/treesitter/loop.py b/scaffold/src/parser_security_eval/treesitter/loop.py index 0631a16..0b42807 100644 --- a/scaffold/src/parser_security_eval/treesitter/loop.py +++ b/scaffold/src/parser_security_eval/treesitter/loop.py @@ -50,6 +50,7 @@ def run_loop( max_len: int = 65536, fixer: Fixer | None = None, seed_corpus: bool = True, + max_repair_iters: int = 1, ) -> TSLoopResult: """Run the fuzz→fix loop for a single grammar; stream JSONL to ``out_dir``.""" configure_telemetry() # Logfire: instruments pydantic-ai + provider SDKs @@ -181,7 +182,14 @@ def run_loop( # The crash triggers the fix phase. attempt, new_build = fixer_mod.run_fix( - crash, crash_file, target, spec, build_result, fixer, report + crash, + crash_file, + target, + spec, + build_result, + fixer, + report, + max_repair_iters=max_repair_iters, ) it.fix = attempt logfire.info( @@ -308,14 +316,26 @@ def run_hunt( for i, target in enumerate(targets): if progress is not None: progress(i + 1, len(targets), target.name) - results.append( - run_loop( - target, - iterations=1, - fuzz_seconds=fuzz_seconds, - model=model, - out_dir=out_dir, - cache_dir=cache_dir, + try: + results.append( + run_loop( + target, + iterations=1, + fuzz_seconds=fuzz_seconds, + model=model, + out_dir=out_dir, + cache_dir=cache_dir, + ) ) - ) + except Exception as exc: # noqa: BLE001 — one grammar must not kill the campaign + logfire.warn( + "hunt grammar failed {grammar}: {error}", + grammar=target.name, + error=str(exc), + ) + failed = TSLoopResult( + grammar=target.name, tier=target.tier, language=target.language + ) + failed.build_error = f"run_loop raised: {type(exc).__name__}: {exc}" + results.append(failed) return results diff --git a/scaffold/src/parser_security_eval/treesitter/models.py b/scaffold/src/parser_security_eval/treesitter/models.py index f096dd9..a56429a 100644 --- a/scaffold/src/parser_security_eval/treesitter/models.py +++ b/scaffold/src/parser_security_eval/treesitter/models.py @@ -103,6 +103,7 @@ class TSFixAttempt(BaseModel): # verified == reproduced_original AND the patched build never crashed on replay. reproduced_original: bool = False # the bug reproduced on the pre-fix build verify_attempts: int = 0 # replays used to confirm repro / clearance (flaky-safe) + iters_used: int = 0 # LLM repair iterations attempted (>1 = retried with feedback) verified: bool = False # crashing input no longer reproduces after the fix error: str = "" input_tokens: int = 0 diff --git a/scaffold/src/tests/test_treesitter_fixer.py b/scaffold/src/tests/test_treesitter_fixer.py index b433b0d..66463b5 100644 --- a/scaffold/src/tests/test_treesitter_fixer.py +++ b/scaffold/src/tests/test_treesitter_fixer.py @@ -94,7 +94,16 @@ def test_select_fix_target_runtime_crash_returns_none(tmp_path: Path) -> None: class _StubFixer: name = "stub" - def propose(self, crash, target, source, implicated_file, asan_report, lang_hint): # noqa: ANN001, ANN201, PLR0913 + def propose( + self, + crash, + target, + source, + implicated_file, + asan_report, + lang_hint, + feedback="", + ): # noqa: ANN001, ANN201, PLR0913 return FixProposal( new_content="int scan(void){return 0;}\n", rationale="add guard", @@ -247,6 +256,93 @@ def test_run_fix_patch_that_does_not_eliminate_is_not_verified( assert "did not eliminate" in attempt.error +def test_run_fix_retries_with_feedback_until_verified( + tmp_path: Path, monkeypatch +) -> None: + # First patch fails to eliminate the crash; the retry (with feedback) succeeds. + grammar = tmp_path / "g" + src = grammar / "src" + src.mkdir(parents=True) + scanner = src / "scanner.c" + scanner.write_text("int scan(void){buf[i];}\n") + (src / "parser.c").write_text("// parser") + target = _target() + spec = runtime.BuildSpec( + target=target, + runtime_dir=tmp_path / "rt", + grammar_dir=grammar, + work_dir=tmp_path / "work", + ) + build = TSBuildResult( + built=True, binary_path=str(tmp_path / "fuzz_orig"), scanner_file=str(scanner) + ) + crash = TSCrash( + bug_class=BugClass.heap_buffer_overflow, + in_scanner=True, + implicated_file="scanner.c", + ) + crash_file = tmp_path / "crash-1" + crash_file.write_bytes(b"bad") + + builds = {"n": 0} + + def fake_build(_spec): # noqa: ANN001, ANN202 + builds["n"] += 1 + return TSBuildResult( + built=True, binary_path=str(tmp_path / f"fuzz_patched_{builds['n']}") + ) + + def fake_reproduce(binary, cf, timeout=60): # noqa: ANN001, ANN202 + b = str(binary) + # original crashes; first patched build still crashes; second is clean. + if "orig" in b or b.endswith("_1"): + return 1, "==ERROR: AddressSanitizer: heap-buffer-overflow" + return 0, "clean" + + monkeypatch.setattr(runtime, "build", fake_build) + monkeypatch.setattr(runtime, "reproduce", fake_reproduce) + + seen_feedback: list[bool] = [] + + class _TwoTryFixer: + name = "two-try" + + def propose( + self, + crash, + target, + source, + implicated_file, + asan_report, + lang_hint, + feedback="", + ): # noqa: ANN001, ANN201, PLR0913 + seen_feedback.append(bool(feedback)) + # First call (no feedback) → weak fix; retry (with feedback) → real fix. + body = "return 1;" if feedback else "return 0;" + return FixProposal( + new_content=f"int scan(void){{{body}}}\n", + rationale="attempt", + input_tokens=1, + output_tokens=1, + ) + + attempt, _ = run_fix( + crash, + crash_file, + target, + spec, + build, + _TwoTryFixer(), + "report", + max_repair_iters=3, + ) + assert attempt.iters_used == 2 # succeeded on the 2nd try + assert attempt.verified is True + assert attempt.error == "" + assert seen_feedback == [False, True] # retry received failure feedback + + def test_run_fix_skips_runtime_crash(tmp_path: Path) -> None: target = _target() grammar = tmp_path / "g" diff --git a/scaffold/src/tests/test_treesitter_hunt.py b/scaffold/src/tests/test_treesitter_hunt.py index 6ba9c6e..156e048 100644 --- a/scaffold/src/tests/test_treesitter_hunt.py +++ b/scaffold/src/tests/test_treesitter_hunt.py @@ -115,3 +115,24 @@ def fake_run_loop(target, **kwargs): # noqa: ANN001, ANN202 results = run_hunt(reg, top_n=10, fuzz_seconds=99, out_dir=tmp_path / "hunt") assert calls == ["uyha-cmake", "ram02z-fish"] assert len(results) == 2 + + +def test_run_hunt_survives_a_failing_grammar(tmp_path: Path, monkeypatch) -> None: + # One grammar raising (e.g. clone/build/toolchain error) must not abort the + # whole overnight campaign. + reg = _write_registry(tmp_path) + monkeypatch.setattr(loop_mod, "configure_telemetry", lambda *a, **k: False) + + def flaky_run_loop(target, **kwargs): # noqa: ANN001, ANN202 + if target.name == "uyha-cmake": + raise RuntimeError("clone failed") + return TSLoopResult( + grammar=target.name, tier=target.tier, language=target.language + ) + + monkeypatch.setattr(loop_mod, "run_loop", flaky_run_loop) + results = run_hunt(reg, top_n=10, fuzz_seconds=1, out_dir=tmp_path / "hunt") + assert len(results) == 2 # both candidates still produce a result row + failed = next(r for r in results if r.grammar == "uyha-cmake") + assert "run_loop raised" in failed.build_error + assert "clone failed" in failed.build_error diff --git a/scaffold/src/tests/test_treesitter_loop.py b/scaffold/src/tests/test_treesitter_loop.py index 0bfa7b6..f0a696b 100644 --- a/scaffold/src/tests/test_treesitter_loop.py +++ b/scaffold/src/tests/test_treesitter_loop.py @@ -67,7 +67,7 @@ def fake_run(self: runtime.LibFuzzerRunner, duration: int): # noqa: ANN202 fix_calls = {"n": 0} - def fake_run_fix(crash, crash_file, target, spec, build_result, fixer, report): # noqa: ANN001, ANN202, PLR0913 + def fake_run_fix(crash, crash_file, target, spec, build_result, fixer, report, **kwargs): # noqa: ANN001, ANN003, ANN202, PLR0913 fix_calls["n"] += 1 return ( TSFixAttempt( From 3f84d3a0fecbcb3cdc0992d6a60dbe742007f2ab Mon Sep 17 00:00:00 2001 From: Matthew Doty Date: Tue, 7 Jul 2026 11:16:39 -0400 Subject: [PATCH 5/6] chore(treesitter): defaulting to sonnet --- scaffold/src/parser_security_eval/treesitter/fixer.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scaffold/src/parser_security_eval/treesitter/fixer.py b/scaffold/src/parser_security_eval/treesitter/fixer.py index 8d43e8d..9b1f2d6 100644 --- a/scaffold/src/parser_security_eval/treesitter/fixer.py +++ b/scaffold/src/parser_security_eval/treesitter/fixer.py @@ -30,7 +30,8 @@ # Model-agnostic: a pydantic-ai "provider:model" string. Swap the provider freely # (e.g. "openai:gpt-4o", "google-gla:gemini-2.5-pro"); keys are read from the env. -DEFAULT_FIX_MODEL = "anthropic:claude-opus-4-8" +# Defaults to Sonnet; override per-run with `--model`/`-m` on any `treesitter` command. +DEFAULT_FIX_MODEL = "anthropic:claude-sonnet-5" # Replays used to confirm a crash reproduces (pre-fix) and is gone (post-fix). # >1 because some scanner crashes are flaky (e.g. heap-layout-dependent SEGVs). VERIFY_ATTEMPTS = 20 From 3b28676a4db88522c570e89265ffb7fec10204cf Mon Sep 17 00:00:00 2001 From: Matthew Doty Date: Tue, 7 Jul 2026 11:29:34 -0400 Subject: [PATCH 6/6] formatting(treesitter): uv ruff --- scaffold/src/tests/test_treesitter_loop.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scaffold/src/tests/test_treesitter_loop.py b/scaffold/src/tests/test_treesitter_loop.py index f0a696b..3b429bd 100644 --- a/scaffold/src/tests/test_treesitter_loop.py +++ b/scaffold/src/tests/test_treesitter_loop.py @@ -67,7 +67,9 @@ def fake_run(self: runtime.LibFuzzerRunner, duration: int): # noqa: ANN202 fix_calls = {"n": 0} - def fake_run_fix(crash, crash_file, target, spec, build_result, fixer, report, **kwargs): # noqa: ANN001, ANN003, ANN202, PLR0913 + def fake_run_fix( + crash, crash_file, target, spec, build_result, fixer, report, **kwargs + ): # noqa: ANN001, ANN003, ANN202, PLR0913 fix_calls["n"] += 1 return ( TSFixAttempt(