Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 28 additions & 2 deletions .claude/scripts/audit_xfails.py
Original file line number Diff line number Diff line change
Expand Up @@ -159,6 +159,16 @@ def parse_conftest_xfail_tags(conftest_path: Path) -> dict[str, tuple[str, str]]
# ── Step-level xfail detector ────────────────────────────────────────


def _mentions_identifier(haystack: str, name: str) -> bool:
"""True when ``name`` appears as an identifier boundary in ``haystack``.

Avoids bare substring false positives (``check_x`` matching ``check_xy``).
"""
if not name:
return False
return bool(re.search(rf"(?<![A-Za-z0-9_]){re.escape(name)}(?![A-Za-z0-9_])", haystack))


def find_premature_xfails(steps_dir: Path) -> set[str]:
"""Find step functions that call pytest.xfail() before any production code.

Expand Down Expand Up @@ -190,8 +200,12 @@ def find_premature_xfails(steps_dir: Path) -> set[str]:

# Check if first meaningful statement is pytest.xfail()
for stmt in node.body:
# Skip docstrings
if isinstance(stmt, ast.Expr) and isinstance(stmt.value, (ast.Constant, ast.Str)):
# Skip string docstrings only (ast.Str removed in Python 3.12)
if (
isinstance(stmt, ast.Expr)
and isinstance(stmt.value, ast.Constant)
and isinstance(stmt.value.value, str)
):
continue
# Skip imports
if isinstance(stmt, (ast.Import, ast.ImportFrom)):
Expand Down Expand Up @@ -279,6 +293,18 @@ def classify_xfail(
tags=tags,
)

# Priority 0: Premature step-level pytest.xfail() (AST scan)
# Match on identifier boundaries so e.g. check_x does not hit check_xy.
call = test.get("call") or {}
longrepr = str(call.get("longrepr") or "") if isinstance(call, dict) else ""
haystack = f"{wasxfail}\n{longrepr}\n{nodeid}"
for fname in premature_xfails:
if fname and _mentions_identifier(haystack, fname):
entry.category = "PREMATURE_XFAIL"
entry.reason = f"step {fname} calls pytest.xfail() before production code"
entry.xfail_source = f"step:{fname}"
return entry

# Priority 1: Missing step definition (auto-xfail from pytest hook)
if "Step definition not found" in wasxfail:
entry.category = "MISSING_STEP"
Expand Down
31 changes: 17 additions & 14 deletions .claude/scripts/bdd_full_audit.py
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@
FIX_NOW = {
"STALE_STRICT_XFAIL": "Test passes but strict xfail tag rejects it — remove tag",
"GRADUATE": "All transports pass — remove xfail tag",
"PARTIAL_XPASS": "Passes some transports — investigate remaining gaps before graduating",
"FIXTURE_GAP": "Strengthened assertion exposes missing test fixture data — fix factory",
"STEP_BUG": "Step implementation has a bug — fix step code",
"WEAK_ASSERTION": "Inspector-flagged: assertion doesn't match scenario intent",
Expand Down Expand Up @@ -298,7 +299,9 @@ def classify_xfail(entry: TestEntry, tag_reasons: dict[str, str]) -> tuple[str,
def classify_xpass(entry: TestEntry, all_entries: list[TestEntry]) -> tuple[str, str, str]:
"""Classify an xpassed test. Returns (bucket, category, detail).

All xpasses are FIX_NOW — remove the stale xfail tag.
FIX_NOW either way: GRADUATE when all four transports pass (remove the
stale xfail tag); PARTIAL_XPASS when only a subset passes (investigate
remaining gaps before graduating).
"""
base = extract_scenario_base(entry.nodeid)

Expand All @@ -311,9 +314,8 @@ def classify_xpass(entry: TestEntry, all_entries: list[TestEntry]) -> tuple[str,

if passing_transports >= TRANSPORTS:
return "FIX_NOW", "GRADUATE", f"All transports pass: {sorted(passing_transports)}"
else:
missing = TRANSPORTS - passing_transports
return "FIX_NOW", "GRADUATE", f"Passes: {sorted(passing_transports)}, missing: {sorted(missing)}"
missing = TRANSPORTS - passing_transports
return "FIX_NOW", "PARTIAL_XPASS", f"Passes: {sorted(passing_transports)}, missing: {sorted(missing)}"
Comment thread
mkostromin-sigma marked this conversation as resolved.


# ── Work item generation ─────────────────────────────────────────────
Expand Down Expand Up @@ -378,20 +380,20 @@ def generate_work_items(
)
)

# ── 2. Xpassed tests → FIX_NOW (graduate) ──────────────────────
grad_groups: dict[str, list[TestEntry]] = defaultdict(list)
# ── 2. Xpassed tests → FIX_NOW (graduate only when all 4 transports)
grad_groups: dict[tuple[str, str], list[TestEntry]] = defaultdict(list)
for entry in xpassed:
_, _, detail = classify_xpass(entry, all_entries)
grad_groups[detail].append(entry)
_, cat, detail = classify_xpass(entry, all_entries)
grad_groups[(cat, detail)].append(entry)

for detail, entries in grad_groups.items():
for (cat, detail), entries in grad_groups.items():
uc = extract_uc(entries[0].nodeid) if entries else "MIXED"
all_pass = "All transports pass" in detail
all_pass = cat == "GRADUATE"
items.append(
WorkItem(
title=f"Graduate {'(all transports)' if all_pass else '(partial)'}: {uc}",
title=f"{'Graduate' if all_pass else 'Partial xpass'} ({'all transports' if all_pass else 'gaps remain'}): {uc}",
bucket="FIX_NOW",
category="GRADUATE",
category=cat,
uc=uc,
test_count=len(entries),
details=detail,
Expand Down Expand Up @@ -504,11 +506,12 @@ def generate_report(
for item in fix_now:
by_cat[item.category].append(item)

for cat in ["STALE_STRICT_XFAIL", "GRADUATE", "FIXTURE_GAP", "STEP_BUG", "WEAK_ASSERTION"]:
# Single source of truth: FIX_NOW insertion order (no parallel list).
for cat in FIX_NOW:
cat_items = by_cat.get(cat, [])
if not cat_items:
continue
cat_desc = FIX_NOW.get(cat, cat)
cat_desc = FIX_NOW[cat]
total = sum(i.test_count for i in cat_items)
lines.append(f"### {cat} ({len(cat_items)} items, {total} tests)")
lines.append(f"*{cat_desc}*")
Expand Down
11 changes: 9 additions & 2 deletions .claude/scripts/inspect_parallel.sh
Original file line number Diff line number Diff line change
Expand Up @@ -14,17 +14,24 @@ set -euo pipefail

SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
INSPECTOR="/Users/konst/.claude/plugins/cache/agentic-toolkit/qa-bdd/0.2.0/skills/inspect-steps/scripts/inspect_bdd_steps.py"
# Prefer repo-local inspector; allow override for plugin/vendored copies.
INSPECTOR="${INSPECT_BDD_STEPS:-$SCRIPT_DIR/inspect_bdd_steps.py}"
FEATURES_DIR="$PROJECT_ROOT/tests/bdd/features"
STEPS_DIR="$PROJECT_ROOT/tests/bdd/steps"

if [ ! -f "$INSPECTOR" ]; then
echo "ERROR: inspector not found: $INSPECTOR" >&2
echo "Set INSPECT_BDD_STEPS to an alternate inspect_bdd_steps.py path." >&2
exit 1
fi

# Parse args
OUTPUT_DIR="${1:-.claude/reports/inspect-parallel-$(date +%d%m%y_%H%M)}"
mkdir -p "$OUTPUT_DIR"

# Create patched inspector with timeout=600 and --then-only=False
PATCHED="/tmp/inspect_bdd_parallel.py"
sed 's/timeout=180/timeout=600/;s/"--then-only", action="store_true", default=True/"--then-only", action="store_true", default=False/' \
sed -E 's/timeout=[0-9]+/timeout=600/;s/"--then-only", action="store_true", default=True/"--then-only", action="store_true", default=False/' \
"$INSPECTOR" > "$PATCHED"

echo "=== Parallel BDD Step Inspection ==="
Expand Down
98 changes: 52 additions & 46 deletions .claude/scripts/salvage_audit_output.py
Original file line number Diff line number Diff line change
Expand Up @@ -53,24 +53,48 @@ def parse_raw_output(raw_path: Path) -> dict:
}


def _store_key(kind: str, step: dict, fallback_name: str = "") -> str:
"""Kind-scoped dedupe key so triage and deep traces do not collide."""
name = step.get("function_name") or fallback_name
return f"{kind}:{name}:{step.get('line_number', 0)}"


def _placeholder_step(func_name: str) -> dict:
"""Minimal step record when the step index is unavailable."""
return {
"file_path": "unknown",
"line_number": 0,
"step_type": "then",
"step_text": "",
"function_name": func_name,
"source_text": "",
}


def _load_existing_keys(store_path: Path) -> set[str]:
"""Load kind-scoped keys already present in the JSONL store."""
existing_keys: set[str] = set()
if not store_path.exists():
return existing_keys
for line in store_path.read_text().splitlines():
if not line.strip():
continue
try:
obj = json.loads(line)
step = obj.get("step", {})
existing_keys.add(_store_key(str(obj.get("kind")), step))
except json.JSONDecodeError:
continue
return existing_keys


def write_to_store(parsed: dict, store_path: Path, step_index_path: Path | None) -> None:
"""Write parsed results to the JSONL store.

Since we only have function names (not full BddStepInfo), we write
lightweight records that the resume logic can match against.
"""
# Load existing store to avoid duplicates
existing_keys = set()
if store_path.exists():
for line in store_path.read_text().splitlines():
if not line.strip():
continue
try:
obj = json.loads(line)
step = obj.get("step", {})
existing_keys.add(f"{step.get('function_name')}:{step.get('line_number', 0)}")
except json.JSONDecodeError:
continue
existing_keys = _load_existing_keys(store_path)

# We need the step index to get file/line info.
# If not available, write with func_name only (partial records).
Expand All @@ -93,18 +117,8 @@ def write_to_store(parsed: dict, store_path: Path, step_index_path: Path | None)
with open(store_path, "a") as f:
# Write Pass 1 results
for r in parsed["pass1"]:
step_info = step_lookup.get(
r["func_name"],
{
"file_path": "unknown",
"line_number": 0,
"step_type": "then",
"step_text": "",
"function_name": r["func_name"],
"source_text": "",
},
)
key = f"{step_info.get('function_name', r['func_name'])}:{step_info.get('line_number', 0)}"
step_info = step_lookup.get(r["func_name"], _placeholder_step(r["func_name"]))
key = _store_key("triage", step_info, r["func_name"])
if key not in existing_keys:
record = {
"kind": "triage",
Expand All @@ -119,29 +133,21 @@ def write_to_store(parsed: dict, store_path: Path, step_index_path: Path | None)

# Write Pass 2 results
for r in parsed["pass2"]:
step_info = step_lookup.get(
r["func_name"],
{
"file_path": "unknown",
"line_number": 0,
"step_type": "then",
"step_text": "",
"function_name": r["func_name"],
"source_text": "",
},
)
key = f"{step_info.get('function_name', r['func_name'])}:{step_info.get('line_number', 0)}"
step_info = step_lookup.get(r["func_name"], _placeholder_step(r["func_name"]))
key = _store_key("deep", step_info, r["func_name"])
# Only write deep trace if not already present
record = {
"kind": "deep",
"step": step_info,
"claims": "salvaged — see full report for details",
"actually_tests": "salvaged — see full report for details",
"recommendation": "salvaged — see full report for details",
"severity": r["severity"],
}
f.write(json.dumps(record) + "\n")
new_deep += 1
if key not in existing_keys:
Comment thread
mkostromin-sigma marked this conversation as resolved.
record = {
"kind": "deep",
"step": step_info,
"claims": "salvaged — see full report for details",
"actually_tests": "salvaged — see full report for details",
"recommendation": "salvaged — see full report for details",
"severity": r["severity"],
}
f.write(json.dumps(record) + "\n")
existing_keys.add(key)
new_deep += 1

print(f"Salvaged to {store_path}:")
print(f" Pass 1: {new_triage} new triage results (of {len(parsed['pass1'])} total)")
Expand Down
20 changes: 20 additions & 0 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -339,6 +339,23 @@ jobs:
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
with:
persist-credentials: false
# #1662: image build + /opt/venv + second full tox env under tox_data
# overflowed ubuntu-latest (~14GB free) → ENOSPC on uv sync into /app/.tox.
# Drop unused preinstalled toolchains and prune builder cache before compose.
- name: Free disk space
run: |
set -eux
df -h
sudo rm -rf \
/usr/share/dotnet \
/usr/local/lib/android \
/opt/ghc \
/opt/hostedtoolcache/CodeQL \
/usr/local/share/boost \
/usr/share/swift \
|| true
docker builder prune -af 2>/dev/null || true
df -h
- name: Clean up lingering containers
run: |
docker compose -f docker-compose.e2e.yml down -v 2>/dev/null || true
Expand All @@ -362,6 +379,9 @@ jobs:
# run_all_tests.sh builds the pinned creative agent host-side; prefer
# the pin-keyed ghcr pull (~5m faster) over building from source.
CREATIVE_AGENT_GHCR_IMAGE: ghcr.io/${{ github.repository }}/adcp-creative-agent
# Serial e2e_rest leg — no xdist fan-out; default 10g tmpfs wastes RAM
# on ubuntu-latest and contributes to ENOSPC pressure (#1662).
PGDATA_TMPFS_SIZE: 2g
# No host uvx in this job (no _setup-env); the dedicated Security
# Audit check owns the uv-secure scan.
RUN_ALL_SKIP_AUDIT: 1
Expand Down
9 changes: 7 additions & 2 deletions .github/workflows/security.yml
Original file line number Diff line number Diff line change
Expand Up @@ -92,7 +92,7 @@ jobs:
persist-credentials: false
- run: |
set -euo pipefail
PINACT_VERSION=3.10.1
PINACT_VERSION=4.1.0
PINACT_TARBALL="pinact_linux_amd64.tar.gz"
curl -sSfL "https://github.com/suzuki-shunsuke/pinact/releases/download/v${PINACT_VERSION}/${PINACT_TARBALL}" -o "${PINACT_TARBALL}"
# Residual risk: checksums file is fetched from the same release tag as the binary.
Expand All @@ -102,7 +102,12 @@ jobs:
curl -sSfL "https://github.com/suzuki-shunsuke/pinact/releases/download/v${PINACT_VERSION}/pinact_${PINACT_VERSION}_checksums.txt" -o pinact_checksums.txt
grep " ${PINACT_TARBALL}$" pinact_checksums.txt | sha256sum -c -
tar -xzf "${PINACT_TARBALL}" -C /usr/local/bin/ pinact
pinact run --check
# Offline SHA-pin check only (-no-api). pinact's default path calls GitHub REST
# for tag resolution and flakes the gate on api.github.com 503s (5× retries
# still failed on amannn/action-semantic-pull-request; #1669).
# Job contract is "actions are full-length SHA-pinned" — syntactic check is enough.
pinact run -fix=false -no-api
# GITHUB_TOKEN unused with -no-api; kept so a future API mode can opt back in.
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

Expand Down
1 change: 1 addition & 0 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@ dependencies = [
"logfire>=4.16.0",
"pydantic-ai-slim[google,openai,anthropic,cohere,mistral,groq]>=1.99.0", # 1.99.0 fixes CVE-2026-46678 / GHSA-cqp8-fcvh-x7r3; use slim to avoid pydantic-ai meta-package pulling fastmcp>=3.3.0
# Security patches for transitive dependencies
"mcp>=1.28.1", # CVE-2026-59950 / GHSA-vj7q-gjh5-988w (via fastmcp)
"pyjwt>=2.13.0", # GHSA-752w-5fwx-jx9f; PYSEC-2026-175/177/178/179
"filelock>=3.20.3", # GHSA-qmgc-5h2g-mvrw
"urllib3>=2.7.0", # GHSA-38jv-5279-wg99 + GHSA-qccp-gfcp-xxvc + GHSA-mf9v-mfxr-j63j
Expand Down
36 changes: 36 additions & 0 deletions tests/unit/test_architecture_ci_suite_coverage.py
Original file line number Diff line number Diff line change
Expand Up @@ -223,6 +223,42 @@ def test_skip_guard_single_source_of_truth_exists(self):
"TestNoSkippedTests.test_no_skip_decorators is the declared single source of truth for skip enforcement."
)

@pytest.mark.arch_guard
def test_bdd_in_network_frees_disk_before_compose(self):
"""In-network e2e_rest must reclaim runner disk before image build (#1662).

Regression: after #1613/#1634, ``uv sync`` into ``tox_data`` hit ENOSPC
on ubuntu-latest (image + /opt/venv + second full tox env). The job must
free preinstalled toolchains and cap PGDATA tmpfs for the serial leg.
"""
job = load_ci_workflow()["jobs"]["bdd-in-network"]
steps = job.get("steps", [])
assert steps, "bdd-in-network must declare steps (empty job is a vacuous pass)."

step_names = [s.get("name") for s in steps]
assert "Free disk space" in step_names, "bdd-in-network must include a 'Free disk space' step before compose."
assert "Run BDD suite in-network" in step_names, "bdd-in-network must run ./run_all_tests.sh bdd_e2e."
free_idx = step_names.index("Free disk space")
run_idx = step_names.index("Run BDD suite in-network")
assert free_idx < run_idx, (
"Free disk space must run before 'Run BDD suite in-network' "
f"(found Free disk at {free_idx}, run at {run_idx})."
)

free_run = str(steps[free_idx].get("run", ""))
assert "/usr/share/dotnet" in free_run, "Free disk space must remove /usr/share/dotnet."
assert "docker builder prune" in free_run, "Free disk space must prune the Docker builder cache."

run_step = steps[run_idx]
env = run_step.get("env") or {}
assert env.get("PGDATA_TMPFS_SIZE") == "2g", (
"bdd-in-network must set PGDATA_TMPFS_SIZE=2g "
"(serial leg; default 10g wastes RAM / contributes to pressure)."
)
assert "run_all_tests.sh bdd_e2e" in str(run_step.get("run", "")), (
"bdd-in-network must invoke ./run_all_tests.sh bdd_e2e."
)

@pytest.mark.arch_guard
def test_bdd_and_e2e_run_on_pull_request(self):
"""The gate is worthless if it doesn't run on PRs.
Expand Down
Loading
Loading