From 851f6d205727b43ae5ed608a2f74c194c48730d9 Mon Sep 17 00:00:00 2001 From: Alice Vinogradova Date: Fri, 2 Oct 2026 15:02:29 +0000 Subject: [PATCH 1/3] ci: gate compiler assertions and backend fuzz regressions Add paired assertion checks, judge self-tests, advisory output and lint reports, and nightly compiler audits with reproducible local wrappers. Report MIR2/oracle fuzz findings while gating Z80/MIR2 mismatches and assembly failures on PRs; preserve strict standalone and nightly reporting. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01Wrwo36SzxKRZYTgpDo7hzo --- .github/workflows/assert-checks.yml | 106 ++++++++++++++++++++++++++++ .github/workflows/ci.yml | 35 +++++++++ .github/workflows/nightly.yml | 22 ++++++ minzc/.golangci.yml | 84 +++------------------- scripts/README.md | 96 ++++++++++++++++++++++++- scripts/ci/build.sh | 16 +++++ scripts/ci/check-regression.sh | 46 ++++++++++++ scripts/ci/lint.sh | 45 ++++++++++++ scripts/ci/metadata.sh | 24 +++++++ scripts/ci/run.sh | 58 +++++++++++++++ scripts/ci/simulate.sh | 37 ++++++++++ scripts/ci/summary.py | 37 ++++++++++ scripts/fuzz_diff.py | 8 ++- scripts/test_judges.py | 20 ++++++ 14 files changed, 555 insertions(+), 79 deletions(-) create mode 100644 .github/workflows/assert-checks.yml create mode 100644 .github/workflows/nightly.yml create mode 100755 scripts/ci/build.sh create mode 100755 scripts/ci/check-regression.sh create mode 100755 scripts/ci/lint.sh create mode 100755 scripts/ci/metadata.sh create mode 100755 scripts/ci/run.sh create mode 100755 scripts/ci/simulate.sh create mode 100644 scripts/ci/summary.py diff --git a/.github/workflows/assert-checks.yml b/.github/workflows/assert-checks.yml new file mode 100644 index 00000000..1a10cfd6 --- /dev/null +++ b/.github/workflows/assert-checks.yml @@ -0,0 +1,106 @@ +name: Compiler checks + +on: + workflow_call: + inputs: + candidate: + type: string + required: true + base: + type: string + required: true + nightly: + type: boolean + default: false + +permissions: + contents: read + +jobs: + compilers: + name: Build judge compilers + runs-on: ubuntu-latest + timeout-minutes: 15 + outputs: + controls: ${{ steps.metadata.outputs.controls }} + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ inputs.candidate }} + fetch-depth: 0 + - uses: actions/setup-go@v5 + with: + go-version-file: minzc/go.mod + cache-dependency-path: minzc/go.sum + - name: Choose baseline and controls + id: metadata + env: + BASE_REV: ${{ inputs.base }} + NIGHTLY: ${{ inputs.nightly }} + run: bash scripts/ci/metadata.sh + - name: Cache baseline compiler + uses: actions/cache@v4 + with: + path: ${{ runner.temp }}/judge-bin/baseline-mz + key: baseline-mz-${{ runner.os }}-${{ runner.arch }}-${{ steps.metadata.outputs.base }}-${{ hashFiles('minzc/go.sum') }} + - name: Build head and baseline + env: + BASE_SHA: ${{ steps.metadata.outputs.base }} + CI_BIN: ${{ runner.temp }}/judge-bin + run: bash scripts/ci/build.sh + - uses: actions/upload-artifact@v4 + with: + name: judge-compilers + path: | + ${{ runner.temp }}/judge-bin/*-mz + ${{ runner.temp }}/judge-bin/base.tar.gz + retention-days: 2 + + checks: + name: ${{ inputs.nightly && matrix.job == 'fuzz' && 'Long fuzz' || matrix.name }} + needs: compilers + runs-on: ubuntu-latest + timeout-minutes: 40 + continue-on-error: ${{ inputs.nightly || matrix.job == 'sweep' }} + strategy: + fail-fast: false + matrix: + include: + - job: matrix + name: Assert matrix + - job: judges + name: Judge self-tests + - job: sweep + name: Default output sweep + - job: fuzz + name: Short fuzz + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ inputs.candidate }} + - uses: actions/setup-go@v5 + if: matrix.job == 'judges' + with: + go-version-file: minzc/go.mod + cache-dependency-path: minzc/go.sum + - uses: actions/download-artifact@v4 + with: + name: judge-compilers + path: ${{ runner.temp }}/judge-bin + - name: Run judge + env: + CI_BIN: ${{ runner.temp }}/judge-bin + REPORT_DIR: ${{ runner.temp }}/reports/${{ matrix.job }} + CONTROLS: ${{ needs.compilers.outputs.controls }} + RUNS: ${{ inputs.nightly && '2' || '1' }} + FUZZ_COUNT: ${{ inputs.nightly && '10000' || '3000' }} + NIGHTLY: ${{ inputs.nightly }} + run: bash scripts/ci/run.sh '${{ matrix.job }}' + - name: Upload report and reproducers + if: always() + uses: actions/upload-artifact@v4 + with: + name: compiler-${{ matrix.job }}-report + path: ${{ runner.temp }}/reports/${{ matrix.job }}/ + if-no-files-found: warn + retention-days: 14 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c4f55a41..6706a046 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -112,3 +112,38 @@ jobs: } else { await github.rest.issues.createComment({ owner, repo, issue_number, body }); } + + compiler_checks: + name: Compiler checks + uses: ./.github/workflows/assert-checks.yml + with: + candidate: ${{ github.event.pull_request.head.sha || github.sha }} + base: ${{ github.event.pull_request.base.sha || 'HEAD~1' }} + + advisory_lint: + name: Advisory Go lint + if: github.event_name == 'pull_request' + runs-on: ubuntu-latest + timeout-minutes: 15 + continue-on-error: true + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.sha }} + fetch-depth: 0 + - uses: actions/setup-go@v5 + with: + go-version: '1.26.8' + cache-dependency-path: minzc/go.sum + - name: Install pinned golangci-lint + run: go install github.com/golangci/golangci-lint/v2/cmd/golangci-lint@v2.13.2 + - name: Report new lint issues and changed-file formatting + env: + BASE_SHA: ${{ github.event.pull_request.base.sha }} + REPORT_DIR: ${{ runner.temp }}/lint-report + run: bash scripts/ci/lint.sh + - uses: actions/upload-artifact@v4 + if: always() + with: + name: advisory-lint-report + path: ${{ runner.temp }}/lint-report/ diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml new file mode 100644 index 00000000..431490b1 --- /dev/null +++ b/.github/workflows/nightly.yml @@ -0,0 +1,22 @@ +name: Nightly compiler audit + +on: + schedule: + - cron: '17 3 * * *' + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: nightly-compiler-audit + cancel-in-progress: false + +jobs: + compiler_checks: + name: Nightly compiler checks + uses: ./.github/workflows/assert-checks.yml + with: + candidate: ${{ github.sha }} + base: HEAD~1 + nightly: true diff --git a/minzc/.golangci.yml b/minzc/.golangci.yml index 3296a14f..9e139d8a 100644 --- a/minzc/.golangci.yml +++ b/minzc/.golangci.yml @@ -1,86 +1,18 @@ +version: "2" run: timeout: 5m - tests: true - skip-dirs: - - vendor - - testdata - skip-files: - - ".*_test.go" - linters: + default: none enable: - - gofmt - - golint - govet - - errcheck - staticcheck - - gosimple + - errcheck - ineffassign - - typecheck - - goconst - - gocyclo - - misspell - - unconvert - - goimports - - prealloc - - nakedret - - lll - - dupl - disable: - - godox # Allow TODO/FIXME comments - - funlen # Z80 code generation can have long functions - - gocognit # Complex optimization logic is acceptable - -linters-settings: - govet: - check-shadowing: true - gocyclo: - min-complexity: 15 - dupl: - threshold: 100 - goconst: - min-len: 3 - min-occurrences: 3 - lll: - line-length: 120 - nakedret: - max-func-lines: 30 - prealloc: - simple: true - range-loops: true - for-loops: true - + - unused issues: - exclude-rules: - # Exclude some linters from running on test files - - path: _test\.go - linters: - - dupl - - gosec - - goconst - - # Exclude generated files - - path: generated\.go - linters: - - golint - - stylecheck - - # Allow complex functions in optimizer package - - path: pkg/optimizer/ - linters: - - gocyclo - - gocognit - - # Allow long lines in code generation - - path: pkg/codegen/ - linters: - - lll - - # Maximum issues count per one linter. Set to 0 to disable max-issues-per-linter: 0 - - # Maximum count of issues with the same text. Set to 0 to disable max-same-issues: 0 - - # Show only new issues created after git revision - new: false \ No newline at end of file +output: + formats: + json: + path: stdout diff --git a/scripts/README.md b/scripts/README.md index 691112b5..c1382159 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -41,7 +41,11 @@ failure, and `seed-N.original.nanz`. This is a local deletion minimum. `--no-red fast smoke triage without deletion reduction. Syntax, compiler, assembly, timeout and oracle errors are counted separately; compiler failures save `seed-N.error.nanz`. `results.json` records all seeds and diagnostics. -Exit status is 1 if any program fails or cannot be checked. `--timeout` bounds +By default exit status is 1 if any program fails or cannot be checked. +`--fail-on backend` exits 1 only for Z80/MIR2 mismatches and assembly failures; +all findings still appear in logs, JSON and reproducers. Tool exceptions exit 2 +under either policy. Assembly failures take precedence even when MIR2 also +disagrees with the oracle. `--timeout` bounds each compiler invocation, including reduction attempts. Run self-tests with `python3 scripts/test_judges.py`; `go test ./pkg/hir` also @@ -135,3 +139,93 @@ errors include `call error:`; top-level messages are unchanged. Run `python3 scripts/test_assert_mutants.py` to build M4 (checks only element 0) and M5 (skips element 0) in temporary copies, and verify that the tuple fixture's matrix exits nonzero on both Z80 and MIR2 for each mutant. + +## CI checks + +The existing required **PR gate** keeps its name and Go/toolchain checks. +The `Compiler checks` reusable workflow builds `mz` from the PR head (not the +merge ref) and `github.event.pull_request.base.sha`, then runs these jobs on +PRs and pushes to main: + +- **Assert matrix** compares separate baseline/head inventories on both Z80 + and MIR2. It preserves `assert_matrix.py`'s exit status, including newly + failing, removed/changed assertions, unexpectedly passing controls, and + known-failure audit errors. JSON, logs and summaries are uploaded even on + failure; the log and step summary include the newly failing locations. +- **Judge self-tests** sets `JUDGE_MZ` to the candidate and runs + `test_judges.py`. When controls are enabled it also builds and checks the + two tuple mutants with `test_assert_mutants.py`. +- **Default output sweep** compares exact default diagnostics and generated + files. It is advisory (`continue-on-error`), because the sweep has no + allowlist and PRs can intentionally change output. Its raw exit status and + differences remain in the report. +- **Short fuzz** checks fixed seeds 0–2999 with `--no-reduce`. The fuzzer has + no baseline mode, so this gate checks the candidate alone with + `--fail-on backend`: Z80/MIR2 mismatches and assembly failures block the PR. + MIR2/Python-oracle disagreements and other findings remain in the job summary + and artifacts without failing the PR. No seeds are skipped. Tool exceptions + still fail the job. Full reproducers and JSON are artifacts. + +Workers default to `nproc`. Baseline binaries are cached by full base SHA plus +candidate `minzc/go.sum` hash, OS and architecture; no fallback cache keys are +used. Both binaries and the exact base archive are passed to the jobs within +that workflow run. These new checks are not made required by workflow code; +Alice can configure branch protection separately. + +Positive checks normally use `--no-controls`. A plain `git diff` against the +base enables controls and tuple mutant tests for changes under `scripts/`, +`minzc/cmd/minzc/`, `minzc/pkg/pipeline/`, or assertion-parsing frontend +packages (`nanz`, `c89`, `pascal`, `plm`, `abap`, `frill`, `lanz`, `lizp`). +It also covers `hir/hir.go`, `hir/assert*.go`, and the assertion-executing +`mir2wasm/runner.go` and `mir2llvm/runner.go`. Deleted and renamed paths count. +The exact filter is in `scripts/ci/metadata.sh`; extend it when assertion +handling moves. `--no-controls` alone cannot catch a weakened checker. + +**Nightly compiler audit** runs at 03:17 UTC and supports `workflow_dispatch`. +It compares main with `HEAD~1`, always enables controls, repeats the matrix +with `--runs 2`, runs both mutant tests, and fuzzes seeds 0–9999 using the +fuzzer's default `--fail-on all` policy. Judge jobs +are report-only; artifacts include their raw statuses and summaries. Build or +infrastructure failures remain visible as workflow failures. + +**Advisory Go lint** runs only on PRs from the `minzc/` module using pinned +`golangci-lint v2.13.2` (built with Go 1.26.8). Its minimal configuration +runs govet, staticcheck, errcheck, ineffassign and unused with +`--new-from-rev=` over `./pkg/... ./cmd/...`. This scope +excludes the intentionally invalid scratch programs in `minzc/scripts/analysis` +that cannot be package-loaded. `gofmt -l` checks only changed Go files. +Warnings are GitHub annotations; JSON, tool errors and counts are artifacts. +The job uses `continue-on-error`, so lint never blocks the PR. + +To reproduce all jobs sequentially with timings and retained logs: + +```sh +# Install tools to a writable directory using Go 1.26.8. +GOBIN=/tmp/minz-ci-tools go install github.com/rhysd/actionlint/cmd/actionlint@v1.7.12 +GOBIN=/tmp/minz-ci-tools go install github.com/golangci/golangci-lint/v2/cmd/golangci-lint@v2.13.2 +PATH=/tmp/minz-ci-tools:$PATH BASE_REV=origin/main JOBS=4 CONTROLS=true \ + bash scripts/ci/simulate.sh +``` + +`BASE_REV` must support assertion inventory, selection and execution receipts; +older pre-J2a binaries cannot be used for a valid comparison. Use a synthetic +base at current `origin/main` to test CI wiring. The script prints its temporary +report directory and each raw exit code; advisory sweep/lint failures do not +change its final status. `CONTROLS` overrides the committed-path filter when +simulating uncommitted changes. For a nightly simulation also set +`NIGHTLY=true RUNS=2 FUZZ_COUNT=10000 BASE_REV=HEAD~1`. To rerun one job, set +`CI_BIN` to the printed compiler directory, `REPORT_DIR` to a writable output +directory, and run `bash scripts/ci/run.sh matrix` (or `judges`, `sweep`, `fuzz`). +`MATRIX_ROOT` selects a disposable checkout; `MATRIX_GLOB` narrows a matrix +run for a regression fixture. CI leaves both unset and audits the full corpus. + +To verify that the CI matrix wrapper rejects a codegen regression after building +both compilers: + +```sh +CI_BIN=/tmp/minz-ci.YOUR_RUN JOBS=4 bash scripts/ci/check-regression.sh +``` + +This creates a tracked fixture and compiler copy under `/tmp`, requires exit 0 +from the original compiler, substitutes OR for XOR in the copy, then requires +matrix exit 1 with a newly failing Z80 assertion and unchanged MIR2 results. diff --git a/scripts/ci/build.sh b/scripts/ci/build.sh new file mode 100755 index 00000000..02a7f3b6 --- /dev/null +++ b/scripts/ci/build.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +set -euo pipefail +: "${BASE_SHA:?}" "${CI_BIN:?}" +mkdir -p "$CI_BIN" +base_root=$(mktemp -d "$CI_BIN/base.XXXXXX") +trap 'rm -rf "$base_root"' EXIT +# Archive rather than worktree: no mutation of the main checkout's Git metadata. +git archive "$BASE_SHA" | gzip > "$CI_BIN/base.tar.gz" +tar -xzf "$CI_BIN/base.tar.gz" -C "$base_root" +export GOFLAGS=-buildvcs=false +if [[ ! -x $CI_BIN/baseline-mz ]]; then + (cd "$base_root/minzc" && go build -o "$CI_BIN/baseline-mz" ./cmd/minzc) +fi +(cd minzc && go build -o "$CI_BIN/candidate-mz" ./cmd/minzc) +# The source archive contains only the base SHA's tracked files. Downstream +# jobs create their own Git index for inventory; no object database is uploaded. diff --git a/scripts/ci/check-regression.sh b/scripts/ci/check-regression.sh new file mode 100755 index 00000000..04a7b0ae --- /dev/null +++ b/scripts/ci/check-regression.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash +# Verify that the exact CI matrix wrapper rejects a real codegen regression. +set -euo pipefail +: "${CI_BIN:?Run build.sh first}" +export JOBS=${JOBS:-4} CONTROLS=false +export MATRIX_GLOB=examples/nanz/ci_xor.nanz +original_bin=$CI_BIN +export CI_BIN +CI_BIN=$(mktemp -d /tmp/minz-ci-regression.XXXXXX) +cp "$original_bin/baseline-mz" "$original_bin/candidate-mz" "$CI_BIN/" +export MATRIX_ROOT="$CI_BIN/fixture" +mkdir -p "$MATRIX_ROOT/examples/nanz" +cat > "$MATRIX_ROOT/$MATRIX_GLOB" <<'NANZ' +fun xor_gate(a: u8, b: u8) -> u8 { return a xor b } +assert xor_gate(3, 1) == 2 via z80 +NANZ +git -C "$MATRIX_ROOT" init -q +git -C "$MATRIX_ROOT" add . +tar -czf "$CI_BIN/base.tar.gz" -C "$MATRIX_ROOT" . +export REPORT_DIR="$CI_BIN/green" +bash scripts/ci/run.sh matrix +cp -a minzc "$CI_BIN/minzc" +python3 - "$CI_BIN/minzc/pkg/mir2/z80codegen_inst.go" <<'PY' +from pathlib import Path +import sys +p = Path(sys.argv[1]) +s = p.read_text() +needle = 'g.genBinOp("XOR", inst)' +assert s.count(needle) == 1 +p.write_text(s.replace(needle, 'g.genBinOp("OR", inst)')) +PY +(cd "$CI_BIN/minzc" && GOCACHE=/tmp/minz-go-cache GOFLAGS=-buildvcs=false go build -o "$CI_BIN/candidate-mz" ./cmd/minzc) +export REPORT_DIR="$CI_BIN/red" +status=0 +bash scripts/ci/run.sh matrix || status=$? +if [[ $status != 1 ]]; then + echo "Expected regression exit 1, got $status" >&2 + exit 1 +fi +python3 - "$REPORT_DIR/matrix.json" <<'PY' +import json, sys +passes = json.load(open(sys.argv[1]))['passes'] +assert passes['z80']['summary']['newly fail'] > 0 +assert passes['mir2']['summary']['newly fail'] == 0 +print('Unmodified exit 0; XOR→OR mutant exit 1; Z80 regression detected; MIR2 unchanged.') +PY diff --git a/scripts/ci/lint.sh b/scripts/ci/lint.sh new file mode 100755 index 00000000..0257fe60 --- /dev/null +++ b/scripts/ci/lint.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash +set -euo pipefail +: "${BASE_SHA:?}" "${REPORT_DIR:?}" +mkdir -p "$REPORT_DIR" +export GOLANGCI_LINT_CACHE=${GOLANGCI_LINT_CACHE:-/tmp/minz-golangci-cache} +start=$SECONDS +set +e +(cd minzc && golangci-lint run --new-from-rev="$BASE_SHA" --show-stats=false ./pkg/... ./cmd/...) > "$REPORT_DIR/lint.json" 2> "$REPORT_DIR/lint.log" +status=$? +set -e +python3 - "$REPORT_DIR" "$status" "$((SECONDS-start))" <<'PY' +import json, os, pathlib, subprocess, sys +root = pathlib.Path(sys.argv[1]) +try: + issues = json.loads((root / 'lint.json').read_text()).get('Issues') or [] +except (ValueError, AttributeError): + issues = [] + print('::warning::golangci-lint produced no JSON; see lint.log') +counts = {} +for issue in issues: + name = issue['FromLinter'] + counts[name] = counts.get(name, 0) + 1 + pos = issue['Pos'] + text = issue['Text'].replace('%', '%25').replace('\r', '%0D').replace('\n', '%0A') + print(f'::warning file=minzc/{pos["Filename"]},line={pos["Line"]},col={pos["Column"]}::{name}: {text}') +# gofmt checks whole changed files, including unchanged lines, but never old files. +paths = subprocess.check_output(['git', 'diff', '--name-only', '--diff-filter=ACMR', '-z', + os.environ['BASE_SHA'], 'HEAD', '--', 'minzc/**/*.go']).split(b'\0') +format_issues = 0 +for path in paths: + if not path: + continue + name = os.fsdecode(path) + result = subprocess.run(['gofmt', '-l', name], capture_output=True, text=True, check=True) + if result.stdout: + format_issues += 1 + print(f'::warning file={name},line=1::gofmt: run gofmt on this changed file') +summary = f'### Advisory lint\n\nExit {sys.argv[2]}; wall {sys.argv[3]}s; new issues {len(issues)}: {counts}; gofmt files {format_issues}.\n' +summary += '\n```text\n' + '\n'.join((root / 'lint.log').read_text().splitlines()[-50:]) + '\n```\n' +(root / 'summary.md').write_text(summary) +print(summary) +with open(os.environ.get('GITHUB_STEP_SUMMARY', os.devnull), 'a') as out: + out.write(summary) +PY +exit "$status" diff --git a/scripts/ci/metadata.sh b/scripts/ci/metadata.sh new file mode 100755 index 00000000..8eca7832 --- /dev/null +++ b/scripts/ci/metadata.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +set -euo pipefail +# Run from the repository root after fetching full history. +base=$(git rev-parse "${BASE_REV:-HEAD~1}^{commit}") +controls=false +if [[ ${NIGHTLY:-false} == true ]]; then + controls=true +else + # Includes deleted/renamed paths. Frontends parse assertions; HIR owns receipts; + # the WASM/LLVM runners execute sandbox assertions. + changed=$(git diff --no-renames --name-only "$base" HEAD) + while IFS= read -r path; do + case "$path" in + scripts/* | minzc/cmd/minzc/* | minzc/pkg/pipeline/* | \ + minzc/pkg/nanz/* | minzc/pkg/c89/* | minzc/pkg/pascal/* | \ + minzc/pkg/plm/* | minzc/pkg/abap/* | minzc/pkg/frill/* | \ + minzc/pkg/lanz/* | minzc/pkg/lizp/* | minzc/pkg/hir/hir.go | \ + minzc/pkg/hir/assert*.go | minzc/pkg/mir2wasm/runner.go | \ + minzc/pkg/mir2llvm/runner.go) + controls=true ;; + esac + done <<< "$changed" +fi +printf 'base=%s\ncontrols=%s\n' "$base" "$controls" | tee -a "${GITHUB_OUTPUT:-/dev/null}" diff --git a/scripts/ci/run.sh b/scripts/ci/run.sh new file mode 100755 index 00000000..8952f040 --- /dev/null +++ b/scripts/ci/run.sh @@ -0,0 +1,58 @@ +#!/usr/bin/env bash +set -euo pipefail +: "${CI_BIN:?}" +job=${1:?matrix, judges, sweep, or fuzz} +report_dir=${REPORT_DIR:-$CI_BIN/reports/$job} +mkdir -p "$report_dir" +chmod +x "$CI_BIN/candidate-mz" "$CI_BIN/baseline-mz" +jobs=${JOBS:-$(nproc)} +controls=--no-controls +[[ ${CONTROLS:-false} == true ]] && controls=--controls +start=$SECONDS +# Capture the original judge status even through tee, then publish a summary +# before returning it. Advisory policy belongs in the workflow, never here. +set +e +case "$job" in + matrix) + baseline_root=$(mktemp -d "$CI_BIN/base.XXXXXX") + trap 'rm -rf "$baseline_root"' EXIT + tar -xzf "$CI_BIN/base.tar.gz" -C "$baseline_root" || exit $? + git -C "$baseline_root" init -q || exit $? + git -C "$baseline_root" add -f . || exit $? + extra=() + [[ -n ${MATRIX_GLOB:-} ]] && extra+=(--glob "$MATRIX_GLOB") + python3 scripts/assert_matrix.py --root "${MATRIX_ROOT:-.}" --baseline "$CI_BIN/baseline-mz" \ + --candidate "$CI_BIN/candidate-mz" --baseline-root "$baseline_root" \ + "$controls" --runs "${RUNS:-1}" -j "$jobs" --json "$report_dir/matrix.json" "${extra[@]}" 2>&1 | tee "$report_dir/log.txt" + status=${PIPESTATUS[0]} + ;; + judges) + JUDGE_MZ="$CI_BIN/candidate-mz" python3 scripts/test_judges.py 2>&1 | tee "$report_dir/log.txt" + status=${PIPESTATUS[0]} + if [[ ${CONTROLS:-false} == true ]]; then + python3 scripts/test_assert_mutants.py 2>&1 | tee -a "$report_dir/log.txt" + mutant_status=${PIPESTATUS[0]} + (( mutant_status > status )) && status=$mutant_status + fi + ;; + sweep) + python3 scripts/compile_sweep.py --baseline "$CI_BIN/baseline-mz" \ + --candidate "$CI_BIN/candidate-mz" -j "$jobs" --json "$report_dir/sweep.json" 2>&1 | tee "$report_dir/log.txt" + status=${PIPESTATUS[0]} + ;; + fuzz) + # No baseline mode: gate backend failures, report all oracle findings. + extra=(--fail-on backend) + [[ ${NIGHTLY:-false} == true ]] && extra=(--fail-on all) + python3 scripts/fuzz_diff.py --mz "$CI_BIN/candidate-mz" --seed 0 \ + --count "${FUZZ_COUNT:-3000}" -j "$jobs" --no-reduce \ + --output "$report_dir" "${extra[@]}" 2>&1 | tee "$report_dir/log.txt" + status=${PIPESTATUS[0]} + ;; + *) echo "Unknown job: $job" >&2; exit 2 ;; +esac +set -e +printf '\n%s: exit %s, wall %ss, workers %s\n' "$job" "$status" "$((SECONDS-start))" "$jobs" | tee -a "$report_dir/log.txt" +python3 scripts/ci/summary.py "$job" "$status" "$((SECONDS-start))" "$report_dir" +cat "$report_dir/summary.md" >> "${GITHUB_STEP_SUMMARY:-/dev/null}" +exit "$status" diff --git a/scripts/ci/simulate.sh b/scripts/ci/simulate.sh new file mode 100755 index 00000000..94ba69df --- /dev/null +++ b/scripts/ci/simulate.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash +# Sequential local reproduction: retain logs, statuses and timings even if a +# gate fails; exit nonzero for blocking jobs. Requires pinned tools on PATH. +set -euo pipefail +export BASE_REV=${BASE_REV:-origin/main} +export CI_BIN=${CI_BIN:-$(mktemp -d /tmp/minz-ci.XXXXXX)} +export JOBS=${JOBS:-4} +export GOCACHE=${GOCACHE:-/tmp/minz-go-cache} +export GOFLAGS=-buildvcs=false +export REPORT_DIR="$CI_BIN/reports" +mkdir -p "$REPORT_DIR" +export GITHUB_OUTPUT="$CI_BIN/metadata.txt" +bash scripts/ci/metadata.sh +export BASE_SHA +BASE_SHA=$(sed -n 's/^base=//p' "$GITHUB_OUTPUT" | tail -1) +export CONTROLS=${CONTROLS:-$(sed -n 's/^controls=//p' "$GITHUB_OUTPUT" | tail -1)} +# Uncommitted changes are not included in the PR path filter; override CONTROLS +# explicitly when testing an uncommitted scripts/pipeline edit. +failed=0 +run_gate() { + local name=$1 + shift + local start=$SECONDS status=0 + "$@" > "$CI_BIN/$name.log" 2>&1 || status=$? + printf '%s: exit %s, wall %ss (log: %s)\n' "$name" "$status" "$((SECONDS-start))" "$CI_BIN/$name.log" | tee -a "$CI_BIN/timings.txt" + if [[ $name != sweep && $name != lint && $status != 0 ]]; then failed=1; fi +} +run_gate actionlint actionlint .github/workflows/*.yml +run_gate build bash scripts/ci/build.sh +for job in matrix judges sweep fuzz; do + export REPORT_DIR="$CI_BIN/reports/$job" + run_gate "$job" bash scripts/ci/run.sh "$job" +done +export REPORT_DIR="$CI_BIN/reports/lint" +run_gate lint bash scripts/ci/lint.sh +run_gate packages bash -c 'cd minzc && go build ./pkg/... ./cmd/...' +exit "$failed" diff --git a/scripts/ci/summary.py b/scripts/ci/summary.py new file mode 100644 index 00000000..ca01b7b1 --- /dev/null +++ b/scripts/ci/summary.py @@ -0,0 +1,37 @@ +#!/usr/bin/env python3 +"""Compact step summaries; complete diagnostics stay in log/JSON artifacts.""" +import json +from pathlib import Path +import sys + +job, status, seconds, directory = sys.argv[1:] +root = Path(directory) +lines = [f'### {job} — exit {status}', '', f'Wall time: {seconds}s.', ''] +if job == 'matrix' and (root / 'matrix.json').exists(): + for backend, report in json.loads((root / 'matrix.json').read_text())['passes'].items(): + lines += [f'#### {backend}', '', '```json', json.dumps(report['summary'], indent=2), '```', '', 'Newly failing assertions:', ''] + failures = [r for r in report['results'] if r['classification'] == 'newly fail'] + lines += [f'- `{r["file"]}:{r["line"]}`: {r["assert"]}' for r in failures] or ['None.'] + lines += ['', 'Known-failure audit issues:', ''] + lines += [f'- `{r["file"]}:{r["line"]}`: {r["status"]}' for r in report.get('known_failure_issues', [])] or ['None.'] + lines += [''] +elif job == 'sweep' and (root / 'sweep.json').exists(): + report = json.loads((root / 'sweep.json').read_text()) + lines += [f'Compared {report["files"]} sources and {report["output_files_compared"]} emitted files.', ''] + for key in ('exit_changes', 'stdout_changes', 'stderr_changes', 'output_file_changes', 'output_changes'): + lines += [f'{key}: {len(report[key])}'] + lines += ['', 'Changed sources (first 100; all details in the artifact):', ''] + lines += [f'- `{r["file"]}`' for r in report['output_changes'][:100]] or ['None.'] +elif job == 'fuzz' and (root / 'results.json').exists(): + counts = {} + failures = [] + for r in json.loads((root / 'results.json').read_text()): + counts[r['status']] = counts.get(r['status'], 0) + 1 + if r['status'] != 'pass': + failures.append(r) + lines += ['```json', json.dumps(counts, indent=2), '```', ''] + lines += [f'- seed {r["seed"]}: {r["status"]}' for r in failures[:100]] +else: + # unittest output is short; bound infrastructure errors to the last lines. + lines += ['```text', '\n'.join((root / 'log.txt').read_text().splitlines()[-100:]), '```'] +(root / 'summary.md').write_text('\n'.join(lines) + '\n') diff --git a/scripts/fuzz_diff.py b/scripts/fuzz_diff.py index 33376cfa..1ea60335 100644 --- a/scripts/fuzz_diff.py +++ b/scripts/fuzz_diff.py @@ -255,8 +255,8 @@ def minimize(src, args, check): def differential_status(mir, z80): - if mismatch(mir): return 'MIR2 != oracle' if re.search(r'(?i)assembl|invalid instruction|unknown instruction', z80['error']): return 'assembly failure' + if mismatch(mir): return 'MIR2 != oracle' if mir['pass'] and mismatch(z80): return 'Z80 != MIR2' if mir['pass'] and z80['pass']: return 'pass' return 'compiler error' @@ -306,6 +306,8 @@ def main(): p.add_argument('--timeout', type=float, default=30) p.add_argument('--output', type=Path, required=True) p.add_argument('--no-reduce', action='store_true', help='save full reproducers and skip deletion reduction for fast triage') + p.add_argument('--fail-on', choices=('all', 'backend'), default='all', + help='all: fail on any finding (default); backend: fail only on Z80/MIR2 mismatches and assembly failures; report all findings') a = p.parse_args() if a.timeout <= 0: p.error('--timeout must be positive') @@ -319,7 +321,9 @@ def main(): for r in results: if r['status'] != 'pass': print(f'seed {r["seed"]}: {r["status"]}: {r["error"]}') - return int(any(r['status'] != 'pass' for r in results)) + blocking = {'Z80 != MIR2', 'assembly failure'} + return int(any(r['status'] != 'pass' if a.fail_on == 'all' else r['status'] in blocking + for r in results)) if __name__ == '__main__': diff --git a/scripts/test_judges.py b/scripts/test_judges.py index 6870b15b..9f660fcb 100644 --- a/scripts/test_judges.py +++ b/scripts/test_judges.py @@ -449,6 +449,25 @@ def test_real_frontends_listing_execution_sandbox_controls(self): class FuzzTests(unittest.TestCase): + def test_cli_exit_policy_preserves_findings(self): + statuses = ('pass', 'MIR2 != oracle', 'Z80 != MIR2', + 'assembly failure', 'compiler error', 'oracle error') + with tempfile.TemporaryDirectory() as temp: + for policy in ('all', 'backend'): + for status in statuses: + with self.subTest(policy=policy, status=status): + finding = {'seed': 56, 'status': status, 'error': 'diagnostic'} + argv = ['fuzz_diff.py', '--mz', 'mz', '--seed', '56', '--count', '1', + '--output', temp, '--fail-on', policy] + with patch.object(sys, 'argv', argv), patch.object(fuzz, 'fuzz_one', return_value=finding) as run, patch('builtins.print') as log: + code = fuzz.main() + expected = status != 'pass' if policy == 'all' else status in ('Z80 != MIR2', 'assembly failure') + self.assertEqual(code, int(expected)) + self.assertEqual(run.call_args.args[0], 56) + self.assertEqual(json.loads((Path(temp) / 'results.json').read_text()), [finding]) + if status != 'pass': + log.assert_any_call(f'seed 56: {status}: diagnostic') + def test_tool_exception_exit(self): with tempfile.TemporaryDirectory() as temp: proc=subprocess.run([sys.executable,fuzz.__file__,'--mz',str(Path(temp)/'missing'),'--count','1','--output',temp],capture_output=True,text=True) @@ -482,6 +501,7 @@ def test_differential_classes(self): self.assertEqual(fuzz.differential_status(wrong, wrong), 'MIR2 != oracle') self.assertEqual(fuzz.differential_status(passed, wrong), 'Z80 != MIR2') self.assertEqual(fuzz.differential_status(passed, {'pass':False,'error':'assembly failed'}), 'assembly failure') + self.assertEqual(fuzz.differential_status(wrong, {'pass':False,'error':'assembly failed'}), 'assembly failure') self.assertEqual(fuzz.differential_status(passed, passed), 'pass') def test_seed_and_parallel_oracle(self): From 45cc265d2537293d7499dd8723894bed638fbbda Mon Sep 17 00:00:00 2001 From: Alice Vinogradova Date: Fri, 2 Oct 2026 15:43:57 +0000 Subject: [PATCH 2/3] ci: compare tested merge trees and expose direct-call fuzz findings Gate compiler errors and low fuzz pass rates, cover assertion execution paths in controls, and retain direct-call findings as report-only pending fixes. Remove baseline caching, bound summaries, and let nightly gates fail visibly. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01Wrwo36SzxKRZYTgpDo7hzo --- .github/workflows/assert-checks.yml | 13 ++--- .github/workflows/ci.yml | 15 ++++-- scripts/README.md | 74 ++++++++++++++++++------- scripts/ci/check-regression.sh | 22 ++++++++ scripts/ci/lint.sh | 6 ++- scripts/ci/metadata.sh | 4 +- scripts/ci/run.sh | 17 +++--- scripts/ci/simulate.sh | 6 ++- scripts/ci/summary.py | 34 ++++++++++-- scripts/fuzz_diff.py | 45 +++++++++++----- scripts/fuzz_findings.md | 44 +++++++++++++++ scripts/test_ci.py | 83 +++++++++++++++++++++++++++++ scripts/test_judges.py | 46 +++++++++++++--- 13 files changed, 342 insertions(+), 67 deletions(-) create mode 100644 scripts/test_ci.py diff --git a/.github/workflows/assert-checks.yml b/.github/workflows/assert-checks.yml index 1a10cfd6..b15b2f5e 100644 --- a/.github/workflows/assert-checks.yml +++ b/.github/workflows/assert-checks.yml @@ -38,12 +38,7 @@ jobs: BASE_REV: ${{ inputs.base }} NIGHTLY: ${{ inputs.nightly }} run: bash scripts/ci/metadata.sh - - name: Cache baseline compiler - uses: actions/cache@v4 - with: - path: ${{ runner.temp }}/judge-bin/baseline-mz - key: baseline-mz-${{ runner.os }}-${{ runner.arch }}-${{ steps.metadata.outputs.base }}-${{ hashFiles('minzc/go.sum') }} - - name: Build head and baseline + - name: Build candidate and baseline env: BASE_SHA: ${{ steps.metadata.outputs.base }} CI_BIN: ${{ runner.temp }}/judge-bin @@ -54,14 +49,14 @@ jobs: path: | ${{ runner.temp }}/judge-bin/*-mz ${{ runner.temp }}/judge-bin/base.tar.gz - retention-days: 2 + retention-days: 7 checks: name: ${{ inputs.nightly && matrix.job == 'fuzz' && 'Long fuzz' || matrix.name }} needs: compilers runs-on: ubuntu-latest timeout-minutes: 40 - continue-on-error: ${{ inputs.nightly || matrix.job == 'sweep' }} + continue-on-error: ${{ !inputs.nightly && matrix.job == 'sweep' }} strategy: fail-fast: false matrix: @@ -74,6 +69,8 @@ jobs: name: Default output sweep - job: fuzz name: Short fuzz + - job: fuzz-direct + name: Direct-call fuzz findings steps: - uses: actions/checkout@v4 with: diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6706a046..067cf022 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -117,8 +117,8 @@ jobs: name: Compiler checks uses: ./.github/workflows/assert-checks.yml with: - candidate: ${{ github.event.pull_request.head.sha || github.sha }} - base: ${{ github.event.pull_request.base.sha || 'HEAD~1' }} + candidate: ${{ github.sha }} + base: HEAD^1 advisory_lint: name: Advisory Go lint @@ -129,17 +129,24 @@ jobs: steps: - uses: actions/checkout@v4 with: - ref: ${{ github.event.pull_request.head.sha }} + ref: ${{ github.sha }} fetch-depth: 0 - uses: actions/setup-go@v5 with: go-version: '1.26.8' cache-dependency-path: minzc/go.sum + - name: Cache pinned golangci-lint binary + id: lint-cache + uses: actions/cache@v4 + with: + path: ~/go/bin/golangci-lint + key: golangci-lint-${{ runner.os }}-${{ runner.arch }}-v2.13.2-go1.26.8 - name: Install pinned golangci-lint + if: steps.lint-cache.outputs.cache-hit != 'true' run: go install github.com/golangci/golangci-lint/v2/cmd/golangci-lint@v2.13.2 - name: Report new lint issues and changed-file formatting env: - BASE_SHA: ${{ github.event.pull_request.base.sha }} + BASE_SHA: HEAD^1 REPORT_DIR: ${{ runner.temp }}/lint-report run: bash scripts/ci/lint.sh - uses: actions/upload-artifact@v4 diff --git a/scripts/README.md b/scripts/README.md index c1382159..ba2821b3 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -42,12 +42,24 @@ fast smoke triage without deletion reduction. Syntax, compiler, assembly, timeout and oracle errors are counted separately; compiler failures save `seed-N.error.nanz`. `results.json` records all seeds and diagnostics. By default exit status is 1 if any program fails or cannot be checked. -`--fail-on backend` exits 1 only for Z80/MIR2 mismatches and assembly failures; +`--fail-on backend` exits 1 for Z80/MIR2 mismatches, assembly failures, and +compiler errors (including crashes, timeouts, and missing or zero-count receipts). +It also fails if fewer than 95% of seeds pass both backends; all findings still appear in logs, JSON and reproducers. Tool exceptions exit 2 under either policy. Assembly failures take precedence even when MIR2 also disagrees with the oracle. `--timeout` bounds each compiler invocation, including reduction attempts. +The default `--mode folded` asserts `g()`, a wrapper calling `f` with constant +arguments. This exercises mostly constant-folded code and does **not** test Z80 +codegen of `f`. Use `--mode direct-call` to assert `f(args)` directly: the +parameterized function runs in the Z80 harness. Direct mode names that function +`fuzz_entry` to avoid the assembler’s `F` register token and retains the wrapper +as an emission root. It does not repair the known argument-loading failures. This mode currently exposes +compiler/backend findings and is report-only in CI. It becomes blocking once +these findings are fixed in a follow-up. Seeds and inputs are identical across +modes; reduction preserves the selected mode. + Run self-tests with `python3 scripts/test_judges.py`; `go test ./pkg/hir` also runs them, skipping when Python 3 is unavailable. @@ -143,11 +155,13 @@ matrix exits nonzero on both Z80 and MIR2 for each mutant. ## CI checks The existing required **PR gate** keeps its name and Go/toolchain checks. -The `Compiler checks` reusable workflow builds `mz` from the PR head (not the -merge ref) and `github.event.pull_request.base.sha`, then runs these jobs on -PRs and pushes to main: +The `Compiler checks` reusable workflow builds the candidate `mz` from +`github.sha` (the +`refs/pull/N/merge` commit, the same tree tested by **PR gate**) and the baseline +from that merge commit’s first parent (`HEAD^1`, the current base tip), then +runs these jobs on PRs and pushes to main: -- **Assert matrix** compares separate baseline/head inventories on both Z80 +- **Assert matrix** compares separate baseline/merge inventories on both Z80 and MIR2. It preserves `assert_matrix.py`'s exit status, including newly failing, removed/changed assertions, unexpectedly passing controls, and known-failure audit errors. JSON, logs and summaries are uploaded even on @@ -161,15 +175,24 @@ PRs and pushes to main: differences remain in the report. - **Short fuzz** checks fixed seeds 0–2999 with `--no-reduce`. The fuzzer has no baseline mode, so this gate checks the candidate alone with - `--fail-on backend`: Z80/MIR2 mismatches and assembly failures block the PR. - MIR2/Python-oracle disagreements and other findings remain in the job summary - and artifacts without failing the PR. No seeds are skipped. Tool exceptions + `--fail-on backend`: Z80/MIR2 mismatches, assembly failures and compiler errors + block the PR, as does a passing-seed rate below 95%. MIR2/Python-oracle + disagreements remain in summaries and artifacts; they can also trip the floor. + This uses the mostly constant-folded mode described above. No seeds are skipped. + Tool exceptions still fail the job. Full reproducers and JSON are artifacts. - -Workers default to `nproc`. Baseline binaries are cached by full base SHA plus -candidate `minzc/go.sum` hash, OS and architecture; no fallback cache keys are -used. Both binaries and the exact base archive are passed to the jobs within -that workflow run. These new checks are not made required by workflow code; +- **Direct-call fuzz findings** runs the same seeds with `--mode direct-call` + and `--fail-on all`. It reports counts and finding classes in the summary, + retains the raw exit code, and returns success for findings pending fixes. + Tool exceptions still fail this report-only job. + +Workers default to `nproc`. Baseline binaries are built fresh; there is no +baseline compiler cache to save from PR runs. Both binaries and the exact base +archive are passed to jobs within the run, with 7-day artifact retention. +Push-to-main comparisons use `HEAD^1` versus `HEAD`. Metadata uses the exact +same parent-to-candidate diff as the matrix. Step summaries cap individual +text and total output at about 64 KiB per job; full details stay in artifacts. +These new checks are not made required by workflow code; Alice can configure branch protection separately. Positive checks normally use `--no-controls`. A plain `git diff` against the @@ -177,21 +200,29 @@ base enables controls and tuple mutant tests for changes under `scripts/`, `minzc/cmd/minzc/`, `minzc/pkg/pipeline/`, or assertion-parsing frontend packages (`nanz`, `c89`, `pascal`, `plm`, `abap`, `frill`, `lanz`, `lizp`). It also covers `hir/hir.go`, `hir/assert*.go`, and the assertion-executing -`mir2wasm/runner.go` and `mir2llvm/runner.go`. Deleted and renamed paths count. +`mir2wasm/runner.go`, `mir2llvm/runner.go` and `mir2gpu/runner.go`. +The filter also includes `minzc/pkg/emulator/**`, `minzc/pkg/mir2/vm*.go`, +`minzc/pkg/z80asm/**`, `minzc/go.mod`, `minzc/go.sum`, and +`.github/workflows/**`. Receipt printing is in `minzc/cmd/minzc/main.go`; +assert execution is in the pipeline and these backend runners (verified by +searching receipt and assertion executor definitions). Deleted and renamed +paths count. The exact filter is in `scripts/ci/metadata.sh`; extend it when assertion handling moves. `--no-controls` alone cannot catch a weakened checker. **Nightly compiler audit** runs at 03:17 UTC and supports `workflow_dispatch`. It compares main with `HEAD~1`, always enables controls, repeats the matrix with `--runs 2`, runs both mutant tests, and fuzzes seeds 0–9999 using the -fuzzer's default `--fail-on all` policy. Judge jobs -are report-only; artifacts include their raw statuses and summaries. Build or -infrastructure failures remain visible as workflow failures. +fuzzer's `--fail-on all` policy, plus direct-call seeds 0–9999 report-only. +Matrix, self-test, sweep and folded-fuzz failures make nightly jobs red; +there is no blanket nightly `continue-on-error`. Direct-call findings retain +raw statuses and summaries while their report-only job succeeds. -**Advisory Go lint** runs only on PRs from the `minzc/` module using pinned +**Advisory Go lint** runs only on PRs from the `minzc/` module using a cached, +pinned `golangci-lint v2.13.2` (built with Go 1.26.8). Its minimal configuration runs govet, staticcheck, errcheck, ineffassign and unused with -`--new-from-rev=` over `./pkg/... ./cmd/...`. This scope +`--new-from-rev=` over `./pkg/... ./cmd/...`. This scope excludes the intentionally invalid scratch programs in `minzc/scripts/analysis` that cannot be package-loaded. `gofmt -l` checks only changed Go files. Warnings are GitHub annotations; JSON, tool errors and counts are artifacts. @@ -215,7 +246,7 @@ change its final status. `CONTROLS` overrides the committed-path filter when simulating uncommitted changes. For a nightly simulation also set `NIGHTLY=true RUNS=2 FUZZ_COUNT=10000 BASE_REV=HEAD~1`. To rerun one job, set `CI_BIN` to the printed compiler directory, `REPORT_DIR` to a writable output -directory, and run `bash scripts/ci/run.sh matrix` (or `judges`, `sweep`, `fuzz`). +directory, and run `bash scripts/ci/run.sh matrix` (or `judges`, `sweep`, `fuzz`, `fuzz-direct`). `MATRIX_ROOT` selects a disposable checkout; `MATRIX_GLOB` narrows a matrix run for a regression fixture. CI leaves both unset and audits the full corpus. @@ -229,3 +260,6 @@ CI_BIN=/tmp/minz-ci.YOUR_RUN JOBS=4 bash scripts/ci/check-regression.sh This creates a tracked fixture and compiler copy under `/tmp`, requires exit 0 from the original compiler, substitutes OR for XOR in the copy, then requires matrix exit 1 with a newly failing Z80 assertion and unchanged MIR2 results. +It also compares 300 direct-call seeds and requires a passing seed to become a +Z80/MIR2 mismatch under the XOR→OR mutant; pre-existing findings cannot satisfy +that check. CI filter/summary tests run with `python3 scripts/test_ci.py`. diff --git a/scripts/ci/check-regression.sh b/scripts/ci/check-regression.sh index 04a7b0ae..31144860 100755 --- a/scripts/ci/check-regression.sh +++ b/scripts/ci/check-regression.sh @@ -44,3 +44,25 @@ assert passes['z80']['summary']['newly fail'] > 0 assert passes['mir2']['summary']['newly fail'] == 0 print('Unmodified exit 0; XOR→OR mutant exit 1; Z80 regression detected; MIR2 unchanged.') PY + +# Existing direct-call findings do not establish mutation sensitivity. Require +# a seed judged correctly by the original to become a Z80-only wrong value. +for build in green red; do + compiler="$original_bin/candidate-mz" + [[ $build == red ]] && compiler="$CI_BIN/candidate-mz" + raw=0 + python3 scripts/fuzz_diff.py --mz "$compiler" --mode direct-call --seed 0 \ + --count 300 -j "$JOBS" --no-reduce --output "$CI_BIN/fuzz-$build" \ + > "$CI_BIN/fuzz-$build.log" 2>&1 || raw=$? + printf 'Direct-call %s: raw exit %s\n' "$build" "$raw" + [[ $raw == 0 || $raw == 1 ]] || exit "$raw" +done +python3 - "$CI_BIN" <<'PYTEST' +import json, pathlib, sys +root = pathlib.Path(sys.argv[1]) +original = {r['seed']: r for r in json.loads((root / 'fuzz-green/results.json').read_text())} +mutant = json.loads((root / 'fuzz-red/results.json').read_text()) +caught = [r['seed'] for r in mutant if original[r['seed']]['status'] == 'pass' and r['status'] == 'Z80 != MIR2'] +assert caught, 'Direct-call fuzz must detect a new XOR-to-OR wrong value' +print(f'Direct-call XOR→OR mutant caught at {len(caught)} previously passing seeds: {caught}') +PYTEST diff --git a/scripts/ci/lint.sh b/scripts/ci/lint.sh index 0257fe60..9464115a 100755 --- a/scripts/ci/lint.sh +++ b/scripts/ci/lint.sh @@ -25,7 +25,7 @@ for issue in issues: print(f'::warning file=minzc/{pos["Filename"]},line={pos["Line"]},col={pos["Column"]}::{name}: {text}') # gofmt checks whole changed files, including unchanged lines, but never old files. paths = subprocess.check_output(['git', 'diff', '--name-only', '--diff-filter=ACMR', '-z', - os.environ['BASE_SHA'], 'HEAD', '--', 'minzc/**/*.go']).split(b'\0') + os.environ['BASE_SHA'], 'HEAD', '--', ':(glob)minzc/**/*.go']).split(b'\0') format_issues = 0 for path in paths: if not path: @@ -36,7 +36,9 @@ for path in paths: format_issues += 1 print(f'::warning file={name},line=1::gofmt: run gofmt on this changed file') summary = f'### Advisory lint\n\nExit {sys.argv[2]}; wall {sys.argv[3]}s; new issues {len(issues)}: {counts}; gofmt files {format_issues}.\n' -summary += '\n```text\n' + '\n'.join((root / 'lint.log').read_text().splitlines()[-50:]) + '\n```\n' +summary += '\n```text\n' + '\n'.join(line[:1024] for line in (root / 'lint.log').read_text().splitlines()[-50:]) + '\n```\n' +if len(summary.encode('utf-8')) > 64 * 1024: + summary = summary.encode('utf-8')[:64 * 1024].decode('utf-8', errors='ignore') + '\n[Truncated; see artifact.]\n' (root / 'summary.md').write_text(summary) print(summary) with open(os.environ.get('GITHUB_STEP_SUMMARY', os.devnull), 'a') as out: diff --git a/scripts/ci/metadata.sh b/scripts/ci/metadata.sh index 8eca7832..3a388c7e 100755 --- a/scripts/ci/metadata.sh +++ b/scripts/ci/metadata.sh @@ -11,7 +11,9 @@ else changed=$(git diff --no-renames --name-only "$base" HEAD) while IFS= read -r path; do case "$path" in - scripts/* | minzc/cmd/minzc/* | minzc/pkg/pipeline/* | \ + .github/workflows/* | minzc/go.mod | minzc/go.sum | \ + minzc/pkg/emulator/* | minzc/pkg/z80asm/* | minzc/pkg/mir2/vm*.go | \ + minzc/pkg/mir2gpu/runner.go | scripts/* | minzc/cmd/minzc/* | minzc/pkg/pipeline/* | \ minzc/pkg/nanz/* | minzc/pkg/c89/* | minzc/pkg/pascal/* | \ minzc/pkg/plm/* | minzc/pkg/abap/* | minzc/pkg/frill/* | \ minzc/pkg/lanz/* | minzc/pkg/lizp/* | minzc/pkg/hir/hir.go | \ diff --git a/scripts/ci/run.sh b/scripts/ci/run.sh index 8952f040..8d808176 100755 --- a/scripts/ci/run.sh +++ b/scripts/ci/run.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash set -euo pipefail : "${CI_BIN:?}" -job=${1:?matrix, judges, sweep, or fuzz} +job=${1:?matrix, judges, sweep, fuzz, or fuzz-direct} report_dir=${REPORT_DIR:-$CI_BIN/reports/$job} mkdir -p "$report_dir" chmod +x "$CI_BIN/candidate-mz" "$CI_BIN/baseline-mz" @@ -10,7 +10,7 @@ controls=--no-controls [[ ${CONTROLS:-false} == true ]] && controls=--controls start=$SECONDS # Capture the original judge status even through tee, then publish a summary -# before returning it. Advisory policy belongs in the workflow, never here. +# before returning it. Direct-call findings become success only after reporting. set +e case "$job" in matrix) @@ -29,6 +29,9 @@ case "$job" in judges) JUDGE_MZ="$CI_BIN/candidate-mz" python3 scripts/test_judges.py 2>&1 | tee "$report_dir/log.txt" status=${PIPESTATUS[0]} + python3 scripts/test_ci.py 2>&1 | tee -a "$report_dir/log.txt" + ci_status=${PIPESTATUS[0]} + (( ci_status > status )) && status=$ci_status if [[ ${CONTROLS:-false} == true ]]; then python3 scripts/test_assert_mutants.py 2>&1 | tee -a "$report_dir/log.txt" mutant_status=${PIPESTATUS[0]} @@ -40,10 +43,10 @@ case "$job" in --candidate "$CI_BIN/candidate-mz" -j "$jobs" --json "$report_dir/sweep.json" 2>&1 | tee "$report_dir/log.txt" status=${PIPESTATUS[0]} ;; - fuzz) - # No baseline mode: gate backend failures, report all oracle findings. - extra=(--fail-on backend) - [[ ${NIGHTLY:-false} == true ]] && extra=(--fail-on all) + fuzz|fuzz-direct) + extra=(--fail-on backend --mode folded) + [[ ${NIGHTLY:-false} == true ]] && extra=(--fail-on all --mode folded) + [[ $job == fuzz-direct ]] && extra=(--fail-on all --mode direct-call) python3 scripts/fuzz_diff.py --mz "$CI_BIN/candidate-mz" --seed 0 \ --count "${FUZZ_COUNT:-3000}" -j "$jobs" --no-reduce \ --output "$report_dir" "${extra[@]}" 2>&1 | tee "$report_dir/log.txt" @@ -55,4 +58,6 @@ set -e printf '\n%s: exit %s, wall %ss, workers %s\n' "$job" "$status" "$((SECONDS-start))" "$jobs" | tee -a "$report_dir/log.txt" python3 scripts/ci/summary.py "$job" "$status" "$((SECONDS-start))" "$report_dir" cat "$report_dir/summary.md" >> "${GITHUB_STEP_SUMMARY:-/dev/null}" +# Direct-call findings are deliberately report-only until compiler fixes land. +[[ $job == fuzz-direct && $status == 1 ]] && exit 0 exit "$status" diff --git a/scripts/ci/simulate.sh b/scripts/ci/simulate.sh index 94ba69df..17ffb871 100755 --- a/scripts/ci/simulate.sh +++ b/scripts/ci/simulate.sh @@ -23,11 +23,13 @@ run_gate() { local start=$SECONDS status=0 "$@" > "$CI_BIN/$name.log" 2>&1 || status=$? printf '%s: exit %s, wall %ss (log: %s)\n' "$name" "$status" "$((SECONDS-start))" "$CI_BIN/$name.log" | tee -a "$CI_BIN/timings.txt" - if [[ $name != sweep && $name != lint && $status != 0 ]]; then failed=1; fi + if [[ $status != 0 && $name != lint ]]; then + if [[ $name != sweep || ${NIGHTLY:-false} == true ]]; then failed=1; fi + fi } run_gate actionlint actionlint .github/workflows/*.yml run_gate build bash scripts/ci/build.sh -for job in matrix judges sweep fuzz; do +for job in matrix judges sweep fuzz fuzz-direct; do export REPORT_DIR="$CI_BIN/reports/$job" run_gate "$job" bash scripts/ci/run.sh "$job" done diff --git a/scripts/ci/summary.py b/scripts/ci/summary.py index ca01b7b1..4a358553 100644 --- a/scripts/ci/summary.py +++ b/scripts/ci/summary.py @@ -4,16 +4,35 @@ from pathlib import Path import sys +# Bound every diagnostic and collection, then bound the whole UTF-8 summary. +# Full listing errors, assertions and seed results remain in JSON artifacts. +MAX_TEXT = 1024 +MAX_ITEMS = 100 +MAX_BYTES = 64 * 1024 + +def compact(value): + if isinstance(value, str): + return value if len(value) <= MAX_TEXT else value[:MAX_TEXT] + '… [truncated]' + if isinstance(value, dict): + entries = list(value.items()) + result = {compact(k): compact(v) for k, v in entries[:MAX_ITEMS]} + if len(entries) > MAX_ITEMS: + result['[truncated entries]'] = len(entries) - MAX_ITEMS + return result + if isinstance(value, list): + return [compact(v) for v in value[:MAX_ITEMS]] + ([f'[truncated {len(value)-MAX_ITEMS} entries]'] if len(value) > MAX_ITEMS else []) + return value + job, status, seconds, directory = sys.argv[1:] root = Path(directory) lines = [f'### {job} — exit {status}', '', f'Wall time: {seconds}s.', ''] if job == 'matrix' and (root / 'matrix.json').exists(): for backend, report in json.loads((root / 'matrix.json').read_text())['passes'].items(): - lines += [f'#### {backend}', '', '```json', json.dumps(report['summary'], indent=2), '```', '', 'Newly failing assertions:', ''] + lines += [f'#### {backend}', '', '```json', json.dumps(compact(report['summary']), indent=2), '```', '', 'Newly failing assertions:', ''] failures = [r for r in report['results'] if r['classification'] == 'newly fail'] - lines += [f'- `{r["file"]}:{r["line"]}`: {r["assert"]}' for r in failures] or ['None.'] + lines += [f'- `{r["file"]}:{r["line"]}`: {compact(r["assert"])}' for r in failures[:MAX_ITEMS]] or ['None.'] lines += ['', 'Known-failure audit issues:', ''] - lines += [f'- `{r["file"]}:{r["line"]}`: {r["status"]}' for r in report.get('known_failure_issues', [])] or ['None.'] + lines += [f'- `{r["file"]}:{r["line"]}`: {r["status"]}' for r in report.get('known_failure_issues', [])[:MAX_ITEMS]] or ['None.'] lines += [''] elif job == 'sweep' and (root / 'sweep.json').exists(): report = json.loads((root / 'sweep.json').read_text()) @@ -22,7 +41,9 @@ lines += [f'{key}: {len(report[key])}'] lines += ['', 'Changed sources (first 100; all details in the artifact):', ''] lines += [f'- `{r["file"]}`' for r in report['output_changes'][:100]] or ['None.'] -elif job == 'fuzz' and (root / 'results.json').exists(): +elif job in ('fuzz', 'fuzz-direct') and (root / 'results.json').exists(): + if job == 'fuzz-direct': + lines += ['Direct-call findings are report-only pending compiler fixes; tool errors still fail the job.', ''] counts = {} failures = [] for r in json.loads((root / 'results.json').read_text()): @@ -34,4 +55,7 @@ else: # unittest output is short; bound infrastructure errors to the last lines. lines += ['```text', '\n'.join((root / 'log.txt').read_text().splitlines()[-100:]), '```'] -(root / 'summary.md').write_text('\n'.join(lines) + '\n') +summary = ('\n'.join(lines) + '\n').encode('utf-8') +if len(summary) > MAX_BYTES: + summary = summary[:MAX_BYTES].decode('utf-8', errors='ignore').encode('utf-8') + b'\n\n[Summary truncated; see artifacts.]\n' +(root / 'summary.md').write_bytes(summary) diff --git a/scripts/fuzz_diff.py b/scripts/fuzz_diff.py index 1ea60335..8979bb20 100644 --- a/scripts/fuzz_diff.py +++ b/scripts/fuzz_diff.py @@ -209,18 +209,30 @@ def generated(seed): return src, args -def with_assert(src, args): +def with_assert(src, args, mode="folded"): + """Direct calls keep f parameterized when Z80 executes the assertion.""" val = Interp(src).call('f', args) - return src + f'\nfun g() -> u16 {{ return f({args[0]}, {args[1]}, {args[2]}) }}\nassert g() == {val} via z80\n' + # Keep the existing wrapper as an emission root in both modes; only the + # assertion target changes, so direct-call executes f rather than folded g. + wrapper = f'\nfun g() -> u16 {{ return f({args[0]}, {args[1]}, {args[2]}) }}\n' + target = f'f({args[0]}, {args[1]}, {args[2]})' if mode == 'direct-call' else 'g()' + program = src + wrapper + f'assert {target} == {val} via z80\n' + # The assembler treats F as the flags register, making CALL f invalid. + # Use an unambiguous symbol so direct mode reaches the function body. + return re.sub(r'\bf(?=\()', 'fuzz_entry', program) if mode == 'direct-call' else program def mismatch(result): # Syntax/codegen errors and timeouts are reported separately, never used # as evidence that a reduced program still reproduces a wrong value. + if 'exit_code' in result and (result['exit_code'] != 1 or + result.get('executed') != 1 or + result.get('passed') != 0 or result.get('failed') != 1): + return False return not result['pass'] and bool(re.search(r'(?i)\bgot\s+-?(?:0x[0-9a-f]+|\d+)[,\s]+(?:want|expected)\b', result['error'])) -def minimize(src, args, check): +def minimize(src, args, check, mode="folded"): """Greedy deletion: 1-minimal over complete helper functions, control blocks and lines. Recompute the oracle for each reduction and accept only wrong-value @@ -244,18 +256,20 @@ def minimize(src, args, check): candidates.append(''.join(lines[:i] + lines[i+1:])) for candidate in candidates: try: - program = with_assert(candidate, args) + program = with_assert(candidate, args, mode) except Exception: continue if mismatch(check(program)): src = candidate changed = True break - return with_assert(src, args) + return with_assert(src, args, mode) def differential_status(mir, z80): if re.search(r'(?i)assembl|invalid instruction|unknown instruction', z80['error']): return 'assembly failure' + # An unjudged backend must not be hidden by the other backend's wrong value. + if any(not r['pass'] and not mismatch(r) for r in (mir, z80)): return 'compiler error' if mismatch(mir): return 'MIR2 != oracle' if mir['pass'] and mismatch(z80): return 'Z80 != MIR2' if mir['pass'] and z80['pass']: return 'pass' @@ -271,10 +285,10 @@ def reduction_check(text, status, check): return check(text) -def fuzz_one(seed, mz, timeout, output, reduce=True): +def fuzz_one(seed, mz, timeout, output, reduce=True, mode="folded"): src, args = generated(seed) try: - program = with_assert(src, args) + program = with_assert(src, args, mode) except Exception as e: return {'seed': seed, 'status': 'oracle error', 'error': str(e)} with tempfile.TemporaryDirectory(prefix='fuzz-diff-') as temp: @@ -285,11 +299,11 @@ def check(text, backend='z80'): mir = check(program, 'mir2') z80 = check(program) status = differential_status(mir, z80) - result = mir if status == 'MIR2 != oracle' else z80 + result = mir if status == 'MIR2 != oracle' or (status == 'compiler error' and not mir['pass'] and not mismatch(mir)) else z80 if status in ('MIR2 != oracle', 'Z80 != MIR2'): def reduce_check(text): return reduction_check(text, status, check) - reduced = minimize(src, args, reduce_check) if reduce else program + reduced = minimize(src, args, reduce_check, mode) if reduce else program (output / f'seed-{seed}.nanz').write_text(reduced) (output / f'seed-{seed}.original.nanz').write_text(program) elif status in ('compiler error', 'assembly failure'): @@ -307,22 +321,27 @@ def main(): p.add_argument('--output', type=Path, required=True) p.add_argument('--no-reduce', action='store_true', help='save full reproducers and skip deletion reduction for fast triage') p.add_argument('--fail-on', choices=('all', 'backend'), default='all', - help='all: fail on any finding (default); backend: fail only on Z80/MIR2 mismatches and assembly failures; report all findings') + help='all: fail on any finding (default); backend: fail on backend/compiler failures or less than 95%% passing seeds; report all findings') + p.add_argument('--mode', choices=('folded', 'direct-call'), default='folded', + help='folded: assert a constant-foldable wrapper; direct-call: execute f with assertion arguments') a = p.parse_args() if a.timeout <= 0: p.error('--timeout must be positive') a.output.mkdir(parents=True, exist_ok=True) mz = str(Path(a.mz).resolve()) with concurrent.futures.ThreadPoolExecutor(max_workers=a.j) as pool: - results = list(pool.map(lambda seed: fuzz_one(seed, mz, a.timeout, a.output, not a.no_reduce), range(a.seed, a.seed+a.count))) + results = list(pool.map(lambda seed: fuzz_one(seed, mz, a.timeout, a.output, not a.no_reduce, a.mode), range(a.seed, a.seed+a.count))) (a.output / 'results.json').write_text(json.dumps(results, indent=2) + '\n') for status in ['pass', 'MIR2 != oracle', 'Z80 != MIR2', 'assembly failure', 'compiler error', 'oracle error']: print(f'{status}: {sum(r["status"] == status for r in results)}') for r in results: if r['status'] != 'pass': print(f'seed {r["seed"]}: {r["status"]}: {r["error"]}') - blocking = {'Z80 != MIR2', 'assembly failure'} - return int(any(r['status'] != 'pass' if a.fail_on == 'all' else r['status'] in blocking + passes = sum(r['status'] == 'pass' for r in results) + floor_failed = passes * 100 < len(results) * 95 + print(f'Passing seeds: {passes}/{len(results)}; minimum 95%; mode: {a.mode}') + blocking = {'Z80 != MIR2', 'assembly failure', 'compiler error'} + return int(floor_failed or any(r['status'] != 'pass' if a.fail_on == 'all' else r['status'] in blocking for r in results)) diff --git a/scripts/fuzz_findings.md b/scripts/fuzz_findings.md index 579b89cb..1420b6b5 100644 --- a/scripts/fuzz_findings.md +++ b/scripts/fuzz_findings.md @@ -62,3 +62,47 @@ reduced 56/399 fixtures, real compiler integration tests also verify that forced MIR2 and Z80 return identical wrong values (15014 and 17814). The old wrong-value count described only completed Z80 evaluations; it must not be interpreted as the count of Z80 codegen discrepancies. + +## CI direct-call verification (C2 FIX1) + +Compiler tree: current main `04e0ba172fbfbb3af08153d66607ab6874faeeba`. This CI change does +not change compiler behavior. At four workers, the PR direct-call job ran +seeds 0–2999 without reduction: raw fuzzer exit 1, report-only wrapper exit 0, +wall 69 seconds. The requested seeds 0–999 subset produced: + +| Primary class | Seeds | +| --- | ---: | +| Pass | 28 | +| Z80≠MIR2 (MIR2 agrees with oracle) | 73 | +| Assembly failure | 835 | +| Compiler/execution error | 59 | +| MIR2≠oracle | 5 | +| Oracle interpreter error | 0 | + +All 835 assembly diagnostics name invalid `LD` operands. Seed 1 reproduces +`assemble errors: line 4: unknown instruction or invalid operands: LD`. +The 59 compiler/execution errors are classified separately from wrong values; +for example, seeds 37, 53 and 57 report `run: execution limit exceeded`. +Assembly errors retain precedence over simultaneous MIR2/oracle findings; +both backend results remain in JSON. + +Direct mode asserts the parameterized function instead of the folded `g()` +wrapper. Its entry symbol is `fuzz_entry`: a probe of the original symbol `f` +also failed with invalid `CALL`, because `F` is an assembler register token. +The wrapper remains as an emission root; inputs and function bodies retain +the seeded generator's semantics. The seed-1 argument-loading failure remains. + +For all 3000 seeds: 89 pass, 207 Z80≠MIR2, 2494 assembly failures, +196 compiler/execution errors, 14 MIR2≠oracle, zero oracle errors. Local full +results, reproducers, summaries and the first-1000 JSON subset are retained +under `/tmp/C2-fix/reports/fuzz-direct/`. Regenerate with: + +```sh +python3 scripts/fuzz_diff.py --mz /tmp/mz --mode direct-call --seed 0 --count 1000 -j 4 --no-reduce --output /tmp/direct-call-findings +``` + +The disposable XOR→OR codegen mutant becomes a new Z80≠MIR2 mismatch at +10 previously passing direct-call seeds in 0–299: 20, 28, 76, 85, 97, 164, +189, 202, 229 and 257. The matrix wrapper also rejects the mutant. These +mutation checks require new wrong values; existing assembly/execution findings +cannot satisfy them. Direct-call CI stays report-only until findings are fixed. diff --git a/scripts/test_ci.py b/scripts/test_ci.py new file mode 100644 index 00000000..d243eff2 --- /dev/null +++ b/scripts/test_ci.py @@ -0,0 +1,83 @@ +#!/usr/bin/env python3 +"""Exercise CI tree selection, controls filtering and bounded summaries.""" +import json +import os +from pathlib import Path +import subprocess +import tempfile +import unittest + +ROOT = Path(__file__).resolve().parents[1] + +class CITests(unittest.TestCase): + def test_merge_parent_and_controls_filter(self): + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + def git(*args): + return subprocess.check_output(['git', '-C', temp, *args], text=True).strip() + git('init', '-q', '-b', 'main') + git('config', 'user.name', 'CI fixture') + git('config', 'user.email', 'fixture@example.invalid') + (root / 'readme').write_text('base') + git('add', '.') + git('commit', '-qm', 'base') + # PR branch and current base both move: merge^1 must select current base. + git('checkout', '-qb', 'pr') + (root / 'feature').write_text('pr') + git('add', '.') + git('commit', '-qm', 'pr') + git('checkout', '-q', 'main') + (root / 'scripts').mkdir() + (root / 'scripts/base-only.py').write_text('base movement') + git('add', '.') + git('commit', '-qm', 'base moved') + current_base = git('rev-parse', 'HEAD') + git('merge', '--no-ff', '-qm', 'merge', 'pr') + env = dict(os.environ, BASE_REV='HEAD^1', NIGHTLY='false', GITHUB_OUTPUT=os.devnull) + def metadata(): + return subprocess.check_output(['bash', str(ROOT / 'scripts/ci/metadata.sh')], cwd=temp, env=env, text=True) + self.assertEqual(metadata(), f'base={current_base}\ncontrols=false\n') + paths = ['scripts/test.py', 'minzc/cmd/minzc/main.go', + 'minzc/pkg/pipeline/pipeline.go', 'minzc/pkg/hir/hir.go', + 'minzc/pkg/hir/assert_stats_test.go', 'minzc/pkg/nanz/parse.go', + 'minzc/pkg/c89/c89.go', 'minzc/pkg/pascal/parse.go', + 'minzc/pkg/plm/lower.go', 'minzc/pkg/abap/lower.go', + 'minzc/pkg/frill/frill.go', 'minzc/pkg/lanz/lower.go', + 'minzc/pkg/lizp/parse.go', 'minzc/pkg/emulator/z80.go', + 'minzc/pkg/mir2/vm.go', 'minzc/pkg/mir2/vm_call.go', + 'minzc/pkg/z80asm/assembler.go', 'minzc/pkg/mir2gpu/runner.go', + 'minzc/pkg/mir2wasm/runner.go', 'minzc/pkg/mir2llvm/runner.go', + 'minzc/go.mod', 'minzc/go.sum', '.github/workflows/ci.yml'] + for path in paths + ['docs/guide.md', 'minzc/pkg/mir2/z80codegen.go']: + with self.subTest(path=path): + target = root / path + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text('change') + git('add', '.') + git('commit', '-qm', 'test path') + self.assertIn(f'controls={str(path in paths).lower()}', metadata()) + git('reset', '--hard', 'HEAD^1') + # Deletions and both sides of a rename retain controls coverage. + git('rm', 'scripts/base-only.py') + git('commit', '-qm', 'delete') + self.assertIn('controls=true', metadata()) + git('reset', '--hard', 'HEAD^1') + git('mv', 'scripts/base-only.py', 'renamed.py') + git('commit', '-qm', 'rename') + self.assertIn('controls=true', metadata()) + + def test_summary_limits(self): + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + report = {'summary': {'listing_errors': {'candidate': {str(i): 'E' * 20000 for i in range(200)}}}, + 'results': [{'classification': 'newly fail', 'file': 'case.nanz', 'line': 1, + 'assert': 'X' * 20000} for _ in range(1000)]} + (root / 'matrix.json').write_text(json.dumps({'passes': {'z80': report, 'mir2': report}})) + subprocess.run(['python3', str(ROOT / 'scripts/ci/summary.py'), 'matrix', '1', '1', temp], check=True) + summary = (root / 'summary.md').read_bytes() + self.assertLess(len(summary), 66 * 1024) + self.assertIn(b'truncated', summary) + self.assertNotIn(b'E' * 1025, summary) + +if __name__ == '__main__': + unittest.main() diff --git a/scripts/test_judges.py b/scripts/test_judges.py index 9f660fcb..4f62bbff 100644 --- a/scripts/test_judges.py +++ b/scripts/test_judges.py @@ -461,13 +461,46 @@ def test_cli_exit_policy_preserves_findings(self): '--output', temp, '--fail-on', policy] with patch.object(sys, 'argv', argv), patch.object(fuzz, 'fuzz_one', return_value=finding) as run, patch('builtins.print') as log: code = fuzz.main() - expected = status != 'pass' if policy == 'all' else status in ('Z80 != MIR2', 'assembly failure') + expected = status != 'pass' # Single failing seed also violates the 95% floor. self.assertEqual(code, int(expected)) self.assertEqual(run.call_args.args[0], 56) self.assertEqual(json.loads((Path(temp) / 'results.json').read_text()), [finding]) if status != 'pass': log.assert_any_call(f'seed 56: {status}: diagnostic') + def test_backend_floor_and_compiler_errors(self): + with tempfile.TemporaryDirectory() as temp: + for bad, count, expected in [('MIR2 != oracle', 1, 0), + ('oracle error', 1, 0), + ('MIR2 != oracle', 2, 1), + ('compiler error', 1, 1)]: + results = ([{'seed': i, 'status': bad, 'error': 'diagnostic'} for i in range(count)] + + [{'seed': i, 'status': 'pass', 'error': ''} for i in range(count, 20)]) + argv = ['fuzz_diff.py', '--mz', 'mz', '--count', '20', '--output', temp, '--fail-on', 'backend'] + with patch.object(sys, 'argv', argv), patch.object(fuzz, 'fuzz_one', side_effect=results), patch('builtins.print'): + self.assertEqual(fuzz.main(), expected) + passed = {'pass': True, 'error': ''} + wrong = {'pass': False, 'error': 'got 1, want 2'} + for error in ('crash', 'timeout after 1s', 'missing execution receipt', + 'ASSERTS: executed=0 passed=0 failed=0'): + unjudged = {'pass': False, 'error': error} + self.assertEqual(fuzz.differential_status(passed, unjudged), 'compiler error') + self.assertEqual(fuzz.differential_status(unjudged, passed), 'compiler error') + self.assertEqual(fuzz.differential_status(wrong, unjudged), 'compiler error') + # A wrong-value-looking diagnostic cannot excuse a crash or absent receipt. + for fields in ({'exit_code': -11}, {'exit_code': None}, {'exit_code': 1}, + {'exit_code': 1, 'executed': 0, 'passed': 0, 'failed': 0}): + unjudged = dict(wrong, **fields) + self.assertEqual(fuzz.differential_status(unjudged, passed), 'compiler error') + + def test_direct_call_generation_and_reduction(self): + src, args = fuzz.generated(1) + program = fuzz.with_assert(src, args, 'direct-call') + self.assertNotIn('assert g()', program) + self.assertIn(f'assert fuzz_entry({args[0]}, {args[1]}, {args[2]}) ==', program) + reduced = fuzz.minimize(src, args, lambda text: {'pass': True, 'error': ''}, 'direct-call') + self.assertEqual(program, reduced) + def test_tool_exception_exit(self): with tempfile.TemporaryDirectory() as temp: proc=subprocess.run([sys.executable,fuzz.__file__,'--mz',str(Path(temp)/'missing'),'--count','1','--output',temp],capture_output=True,text=True) @@ -506,11 +539,12 @@ def test_differential_classes(self): def test_seed_and_parallel_oracle(self): seeds = range(30) - sequential = [fuzz.with_assert(*fuzz.generated(s)) for s in seeds] - with concurrent.futures.ThreadPoolExecutor(4) as pool: - parallel = list(pool.map(lambda s: fuzz.with_assert(*fuzz.generated(s)), seeds)) - self.assertEqual(sequential, parallel) - self.assertNotEqual(sequential[0], sequential[1]) + for mode in ('folded', 'direct-call'): + sequential = [fuzz.with_assert(*fuzz.generated(s), mode) for s in seeds] + with concurrent.futures.ThreadPoolExecutor(4) as pool: + parallel = list(pool.map(lambda s: fuzz.with_assert(*fuzz.generated(s), mode), seeds)) + self.assertEqual(sequential, parallel) + self.assertNotEqual(sequential[0], sequential[1]) def test_interpreter(self): src = '''fun f(a: u16, b: u16, c: u16) -> u16 { From 11a1cbdc997c7e010be15162b08afeee5c718568 Mon Sep 17 00:00:00 2001 From: Alice Vinogradova Date: Fri, 2 Oct 2026 15:54:45 +0000 Subject: [PATCH 3/3] ci: skip advisory Go lint when PR changes no Go source --- scripts/ci/lint.sh | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/scripts/ci/lint.sh b/scripts/ci/lint.sh index 9464115a..9ff5c2d6 100755 --- a/scripts/ci/lint.sh +++ b/scripts/ci/lint.sh @@ -4,10 +4,17 @@ set -euo pipefail mkdir -p "$REPORT_DIR" export GOLANGCI_LINT_CACHE=${GOLANGCI_LINT_CACHE:-/tmp/minz-golangci-cache} start=$SECONDS -set +e -(cd minzc && golangci-lint run --new-from-rev="$BASE_SHA" --show-stats=false ./pkg/... ./cmd/...) > "$REPORT_DIR/lint.json" 2> "$REPORT_DIR/lint.log" -status=$? -set -e +changed_go=$(git diff --name-only --diff-filter=ACMR "$BASE_SHA" HEAD -- ':(glob)minzc/**/*.go') +if [[ -z $changed_go ]]; then + printf '{"Issues": []}\n' > "$REPORT_DIR/lint.json" + printf 'No changed Go source files; Go lint skipped.\n' > "$REPORT_DIR/lint.log" + status=0 +else + set +e + (cd minzc && golangci-lint run --new-from-rev="$BASE_SHA" --show-stats=false --output.json.path stdout --output.text.path /dev/null ./pkg/... ./cmd/...) > "$REPORT_DIR/lint.json" 2> "$REPORT_DIR/lint.log" + status=$? + set -e +fi python3 - "$REPORT_DIR" "$status" "$((SECONDS-start))" <<'PY' import json, os, pathlib, subprocess, sys root = pathlib.Path(sys.argv[1])