Skip to content

Commit 34d7f78

Browse files
dergoeggedergoegge
authored
Merge pull request #146 from dergoegge/benchmark-pipeline
ci: benchmarking pipeline
2 parents ea874f1 + 7d40f2d commit 34d7f78

4 files changed

Lines changed: 1019 additions & 1 deletion

File tree

‎.github/workflows/benchmark.yml‎

Lines changed: 289 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,289 @@
1+
name: Benchmark
2+
3+
# A/B benchmark of two fuzzamoto-libafl versions: run each version N times for a fixed
4+
# duration in parallel, then aggregate and statistically compare the campaigns.
5+
#
6+
# Build stages (so campaigns never recompile - 2 builds total instead of 2*N):
7+
# - build-base: the version-independent base image (Bitcoin Core + AFL++ + toolchain).
8+
# - build-fuzzer: per version, bakes that ref's compiled fuzzer + Nyx share dir on top
9+
# of the base (Dockerfile.libafl.bench), cached per version.
10+
# Bitcoin Core (the target) is identical across both arms, so the comparison isolates the
11+
# fuzzer change. See ci/benchmark-evaluation.py for the metrics.
12+
#
13+
# Runner specs are inlined in each job's `runs-on:` label (runs-on.com syntax); there is
14+
# no .github/runs-on.yml. Campaign runners are pinned to a single exact instance type so
15+
# every campaign runs on identical hardware (see the campaign job).
16+
#
17+
# Triggers:
18+
# - workflow_dispatch: compare two arbitrary refs (full control over N / duration).
19+
# - PR label 'needs benchmark': full run, N=10, 3600s/campaign (PR base vs head).
20+
# - PR label 'needs benchmark smoke': cheap wiring test, N=2, 300s/campaign.
21+
22+
on:
23+
workflow_dispatch:
24+
inputs:
25+
baseline_ref:
26+
description: "Baseline fuzzamoto git ref"
27+
default: "master"
28+
treatment_ref:
29+
description: "Treatment fuzzamoto git ref"
30+
required: true
31+
runs:
32+
description: "Campaigns per version (N)"
33+
default: "10"
34+
duration_secs:
35+
description: "Seconds per campaign"
36+
default: "3600"
37+
pull_request:
38+
# Gated on the 'needs benchmark' / 'needs benchmark smoke' labels (checked in setup's `if:`).
39+
types: [labeled]
40+
41+
permissions:
42+
contents: read
43+
pull-requests: write # for the PR comment
44+
45+
jobs:
46+
setup:
47+
if: github.event_name == 'workflow_dispatch' || github.event.label.name == 'needs benchmark' || github.event.label.name == 'needs benchmark smoke'
48+
runs-on:
49+
- runs-on=${{ github.run_id }}
50+
- cpu=2+4
51+
- family=m7i+c7i
52+
- image=ubuntu22-full-x64
53+
- extras=s3-cache
54+
outputs:
55+
baseline: ${{ steps.refs.outputs.baseline }}
56+
treatment: ${{ steps.refs.outputs.treatment }}
57+
duration: ${{ steps.refs.outputs.duration }}
58+
runs_json: ${{ steps.refs.outputs.runs_json }} # e.g. "[1,2,...,10]"
59+
spot: ${{ steps.refs.outputs.spot }} # true for smoke tests, false for real benchmarks
60+
steps:
61+
- id: refs
62+
run: |
63+
# Real benchmarks run on-demand (spot=false) so an interruption can't void a long
64+
# campaign; the cheap smoke test runs on spot to save money.
65+
SPOT=false
66+
if [ "${{ github.event_name }}" = "pull_request" ]; then
67+
echo "baseline=${{ github.event.pull_request.base.sha }}" >> "$GITHUB_OUTPUT"
68+
echo "treatment=${{ github.event.pull_request.head.sha }}" >> "$GITHUB_OUTPUT"
69+
if [ "${{ github.event.label.name }}" = "needs benchmark smoke" ]; then
70+
N=2
71+
echo "duration=300" >> "$GITHUB_OUTPUT"
72+
SPOT=true
73+
else
74+
N=10
75+
echo "duration=3600" >> "$GITHUB_OUTPUT"
76+
fi
77+
else
78+
echo "baseline=${{ inputs.baseline_ref }}" >> "$GITHUB_OUTPUT"
79+
echo "treatment=${{ inputs.treatment_ref }}" >> "$GITHUB_OUTPUT"
80+
N=${{ inputs.runs }}
81+
echo "duration=${{ inputs.duration_secs }}" >> "$GITHUB_OUTPUT"
82+
fi
83+
echo "spot=$SPOT" >> "$GITHUB_OUTPUT"
84+
echo "runs_json=$(seq -s, 1 "$N" | sed 's/^/[/;s/$/]/')" >> "$GITHUB_OUTPUT"
85+
86+
build-base:
87+
needs: setup
88+
# Fat instance to build the version-independent base image (Bitcoin Core + AFL++ + deps).
89+
runs-on:
90+
- runs-on=${{ github.run_id }}
91+
- cpu=16+32
92+
- family=c7i+m7i+c7a
93+
- image=ubuntu22-full-x64
94+
- volume=120gb:gp3
95+
- extras=s3-cache # S3-backed docker layer cache (no 10GB cap)
96+
- spot=true
97+
steps:
98+
- uses: actions/checkout@v4 # Bitcoin Core build is version-independent
99+
- uses: docker/setup-buildx-action@v3
100+
- name: Build & cache the base image
101+
uses: docker/build-push-action@v5
102+
with:
103+
context: .
104+
file: ./Dockerfile.libafl
105+
push: false
106+
tags: fuzzamoto-libafl:bench
107+
platforms: linux/amd64
108+
cache-from: type=gha,scope=base
109+
cache-to: type=gha,scope=base,mode=max
110+
111+
build-fuzzer:
112+
needs: [setup, build-base]
113+
# Bake each fuzzer version (compiled binary + Nyx share dir) once, cached per version,
114+
# so the campaign jobs reuse it instead of recompiling. Only compiles the fuzzamoto
115+
# crates (the heavy base is already built), so 8 CPUs is plenty and spot is fine - a
116+
# reclaim mid-bake just retries and resumes from the layer cache.
117+
runs-on:
118+
- runs-on=${{ github.run_id }}
119+
- cpu=8
120+
- family=c7i+m7i+c7a
121+
- image=ubuntu22-full-x64
122+
- volume=120gb:gp3
123+
- extras=s3-cache
124+
- spot=true
125+
- retry=when-interrupted
126+
strategy:
127+
matrix:
128+
version: [baseline, treatment]
129+
steps:
130+
- name: Checkout fuzzer version to bake
131+
uses: actions/checkout@v4
132+
with:
133+
ref: ${{ matrix.version == 'baseline' && needs.setup.outputs.baseline || needs.setup.outputs.treatment }}
134+
- uses: docker/setup-buildx-action@v3
135+
- name: Materialize base image from cache
136+
uses: docker/build-push-action@v5
137+
with:
138+
context: .
139+
file: ./Dockerfile.libafl
140+
load: true
141+
tags: fuzzamoto-libafl:bench
142+
cache-from: type=gha,scope=base
143+
- name: Build & cache the per-version fuzzer image
144+
uses: docker/build-push-action@v5
145+
with:
146+
context: .
147+
file: ./Dockerfile.libafl.bench
148+
build-args: BASE_IMAGE=fuzzamoto-libafl:bench
149+
push: false
150+
tags: fuzzamoto-libafl:bench-${{ matrix.version }}
151+
cache-from: type=gha,scope=bench-${{ matrix.version }}
152+
cache-to: type=gha,scope=bench-${{ matrix.version }},mode=max
153+
154+
campaign:
155+
needs: [setup, build-fuzzer]
156+
# nested-virt exposes /dev/kvm (required by Nyx). IMPORTANT: pinned to a single,
157+
# exact instance type so every campaign (both arms, all N runs) runs on identical
158+
# hardware - otherwise a coverage/exec-rate delta could reflect instance variance
159+
# rather than the fuzzer change. spot is false for real benchmarks (an interruption
160+
# would void a long campaign) and true for the cheap smoke test (set by setup).
161+
runs-on:
162+
- runs-on=${{ github.run_id }}
163+
- family=c7i.2xlarge # exact type; adjust to one nested-virt type in your account
164+
- image=ubuntu22-full-x64
165+
- nested-virt
166+
- volume=80gb:gp3
167+
- extras=s3-cache
168+
- spot=${{ needs.setup.outputs.spot }}
169+
timeout-minutes: 90 # 60m campaign + image-restore headroom
170+
strategy:
171+
fail-fast: false # a dead campaign must not cancel the others
172+
matrix:
173+
version: [baseline, treatment]
174+
run: ${{ fromJSON(needs.setup.outputs.runs_json) }}
175+
steps:
176+
- name: Checkout fuzzer version under test
177+
uses: actions/checkout@v4
178+
with:
179+
ref: ${{ matrix.version == 'baseline' && needs.setup.outputs.baseline || needs.setup.outputs.treatment }}
180+
- uses: docker/setup-buildx-action@v3
181+
- name: Materialize base image from cache
182+
uses: docker/build-push-action@v5
183+
with:
184+
context: .
185+
file: ./Dockerfile.libafl
186+
load: true
187+
tags: fuzzamoto-libafl:bench
188+
cache-from: type=gha,scope=base
189+
- name: Restore prebuilt fuzzer image from cache
190+
uses: docker/build-push-action@v5
191+
with:
192+
context: .
193+
file: ./Dockerfile.libafl.bench
194+
build-args: BASE_IMAGE=fuzzamoto-libafl:bench
195+
load: true
196+
tags: fuzzamoto-libafl:bench-${{ matrix.version }}
197+
cache-from: type=gha,scope=bench-${{ matrix.version }} # cache hit incl. compile
198+
- name: Enable KVM vmware backdoor (required by Nyx)
199+
run: |
200+
sudo modprobe -r kvm_intel 2>/dev/null || true
201+
sudo modprobe -r kvm 2>/dev/null || true
202+
sudo modprobe kvm enable_vmware_backdoor=y
203+
sudo modprobe kvm_intel
204+
echo "enable_vmware_backdoor=$(cat /sys/module/kvm/parameters/enable_vmware_backdoor)"
205+
test "$(cat /sys/module/kvm/parameters/enable_vmware_backdoor)" = "Y"
206+
- name: Run campaign
207+
run: |
208+
rm -rf /tmp/out && mkdir -p /tmp/out
209+
# Source/binary are baked into the image (no source mount); only mount the output
210+
# dir. --privileged exposes the host's /dev (incl. /dev/kvm for Nyx), so no
211+
# separate --device=/dev/kvm is needed.
212+
docker run --privileged \
213+
-e BENCH_DURATION=${{ needs.setup.outputs.duration }} \
214+
-v /tmp/out:/tmp/out \
215+
fuzzamoto-libafl:bench-${{ matrix.version }} just -f /ci/libafl.justfile bench_run
216+
- name: Collect results
217+
run: |
218+
mkdir -p artifact
219+
# All per-core snapshot files (one per core; aggregated per campaign downstream).
220+
cp /tmp/out/bench/bench-cpu_*.csv artifact/ \
221+
|| echo "no bench csv (campaign may have died early)"
222+
# Crashes from every core.
223+
for c in /tmp/out/cpu_*/crashes; do
224+
[ -d "$c" ] || continue
225+
core=$(basename "$(dirname "$c")")
226+
cp -r "$c" "artifact/crashes-$core" 2>/dev/null || true
227+
done
228+
- uses: actions/upload-artifact@v4
229+
with:
230+
name: bench-${{ matrix.version }}-${{ matrix.run }}
231+
path: artifact
232+
retention-days: 14
233+
234+
compare:
235+
needs: [setup, campaign]
236+
if: always()
237+
# Lightweight runner for report aggregation; does not affect measurements.
238+
runs-on:
239+
- runs-on=${{ github.run_id }}
240+
- cpu=2+4
241+
- family=m7i+c7i
242+
- image=ubuntu22-full-x64
243+
- extras=s3-cache
244+
steps:
245+
- uses: actions/checkout@v4
246+
- uses: actions/download-artifact@v4
247+
with:
248+
pattern: bench-*
249+
path: dl
250+
- name: Arrange into baseline/treatment layout
251+
run: |
252+
shopt -s nullglob
253+
for d in dl/bench-baseline-*; do
254+
n=${d##*-}; mkdir -p results/baseline/ir/$n
255+
cp "$d"/bench-cpu_*.csv results/baseline/ir/$n/ 2>/dev/null || true
256+
done
257+
for d in dl/bench-treatment-*; do
258+
n=${d##*-}; mkdir -p results/treatment/ir/$n
259+
cp "$d"/bench-cpu_*.csv results/treatment/ir/$n/ 2>/dev/null || true
260+
done
261+
- name: Install evaluation dependencies
262+
run: pip install numpy pandas matplotlib seaborn scipy statsmodels tabulate
263+
- name: Evaluate
264+
run: |
265+
python3 ci/benchmark-evaluation.py results --out report --hours min
266+
{
267+
echo "## fuzzamoto-libafl benchmark"
268+
echo
269+
echo "Baseline: \`${{ needs.setup.outputs.baseline }}\` • Treatment: \`${{ needs.setup.outputs.treatment }}\`"
270+
echo
271+
cat report/evaluation_report.md
272+
} >> "$GITHUB_STEP_SUMMARY"
273+
- uses: actions/upload-artifact@v4
274+
with:
275+
name: benchmark-report
276+
path: report
277+
retention-days: 90
278+
- name: Comment on PR
279+
if: github.event_name == 'pull_request'
280+
uses: actions/github-script@v7
281+
with:
282+
script: |
283+
const fs = require('fs');
284+
const body = fs.readFileSync('report/evaluation_report.md', 'utf8');
285+
await github.rest.issues.createComment({
286+
...context.repo,
287+
issue_number: context.issue.number,
288+
body,
289+
});

‎Dockerfile.libafl.bench‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
# Bakes a specific fuzzamoto-libafl build (and the Nyx share dir) on top of the base
2+
# libafl image, so benchmark campaigns run the prebuilt binary instead of recompiling.
3+
#
4+
# The base image (Bitcoin Core + AFL++ + toolchain) is version-independent and built once
5+
# by the benchmark pipeline's build-base job; only this thin layer differs per fuzzer
6+
# version. Used by .github/workflows/benchmark.yml.
7+
ARG BASE_IMAGE=fuzzamoto-libafl:bench
8+
FROM ${BASE_IMAGE}
9+
10+
# Bring in the fuzzer source for the version under test (the build context is a checkout
11+
# of that ref) and compile it once here, plus initialise the Nyx share dir.
12+
COPY . /fuzzamoto
13+
WORKDIR /fuzzamoto
14+
RUN git config --global --add safe.directory /fuzzamoto
15+
RUN just -f /ci/libafl.justfile compile_bench nyx_init

0 commit comments

Comments
 (0)