Skip to content

Commit c0aae10

Browse files
Merge remote-tracking branch 'upstream/develop' into chore/remove-marf-node-caches
2 parents 1d48b2f + b17d3a1 commit c0aae10

636 files changed

Lines changed: 69187 additions & 54922 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.
Lines changed: 112 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,112 @@
1+
name: Nextest Archive Artifact
2+
description: |
3+
Nextest binary archive artifact management.
4+
5+
inputs:
6+
action:
7+
description: "Type of operation (restore,create)"
8+
required: true
9+
archive-file:
10+
description: "The nextest archive file"
11+
required: true
12+
debug-binary-path:
13+
description: "Debug binary path"
14+
required: true
15+
16+
# Set default env vars
17+
env:
18+
RUSTFLAGS: "-C instrument-coverage -A warnings"
19+
LLVM_PROFILE_FILE: "stacks-core-%p-%m.profraw"
20+
21+
runs:
22+
using: "composite"
23+
steps:
24+
- name: Restore Nextest Archive Artifact
25+
if: |
26+
inputs.action == 'restore'
27+
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # 8.0.1
28+
with:
29+
name: "nextest-archive"
30+
31+
# Checkout the code
32+
- name: Checkout the latest code
33+
if: |
34+
inputs.action == 'create'
35+
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
36+
with:
37+
ref: ${{ github.event.pull_request.head.sha || github.ref }}
38+
39+
# Setup rust toolchain
40+
- name: Setup Rust Toolchain
41+
if: |
42+
inputs.action == 'create'
43+
uses: ./.github/actions/setup-rust-toolchain
44+
with:
45+
components: llvm-tools-preview
46+
47+
# Install nextest
48+
- name: Install nextest
49+
if: |
50+
inputs.action == 'create'
51+
uses: ./.github/actions/install-tool
52+
with:
53+
tools: nextest
54+
55+
# Install mold linker and clang (faster than default)
56+
- name: Install mold Linker (and clang)
57+
if: |
58+
inputs.action == 'create'
59+
shell: bash
60+
run: |
61+
sudo apt-get update
62+
sudo apt-get install -y mold clang
63+
64+
# Restore the cargo cache
65+
- name: Restore Cargo Cache
66+
if: |
67+
inputs.action == 'create'
68+
uses: ./.github/actions/cache/cargo
69+
with:
70+
action: restore
71+
72+
# Build the nextest archive
73+
- name: Build Nextest Archive
74+
if: |
75+
inputs.action == 'create'
76+
shell: bash
77+
env:
78+
RUSTFLAGS: "-C instrument-coverage -C linker=clang -C link-arg=-fuse-ld=mold -A warnings"
79+
CARGO_INCREMENTAL: 0
80+
# This command should be kept in sync with the same command in cache/cargo action
81+
run: |
82+
# Capture the archive file input, then expand the leading ~ to $HOME
83+
INPUT_FILE="${{ inputs.archive-file }}"
84+
ARCHIVE_FILE="${INPUT_FILE/#\~/$HOME}"
85+
86+
cargo nextest archive \
87+
--workspace \
88+
--tests \
89+
--locked \
90+
--features monitoring_prom \
91+
--archive-file "$ARCHIVE_FILE"
92+
93+
- name: Create Nextest Archive Artifact
94+
if: |
95+
inputs.action == 'create'
96+
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # 7.0.1
97+
with:
98+
name: "nextest-archive"
99+
path: |
100+
${{ inputs.archive-file }}
101+
${{ inputs.debug-binary-path }}
102+
!${{ inputs.debug-binary-path }}/examples/**
103+
!${{ inputs.debug-binary-path }}/incremental/**
104+
!${{ inputs.debug-binary-path }}/.fingerprint/**
105+
!${{ inputs.debug-binary-path }}/build/**
106+
!${{ inputs.debug-binary-path }}/deps/**/*.d
107+
!${{ inputs.debug-binary-path }}/deps/**/*.rlib
108+
!${{ inputs.debug-binary-path }}/deps/**/*.rmeta
109+
if-no-files-found: error
110+
retention-days: 1
111+
compression-level: 9 # 0 (none) - 9 (most)
112+
overwrite: true
Lines changed: 38 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -1,58 +1,70 @@
11
name: Bitcoin Cache
22
description: |
3-
Bitcoin binary cache
3+
Bitcoin binary cache management.
44
55
inputs:
66
action:
7-
description: "Type of operation (check,restore,save)"
7+
description: "Type of operation (check,restore,create)"
88
required: false
99
default: "check"
1010
btc-version:
11-
description: "Bitcoin version"
12-
required: true
11+
description: "The Bitcoin binary version"
12+
required: false
13+
default: "25.0"
1314

1415
outputs:
15-
cache-hit:
16-
description: "Cache Hit"
17-
value: ${{ steps.check_cache.outputs.cache-hit }}
16+
cache-key:
17+
description: "Cache Key"
18+
value: ${{ steps.generate-keys.outputs.bitcoin-cache-key }}
19+
cache-matched-key:
20+
description: "Key of the cache that was checked or restored, it could either be the primary key on cache-hit or a partial/complete match of one of the restore keys."
21+
value: ${{ steps.check-cache.outputs.cache-matched-key || steps.restore-cache.outputs.cache-matched-key }}
1822

1923
runs:
2024
using: "composite"
2125
steps:
22-
- name: Generate Cache Key
23-
shell: bash
24-
run: |
25-
echo "BITCOIN_CACHE_KEY=cache-bitcoin-binary--${{ inputs.btc-version }}" >> $GITHUB_ENV
26+
# Generate cache keys
27+
- name: Generate Cache Keys
28+
id: generate-keys
29+
uses: ./.github/actions/cache/generate-keys
30+
with:
31+
btc-version: ${{ inputs.btc-version }}
2632

2733
# Check if cache data exists
2834
- name: Check Cache
29-
uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
30-
id: check_cache
35+
if: |
36+
inputs.action == 'check'
37+
uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
38+
id: check-cache
3139
with:
3240
lookup-only: true
3341
path: ~/bitcoin
34-
key: ${{ env.BITCOIN_CACHE_KEY }}
42+
key: ${{ steps.generate-keys.outputs.bitcoin-cache-key }}
43+
# Allows cache to "fallback" to `master` generated cache, using a prefix. The most recently created cache will be selected.
44+
restore-keys: ${{ steps.generate-keys.outputs.bitcoin-restore-key }}
3545

3646
# Restore cache data
3747
- name: Restore Cache
3848
if: |
39-
inputs.action == 'restore' &&
40-
steps.check_cache.outputs.cache-hit == 'true'
41-
uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
49+
inputs.action == 'restore'
50+
uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
51+
id: restore-cache
4252
with:
4353
path: ~/bitcoin
44-
key: ${{ env.BITCOIN_CACHE_KEY }}
54+
key: ${{ steps.generate-keys.outputs.bitcoin-cache-key }}
55+
# Allows cache to "fallback" to base branch or `master`
56+
restore-keys: ${{ steps.generate-keys.outputs.bitcoin-restore-key }}
57+
fail-on-cache-miss: true
4558

4659
# Install bitcoin binary on cache-miss
4760
- name: Install Bitcoin Binary
4861
if: |
49-
inputs.action == 'save' &&
50-
steps.check_cache.outputs.cache-hit != 'true'
62+
inputs.action == 'create'
5163
shell: bash
5264
run: |
5365
# 1. Hardcode the trusted SHA-256 checksums for specific bitcoin versions
5466
# (Add more as needed in the future: generate this locally using `sha256sum bitcoin.tar.gz`)
55-
case "${{ inputs.btc-version }}" in
67+
case "${{ steps.generate-keys.outputs.bitcoin-binary-version }}" in
5668
"25.0") EXPECTED_SHA="33930d432593e49d58a9bff4c30078823e9af5d98594d2935862788ce8a20aec" ;;
5769
"26.0") EXPECTED_SHA="23e5ab226d9e01ffaadef5ffabe8868d0db23db952b90b0593652993680bb8ab" ;;
5870
"27.0") EXPECTED_SHA="2a6974c5486f528793c79d42694b5987401e4a43c97f62b1383abf35bcee44a8" ;;
@@ -61,7 +73,7 @@ runs:
6173
esac
6274
6375
# 2. Download the binary to a explicit filename
64-
URL="https://github.com/stacks-network/bitcoin/releases/download/v${{ inputs.btc-version }}/bitcoin-${{ inputs.btc-version }}-x86_64-linux-gnu.tar.gz"
76+
URL="https://github.com/stacks-network/bitcoin/releases/download/v${{ steps.generate-keys.outputs.bitcoin-binary-version }}/bitcoin-${{ steps.generate-keys.outputs.bitcoin-binary-version }}-x86_64-linux-gnu.tar.gz"
6577
curl -LSf -# -o bitcoin.tar.gz "$URL"
6678
6779
# 3. Enforce the checksum validation. If this fails, the script exits immediately.
@@ -72,14 +84,13 @@ runs:
7284
7385
# 4. Safely extract
7486
tar zxf bitcoin.tar.gz -C ~
75-
mv ~/bitcoin-${{ inputs.btc-version }} ~/bitcoin
87+
mv ~/bitcoin-${{ steps.generate-keys.outputs.bitcoin-binary-version }} ~/bitcoin
7688
7789
# Save cache data
7890
- name: Save Cache
7991
if: |
80-
inputs.action == 'save' &&
81-
steps.check_cache.outputs.cache-hit != 'true'
82-
uses: actions/cache/save@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
92+
inputs.action == 'create'
93+
uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
8394
with:
8495
path: ~/bitcoin
85-
key: ${{ env.BITCOIN_CACHE_KEY }}
96+
key: ${{ steps.generate-keys.outputs.bitcoin-cache-key }}
Lines changed: 99 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -1,31 +1,34 @@
11
name: Cargo Cache
22
description: |
3-
Rust Cargo cache used only to build the test archive.
4-
All crates should be cached based on runner OS and Cargo.lock hash.
5-
This speeds up test archive generation but this cache should not be used for executing the tests.
6-
See: https://github.com/actions/cache/blob/main/examples.md#rust---cargo
3+
Cargo crate cache management.
74
85
inputs:
96
action:
10-
description: "Type of operation (restore,save)"
7+
description: "Type of operation (check,restore,create)"
118
required: false
12-
default: ""
13-
cache-key:
14-
description: "The cache key to use. Should be based on runner OS, Cargo.lock hash, and Cargo.toml hash."
15-
required: true
9+
default: "check"
1610

1711
outputs:
18-
cache-hit:
19-
description: "Cache Hit"
20-
value: ${{ steps.check-cache.outputs.cache-hit }}
12+
cache-key:
13+
description: "Cache Key"
14+
value: ${{ steps.generate-keys.outputs.cargo-cache-key }}
15+
cache-matched-key:
16+
description: "Key of the cache that was checked or restored, it could either be the primary key on cache-hit or a partial/complete match of one of the restore keys."
17+
value: ${{ steps.check-cache.outputs.cache-matched-key || steps.restore-cache.outputs.cache-matched-key }}
2118

2219
runs:
2320
using: "composite"
2421
steps:
25-
22+
# Generate cache keys
23+
- name: Generate Cache Keys
24+
id: generate-keys
25+
uses: ./.github/actions/cache/generate-keys
26+
2627
# Check if cache data exists
2728
- name: Check Cache
28-
uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
29+
if: |
30+
inputs.action == 'check'
31+
uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
2932
id: check-cache
3033
with:
3134
lookup-only: true
@@ -35,34 +38,107 @@ runs:
3538
~/.cargo/registry/cache/
3639
~/.cargo/git/db/
3740
target/
38-
key: ${{ inputs.cache-key }}
41+
key: ${{ steps.generate-keys.outputs.cargo-cache-key }}
42+
# Allows cache to "fallback" to `master` generated cache, using a prefix. The most recently created cache will be selected.
43+
restore-keys: ${{ steps.generate-keys.outputs.cargo-restore-key }}
3944

4045
# Restore cache data
4146
- name: Restore Cache
4247
if: |
43-
inputs.action == 'restore' &&
44-
steps.check-cache.outputs.cache-hit == 'true'
45-
uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
48+
inputs.action == 'restore'
49+
uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
50+
id: restore-cache
4651
with:
4752
path: |
4853
~/.cargo/bin/
4954
~/.cargo/registry/index/
5055
~/.cargo/registry/cache/
5156
~/.cargo/git/db/
5257
target/
53-
key: ${{ inputs.cache-key }}
58+
key: ${{ steps.generate-keys.outputs.cargo-cache-key }}
59+
# Allows cache to "fallback" to base branch or `master`
60+
restore-keys: ${{ steps.generate-keys.outputs.cargo-restore-key }}
61+
fail-on-cache-miss: true
5462

63+
# Checkout the code
64+
- name: Checkout the latest code
65+
if: |
66+
inputs.action == 'create'
67+
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
68+
with:
69+
ref: ${{ github.ref }}
70+
71+
# Setup rust toolchain
72+
- name: Setup Rust Toolchain
73+
if: |
74+
inputs.action == 'create'
75+
uses: ./.github/actions/setup-rust-toolchain
76+
with:
77+
components: llvm-tools-preview
78+
79+
# Install nextest
80+
- name: Install nextest
81+
if: |
82+
inputs.action == 'create'
83+
uses: ./.github/actions/install-tool
84+
with:
85+
tools: nextest
86+
87+
# Install mold linker and clang (faster than default)
88+
- name: Install mold Linker (and clang)
89+
if: |
90+
inputs.action == 'create'
91+
shell: bash
92+
run: |
93+
sudo apt-get update
94+
sudo apt-get install -y mold clang
95+
96+
# Build the cargo crates
97+
- name: Build Cargo Crates
98+
if: |
99+
inputs.action == 'create'
100+
shell: bash
101+
env:
102+
RUSTFLAGS: "-C instrument-coverage -C linker=clang -C link-arg=-fuse-ld=mold -A warnings"
103+
CARGO_INCREMENTAL: 0
104+
# This command should be kept in sync with the same command in artifacts/nextest-archive action
105+
run: |
106+
cargo nextest archive \
107+
--workspace \
108+
--tests \
109+
--locked \
110+
--features monitoring_prom \
111+
--archive-file "not_used.tar.zst"
112+
113+
# Dynamic cleanup of local workspace crates right after building. We never want them to exist in the produced cache.
114+
# These are things like stacks-signer, stacks-inspect, etc. built locally from our codebase.
115+
# We must always compile them on-the-fly from current code, as they aren't bumped in version when they are changed.
116+
- name: Clean Local Workspace Crates
117+
if: |
118+
inputs.action == 'create'
119+
shell: bash
120+
run: |
121+
if [ -f Cargo.toml ]; then
122+
echo "Identifying local workspace crates to exclude from restored cache..."
123+
cargo metadata --format-version 1 --no-deps | jq -r '.packages[].name' | while read -r crate; do
124+
echo "Wiping build artifacts for local crate: $crate"
125+
cargo clean -p "$crate"
126+
done
127+
else
128+
echo "ERROR: No Cargo.toml found in the current working directory."
129+
exit 1
130+
fi
131+
55132
# Save cache data
56133
- name: Save Cache
57134
if: |
58-
inputs.action == 'save' &&
59-
steps.check-cache.outputs.cache-hit != 'true'
60-
uses: actions/cache/save@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
135+
inputs.action == 'create'
136+
uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
61137
with:
62138
path: |
63139
~/.cargo/bin/
64140
~/.cargo/registry/index/
65141
~/.cargo/registry/cache/
66142
~/.cargo/git/db/
67143
target/
68-
key: ${{ inputs.cache-key }}
144+
key: ${{ steps.generate-keys.outputs.cargo-cache-key }}

0 commit comments

Comments
 (0)