[CI]: migrate sync committee devnet from docker-compose to Kurtosis

.github/workflows/ci.yml

@@ -492,13 +492,48 @@ jobs:
 
             - uses: Swatinem/rust-cache@v2
 
+            # Authenticate to Docker Hub so the many images Kurtosis/ethereum-package pull are
+            # not subject to the anonymous per-IP rate limit (shared across GitHub runners),
+            # which otherwise surfaces as image-pull timeouts during `kurtosis engine start`.
+            - name: Login to Docker Hub
+              uses: docker/login-action@v3
+              with:
+                  username: ${{ secrets.DOCKER_USERNAME }}
+                  password: ${{ secrets.DOCKER_ACCESS_TOKEN }}
+
+            - name: Install Kurtosis
+              run: |
+                  # The legacy Gemfury apt repo only serves old Kurtosis (<=1.15.x), which cannot
+                  # run current ethereum-package (it uses the newer GpuConfig Starlark API). Install
+                  # a pinned modern release directly from the official release artifacts instead.
+                  KURTOSIS_VERSION=1.20.0
+                  curl -fsSL -o /tmp/kurtosis-cli.deb \
+                    "https://github.com/kurtosis-tech/kurtosis-cli-release-artifacts/releases/download/${KURTOSIS_VERSION}/kurtosis-cli_${KURTOSIS_VERSION}_linux_amd64.deb"
+                  sudo dpkg -i /tmp/kurtosis-cli.deb
+                  kurtosis version
 
             - name: Setup Eth POS Devnet
               run: |
-                  git clone --branch david/electra https://github.com/polytope-labs/eth-pos-devnet.git
-                  DOCKER_CLIENT_TIMEOUT=300 COMPOSE_HTTP_TIMEOUT=300 docker compose -f ./eth-pos-devnet/docker-compose.yml up -d
-                  ./scripts/wait_for_tcp_port_opening.sh localhost 3500
-                  ./scripts/wait_for_tcp_port_opening.sh localhost 8545
+                  # Pin ethereum-package to a known-good commit so the devnet (and its
+                  # genesis_validators_root, which KurtosisDevnet hardcodes) stays reproducible.
+                  ETH_PKG=github.com/ethpandaops/ethereum-package@d4bd739b78565ca5a9768461a59ad59942e4e941
+
+                  # Image pulls can still time out transiently; retry a few times before failing.
+                  for i in 1 2 3; do
+                    kurtosis engine start && break
+                    [ "$i" = 3 ] && { echo "kurtosis engine start failed after 3 attempts"; exit 1; }
+                    echo "engine start attempt $i failed; retrying in 15s"; sleep 15
+                  done
+
+                  for i in 1 2 3; do
+                    kurtosis run "$ETH_PKG" --args-file ./sync-committee-devnet.yaml --enclave sync-committee-devnet && break
+                    [ "$i" = 3 ] && { echo "kurtosis run failed after 3 attempts"; exit 1; }
+                    echo "devnet run attempt $i failed; cleaning up and retrying in 15s"
+                    kurtosis enclave rm -f sync-committee-devnet || true
+                    sleep 15
+                  done
+
+                  ./scripts/wait_for_tcp_port_opening.sh localhost 53001
 
             - name: Run Sync Committee Tests
               run: |
@@ -507,8 +542,8 @@ jobs:
             - name: Cleanup
               if: always()
               run: |
-                  docker-compose -f ./eth-pos-devnet/docker-compose.yml down || true
-                  sudo rm -rf ./eth-pos-devnet
+                  kurtosis enclave rm -f sync-committee-devnet || true
+                  kurtosis engine stop || true
 
     bsc-tests:
         name: Binance Smart Chain Tests

modules/consensus/sync-committee/primitives/src/constants.rs

@@ -265,23 +265,31 @@ pub mod devnet {
 	use super::*;
 	use hex_literal::hex;
 
+	/// Config for the Kurtosis-based devnet (`ethpandaops/ethereum-package`) which activates
+	/// every fork at genesis, including Fulu. The genesis root and fork version bytes come from
+	/// the devnet's `/eth/v1/beacon/genesis` and `/eth/v1/config/spec` endpoints; the fork
+	/// epochs are all 0 and the generalized indices are fixed by the Electra+Fulu SSZ schema.
 	#[derive(Default)]
-	pub struct ElectraDevnet;
+	pub struct KurtosisDevnet;
 
-	impl Config for ElectraDevnet {
+	impl Config for KurtosisDevnet {
 		const SLOTS_PER_EPOCH: Slot = 32;
 		const GENESIS_VALIDATORS_ROOT: [u8; 32] =
-			hex_literal::hex!("83431ec7fcf92cfc44947fc0418e831c25e1d0806590231c439830db7ad54fda");
-		const BELLATRIX_FORK_VERSION: Version = hex!("52525502");
-		const ALTAIR_FORK_VERSION: Version = hex!("52525501");
-		const GENESIS_FORK_VERSION: Version = hex!("52525500");
+			hex_literal::hex!("d1ec305b97bf6336571c2348e4a8bf173684b0cdb7e55f7e6554d51f8478b5a3");
+		const GENESIS_FORK_VERSION: Version = hex!("10000038");
+		const ALTAIR_FORK_VERSION: Version = hex!("20000038");
+		const BELLATRIX_FORK_VERSION: Version = hex!("30000038");
+		const CAPELLA_FORK_VERSION: Version = hex!("40000038");
+		const DENEB_FORK_VERSION: Version = hex!("50000038");
+		const ELECTRA_FORK_VERSION: Version = hex!("60000038");
+		const FULU_FORK_VERSION: Version = hex!("70000038");
 		const ALTAIR_FORK_EPOCH: Epoch = 0;
 		const BELLATRIX_FORK_EPOCH: Epoch = 0;
 		const CAPELLA_FORK_EPOCH: Epoch = 0;
-		const CAPELLA_FORK_VERSION: Version = hex!("52525503");
 		const DENEB_FORK_EPOCH: Epoch = 0;
-		const DENEB_FORK_VERSION: Version = hex!("52525504");
-		const EPOCHS_PER_SYNC_COMMITTEE_PERIOD: Epoch = 4;
+		const ELECTRA_FORK_EPOCH: Epoch = 0;
+		const FULU_FORK_EPOCH: Epoch = 0;
+		const EPOCHS_PER_SYNC_COMMITTEE_PERIOD: Epoch = 256;
 		const EXECUTION_PAYLOAD_STATE_ROOT_INDEX: u64 = 34;
 		const EXECUTION_PAYLOAD_BLOCK_NUMBER_INDEX: u64 = 38;
 		const EXECUTION_PAYLOAD_TIMESTAMP_INDEX: u64 = 41;
@@ -291,10 +299,6 @@ pub mod devnet {
 		const FINALIZED_ROOT_INDEX_LOG2: u64 = 6;
 		const EXECUTION_PAYLOAD_INDEX_LOG2: u64 = 6;
 		const NEXT_SYNC_COMMITTEE_INDEX_LOG2: u64 = 6;
-		const ELECTRA_FORK_VERSION: Version = hex_literal::hex!("52525505");
-		const ELECTRA_FORK_EPOCH: Epoch = 0;
-		const FULU_FORK_EPOCH: Epoch = u64::MAX;
-		const FULU_FORK_VERSION: Version = hex_literal::hex!("52525506");
 		const ID: [u8; 4] = BEACON_CONSENSUS_ID;
 	}
 }

modules/consensus/sync-committee/prover/Cargo.toml

@@ -9,7 +9,7 @@ sync-committee-primitives = { workspace = true, default-features = true }
 sync-committee-verifier = { workspace = true, default-features = true }
 serde = { workspace = true, default-features = true, features = ["derive"] }
 anyhow = { workspace = true, default-features = true }
-tokio = { workspace = true, features = ["sync"] }
+tokio = { workspace = true, features = ["sync", "time"] }
 tokio-stream = { workspace = true }
 log = { workspace = true, default-features = true }
 hex = { workspace = true, default-features = true }
@@ -28,8 +28,7 @@ tracing = "0.1.40"
 
 [dev-dependencies]
 env_logger = "0.10.0"
-# feature flag needed so sync-committee tests pass, eth pos server in CI does not support fusaka
-sync-committee-primitives = { path= "../primitives", features = ["nofulu"] }
+sync-committee-primitives = { path= "../primitives" }
 sync-committee-verifier = { path= "../verifier" }
 tokio = { workspace = true, features = ["macros", "rt-multi-thread"]}
 parity-scale-codec = "3.2.2"

modules/consensus/sync-committee/prover/src/lib.rs

@@ -327,8 +327,9 @@ impl<C: Config, const ETH1_DATA_VOTES_BOUND: usize, const PROPOSER_LOOK_AHEAD_LI
 		}
 		let finalized_block_id = get_block_id(attested_state.finalized_checkpoint.root);
 		let finalized_header = self.fetch_header(&finalized_block_id).await?;
+		// Fetch the finalized state by slot rather than by state root.
 		let mut finalized_state =
-			self.fetch_beacon_state(&get_block_id(finalized_header.state_root)).await?;
+			self.fetch_beacon_state(&finalized_header.slot.to_string()).await?;
 		let finality_proof = FinalityProof {
 			epoch: attested_state.finalized_checkpoint.epoch,
 			finality_branch: prove_finalized_header::<
@@ -428,8 +429,9 @@ impl<C: Config, const ETH1_DATA_VOTES_BOUND: usize, const PROPOSER_LOOK_AHEAD_LI
 			self.fetch_beacon_state(&get_block_id(attested_header.state_root)).await?;
 		let finalized_block_id = get_block_id(attested_state.finalized_checkpoint.root);
 		let finalized_header = self.fetch_header(&finalized_block_id).await?;
+		// Fetch the finalized state by slot rather than by state root.
 		let mut finalized_state =
-			self.fetch_beacon_state(&get_block_id(finalized_header.state_root)).await?;
+			self.fetch_beacon_state(&finalized_header.slot.to_string()).await?;
 		let finality_proof = FinalityProof {
 			epoch: attested_state.finalized_checkpoint.epoch,
 			finality_branch: prove_finalized_header::<

modules/consensus/sync-committee/prover/src/test.rs

@@ -8,7 +8,7 @@ use sync_committee_primitives::{
 	util::compute_epoch_at_slot,
 };
 
-use sync_committee_primitives::constants::devnet::ElectraDevnet;
+use sync_committee_primitives::constants::devnet::KurtosisDevnet;
 
 use sync_committee_verifier::verify_sync_committee_attestation;
 use tokio_stream::StreamExt;
@@ -82,7 +82,7 @@ async fn test_finalized_header() {
 	let mut state = sync_committee_prover.fetch_beacon_state("head").await.unwrap();
 
 	let proof =
-		ssz_rs::generate_proof(&mut state, &vec![ElectraDevnet::FINALIZED_ROOT_INDEX as usize])
+		ssz_rs::generate_proof(&mut state, &vec![KurtosisDevnet::FINALIZED_ROOT_INDEX as usize])
 			.unwrap();
 
 	let leaves = vec![Node::from_bytes(
@@ -97,7 +97,7 @@ async fn test_finalized_header() {
 	let root = calculate_multi_merkle_root(
 		&leaves,
 		&proof,
-		&[GeneralizedIndex(ElectraDevnet::FINALIZED_ROOT_INDEX as usize)],
+		&[GeneralizedIndex(KurtosisDevnet::FINALIZED_ROOT_INDEX as usize)],
 	);
 	assert_eq!(root, state.hash_tree_root().unwrap());
 }
@@ -111,7 +111,7 @@ async fn test_execution_payload_proof() {
 	let mut finalized_state = sync_committee_prover.fetch_beacon_state("head").await.unwrap();
 	let block_id = finalized_state.slot.to_string();
 	let execution_payload_proof = prove_execution_payload::<
-		ElectraDevnet,
+		KurtosisDevnet,
 		ETH1_DATA_VOTES_BOUND_ETH,
 		PROPOSER_LOOK_AHEAD_LIMIT_ETHEREUM,
 	>(&mut finalized_state)
@@ -130,9 +130,9 @@ async fn test_execution_payload_proof() {
 		],
 		&multi_proof_vec,
 		&[
-			GeneralizedIndex(ElectraDevnet::EXECUTION_PAYLOAD_STATE_ROOT_INDEX as usize),
-			GeneralizedIndex(ElectraDevnet::EXECUTION_PAYLOAD_BLOCK_NUMBER_INDEX as usize),
-			GeneralizedIndex(ElectraDevnet::EXECUTION_PAYLOAD_TIMESTAMP_INDEX as usize),
+			GeneralizedIndex(KurtosisDevnet::EXECUTION_PAYLOAD_STATE_ROOT_INDEX as usize),
+			GeneralizedIndex(KurtosisDevnet::EXECUTION_PAYLOAD_BLOCK_NUMBER_INDEX as usize),
+			GeneralizedIndex(KurtosisDevnet::EXECUTION_PAYLOAD_TIMESTAMP_INDEX as usize),
 		],
 	);
 
@@ -149,8 +149,8 @@ async fn test_execution_payload_proof() {
 	let is_merkle_branch_valid = is_valid_merkle_branch(
 		&execution_payload_root,
 		execution_payload_branch,
-		ElectraDevnet::EXECUTION_PAYLOAD_INDEX_LOG2 as usize,
-		ElectraDevnet::EXECUTION_PAYLOAD_INDEX as usize,
+		KurtosisDevnet::EXECUTION_PAYLOAD_INDEX_LOG2 as usize,
+		KurtosisDevnet::EXECUTION_PAYLOAD_INDEX as usize,
 		&finalized_header.state_root,
 	);
 
@@ -161,7 +161,7 @@ async fn test_execution_payload_proof() {
 #[tokio::test]
 #[ignore]
 async fn test_sync_committee_update_proof() {
-	use sync_committee_primitives::constants::devnet::ElectraDevnet;
+	use sync_committee_primitives::constants::devnet::KurtosisDevnet;
 
 	let sync_committee_prover = setup_prover();
 
@@ -170,7 +170,7 @@ async fn test_sync_committee_update_proof() {
 	let finalized_header = sync_committee_prover.fetch_header(&block_id).await.unwrap();
 
 	let sync_committee_proof = prove_sync_committee_update::<
-		ElectraDevnet,
+		KurtosisDevnet,
 		ETH1_DATA_VOTES_BOUND_ETH,
 		PROPOSER_LOOK_AHEAD_LIMIT_ETHEREUM,
 	>(&mut finalized_state)
@@ -181,16 +181,16 @@ async fn test_sync_committee_update_proof() {
 	let calculated_finalized_root = calculate_multi_merkle_root(
 		&[sync_committee.hash_tree_root().unwrap()],
 		&sync_committee_proof,
-		&[GeneralizedIndex(ElectraDevnet::NEXT_SYNC_COMMITTEE_INDEX as usize)],
+		&[GeneralizedIndex(KurtosisDevnet::NEXT_SYNC_COMMITTEE_INDEX as usize)],
 	);
 
 	assert_eq!(calculated_finalized_root.as_bytes(), finalized_header.state_root.as_bytes());
 
 	let is_merkle_branch_valid = is_valid_merkle_branch(
 		&sync_committee.hash_tree_root().unwrap(),
 		sync_committee_proof.iter(),
-		ElectraDevnet::NEXT_SYNC_COMMITTEE_INDEX_LOG2 as usize,
-		ElectraDevnet::NEXT_SYNC_COMMITTEE_INDEX as usize,
+		KurtosisDevnet::NEXT_SYNC_COMMITTEE_INDEX_LOG2 as usize,
+		KurtosisDevnet::NEXT_SYNC_COMMITTEE_INDEX as usize,
 		&finalized_header.state_root,
 	);
 
@@ -203,7 +203,7 @@ async fn test_sync_committee_update_proof() {
 async fn test_prover() {
 	use log::LevelFilter;
 	use parity_scale_codec::{Decode, Encode};
-	use sync_committee_primitives::constants::devnet::ElectraDevnet;
+	use sync_committee_primitives::constants::devnet::KurtosisDevnet;
 	env_logger::builder()
 		.filter_module("prover", LevelFilter::Debug)
 		.format_module_path(false)
@@ -221,10 +221,10 @@ async fn test_prover() {
 
 	let mut client_state = VerifierState {
 		finalized_header: block_header.clone(),
-		latest_finalized_epoch: compute_epoch_at_slot::<ElectraDevnet>(block_header.slot),
+		latest_finalized_epoch: compute_epoch_at_slot::<KurtosisDevnet>(block_header.slot),
 		current_sync_committee: state.current_sync_committee,
 		next_sync_committee: state.next_sync_committee,
-		state_period: compute_sync_committee_period_at_slot::<ElectraDevnet>(block_header.slot),
+		state_period: compute_sync_committee_period_at_slot::<KurtosisDevnet>(block_header.slot),
 	};
 
 	let mut count = 0;
@@ -236,14 +236,27 @@ async fn test_prover() {
 				let message: EventResponse = json::from_str(&msg.data).unwrap();
 				let checkpoint =
 					Checkpoint { epoch: message.epoch.parse().unwrap(), root: message.block };
-				let light_client_update = if let Some(update) = sync_committee_prover
+				// The SSE fires at the last slot of the finalizing epoch. The prover needs
+				// the *next* block (first slot of the following epoch) so that its parent
+				// state already reflects the new finalized checkpoint. Sleep one slot before
+				// fetching the update.
+				tokio::time::sleep(std::time::Duration::from_secs(5)).await;
+				let light_client_update = match sync_committee_prover
 					.fetch_light_client_update(client_state.clone(), checkpoint, None)
 					.await
-					.unwrap()
 				{
-					update
-				} else {
-					continue;
+					Ok(Some(update)) => update,
+					Ok(None) => continue,
+					// Lighthouse can transiently 404 the finalized state while it migrates its
+					// hot DB on finalization. That is a devnet/node artifact, not a prover error,
+					// so skip this event and wait for the next finalization.
+					Err(e) if e.to_string().contains("404") => {
+						println!(
+							"transient 404 fetching light client update; waiting for next finalization: {e}"
+						);
+						continue;
+					},
+					Err(e) => panic!("fetch_light_client_update failed: {e}"),
 				};
 
 				if light_client_update.sync_committee_update.is_some() {
@@ -256,7 +269,7 @@ async fn test_prover() {
 				let decoded = VerifierStateUpdate::decode(&mut &*encoded).unwrap();
 				assert_eq!(light_client_update, decoded);
 
-				client_state = verify_sync_committee_attestation::<ElectraDevnet>(
+				client_state = verify_sync_committee_attestation::<KurtosisDevnet>(
 					client_state.clone(),
 					light_client_update,
 				)
@@ -286,11 +299,11 @@ async fn test_switch_provider_middleware() {
 	let providers = vec![
 		"http://localhost:3505".to_string(),
 		"http://localhost:3510".to_string(),
-		"http://localhost:3500".to_string(),
+		"http://localhost:53001".to_string(),
 	];
 
 	let prover = SyncCommitteeProver::<
-		ElectraDevnet,
+		KurtosisDevnet,
 		ETH1_DATA_VOTES_BOUND_ETH,
 		PROPOSER_LOOK_AHEAD_LIMIT_ETHEREUM,
 	>::new(providers);
@@ -306,14 +319,16 @@ pub struct EventResponse {
 	pub execution_optimistic: bool,
 }
 
-fn setup_prover(
-) -> SyncCommitteeProver<ElectraDevnet, ETH1_DATA_VOTES_BOUND_ETH, PROPOSER_LOOK_AHEAD_LIMIT_ETHEREUM>
-{
+fn setup_prover() -> SyncCommitteeProver<
+	KurtosisDevnet,
+	ETH1_DATA_VOTES_BOUND_ETH,
+	PROPOSER_LOOK_AHEAD_LIMIT_ETHEREUM,
+> {
 	dotenv::dotenv().ok();
 	let consensus_url =
-		std::env::var("CONSENSUS_NODE_URL").unwrap_or("http://localhost:3500".to_string());
+		std::env::var("CONSENSUS_NODE_URL").unwrap_or("http://localhost:53001".to_string());
 	SyncCommitteeProver::<
-		ElectraDevnet,
+		KurtosisDevnet,
 		ETH1_DATA_VOTES_BOUND_ETH,
 		PROPOSER_LOOK_AHEAD_LIMIT_ETHEREUM,
 	>::new(vec![consensus_url])

sync-committee-devnet.yaml

@@ -0,0 +1,26 @@
+participants:
+  - el_type: geth
+    cl_type: lighthouse
+    count: 1
+
+network_params:
+  preset: mainnet
+  genesis_delay: 20
+  seconds_per_slot: 4
+  # All forks active at epoch 0 so the devnet starts on Electra (Pectra CL) immediately
+  altair_fork_epoch: 0
+  bellatrix_fork_epoch: 0
+  capella_fork_epoch: 0
+  deneb_fork_epoch: 0
+  electra_fork_epoch: 0
+
+# Expose beacon and execution ports on fixed local ports for easy access
+port_publisher:
+  cl:
+    enabled: true
+    public_port_start: 53000
+  el:
+    enabled: true
+    public_port_start: 52000
+
+additional_services: []