diff --git a/docker/Dockerfile b/docker/Dockerfile index 89640739c..17b6d1734 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -65,7 +65,7 @@ RUN rust_version=$(grep -oP '(?<=channel = ")[^"]+' /usr/src/rust-toolchain.toml curl --fail https://sh.rustup.rs -sSf \ | sh -s -- -y --default-toolchain ${rust_version}-x86_64-unknown-linux-gnu --no-modify-path && \ rustup default ${rust_version}-x86_64-unknown-linux-gnu && \ - rustup component add rls && \ + rustup component add rust-analyzer && \ chown -R ubuntu.ubuntu ${RUSTUP_HOME} ${CARGO_HOME} # Cargo maintains a local cache of the registry index and of git checkouts of crates at CARGO_HOME @@ -75,9 +75,10 @@ ENV CARGO_HOME=/cargo # Used to detect the Dockerfile changes and automatically rebuild the image COPY docker /docker -# Download ic-admin -ARG ic_git_revision=89cc1c20223532c900b94de5bc6bd8cbde278797 -RUN curl --fail https://download.dfinity.systems/ic/${ic_git_revision}/release/ic-admin.gz -o - | gunzip -c >| /usr/bin/ic-admin && \ +# Download ic-admin from GitHub releases (download.dfinity.systems no longer serves it). +# The image is x86_64 linux, so we fetch the matching asset from a pinned IC release tag. +ARG ic_release_tag=release-2026-06-04_04-52-base +RUN curl --fail -L https://github.com/dfinity/ic/releases/download/${ic_release_tag}/ic-admin-x86_64-linux.gz -o - | gunzip -c >| /usr/bin/ic-admin && \ chmod +x /usr/bin/ic-admin USER ubuntu diff --git a/rs/cli/src/ctx/mod.rs b/rs/cli/src/ctx/mod.rs index 06bcdcc36..ca232df39 100644 --- a/rs/cli/src/ctx/mod.rs +++ b/rs/cli/src/ctx/mod.rs @@ -8,7 +8,7 @@ use ic_management_backend::{ proposal::{ProposalAgent, ProposalAgentImpl}, }; use ic_management_types::Network; -use log::warn; +use log::{info, warn}; use crate::{ artifact_downloader::{ArtifactDownloader, ArtifactDownloaderImpl}, @@ -45,6 +45,9 @@ pub struct DreContext { artifact_downloader: Arc, neuron: RefCell>, version: IcAdminVersion, + /// Explicit ic-admin binary path/command (from `--ic-admin`). When set, it + /// takes precedence over `version` and no download is performed. + ic_admin_path_override: Option, neuron_opts: NeuronOpts, cordoned_features_fetcher: Arc, health_client: Arc, @@ -60,6 +63,7 @@ impl DreContext { verbose: bool, auth_requirement: AuthRequirement, ic_admin_version: IcAdminVersion, + ic_admin_path_override: Option, cordoned_features_fetcher: Arc, health_client: Arc, store: Store, @@ -76,6 +80,7 @@ impl DreContext { artifact_downloader: Arc::new(ArtifactDownloaderImpl {}) as Arc, neuron: RefCell::new(None), version: ic_admin_version, + ic_admin_path_override, neuron_opts: NeuronOpts { auth_opts: auth, requirement: auth_requirement, @@ -114,6 +119,7 @@ impl DreContext { args.verbose, require_auth, args.ic_admin_version.clone(), + args.ic_admin.clone(), store.cordoned_features_fetcher(args.cordoned_features_file.clone())?, store.health_client(&network)?, store, @@ -155,7 +161,20 @@ impl DreContext { return Ok(a.clone()); } - let ic_admin = self.store.ic_admin(&self.version, self.network(), self.neuron().await?).await?; + let ic_admin: Arc = match &self.ic_admin_path_override { + // An explicit `--ic-admin` binary was provided: use it directly and skip + // any download/version resolution. + Some(path) => { + // If a filesystem path was given (rather than a bare command resolved + // via $PATH), fail early with a clear message when it's missing. + if (path.contains(std::path::MAIN_SEPARATOR) || path.contains('/')) && !std::path::Path::new(path).exists() { + return Err(anyhow::anyhow!("ic-admin binary specified via --ic-admin not found at `{}`", path)); + } + info!("Using explicit ic-admin: {}", path); + Arc::new(IcAdminImpl::new(self.network().clone(), Some(path.clone()), self.neuron().await?)) + } + None => self.store.ic_admin(&self.version, self.network(), self.neuron().await?).await?, + }; *self.ic_admin.borrow_mut() = Some(ic_admin.clone()); Ok(ic_admin) } @@ -314,6 +333,7 @@ pub mod tests { artifact_downloader, neuron: RefCell::new(Some(neuron.clone())), version: IcAdminVersion::Strict("Shouldn't reach this because of mock".to_string()), + ic_admin_path_override: None, neuron_opts: super::NeuronOpts { auth_opts: AuthOpts { private_key_pem: None, diff --git a/rs/cli/src/ctx/unit_tests.rs b/rs/cli/src/ctx/unit_tests.rs index 4400e5bc6..c209acca4 100644 --- a/rs/cli/src/ctx/unit_tests.rs +++ b/rs/cli/src/ctx/unit_tests.rs @@ -43,6 +43,7 @@ async fn get_context(network: &Network, version: IcAdminVersion) -> anyhow::Resu false, AuthRequirement::Anonymous, version, + None, Arc::new(MockCordonedFeatureFetcher::new()), Arc::new(MockHealthStatusQuerier::new()), Store::new(false)?, @@ -55,7 +56,9 @@ struct AdminVersionTestScenario<'a> { name: &'static str, version: IcAdminVersion, should_delete_status_file: bool, - should_contain: Option<&'a str>, + // Acceptable substrings for the resolved ic-admin path. Empty means the + // resolution is expected to fail. Multiple values means "any of". + should_contain: Vec<&'a str>, } impl<'a> AdminVersionTestScenario<'a> { @@ -64,7 +67,7 @@ impl<'a> AdminVersionTestScenario<'a> { name, version: IcAdminVersion::FromRegistry, should_delete_status_file: false, - should_contain: None, + should_contain: vec![], } } @@ -81,7 +84,14 @@ impl<'a> AdminVersionTestScenario<'a> { fn should_contain(self, ver: &'a str) -> Self { Self { - should_contain: Some(ver), + should_contain: vec![ver], + ..self + } + } + + fn should_contain_any(self, vers: Vec<&'a str>) -> Self { + Self { + should_contain: vers, ..self } } @@ -90,21 +100,25 @@ impl<'a> AdminVersionTestScenario<'a> { #[test] fn init_tests_ic_admin_version() { let runtime = tokio::runtime::Runtime::new().unwrap(); - let version_on_s3 = "e47293c0bd7f39540245913f7f75be3d6863183c"; + // A real commit that has a published IC release (release-2026-06-04_04-52-canister-logging). + let released_version = "fb721da900b9e9219773ee312f987971338f7c62"; let mainnet = Network::mainnet_unchecked().unwrap(); let registry_version = runtime.block_on(registry_canister_version(mainnet.get_nns_urls()[0].clone())).unwrap(); let tests = &[ + // The registry/NNS canister commit is usually not a tagged release, so dre + // either matches it directly (if it happens to be released) or falls back to + // the embedded version. Both are valid outcomes. AdminVersionTestScenario::new("match registry canister") .delete_status_file() - .should_contain(®istry_version.stringified_hash), + .should_contain_any(vec![®istry_version.stringified_hash, FALLBACK_IC_ADMIN_VERSION]), AdminVersionTestScenario::new("use default version") .version(IcAdminVersion::Fallback) .should_contain(FALLBACK_IC_ADMIN_VERSION), - AdminVersionTestScenario::new("existing version on s3") - .version(IcAdminVersion::Strict(version_on_s3.to_string())) - .should_contain(version_on_s3), - AdminVersionTestScenario::new("random version not present on s3").version(IcAdminVersion::Strict("random-version".to_string())), + AdminVersionTestScenario::new("strict released version") + .version(IcAdminVersion::Strict(released_version.to_string())) + .should_contain(released_version), + AdminVersionTestScenario::new("strict unreleased version errors").version(IcAdminVersion::Strict("random-version".to_string())), ]; for test in tests { @@ -115,7 +129,7 @@ fn init_tests_ic_admin_version() { let maybe_ctx = runtime.block_on(get_context(&mainnet, test.version.clone())); - if let Some(ver) = test.should_contain { + if !test.should_contain.is_empty() { assert!( maybe_ctx.is_ok(), "Test `{}`: expected to create DreContext, but got error: {:?}", @@ -128,11 +142,11 @@ fn init_tests_ic_admin_version() { assert!(ic_admin_path.is_ok(), "Expected Ok, but was: {:?}", ic_admin_path); let ic_admin_path = ic_admin_path.unwrap().ic_admin_path().unwrap_or_default(); assert!( - ic_admin_path.contains(ver), - "Test `{}`: ic_admin_path `{}`, expected version `{}`", + test.should_contain.iter().any(|ver| ic_admin_path.contains(ver)), + "Test `{}`: ic_admin_path `{}`, expected to contain one of `{:?}`", test.name, ic_admin_path, - ver + test.should_contain ) } else { assert!( @@ -176,6 +190,7 @@ async fn get_ctx_for_neuron_test( true, requirement, IcAdminVersion::Strict("Shouldn't get to here".to_string()), + None, Arc::new(MockCordonedFeatureFetcher::new()), Arc::new(MockHealthStatusQuerier::new()), Store::new(offline)?, diff --git a/rs/cli/src/exe/args.rs b/rs/cli/src/exe/args.rs index 8ecbb54b5..c9a287a53 100644 --- a/rs/cli/src/exe/args.rs +++ b/rs/cli/src/exe/args.rs @@ -13,15 +13,16 @@ pub struct GlobalArgs { #[clap(long, global = true, env = "NEURON_ID", visible_aliases = &["neuron", "proposer"])] pub neuron_id: Option, - /// Path to explicitly state ic-admin path to use + /// Path to an explicit ic-admin binary to use. When set, dre will not + /// download ic-admin and will use this binary instead (overrides + /// --ic-admin-version). A bare command name is resolved via $PATH. #[clap(long, global = true, env = "IC_ADMIN")] pub ic_admin: Option, - #[clap(long, global = true, env = "IC_ADMIN_VERSION", default_value = "from-registry", value_parser = clap::value_parser!(IcAdminVersion), help = r#"Specify the version of ic admin to use -Options: - 1. from-governance, governance, govn, g => same as governance canister - 2. default, d => strict default version, embedded at build time - 3. => specific commit"#)] + #[clap(long, global = true, env = "IC_ADMIN_VERSION", default_value = "from-registry", value_parser = clap::value_parser!(IcAdminVersion), help = r#"Specify the version of ic-admin to use: + 1. from-registry, registry, reg, r, from-governance, governance, govn, g => version matching the NNS registry canister (default) + 2. fallback, default, f, d => fallback version embedded at build time + 3. => specific commit/release (scans all IC releases)"#)] pub ic_admin_version: IcAdminVersion, #[clap( @@ -75,9 +76,39 @@ pub enum IcAdminVersion { impl From<&str> for IcAdminVersion { fn from(value: &str) -> Self { match value { - "from-registry" | "registry" | "r" | "reg" => Self::FromRegistry, - "fallback" | "f" => Self::Fallback, + // The registry canister version is the version the NNS/governance has + // deployed, hence the `governance` aliases are accepted here too. + "from-registry" | "registry" | "reg" | "r" | "from-governance" | "governance" | "govn" | "g" => Self::FromRegistry, + "fallback" | "f" | "default" | "d" => Self::Fallback, s => Self::Strict(s.to_string()), } } } + +#[cfg(test)] +mod tests { + use super::IcAdminVersion; + + #[test] + fn ic_admin_version_aliases() { + for alias in ["from-registry", "registry", "reg", "r", "from-governance", "governance", "govn", "g"] { + assert!( + matches!(IcAdminVersion::from(alias), IcAdminVersion::FromRegistry), + "`{}` should parse as FromRegistry", + alias + ); + } + for alias in ["fallback", "f", "default", "d"] { + assert!( + matches!(IcAdminVersion::from(alias), IcAdminVersion::Fallback), + "`{}` should parse as Fallback", + alias + ); + } + // Anything else is treated as a specific commit/release. + assert!(matches!( + IcAdminVersion::from("fb721da900b9e9219773ee312f987971338f7c62"), + IcAdminVersion::Strict(c) if c == "fb721da900b9e9219773ee312f987971338f7c62" + )); + } +} diff --git a/rs/cli/src/store.rs b/rs/cli/src/store.rs index bd51c897f..0c79735ab 100644 --- a/rs/cli/src/store.rs +++ b/rs/cli/src/store.rs @@ -12,6 +12,92 @@ use log::{debug, info, warn}; use std::os::unix::fs::PermissionsExt; use std::{io::Read, path::PathBuf, sync::Arc, time::Duration}; +#[derive(serde::Deserialize)] +struct GitRefObject { + sha: String, +} + +#[derive(serde::Deserialize)] +struct GitRef { + #[serde(rename = "ref")] + ref_name: String, + object: GitRefObject, +} + +// The earliest year for which dfinity/ic publishes dated `release-*` tags. +const EARLIEST_IC_RELEASE_YEAR: i32 = 2024; + +/// Resolves an IC commit hash to its GitHub release tag (e.g. +/// `release-2026-06-04_04-52-base`). +/// +/// IC release tags are named `release-YYYY-MM-DD_HH-MM-`, so they are +/// grouped per year via the `matching-refs` API. For the common case (version +/// derived from the registry canister or the embedded fallback) the commit is +/// always recent, so we only look at the current year's releases. A full scan +/// back to [`EARLIEST_IC_RELEASE_YEAR`] is only performed when the user pins an +/// arbitrary `--ic-admin-version `, which may point to an older release. +/// +/// Returns `Ok(None)` when no release tag matches the commit (the commit exists +/// but isn't a published IC release); network/rate-limit problems are returned +/// as `Err`. +async fn find_github_release_tag_for_commit(commit: &str, deep_scan: bool) -> anyhow::Result> { + use chrono::Datelike; + let client = reqwest::Client::builder().user_agent("dre-cli").build()?; + let current_year = chrono::Utc::now().year(); + + // Always include the previous year as well, so lookups keep working in early + // January before the first release of the new year is published. + let oldest_year = if deep_scan { + EARLIEST_IC_RELEASE_YEAR + } else { + (current_year - 1).max(EARLIEST_IC_RELEASE_YEAR) + }; + + // Authenticated requests get a much higher rate limit (5000/hour vs 60/hour). + let github_token = std::env::var("GITHUB_TOKEN").ok().filter(|t| !t.is_empty()); + + for year in (oldest_year..=current_year).rev() { + let mut page = 1u32; + loop { + let url = format!("https://api.github.com/repos/dfinity/ic/git/matching-refs/tags/release-{}", year); + let mut request = client.get(&url).query(&[("per_page", "100"), ("page", page.to_string().as_str())]); + if let Some(token) = &github_token { + request = request.bearer_auth(token); + } + let response = request.send().await?; + + // Surface GitHub rate-limiting explicitly, since unauthenticated requests + // are capped at 60/hour per IP and the raw 403 is otherwise cryptic. + if response.status() == reqwest::StatusCode::FORBIDDEN + && response.headers().get("x-ratelimit-remaining").and_then(|v| v.to_str().ok()) == Some("0") + { + return Err(anyhow::anyhow!( + "GitHub API rate limit exceeded while resolving the ic-admin release for commit {}. \ + Set the GITHUB_TOKEN environment variable to raise the limit, or pass an explicit \ + binary with --ic-admin .", + commit + )); + } + let refs: Vec = response.error_for_status()?.json().await?; + + let count = refs.len(); + for git_ref in refs { + if git_ref.object.sha == commit { + let tag = git_ref.ref_name.trim_start_matches("refs/tags/").to_string(); + return Ok(Some(tag)); + } + } + + if count < 100 { + break; + } + page += 1; + } + } + + Ok(None) +} + use crate::{ auth::Neuron, cordoned_feature_fetcher::{CordonedFeatureFetcher, CordonedFeatureFetcherImpl}, @@ -26,7 +112,7 @@ pub struct Store { } const DURATION_BETWEEN_CHECKS_FOR_NEW_IC_ADMIN: Duration = Duration::from_secs(60 * 60 * 24); -pub const FALLBACK_IC_ADMIN_VERSION: &str = "1a1cb8cbff5e5c5c1fd01ec37e3c22e5119f12c3"; +pub const FALLBACK_IC_ADMIN_VERSION: &str = "b95f4a32b41798de115aac9298b51dd1662f1da5"; impl Store { #[cfg(not(test))] @@ -168,15 +254,28 @@ impl Store { Ok(self.ic_admin_revision_dir()?.join(version).join("ic-admin")) } - async fn download_ic_admin(&self, version: &str, path: &PathBuf) -> anyhow::Result<()> { - let url = if std::env::consts::OS == "macos" { - // Apple Silicon will emulate x86 architecture. - format!("https://download.dfinity.systems/ic/{version}/binaries/x86_64-darwin/ic-admin.gz") - } else { - format!("https://download.dfinity.systems/ic/{version}/binaries/x86_64-linux/ic-admin.gz") + async fn download_ic_admin(&self, tag: &str, path: &PathBuf) -> anyhow::Result<()> { + // dfinity/ic only publishes these ic-admin assets. Notably there is no + // x86_64 (Intel) macOS build, so that platform is unsupported. + let asset_name = match (std::env::consts::OS, std::env::consts::ARCH) { + ("macos", "aarch64") => "ic-admin-arm64-darwin.gz", + ("linux", "aarch64") => "ic-admin-arm64-linux.gz", + ("linux", "x86_64") => "ic-admin-x86_64-linux.gz", + ("macos", arch) => { + return Err(anyhow::anyhow!( + "ic-admin is not published for macOS on {arch}; only Apple Silicon (arm64) is supported. \ + Provide a binary explicitly with --ic-admin ." + )); + } + (os, arch) => { + return Err(anyhow::anyhow!( + "ic-admin is not published for {os}/{arch}. Provide a binary explicitly with --ic-admin ." + )); + } }; - info!("Downloading ic-admin version: {} from {}", version, url); - let body = reqwest::get(url).await?.error_for_status()?.bytes().await?; + let url = format!("https://github.com/dfinity/ic/releases/download/{tag}/{asset_name}"); + info!("Downloading ic-admin release {} from {}", tag, url); + let body = reqwest::get(&url).await?.error_for_status()?.bytes().await?; let mut decoded = GzDecoder::new(body.as_ref()); let path_parent = path.parent().ok_or(anyhow::anyhow!("Failed to get parent for ic admin revision dir"))?; @@ -187,15 +286,60 @@ impl Store { Ok(()) } - async fn init_ic_admin(&self, version: &str, network: &Network, neuron: Neuron) -> anyhow::Result> { - let path = self.ic_admin_path_for_version(version)?; - - if !path.exists() { - self.download_ic_admin(version, &path).await?; - } + /// Ensures an ic-admin binary for `version` is available locally, downloading + /// it from the matching GitHub release if needed. + /// + /// When `allow_fallback` is set and `version` has no published release (which + /// is common for the registry/NNS-derived version, since only elected releases + /// are tagged), it transparently falls back to [`FALLBACK_IC_ADMIN_VERSION`]. + /// Returns the IcAdmin handle together with the version actually used. + async fn init_ic_admin( + &self, + version: &str, + network: &Network, + neuron: Neuron, + deep_scan: bool, + allow_fallback: bool, + ) -> anyhow::Result<(Arc, String)> { + let mut effective_version = version.to_string(); + let path = self.ic_admin_path_for_version(&effective_version)?; + + let path = if path.exists() { + path + } else { + match find_github_release_tag_for_commit(version, deep_scan).await? { + Some(tag) => { + self.download_ic_admin(&tag, &path).await?; + path + } + None if allow_fallback => { + warn!( + "Version {} has no published IC release; falling back to embedded ic-admin {}", + version, FALLBACK_IC_ADMIN_VERSION + ); + effective_version = FALLBACK_IC_ADMIN_VERSION.to_string(); + let fallback_path = self.ic_admin_path_for_version(&effective_version)?; + if !fallback_path.exists() { + let tag = find_github_release_tag_for_commit(FALLBACK_IC_ADMIN_VERSION, false) + .await? + .ok_or_else(|| anyhow::anyhow!("Fallback ic-admin version {} has no published release", FALLBACK_IC_ADMIN_VERSION))?; + self.download_ic_admin(&tag, &fallback_path).await?; + } + fallback_path + } + None => { + let hint = if deep_scan { + "The commit may not have an associated IC release. Pass an explicit binary with --ic-admin if needed." + } else { + "Only recent releases were checked. If this is an older commit, pin it explicitly with --ic-admin-version ." + }; + return Err(anyhow::anyhow!("No GitHub release tag found for commit {}. {}", version, hint)); + } + } + }; info!("Using ic-admin: {}", path.display()); - Ok(Arc::new(IcAdminImpl::new( + let ic_admin = Arc::new(IcAdminImpl::new( network.clone(), Some( path.to_str() @@ -203,13 +347,17 @@ impl Store { .to_string(), ), neuron, - ))) + )); + Ok((ic_admin, effective_version)) } pub async fn ic_admin(&self, version: &IcAdminVersion, network: &Network, neuron: Neuron) -> anyhow::Result> { match version { - IcAdminVersion::Fallback => self.init_ic_admin(FALLBACK_IC_ADMIN_VERSION, network, neuron).await, - IcAdminVersion::Strict(ver) => self.init_ic_admin(ver, network, neuron).await, + // The fallback version is recent, so a shallow scan suffices. + IcAdminVersion::Fallback => Ok(self.init_ic_admin(FALLBACK_IC_ADMIN_VERSION, network, neuron, false, false).await?.0), + // The user pinned an arbitrary commit which may be old; scan all releases + // and don't silently substitute a different version. + IcAdminVersion::Strict(ver) => Ok(self.init_ic_admin(ver, network, neuron, true, false).await?.0), // This is the most probable way of running IcAdminVersion::FromRegistry => { let mut status_file = fs_err::File::open(&self.ic_admin_status_file()?)?; @@ -258,11 +406,13 @@ impl Store { } }; - let ic_admin = self.init_ic_admin(&version, network, neuron).await?; + // The registry/NNS canister version is a recent commit but is often + // not a tagged release, so allow falling back to the embedded version. + let (ic_admin, effective_version) = self.init_ic_admin(&version, network, neuron, false, true).await?; - // Only update file when the sync - // with governance has been performed - fs_err::write(self.ic_admin_status_file()?, version)?; + // Cache the version actually used (post-fallback), so subsequent runs + // don't repeatedly try to resolve an unreleased registry commit. + fs_err::write(self.ic_admin_status_file()?, effective_version)?; Ok(ic_admin) } }