diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b40098b..1171e85 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,5 +1,11 @@ on: [push, pull_request] +# https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#concurrency +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + + name: CI jobs: @@ -40,4 +46,4 @@ jobs: - name: Test wasm32-wasip1 build for wasm-pkg-common if: matrix.test_wasm_build - run: cargo build -p wasm-pkg-common --target wasm32-wasip1 \ No newline at end of file + run: cargo build -p wasm-pkg-common --target wasm32-wasip1 diff --git a/Cargo.lock b/Cargo.lock index f577fdf..9d3578c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -107,9 +107,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.102" +version = "1.0.103" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" +checksum = "2a4385e2e34eb35d6b3efe798b9eb88096925d87726c0798709bf56d9ed84af3" [[package]] name = "arraydeque" @@ -1360,6 +1360,12 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" +[[package]] +name = "fixedbitset" +version = "0.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" + [[package]] name = "flate2" version = "1.1.9" @@ -2143,9 +2149,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.102" +version = "0.3.103" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03d04c30968dffe80775bd4d7fb676131cd04a1fb46d2686dbffbaec2d9dfd31" +checksum = "53b44bfcdb3f8d5837a46dae1ca9660a837176eee74a28b229bc626816589102" dependencies = [ "cfg-if", "futures-util", @@ -2292,9 +2298,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.32" +version = "0.4.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "953f07c43838f8e6f9758cab68bf5bed85465e7587ebe0b823f1bcd81978ad3a" +checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" [[package]] name = "logos" @@ -2862,8 +2868,20 @@ version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4c5cc86750666a3ed20bdaf5ca2a0344f9c67674cae0515bec2da16fbaa47db" dependencies = [ - "fixedbitset", + "fixedbitset 0.4.2", + "indexmap 2.14.0", +] + +[[package]] +name = "petgraph" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8701b58ea97060d5e5b155d383a69952a60943f0e6dfe30b04c287beb0b27455" +dependencies = [ + "fixedbitset 0.5.7", + "hashbrown 0.15.5", "indexmap 2.14.0", + "serde", ] [[package]] @@ -3032,7 +3050,7 @@ dependencies = [ "log", "multimap", "once_cell", - "petgraph", + "petgraph 0.6.5", "prettyplease", "prost 0.12.6", "prost-types 0.12.6", @@ -3134,7 +3152,7 @@ dependencies = [ "anstyle", "config", "directories", - "petgraph", + "petgraph 0.6.5", "serde", "serde-value", "tint", @@ -3142,9 +3160,9 @@ dependencies = [ [[package]] name = "quinn" -version = "0.11.9" +version = "0.11.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9e20a958963c291dc322d98411f541009df2ced7b5a4f2bd52337638cfccf20" +checksum = "0c1a41e437b6bbd489372cd4971de128e85c855f56c57f283d20ff016cf7c0a8" dependencies = [ "bytes", "cfg_aliases", @@ -3162,9 +3180,9 @@ dependencies = [ [[package]] name = "quinn-proto" -version = "0.11.14" +version = "0.11.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "434b42fec591c96ef50e21e886936e66d3cc3f737104fdb9b737c40ffb94c098" +checksum = "4fcb935c5bec503c2f0e306bdd3e58bb9029dcb14fa8d9ac76e3a5256ac0763e" dependencies = [ "aws-lc-rs", "bytes", @@ -3198,9 +3216,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.45" +version = "1.0.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" +checksum = "dfbc457d0c7a0759a614551b11a6409e5951f6c7537be1f1b7682b9ae9230368" dependencies = [ "proc-macro2", ] @@ -3574,9 +3592,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.40" +version = "0.23.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef86cd5876211988985292b91c96a8f2d298df24e75989a43a3c73f2d4d8168b" +checksum = "6b92b125634d9b795e7beca796cc790df15a7fb38323bf3196fda83292d06b1f" dependencies = [ "aws-lc-rs", "log", @@ -4306,9 +4324,9 @@ dependencies = [ [[package]] name = "time" -version = "0.3.49" +version = "0.3.51" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "711a53c2d47bbd818258c498c8dbfe186a2526c631495cfe7e078567f86b8469" +checksum = "85c17d80feb7334b40c484e45ed1a5273dfd8bfda537c3be2e74a06a6686f327" dependencies = [ "deranged", "num-conv", @@ -4326,9 +4344,9 @@ checksum = "9e1c906769ad99c88eaa54e728060edef082f8e358ff32030cb7c7d315e81109" [[package]] name = "time-macros" -version = "0.2.29" +version = "0.2.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "71c652a3727a9cbb9a02f707f530b618ce00d0ccd762009c8c23bd191df3c17d" +checksum = "dcef1a61bdb119096e153208ec5cbec23944ce8bca13be5c7f60c634f7403935" dependencies = [ "num-conv", "time-core", @@ -4987,9 +5005,9 @@ dependencies = [ [[package]] name = "wasm-bindgen" -version = "0.2.125" +version = "0.2.126" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ddb3f79143bced6de84270411622a2699cee572fc0875aeaf1e7867cf9fca1a" +checksum = "4b067c0c11094aef6b7a801c1e34a26affafdf3d051dba08456b868789aaf9a4" dependencies = [ "cfg-if", "once_cell", @@ -5000,9 +5018,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.75" +version = "0.4.76" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "503b14d284f2c8dac03b819967e155ea753f573586193b2b2c95990cb5d69280" +checksum = "c62df1340f32221cb9c54d6a27b030e3dba64361d4a95bed55f9aacb44da291d" dependencies = [ "js-sys", "wasm-bindgen", @@ -5010,9 +5028,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.125" +version = "0.2.126" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e21a184b13fb19e157296e2c46056aec9092264fab83e4ba59e68c61b323c3d" +checksum = "167ce5e579f6bcf889c4f7175a8a5a585de84e8ff93976ce393efa5f2837aab1" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -5020,9 +5038,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.125" +version = "0.2.126" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fecefd9c35bd935a20fc3fc344b5f29138961e4f47fb03297d88f2587afb5ebd" +checksum = "f3997c7839262f4ef12cf90b818d6340c18e80f263f1a94bf157d0ec4420380e" dependencies = [ "bumpalo", "proc-macro2", @@ -5033,9 +5051,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.125" +version = "0.2.126" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23939e44bb9a5d7576fa2b563dc2e136628f1224e88a8deed09e04858b77871f" +checksum = "dc1b4cb0cc549fcf58d7dfc081778139b3d283a081644e833e84682ad71cea24" dependencies = [ "unicode-ident", ] @@ -5051,7 +5069,7 @@ dependencies = [ "im-rc", "indexmap 2.14.0", "log", - "petgraph", + "petgraph 0.6.5", "serde", "serde_derive", "serde_yaml", @@ -5112,7 +5130,7 @@ dependencies = [ [[package]] name = "wasm-pkg-client" -version = "0.15.1" +version = "0.16.0" dependencies = [ "anyhow", "async-trait", @@ -5123,6 +5141,7 @@ dependencies = [ "futures-util", "oci-client", "oci-wasm", + "petgraph 0.8.3", "rcgen", "reqwest 0.12.28", "secrecy", @@ -5148,7 +5167,7 @@ dependencies = [ [[package]] name = "wasm-pkg-common" -version = "0.15.1" +version = "0.16.0" dependencies = [ "anyhow", "bytes", @@ -5167,12 +5186,14 @@ dependencies = [ [[package]] name = "wasm-pkg-core" -version = "0.15.1" +version = "0.16.0" dependencies = [ "anyhow", "futures-util", + "glob", "indexmap 2.14.0", "libc", + "petgraph 0.8.3", "rstest", "semver", "serde", @@ -5284,9 +5305,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.102" +version = "0.3.103" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6430a72df5eb332242960fe84b3002a241163998241eb596d4f739b9757061d" +checksum = "8622dcb61c0bcc9fffa6938bed81210af2da9a7e4a1a834b2e37a59b6dfb6141" dependencies = [ "js-sys", "wasm-bindgen", @@ -5715,7 +5736,7 @@ dependencies = [ [[package]] name = "wkg" -version = "0.15.1" +version = "0.16.0" dependencies = [ "anyhow", "base64 0.22.1", diff --git a/Cargo.toml b/Cargo.toml index 7c4bd80..03eef6f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,7 +4,7 @@ resolver = "2" [workspace.package] edition = "2021" -version = "0.15.1" +version = "0.16.0" authors = ["The Wasmtime Project Developers"] license = "Apache-2.0 WITH LLVM-exception" @@ -21,6 +21,7 @@ oci-client = { version = "0.16", default-features = false, features = [ oci-wasm = { version = "0.4", default-features = false, features = [ "rustls-tls", ] } +petgraph = "0.8.3" rcgen = "0.14.8" semver = "1.0.23" serde = { version = "1.0", features = ["derive"] } @@ -37,9 +38,22 @@ tracing-subscriber = { version = "0.3.20", default-features = false, features = "fmt", "env-filter", ] } -wasm-pkg-common = { version = "0.15.1", path = "crates/wasm-pkg-common" } -wasm-pkg-client = { version = "0.15.1", path = "crates/wasm-pkg-client" } -wasm-pkg-core = { version = "0.15.1", path = "crates/wasm-pkg-core" } +wasm-pkg-common = { version = "0.16.0", path = "crates/wasm-pkg-common" } +wasm-pkg-client = { version = "0.16.0", path = "crates/wasm-pkg-client" } +wasm-pkg-core = { version = "0.16.0", path = "crates/wasm-pkg-core" } wasm-metadata = "0.244" wit-component = "0.244" wit-parser = "0.244" + +# https://github.com/crate-ci/typos/blob/master/docs/reference.md +[workspace.metadata.typos.default] +extend-ignore-re = [ + "\\d\\w{4,}\\d", +] +extend-ignore-words-re = [ + '^[a-zA-Z]{1,3}$' # ignore words up to length 3 +] + +[workspace.metadata.typos.default.extend-words] +# strategy shorthand +strat = "strat" diff --git a/crates/wasm-pkg-client/Cargo.toml b/crates/wasm-pkg-client/Cargo.toml index 3fa5f82..d10fc69 100644 --- a/crates/wasm-pkg-client/Cargo.toml +++ b/crates/wasm-pkg-client/Cargo.toml @@ -47,8 +47,9 @@ wasm-metadata = { workspace = true } warg-protocol = "0.9.2" wasm-pkg-common = { workspace = true, features = ["registry-config"] } wit-component = { workspace = true } +petgraph = "0.8.3" +tempfile = { workspace = true } [dev-dependencies] rcgen = { workspace = true } -tempfile = { workspace = true } testcontainers = { workspace = true } diff --git a/crates/wasm-pkg-client/src/caching/file.rs b/crates/wasm-pkg-client/src/caching/file.rs index 7c65332..e54a1e8 100644 --- a/crates/wasm-pkg-client/src/caching/file.rs +++ b/crates/wasm-pkg-client/src/caching/file.rs @@ -16,6 +16,7 @@ use crate::{ContentStream, Release}; use super::Cache; +#[derive(Clone)] pub struct FileCache { root: PathBuf, } diff --git a/crates/wasm-pkg-client/src/caching/mod.rs b/crates/wasm-pkg-client/src/caching/mod.rs index 59c0d93..e90a161 100644 --- a/crates/wasm-pkg-client/src/caching/mod.rs +++ b/crates/wasm-pkg-client/src/caching/mod.rs @@ -45,20 +45,12 @@ pub trait Cache { /// A client that caches response data using the given cache implementation. Can be used without an /// underlying client to be used as a read-only cache. +#[derive(Clone)] pub struct CachingClient { client: Option, cache: Arc, } -impl Clone for CachingClient { - fn clone(&self) -> Self { - Self { - client: self.client.clone(), - cache: self.cache.clone(), - } - } -} - impl CachingClient { /// Creates a new caching client from the given client and cache implementation. If no client is /// given, the client will be in offline or read-only mode, meaning it will only be able to return diff --git a/crates/wasm-pkg-client/src/lib.rs b/crates/wasm-pkg-client/src/lib.rs index b093b88..3d1c85c 100644 --- a/crates/wasm-pkg-client/src/lib.rs +++ b/crates/wasm-pkg-client/src/lib.rs @@ -42,10 +42,10 @@ use std::{collections::HashMap, pin::Pin}; use anyhow::anyhow; use bytes::Bytes; use futures_util::Stream; -use publisher::PackagePublisher; use tokio::io::AsyncSeekExt; use tokio::sync::RwLock; use tokio_util::io::SyncIoBridge; +use wasm_pkg_common::metadata::{LOCAL_PROTOCOL, OCI_PROTOCOL, WARG_PROTOCOL}; pub use wasm_pkg_common::{ config::{Config, CustomConfig, RegistryMapping}, digest::ContentDigest, @@ -56,8 +56,10 @@ pub use wasm_pkg_common::{ }; use wit_component::DecodedWasm; +use crate::local::LocalBackend; use crate::metadata::RegistryMetadataExt; -use crate::{loader::PackageLoader, local::LocalBackend, oci::OciBackend, warg::WargBackend}; +pub use crate::{loader::PackageLoader, publisher::PackagePublisher}; +use crate::{oci::OciBackend, warg::WargBackend}; pub use release::{Release, VersionInfo}; @@ -69,6 +71,7 @@ pub type PublishingSource = Pin>; /// A supertrait combining tokio's AsyncRead and AsyncSeek. pub trait ReaderSeeker: tokio::io::AsyncRead + tokio::io::AsyncSeek {} + impl ReaderSeeker for T where T: tokio::io::AsyncRead + tokio::io::AsyncSeek {} trait LoaderPublisher: PackageLoader + PackagePublisher {} @@ -158,7 +161,7 @@ impl Client { .await } - /// Publishes the given reader as a package release. TThe package name and version will be read + /// Publishes the given reader as a package release. The package name and version will be read /// from the component if not given as part of `additional_options`. Returns the package name /// and version of the published release. pub async fn publish_release_data( @@ -191,111 +194,119 @@ impl Client { .map(|_| (package, version)) } + fn resolve_registry( + &self, + package: &PackageRef, + registry_override: Option, + ) -> Result { + if let Some(registry) = registry_override { + return Ok(registry); + } + self.config + .resolve_registry(package) + .cloned() + .ok_or_else(|| Error::NoRegistryForNamespace(package.namespace().clone())) + } + async fn resolve_source( &self, package: &PackageRef, registry_override: Option, ) -> Result, Error> { let is_override = registry_override.is_some(); - let registry = if let Some(registry) = registry_override { - registry + let registry = self.resolve_registry(package, registry_override)?; + tracing::debug!(?registry, "resolved registry"); + + if let Some(source) = self.sources.read().await.get(®istry) { + return Ok(source.clone()); + } + + let registry_config = self + .config + .registry_config(®istry) + .cloned() + .unwrap_or_default(); + + let maybe_metadata = self + .config + .package_registry_override(package) + .and_then(|mapping| match mapping { + RegistryMapping::Custom(custom) => Some(custom.metadata.clone()), + _ => None, + }) + .or_else(|| { + self.config + .namespace_registry(package.namespace()) + .and_then(|meta| { + // If the overridden registry matches the registry we are trying to resolve, we + // should use the metadata, otherwise we'll need to fetch the metadata from the + // registry + match (meta, is_override) { + (RegistryMapping::Custom(custom), true) + if custom.registry == registry => + { + Some(custom.metadata.clone()) + } + (RegistryMapping::Custom(custom), false) => { + Some(custom.metadata.clone()) + } + _ => None, + } + }) + }); + + let registry_meta = if let Some(meta) = maybe_metadata { + meta + } else if registry_config.default_backend() == LOCAL_PROTOCOL.into() { + // Skip fetching metadata for "local" source + RegistryMetadata::default() } else { - self.config - .resolve_registry(package) - .ok_or_else(|| Error::NoRegistryForNamespace(package.namespace().clone()))? - .to_owned() - }; - let has_key = { - let sources = self.sources.read().await; - sources.contains_key(®istry) + RegistryMetadata::fetch_or_default(®istry).await }; - if !has_key { - let registry_config = self - .config - .registry_config(®istry) - .cloned() - .unwrap_or_default(); - // Skip fetching metadata for "local" source - let should_fetch_meta = registry_config.default_backend() != Some("local"); - let maybe_metadata = self - .config - .package_registry_override(package) - .and_then(|mapping| match mapping { - RegistryMapping::Custom(custom) => Some(custom.metadata.clone()), - _ => None, - }) - .or_else(|| { - self.config - .namespace_registry(package.namespace()) - .and_then(|meta| { - // If the overridden registry matches the registry we are trying to resolve, we - // should use the metadata, otherwise we'll need to fetch the metadata from the - // registry - match (meta, is_override) { - (RegistryMapping::Custom(custom), true) - if custom.registry == registry => - { - Some(custom.metadata.clone()) - } - (RegistryMapping::Custom(custom), false) => { - Some(custom.metadata.clone()) - } - _ => None, - } - }) - }); - - let registry_meta = if let Some(meta) = maybe_metadata { - meta - } else if should_fetch_meta { - RegistryMetadata::fetch_or_default(®istry).await - } else { - RegistryMetadata::default() - }; - - // Resolve backend type - let backend_type = match registry_config.default_backend() { - // If the local config specifies a backend type, use it - Some(backend_type) => Some(backend_type), - None => { - // If the registry metadata indicates a preferred protocol, use it - let preferred_protocol = registry_meta.preferred_protocol(); - // ...except registry metadata cannot force a local backend - if preferred_protocol == Some("local") { - return Err(Error::InvalidRegistryMetadata(anyhow!( - "registry metadata with 'local' protocol not allowed" - ))); - } - preferred_protocol - } - } - // Otherwise use the default backend - .unwrap_or("oci"); - tracing::debug!(?backend_type, "Resolved backend type"); - - let source: InnerClient = match backend_type { - "local" => Box::new(LocalBackend::new(registry_config)?), - "oci" => Box::new(OciBackend::new( - ®istry, - ®istry_config, - ®istry_meta, - )?), - "warg" => { - Box::new(WargBackend::new(®istry, ®istry_config, ®istry_meta).await?) - } - other => { - return Err(Error::InvalidConfig(anyhow!( - "unknown backend type {other:?}" + // Resolve backend type + let backend_type = match registry_config.default_backend() { + // If the local config specifies a backend type, use it + Some(backend_type) => Some(backend_type), + None => { + // If the registry metadata indicates a preferred protocol, use it + let preferred_protocol = registry_meta.preferred_protocol(); + // ...except registry metadata cannot force a local backend + if preferred_protocol == Some(LOCAL_PROTOCOL) { + return Err(Error::InvalidRegistryMetadata(anyhow!( + "registry metadata with 'local' protocol not allowed" ))); } - }; - self.sources - .write() - .await - .insert(registry.clone(), Arc::new(source)); + preferred_protocol + } } - Ok(self.sources.read().await.get(®istry).unwrap().clone()) + // Otherwise use the default backend + .unwrap_or(OCI_PROTOCOL); + tracing::debug!(?backend_type, "Resolved backend type"); + + let source: InnerClient = match backend_type { + LOCAL_PROTOCOL => Box::new(LocalBackend::new(registry_config)?), + OCI_PROTOCOL => Box::new(OciBackend::new( + ®istry, + ®istry_config, + ®istry_meta, + )?), + WARG_PROTOCOL => { + Box::new(WargBackend::new(®istry, ®istry_config, ®istry_meta).await?) + } + other => { + return Err(Error::InvalidConfig(anyhow!( + "unknown backend type {other:?}" + ))); + } + }; + let source = Arc::new(source); + self.sources + .write() + .await + .insert(registry.clone(), source.clone()); + + Ok(source) } } @@ -342,8 +353,9 @@ fn resolve_package( })?; let version = version.ok_or_else(|| { - crate::Error::InvalidComponent(anyhow::anyhow!( - "component package version not found in the Wasm binary\n\ + crate::Error::InvalidComponent( + anyhow::anyhow!( + "component package version not found in the Wasm binary\n\ \n\ The Wasm file was built without a version in the WIT `package` statement.\n\ Add a version to the `package` statement in your .wit file, e.g.:\n\ @@ -353,7 +365,9 @@ fn resolve_package( Alternatively, specify the package and version explicitly with the --package flag:\n\ \n\ \twkg publish --package :@" - )) + ) + .context(format!("package: {package}")), + ) })?; Ok((data.into_inner(), package, version)) } diff --git a/crates/wasm-pkg-client/src/local.rs b/crates/wasm-pkg-client/src/local.rs index acc8440..4de9b46 100644 --- a/crates/wasm-pkg-client/src/local.rs +++ b/crates/wasm-pkg-client/src/local.rs @@ -5,17 +5,20 @@ use std::{ io, path::{Path, PathBuf}, + sync::Arc, }; use anyhow::anyhow; use async_trait::async_trait; use futures_util::{StreamExt, TryStreamExt}; -use serde::Deserialize; +use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256}; +use tempfile::TempDir; use tokio_util::io::ReaderStream; use wasm_pkg_common::{ config::RegistryConfig, digest::ContentDigest, + metadata::LOCAL_PROTOCOL, package::{PackageRef, Version}, Error, }; @@ -27,13 +30,32 @@ use crate::{ ContentStream, PublishingSource, }; -#[derive(Clone, Debug, Deserialize)] +#[derive(Clone, Debug, Deserialize, Serialize)] pub struct LocalConfig { pub root: PathBuf, + // NOTE: set by [`Self::temp_dir`] to avoid holding onto a separate `TempDir` handle. + #[serde(skip)] + #[doc(hidden)] + _temp_handle: Arc>, } +impl LocalConfig { + /// Creates a [`Self`] with a new temporary directory. + /// The returned config owns the directory and removes the config upon drop. + pub fn temp_dir() -> Result { + let handle = TempDir::new()?; + let root = handle.path().to_path_buf(); + tracing::debug!(registry_dir = %root.display(), "created temporary directory"); + Ok(Self { + root, + _temp_handle: Arc::new(Some(handle)), + }) + } +} + +#[derive(Clone)] pub(crate) struct LocalBackend { - root: PathBuf, + pub(crate) root: PathBuf, } fn registry_path_context(err: io::Error, path: &Path) -> Error { @@ -44,7 +66,7 @@ fn registry_path_context(err: io::Error, path: &Path) -> Error { impl LocalBackend { pub fn new(registry_config: RegistryConfig) -> Result { let config = registry_config - .backend_config::("local")? + .backend_config::(LOCAL_PROTOCOL)? .ok_or_else(|| { Error::InvalidConfig(anyhow!("'local' backend requires configuration")) })?; diff --git a/crates/wasm-pkg-client/src/metadata.rs b/crates/wasm-pkg-client/src/metadata.rs index 27827c3..d88d66d 100644 --- a/crates/wasm-pkg-client/src/metadata.rs +++ b/crates/wasm-pkg-client/src/metadata.rs @@ -37,6 +37,7 @@ impl RegistryMetadataExt for RegistryMetadata { } async fn fetch(registry: &Registry) -> Result, Error> { + // TODO use `core::net::Ipv4Addr::is_loopback()` let scheme = if registry.host() == "localhost" || registry.host() == "127.0.0.1" { "http" } else { diff --git a/crates/wasm-pkg-common/src/config.rs b/crates/wasm-pkg-common/src/config.rs index a20da34..809d194 100644 --- a/crates/wasm-pkg-common/src/config.rs +++ b/crates/wasm-pkg-common/src/config.rs @@ -7,7 +7,11 @@ use std::{ use serde::{Deserialize, Serialize}; use crate::{ - label::Label, metadata::RegistryMetadata, package::PackageRef, registry::Registry, Error, + label::Label, + metadata::{RegistryMetadata, LOCAL_PROTOCOL}, + package::PackageRef, + registry::Registry, + Error, }; mod toml; @@ -194,14 +198,7 @@ impl Config { /// - Hard-coded fallbacks for certain well-known namespaces pub fn resolve_registry(&self, package: &PackageRef) -> Option<&Registry> { let namespace = package.namespace(); - // look in `self.package_registry_overrides ` - // then in `self.namespace_registries` - if let Some(reg) = self - .package_registry_overrides - .get(package) - .or_else(|| self.namespace_registries.get(namespace)) - .map(|pkg| pkg.registry()) - { + if let Some(reg) = self.resolve_mapping(package).map(|ns| ns.registry()) { return Some(reg); } else if let Some(reg) = self.default_registry.as_ref() { return Some(reg); @@ -210,6 +207,15 @@ impl Config { self.fallback_namespace_registries.get(namespace) } + fn resolve_mapping(&self, package: &PackageRef) -> Option<&RegistryMapping> { + let namespace = package.namespace(); + // look in `self.package_registry_overrides` + // then in `self.namespace_registries` + self.package_registry_overrides + .get(package) + .or_else(|| self.namespace_registries.get(namespace)) + } + /// Returns the default registry. pub fn default_registry(&self) -> Option<&Registry> { self.default_registry.as_ref() @@ -278,6 +284,17 @@ pub struct RegistryConfig { } impl RegistryConfig { + // Created a [`Self`] a default local backend associated with the provided `backend_name` label. + pub fn with_default_backend( + mut self, + backend_name: &str, + backend_config: T, + ) -> Result { + self.default_backend = Some(backend_name.to_string()); + self.set_backend_config(LOCAL_PROTOCOL, backend_config)?; + Ok(self) + } + /// Merges the given other config into this one. pub fn merge(&mut self, other: Self) { let Self { @@ -302,13 +319,10 @@ impl RegistryConfig { pub fn default_backend(&self) -> Option<&str> { match self.default_backend.as_deref() { Some(ty) => Some(ty), - None => { - if self.backend_configs.len() == 1 { - self.backend_configs.keys().next().map(|ty| ty.as_str()) - } else { - None - } + _ if self.backend_configs.len() == 1 => { + self.backend_configs.keys().next().map(|ty| ty.as_str()) } + None => None, } } diff --git a/crates/wasm-pkg-common/src/metadata.rs b/crates/wasm-pkg-common/src/metadata.rs index c3fd663..dda8332 100644 --- a/crates/wasm-pkg-common/src/metadata.rs +++ b/crates/wasm-pkg-common/src/metadata.rs @@ -36,8 +36,12 @@ pub struct RegistryMetadata { warg_url: Option, } -const OCI_PROTOCOL: &str = "oci"; -const WARG_PROTOCOL: &str = "warg"; +/// OCI registry try +pub const OCI_PROTOCOL: &str = "oci"; +/// Warg registry key +pub const WARG_PROTOCOL: &str = "warg"; +/// Local filesystem key +pub const LOCAL_PROTOCOL: &str = "local"; impl RegistryMetadata { /// Returns the registry's preferred protocol. diff --git a/crates/wasm-pkg-common/src/package.rs b/crates/wasm-pkg-common/src/package.rs index 8360c97..00a816e 100644 --- a/crates/wasm-pkg-common/src/package.rs +++ b/crates/wasm-pkg-common/src/package.rs @@ -23,7 +23,7 @@ impl PackageRef { } /// Returns the namespace of the package. - pub fn namespace(&self) -> &Label { + pub const fn namespace(&self) -> &Label { &self.namespace } @@ -68,14 +68,30 @@ impl FromStr for PackageRef { s.to_string().try_into() } } +impl std::fmt::Display for PackageSpec { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + if let Some(version) = &self.version { + write!(f, "{}@{}", self.package, version) + } else { + write!(f, "{}", self.package) + } + } +} /// A package spec combines a [`PackageRef`] with an optional version. -#[derive(Clone, Debug)] +#[derive(Clone, Debug, Eq, PartialEq, PartialOrd, Ord)] pub struct PackageSpec { pub package: PackageRef, pub version: Option, } +impl PartialEq for PackageSpec { + fn eq(&self, other: &str) -> bool { + // clippy --fix will create a recursive callsite here if `self.to_string()` is used instead + format!("{self}") == other + } +} + impl FromStr for PackageSpec { type Err = Error; diff --git a/crates/wasm-pkg-core/Cargo.toml b/crates/wasm-pkg-core/Cargo.toml index a43e021..1e8818b 100644 --- a/crates/wasm-pkg-core/Cargo.toml +++ b/crates/wasm-pkg-core/Cargo.toml @@ -22,6 +22,7 @@ wasm-pkg-common = { workspace = true } wasm-pkg-client = { workspace = true } wit-component = { workspace = true } wit-parser = { workspace = true } +petgraph = { workspace = true } [target.'cfg(unix)'.dependencies.libc] version = "0.2" @@ -42,3 +43,4 @@ features = [ tempfile = { workspace = true } sha2 = { workspace = true } rstest = "0.23" +glob = "0.3.3" diff --git a/crates/wasm-pkg-core/src/config.rs b/crates/wasm-pkg-core/src/config.rs index bfc8ac3..ef183df 100644 --- a/crates/wasm-pkg-core/src/config.rs +++ b/crates/wasm-pkg-core/src/config.rs @@ -58,6 +58,16 @@ impl Config { .await .context("unable to write config to path") } + + /// Returns a matching override name and value for the input path + pub(crate) fn has_override(&self, path: impl AsRef) -> bool { + let path = path.as_ref().canonicalize().ok(); + self.overrides + .iter() + .flat_map(|map| map.iter()) + .find(|(_, o)| o.path.as_ref().and_then(|p| p.canonicalize().ok()) == path) + .is_some() + } } #[derive(Debug, Clone, Serialize, Deserialize, Default, PartialEq, Eq)] diff --git a/crates/wasm-pkg-core/src/resolver.rs b/crates/wasm-pkg-core/src/resolver.rs index 51a41eb..af6f2f2 100644 --- a/crates/wasm-pkg-core/src/resolver.rs +++ b/crates/wasm-pkg-core/src/resolver.rs @@ -2,7 +2,7 @@ // NOTE(thomastaylor312): This is copied and adapted from the `cargo-component` crate: https://github.com/bytecodealliance/cargo-component/blob/f0be1c7d9917aa97e9102e69e3b838dae38d624b/crates/core/src/registry.rs use std::{ - collections::{hash_map, HashMap, HashSet}, + collections::{hash_map, BTreeSet, HashMap, HashSet}, fmt::Debug, ops::{Deref, DerefMut}, path::{Path, PathBuf}, @@ -12,16 +12,21 @@ use std::{ use anyhow::{bail, Context, Result}; use futures_util::TryStreamExt; use indexmap::{IndexMap, IndexSet}; +use petgraph::{acyclic::Acyclic, graph::NodeIndex, stable_graph::StableDiGraph, Direction}; use semver::{Comparator, Op, Version, VersionReq}; use tokio::io::{AsyncRead, AsyncReadExt}; use wasm_pkg_client::{ caching::{CachingClient, FileCache}, Client, Config, ContentDigest, Error as WasmPkgError, PackageRef, Release, VersionInfo, }; +use wasm_pkg_common::package::PackageSpec; use wit_component::DecodedWasm; use wit_parser::{PackageId, PackageName, Resolve, UnresolvedPackageGroup, WorldId}; -use crate::{lock::LockFile, wit::get_packages}; +use crate::{ + lock::LockFile, + wit::{get_local_dependencies, get_packages}, +}; /// The name of the default registry. pub const DEFAULT_REGISTRY_NAME: &str = "default"; @@ -834,3 +839,232 @@ fn visit<'a>( Ok(()) } + +/// Graph of publishable packages with the [`petgraph::Direction`] edges describing the dependency direction. +pub type DependencyGraph = Acyclic>; + +/// Mapping of [`PackageRef`]s to the respective index inside the dependency graph. +pub type LocalPackageIndex = HashMap; + +/// State for tracking dependencies during upload. +pub struct PublishPlan { + /// Graph of publishable packages where the edges are `(dependency -DependencyOf->) dependent)` + dependents: DependencyGraph, + // TODO look at using cargo's `InternedString` type for `PackageRef`: + // https://docs.rs/cargo/latest/cargo/util/interning/struct.InternedString.html + indices: LocalPackageIndex, +} + +impl PublishPlan { + /// Generate [`Self`] from a list of WIT package paths (files or directories). + pub fn from_paths(paths: &[impl AsRef]) -> Result { + let (graph, indices) = get_local_dependencies(paths)?; + { + // collection of local packages that have no version + let missing_version = graph + .nodes_iter() + .map(|f| graph[f].clone()) + .filter(|pkg| pkg.version.is_none()) + .collect::>(); + if !missing_version.is_empty() { + return Err(anyhow::anyhow!( + "Unable to publish packages without a version specified" + ) + .context(format!( + "packages: {}", + package_list(missing_version.iter(), None) + ))); + } + } + let mut dependents = graph.into_inner(); + + dependents.reverse(); + // graph was already found to be acyclic + let dependents = DependencyGraph::try_from(dependents).unwrap(); + + Ok(Self { + dependents, + indices, + }) + } + + pub fn iter<'a>(&'a self) -> impl Iterator + 'a { + self.dependents + .nodes_iter() + .map(|id| &(self.dependents[id])) + } + + pub fn is_empty(&self) -> bool { + self.indices.is_empty() + } + + pub fn len(&self) -> usize { + self.indices.len() + } + + /// Returns the set of packages that are ready for publishing (i.e. have no outstanding dependencies). + /// + /// These will not be returned in future calls. + pub fn take_ready(&self) -> BTreeSet { + self.dependents + .nodes_iter() + // there are no dependents on `self.dependents[id]` + .filter(|id| { + self.dependents + .neighbors_directed(*id, Direction::Incoming) + .count() + == 0 + }) + .map(|id| self.dependents[id].clone()) + .collect() + } + + /// Return the path associated with a local package. + pub fn get_path(&self, pkg: &PackageRef) -> Option<&Path> { + self.indices.get(pkg).map(|(_, p)| p.as_ref()) + } + + /// Return the [`NodeIndex`] associated with a local package. + pub fn get_node_index(&self, pkg: &PackageRef) -> Option { + self.indices.get(pkg).map(|(id, _)| *id) + } + + /// Packages confirmed to be available in the registry, potentially allowing additional + /// packages to be "ready". + pub fn mark_confirmed(&mut self, published: impl IntoIterator) { + for spec in published { + let (id, _) = self + .indices + .remove(&spec.package) + .expect("PackageSpec has no associated index"); + // NOTE: nodes without edges will return None here + self.dependents.remove_node(id); + } + } +} + +impl std::fmt::Display for PublishPlan { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + // TODO(mkatychev): handle with anstyle and anstream, passing in `anstream::AutoStream` for colour choice + for id in self.dependents.nodes_iter() { + let dep = &self.dependents[id]; + // initial dependency graph visualization + let mut neighbors = self + .dependents + .neighbors_directed(id, Direction::Outgoing) + .peekable(); + + if neighbors.peek().is_none() { + // tracing::debug!("{dep} has no dependents"); + writeln!(f, "[{dep} has no dependents]")?; + } else { + writeln!(f, "[{dep}]")?; + } + + while let Some(id) = neighbors.next() { + let pkg = &self.dependents[id]; + let separator = if neighbors.peek().is_some() { + "├─" + } else { + "╰─" + }; + + writeln!(f, "{separator}─▶ {pkg}")?; + } + } + Ok(()) + } +} + +/// Format a collection of packages as a list +/// +/// e.g. "foo:a@0.1.0, bar:b@0.2.0, and baz:c@0.3.0". +/// +/// Note: the final separator (e.g. "and" in the previous example) can be chosen. +fn package_list<'a>( + pkgs: impl IntoIterator, + final_sep: Option<&str>, +) -> String { + let final_sep = final_sep.unwrap_or("and"); + let mut names: Vec<_> = pkgs.into_iter().map(|pkg| pkg.to_string()).collect(); + names.sort(); + + match &names[..] { + [] => String::new(), + [a] => a.clone(), + [a, b] => format!("{a} {final_sep} {b}"), + [names @ .., last] => { + format!("{}, {final_sep} {last}", names.join(", ")) + } + } +} + +#[cfg(test)] +mod tests { + + use std::path::PathBuf; + + use super::*; + use glob::glob; + + fn transitive_local_paths() -> Vec { + let fixtures_root = + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/transitive-local"); + // pass all fixture dirs to a single `wkg publish` invocation + let mut paths: Vec = glob(fixtures_root.join("**/*.wit").to_str().unwrap()) + .unwrap() + .map(|p| p.expect("glob path error")) + .map(|p| p.parent().unwrap().to_path_buf()) + .collect(); + paths.sort(); + paths.dedup(); + paths + } + + #[test] + fn publish_plan_iter() { + let paths = transitive_local_paths(); + let plan = PublishPlan::from_paths(&paths).unwrap(); + assert_eq!( + plan.iter().count(), + 5, + "unexpected package count\npackages found: {}", + package_list(plan.iter(), None) + ); + } + + #[test] + fn publish_plan_chunks() { + let paths = transitive_local_paths(); + let mut plan = PublishPlan::from_paths(&paths).unwrap(); + let mut ready_for_publish = plan.take_ready(); + assert_eq!( + ready_for_publish.iter().collect::>(), + ["example-c:nested@0.1.0", "example-d:foo@0.1.0",], + ); + plan.mark_confirmed(ready_for_publish); + + ready_for_publish = plan.take_ready(); + assert_eq!( + ready_for_publish.iter().collect::>(), + ["example-c:baz@0.1.0"], + ); + plan.mark_confirmed(ready_for_publish); + + ready_for_publish = plan.take_ready(); + assert_eq!( + ready_for_publish.iter().collect::>(), + ["example-b:bar@0.1.0"], + ); + plan.mark_confirmed(ready_for_publish); + + ready_for_publish = plan.take_ready(); + assert_eq!( + ready_for_publish.iter().collect::>(), + ["example-a:foo@0.1.0"], + ); + plan.mark_confirmed(ready_for_publish); + + assert!(plan.is_empty()); + } +} diff --git a/crates/wasm-pkg-core/src/wit.rs b/crates/wasm-pkg-core/src/wit.rs index 38bd70d..6fbec3c 100644 --- a/crates/wasm-pkg-core/src/wit.rs +++ b/crates/wasm-pkg-core/src/wit.rs @@ -1,14 +1,20 @@ //! Functions for building WIT packages and fetching their dependencies. -use std::{collections::HashSet, path::Path, str::FromStr}; +use std::{ + collections::{HashMap, HashSet}, + path::Path, + str::FromStr, +}; use anyhow::{Context, Result}; +use petgraph::{data::Build, Direction}; use semver::{Version, VersionReq}; use wasm_metadata::{AddMetadata, AddMetadataField}; use wasm_pkg_client::{ caching::{CachingClient, FileCache}, PackageRef, }; +use wasm_pkg_common::package::PackageSpec; use wit_component::WitPrinter; use wit_parser::{PackageId, PackageName, Resolve}; @@ -16,8 +22,9 @@ use crate::{ config::Config, lock::LockFile, resolver::{ - DecodedDependency, Dependency, DependencyResolution, DependencyResolutionMap, - DependencyResolver, LocalResolution, RegistryPackage, + DecodedDependency, Dependency, DependencyGraph, DependencyResolution, + DependencyResolutionMap, DependencyResolver, LocalPackageIndex, LocalResolution, + RegistryPackage, }, }; @@ -54,12 +61,20 @@ pub async fn build_package( ) -> Result<(PackageRef, Option, Vec)> { let dependencies = resolve_dependencies(config, &wit_dir, Some(lock_file), client) .await - .with_context(|| format!("wit_dir: {}", wit_dir.as_ref().display())) .context("Unable to resolve dependencies")?; lock_file.update_dependencies(&dependencies); - let (resolve, pkg_id) = dependencies.generate_resolve(wit_dir).await?; + let (resolve, pkg_id) = dependencies + .generate_resolve(wit_dir.as_ref()) + .await + .map_err(|e| { + if config.has_override(wit_dir.as_ref()) { + e.context("hint: override present for WIT directory".to_string()) + } else { + e + } + })?; let bytes = wit_component::encode(&resolve, pkg_id)?; let pkg = &resolve.packages[pkg_id]; @@ -134,11 +149,11 @@ pub async fn fetch_dependencies( /// for resolving dependencies. pub fn get_packages( path: impl AsRef, -) -> Result<(PackageRef, HashSet<(PackageRef, VersionReq)>)> { +) -> Result<(PackageSpec, HashSet<(PackageRef, VersionReq)>)> { let group = wit_parser::UnresolvedPackageGroup::parse_path(path).context("Couldn't parse package")?; - let name = PackageRef::new( + let package = PackageRef::new( group .main .name @@ -152,6 +167,10 @@ pub fn get_packages( .parse() .context("Invalid name found in package")?, ); + let package = PackageSpec { + package, + version: group.main.name.version.clone(), + }; // Get all package refs from the main package and then from any nested packages let packages: HashSet<(PackageRef, VersionReq)> = @@ -164,7 +183,62 @@ pub fn get_packages( ) .collect(); - Ok((name, packages)) + Ok((package, packages)) +} + +/// Build an acyclic [`DependencyGraph`] for the provided WIT package paths alongside a [`LocalPackageIndex`]. +/// +/// # Errors +/// +/// The function will return an error if there are duplicate references to the same package or if +/// there are cyclical dependencies between packages. +pub(crate) fn get_local_dependencies( + paths: &[impl AsRef], +) -> Result<(DependencyGraph, LocalPackageIndex)> { + let pkg_trees = paths + .iter() + .map(|path| get_packages(path).map(|(pkg, deps)| ((pkg, path), deps))) + .collect::, _>>()?; + let mut graph = DependencyGraph::new(); + let mut indices = HashMap::new(); + // establish all nodes + for ((spec, path), _) in &pkg_trees { + let id = graph.add_node(spec.clone()); + if indices + .insert(spec.package.clone(), (id, path.as_ref().to_owned())) + .is_some() + { + anyhow::bail!("duplicate references to package detected: {spec}"); + } + } + for ((spec, _), deps) in pkg_trees { + // TODO handle version matching for dependencies + for (dep, _version) in deps { + if let Some(&(dep, _)) = indices.get(&dep) { + let pkg = &spec.package; + let (id, _) = indices[pkg]; + // // pkg <=DependsOn= dep + graph + .try_update_edge(id, dep, Direction::Incoming) + .map_err(|e| { + match e { + petgraph::acyclic::AcyclicEdgeError::Cycle(cycle) => { + anyhow::anyhow!("cyclical dependency detected") + .context(format!("other package: {}", graph[cycle.node_id()])) + } + petgraph::acyclic::AcyclicEdgeError::SelfLoop => { + anyhow::anyhow!("Package is declaring self as a dependency.") + } + petgraph::acyclic::AcyclicEdgeError::InvalidEdge => anyhow::anyhow!( + "Could not successfully add the edge to the underlying graph." + ), + } + .context(format!("package: {pkg}")) + })?; + } + } + } + Ok((graph, indices)) } /// Builds a list of resolved dependencies loaded from the component or path containing the WIT. @@ -211,7 +285,7 @@ pub async fn resolve_dependencies( .with_context(|| format!("unable to add dependency {dep}"))?; } } - let (_name, packages) = get_packages(path)?; + let (_spec, packages) = get_packages(path)?; resolver.add_packages(packages).await?; resolver.resolve().await } diff --git a/crates/wasm-pkg-core/tests/fetch.rs b/crates/wasm-pkg-core/tests/fetch.rs index 2b8fe1b..25a093f 100644 --- a/crates/wasm-pkg-core/tests/fetch.rs +++ b/crates/wasm-pkg-core/tests/fetch.rs @@ -90,6 +90,7 @@ async fn test_transitive_local(#[values(OutputType::Wasm, OutputType::Wit)] outp // [overrides] // "example-b:bar" = { "path" = "../example-b/wit" } // "example-c:baz" = { "path" = "../example-c/wit" } + // "example-c:nested" = { "path" = "../example-c/wit/nested" } // ``` let config = Config { overrides: Some(HashMap::from([ @@ -107,18 +108,22 @@ async fn test_transitive_local(#[values(OutputType::Wasm, OutputType::Wit)] outp version: None, }, ), + ( + "example-c:nested".to_string(), + Override { + path: Some(fixture_path.join("example-c").join("wit/nested")), + version: None, + }, + ), ])), ..Default::default() }; let (_temp_cache, client) = common::get_client().await.unwrap(); - assert!( - // If overrides didn't properly resolve, this will fail - wit::fetch_dependencies(&config, project_path.join("wit"), &mut lock, client, output) - .await - .is_ok(), - "Should be able to fetch the dependencies" - ); + // If overrides didn't properly resolve, this will fail + wit::fetch_dependencies(&config, project_path.join("wit"), &mut lock, client, output) + .await + .unwrap_or_else(|e| panic!("Should be able to fetch the dependencies: {e:#}")); // Ensure that the deps directory contains the correct dependencies let mut deps_dir = tokio::fs::read_dir(project_path.join("wit").join("deps")) @@ -128,9 +133,10 @@ async fn test_transitive_local(#[values(OutputType::Wasm, OutputType::Wit)] outp while let Ok(Some(entry)) = deps_dir.next_entry().await { deps.push(entry.file_name().to_string_lossy().to_string()); } - assert_eq!(deps.len(), 2); + assert_eq!(deps.len(), 3); assert!(deps.contains(&"example-b-bar-0.1.0".to_string())); assert!(deps.contains(&"example-c-baz-0.1.0".to_string())); + assert!(deps.contains(&"example-c-nested-0.1.0".to_string())); // All dependencies are local, so the lock file should be empty assert_eq!( diff --git a/crates/wasm-pkg-core/tests/fixtures/transitive-local/example-c/wit/nested/nested.wit b/crates/wasm-pkg-core/tests/fixtures/transitive-local/example-c/wit/nested/nested.wit new file mode 100644 index 0000000..667aff4 --- /dev/null +++ b/crates/wasm-pkg-core/tests/fixtures/transitive-local/example-c/wit/nested/nested.wit @@ -0,0 +1,7 @@ +package example-c:nested@0.1.0; + +interface nested { + record meta { + label: string, + } +} diff --git a/crates/wasm-pkg-core/tests/fixtures/transitive-local/example-c/wit/world.wit b/crates/wasm-pkg-core/tests/fixtures/transitive-local/example-c/wit/world.wit index 98db971..15655fb 100644 --- a/crates/wasm-pkg-core/tests/fixtures/transitive-local/example-c/wit/world.wit +++ b/crates/wasm-pkg-core/tests/fixtures/transitive-local/example-c/wit/world.wit @@ -1,12 +1,14 @@ package example-c:baz@0.1.0; interface baz { + use example-c:nested/nested@0.1.0.{meta}; + record requestresponse { baz: list, + meta: meta, } } world handler { - // No dependencies to import export baz; } diff --git a/crates/wasm-pkg-core/tests/fixtures/transitive-local/example-d/wit/world.wit b/crates/wasm-pkg-core/tests/fixtures/transitive-local/example-d/wit/world.wit new file mode 100644 index 0000000..427a9a5 --- /dev/null +++ b/crates/wasm-pkg-core/tests/fixtures/transitive-local/example-d/wit/world.wit @@ -0,0 +1,5 @@ +package example-d:foo@0.1.0; + +interface bar { + type baz = string; +} diff --git a/crates/wkg/src/main.rs b/crates/wkg/src/main.rs index 0475884..bb01307 100644 --- a/crates/wkg/src/main.rs +++ b/crates/wkg/src/main.rs @@ -1,21 +1,27 @@ -use std::{io::Seek, path::PathBuf}; +use std::{ + collections::HashMap, + io::{Cursor, Seek}, + path::PathBuf, +}; -use anyhow::{ensure, Context}; +use anyhow::{anyhow, ensure, Context}; use clap::{Args, Parser, Subcommand, ValueEnum}; use futures_util::TryStreamExt; -use tokio::io::AsyncWriteExt; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; use tracing::level_filters::LevelFilter; use wasm_pkg_client::{ caching::{CachingClient, FileCache}, + local::LocalConfig, Client, PublishOpts, }; use wasm_pkg_common::{ self, - config::{Config, RegistryMapping}, + config::{Config, RegistryConfig, RegistryMapping}, + metadata::LOCAL_PROTOCOL, package::PackageSpec, registry::Registry, }; -use wasm_pkg_core::lock::LockFile; +use wasm_pkg_core::{lock::LockFile, resolver::PublishPlan}; use wit_component::DecodedWasm; mod oci; @@ -40,7 +46,7 @@ struct RegistryArgs { registry: Option, } -#[derive(Args, Debug)] +#[derive(Args, Debug, Default)] struct Common { /// The path to the configuration file. #[arg(long = "config", value_name = "CONFIG", env = "WKG_CONFIG_FILE")] @@ -124,8 +130,7 @@ impl ConfigArgs { let path = if let Some(path) = self.common.config { path } else { - Config::global_config_path() - .ok_or(anyhow::anyhow!("global config path not available"))? + Config::global_config_path().ok_or(anyhow!("global config path not available"))? }; // Check if the parent directory exists, if not create it @@ -147,7 +152,7 @@ impl ConfigArgs { } if self.edit { - let editor = std::env::var("EDITOR").or(Err(anyhow::anyhow!( + let editor = std::env::var("EDITOR").or(Err(anyhow!( "failed to read `$EDITOR` environment variable" )))?; @@ -170,7 +175,7 @@ impl ConfigArgs { let mut config = match tokio::fs::read_to_string(&path).await { Ok(contents) => Config::from_toml(&contents)?, Err(err) if err.kind() == std::io::ErrorKind::NotFound => Config::default(), - Err(err) => return Err(anyhow::anyhow!("error reading config file: {0}", err)), + Err(err) => return Err(anyhow!("error reading config file: {0}", err)), }; if let Some(default) = self.default_registry { @@ -179,14 +184,14 @@ impl ConfigArgs { // write config file config.to_file(&path).await?; - println!("Updated config file: {path}", path = path.display()); + eprintln!("Updated config file: {path}", path = path.display()); } // print config if let Some(registry) = config.default_registry() { - println!("Default registry: {}", registry); + eprintln!("Default registry: {}", registry); } else { - println!("Default registry is not set"); + eprintln!("Default registry is not set"); } Ok(()) @@ -230,9 +235,9 @@ struct GetArgs { #[derive(Args, Debug)] struct PublishArgs { - /// The file or directory to publish. + /// The files and directories to publish. /// If a directory is provided, the package is built to a tempfile before publishing. - path: PathBuf, + paths: Vec, #[command(flatten)] registry_args: RegistryArgs, @@ -252,37 +257,128 @@ struct PublishArgs { impl PublishArgs { pub async fn run(self) -> anyhow::Result<()> { - let client = self.common.get_client().await?; + let publish_opts = self.publish_opts()?; + let path = match &self.paths[..] { + [path] => path, + paths => { + let mut overlay_config = self.common.load_config().await?; + let cache = self.common.load_cache().await?; + + let local_config = LocalConfig::temp_dir()?; + let reg_config = RegistryConfig::default() + .with_default_backend(LOCAL_PROTOCOL, &local_config)?; + // Route every package in the plan to the local overlay registry + // backed by `reg_config`, so the client used in `build_wit_dir` + // resolves these packages against the local overlay instead of + // an upstream remote. + let local_registry: Registry = "tmp_local_publish".parse()?; + overlay_config + .get_or_insert_registry_config_mut(&local_registry) + .merge(reg_config); + + let mut plan = PublishPlan::from_paths(paths)?; + eprintln!("{plan}"); + + // TODO(mkatychev): Add support for `PackageLoader::get_release` to handle + // querying on a per package, namespace, and registry level + // to handle cargo style overlays. + // see this reference of `cargo::core::Dependency` usage for local overlays in Cargo: + // https://github.com/rust-lang/cargo/blob/d6900d00af2644ea1c0068c5694d9dbe11a3ab39/src/cargo/sources/overlay.rs#L47 + // this is still needed for `wit::build_wit_dir` + for spec in plan.iter() { + overlay_config.set_package_registry_override( + spec.package.clone(), + RegistryMapping::Registry(local_registry.clone()), + ); + } + let client = CachingClient::new(Some(Client::new(overlay_config)), cache); + + let mut lock_file = LockFile::load(false).await?; + // these are packages that have been successfully pushed to our "tmp_local_publish" + let mut validated_packages = HashMap::new(); + + // 1. Publish our packages to "tmp_local_publish" ensuring all dependencies are + // resolved by the local backend + for spec in plan.iter() { + let path = plan.get_path(&spec.package).unwrap(); + let data = if path.is_dir() { + let _prev_lock_ref = (lock_file.version, lock_file.packages.clone()); + let (_pkg_ref, _version, bytes) = + wit::build_wit_dir(&path, client.clone(), &mut lock_file).await?; + bytes + } else { + let mut file = tokio::fs::OpenOptions::new().read(true).open(path).await?; + let mut buf = Vec::new(); + file.read_exact(&mut buf).await?; + buf + }; + + let source = Box::pin(Cursor::new(data.clone())); + client + .client()? + .publish_release_data( + source, + PublishOpts { + package: None, + registry: Some(local_registry.clone()), + // we want to publish to "tmp_local_publish" regardless of flags passed in + dry_run: false, + }, + ) + .await?; + + let id = plan + .get_node_index(&spec.package) + .expect("missing node index"); + validated_packages.insert(id, data); + } - let package = match self.package { - Some(PackageSpec { - package, - version: Some(v), - }) => Some((package, v)), - Some(PackageSpec { version: None, .. }) => { - anyhow::bail!("version is required when manually overriding the package ID"); + let client = self.common.get_client().await?; + // 2. Publish our packages in "waves" to the actual registries ensuring all + // possible dependency free packages are published in the same group + while !plan.is_empty() { + let ready_for_publish = plan.take_ready(); + for spec in &ready_for_publish { + let id = plan + .get_node_index(&spec.package) + .expect("missing node index"); + let source = Box::pin(Cursor::new(validated_packages[&id].clone())); + let (package, version) = client + .client()? + .publish_release_data(source, publish_opts.clone()) + .await?; + if self.dry_run { + eprintln!("Aborting publish due to dry run: {}@{}", package, version); + } else { + eprintln!("Published {}@{}", package, version); + } + } + plan.mark_confirmed(ready_for_publish); + } + return Ok(()); } - None => None, }; + let client = self.common.get_client().await?; + // If the input is a directory, build a WIT package from it into a temp // file first. _tmp is held until the publish completes so the file // isn't deleted out from under us. - let (publish_path, _tmp) = if self.path.is_dir() { + let (publish_path, _tmp) = if path.is_dir() { let mut lock_file = LockFile::load(true).await?; let prev_lock_ref = (lock_file.version, lock_file.packages.clone()); let (pkg_ref, _, bytes) = - wit::build_wit_dir(&self.path, client.clone(), &mut lock_file).await?; + wit::build_wit_dir(&path.clone(), client.clone(), &mut lock_file).await?; // There is no way to check if we are in a git repository unlike `cargo publish --allow-dirty` so // check against previous values. if lock_file != prev_lock_ref && !self.dry_run { - return Err(anyhow::anyhow!( + return Err(anyhow!( "wkg.lock would be updated during publish, aborting" )) .with_context(|| { format!( "Run `wkg wit build {}` before attempting to publish", - self.path.to_string_lossy() + path.display() ) }); } @@ -291,27 +387,41 @@ impl PublishArgs { (tmp.path().to_path_buf(), Some(tmp)) } else { - (self.path.clone(), None) + (path.clone(), None) }; let (package, version) = client .client()? - .publish_release_file( - &publish_path, - PublishOpts { - package, - registry: self.registry_args.registry, - dry_run: self.dry_run, - }, - ) + .publish_release_file(&publish_path, publish_opts) .await?; if self.dry_run { - println!("Aborting publish due to dry run: {}@{}", package, version); + eprintln!("Aborting publish due to dry run: {}@{}", package, version); } else { - println!("Published {}@{}", package, version); + eprintln!("Published {}@{}", package, version); } Ok(()) } + + fn publish_opts(&self) -> anyhow::Result { + let package = match self.package.clone() { + Some(_) if self.paths.len() > 2 => { + anyhow::bail!("`--package` is currently unsupported when providing more than one path argument"); + } + Some(PackageSpec { + package, + version: Some(v), + }) => Some((package, v)), + Some(PackageSpec { version: None, .. }) => { + anyhow::bail!("version is required when manually overriding the package ID"); + } + None => None, + }; + Ok(PublishOpts { + package, + registry: self.registry_args.registry.clone(), + dry_run: self.dry_run, + }) + } } #[derive(ValueEnum, Clone, Debug, PartialEq)] @@ -339,7 +449,7 @@ impl GetArgs { let version = match version { Some(ver) => ver, None => { - println!("No version specified; fetching version list..."); + eprintln!("No version specified; fetching version list..."); let versions = client.list_all_versions(&package).await?; tracing::trace!(?versions, "Fetched version list"); versions @@ -350,7 +460,7 @@ impl GetArgs { } }; - println!("Getting {package}@{version}..."); + eprintln!("Getting {package}@{version}..."); let release = client .get_release(&package, &version) .await @@ -391,7 +501,7 @@ impl GetArgs { "wasm" => Format::Wasm, "wit" => Format::Wit, _ => { - println!( + eprintln!( "Couldn't infer output format from file name {:?}", self.output.file_name().unwrap_or_default() ); @@ -418,7 +528,7 @@ impl GetArgs { if format == Format::Wit { return Err(err); } - println!("Failed to detect package content type: {err:#}"); + eprintln!("Failed to detect package content type: {err:#}"); None } } @@ -464,7 +574,7 @@ impl GetArgs { .persist(&output_path) .with_context(|| format!("Failed to persist WASM to {output_path:?}"))? } - println!("Wrote '{}'", output_path.display()); + eprintln!("Wrote '{}'", output_path.display()); } Ok(()) } diff --git a/crates/wkg/src/wit.rs b/crates/wkg/src/wit.rs index 9d9e4ce..18d5d37 100644 --- a/crates/wkg/src/wit.rs +++ b/crates/wkg/src/wit.rs @@ -106,7 +106,7 @@ impl BuildArgs { tokio::fs::write(&output_path, bytes).await?; // Now write out the lock file since everything else succeeded lock_file.write().await?; - println!("WIT package written to {}", output_path.display()); + eprintln!("WIT package written to {}", output_path.display()); Ok(()) } } @@ -120,7 +120,9 @@ pub async fn build_wit_dir( ) -> anyhow::Result<(PackageRef, Option, Vec)> { check_dir(&dir).await?; let wkg_config = wasm_pkg_core::config::Config::load().await?; - let result = wit::build_package(&wkg_config, dir, lock_file, client).await?; + let result = wit::build_package(&wkg_config, dir.as_ref(), lock_file, client) + .await + .with_context(|| format!("failed to build WIT directory `{}`", dir.as_ref().display()))?; Ok(result) } diff --git a/crates/wkg/tests/common.rs b/crates/wkg/tests/common.rs index 8ae6b49..ef222d2 100644 --- a/crates/wkg/tests/common.rs +++ b/crates/wkg/tests/common.rs @@ -135,8 +135,10 @@ pub async fn load_fixture(fixture: &str) -> Fixture { } } -#[allow(dead_code)] -async fn copy_dir(source: impl AsRef, destination: impl AsRef) -> anyhow::Result<()> { +pub async fn copy_dir( + source: impl AsRef, + destination: impl AsRef, +) -> anyhow::Result<()> { tokio::fs::create_dir_all(&destination).await?; let mut entries = tokio::fs::read_dir(source).await?; while let Some(entry) = entries.next_entry().await? { diff --git a/crates/wkg/tests/e2e.rs b/crates/wkg/tests/e2e.rs index 658027e..cbe7dca 100644 --- a/crates/wkg/tests/e2e.rs +++ b/crates/wkg/tests/e2e.rs @@ -1,3 +1,7 @@ +use wasm_pkg_client::{Version, VersionInfo}; + +use crate::common::copy_dir; + mod common; #[cfg(feature = "docker-tests")] @@ -85,6 +89,71 @@ async fn build_and_publish_with_metadata() { ); } +#[cfg(feature = "docker-tests")] +#[tokio::test] +async fn publish_multiple_transitive_local_packages() { + use std::path::PathBuf; + + let (config, registry, _container) = common::start_registry().await; + let namespaces = ["example-a", "example-b", "example-c", "example-d"]; + + // copy the transitive-local fixtures from wasm-pkg-core into a temp dir + let temp_dir = tempfile::tempdir().expect("Failed to create tempdir"); + let src_root = PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("../wasm-pkg-core/tests/fixtures/transitive-local"); + let fixture_root = temp_dir.path().join("transitive-local"); + copy_dir(&src_root, &fixture_root).await.unwrap(); + + let mut mapped = config.clone(); + for ns in namespaces { + mapped = common::map_namespace(&mapped, ns, ®istry); + } + let config_path = temp_dir.path().join("config.toml"); + mapped.to_file(&config_path).await.expect("write config"); + + // pass all fixture dirs to a single `wkg publish` invocation + let mut dirs: Vec = namespaces + .iter() + .map(|name| fixture_root.join(name).join("wit")) + .collect(); + // TODO use glob suchas in `wasm_pkgs_core::resolver::tests::transitive_local_paths` + dirs.push(fixture_root.join("example-c/wit/nested")); + + let mut publish = tokio::process::Command::new(env!("CARGO_BIN_EXE_wkg")); + publish + .current_dir(temp_dir.path()) + .env("WKG_CACHE_DIR", temp_dir.path().join("cache")) + .env("WKG_CONFIG_FILE", &config_path) + .arg("publish"); + for dir in &dirs { + publish.arg(dir); + } + let status = publish.status().await.expect("spawn wkg publish"); + assert!(status.success(), "wkg publish should succeed"); + + let client = wasm_pkg_client::Client::new(mapped); + + let expected_version = "0.1.0".parse::().unwrap(); + for name in [ + "example-a:foo", + "example-b:bar", + "example-c:baz", + "example-c:nested", + "example-d:foo", + ] { + let pkg = name.parse().unwrap(); + let versions = client + .list_all_versions(&pkg) + .await + .unwrap_or_else(|e| panic!("list versions for {name}: {e:#}")); + std::assert_matches!( + &versions[..], + [VersionInfo { version, .. }] if version == &expected_version, + "{name} should have exactly one published version", + ); + } +} + #[tokio::test] pub async fn check() { let fixture = common::load_fixture("wasi-http").await;