From b3d359f0efbdf15167a22d0a5e12abce055d8a30 Mon Sep 17 00:00:00 2001 From: Mikhail Katychev Date: Tue, 2 Jun 2026 14:35:02 -0500 Subject: [PATCH 01/61] feat(publish): add ability to pass in directory --- crates/wasm-pkg-core/src/lock.rs | 5 ++++ crates/wkg/src/main.rs | 41 +++++++++++++++++++++++++++++--- crates/wkg/src/wit.rs | 26 ++++++++++++++++---- 3 files changed, 64 insertions(+), 8 deletions(-) diff --git a/crates/wasm-pkg-core/src/lock.rs b/crates/wasm-pkg-core/src/lock.rs index bd7efb9..7a864ee 100644 --- a/crates/wasm-pkg-core/src/lock.rs +++ b/crates/wasm-pkg-core/src/lock.rs @@ -48,6 +48,11 @@ impl PartialEq for LockFile { self.packages == other.packages && self.version == other.version } } +impl PartialEq<(u64, BTreeSet)> for LockFile { + fn eq(&self, other: &(u64, BTreeSet)) -> bool { + self.packages == other.1 && self.version == other.0 + } +} impl Eq for LockFile {} diff --git a/crates/wkg/src/main.rs b/crates/wkg/src/main.rs index 7272062..6ad01fa 100644 --- a/crates/wkg/src/main.rs +++ b/crates/wkg/src/main.rs @@ -15,6 +15,7 @@ use wasm_pkg_common::{ package::PackageSpec, registry::Registry, }; +use wasm_pkg_core::lock::LockFile; use wit_component::DecodedWasm; mod oci; @@ -227,8 +228,9 @@ struct GetArgs { #[derive(Args, Debug)] struct PublishArgs { - /// The file to publish - file: PathBuf, + /// The file or directory to publish. + /// If a directory is provided, the package is built to a tempfile before publishing. + path: PathBuf, #[command(flatten)] registry_args: RegistryArgs, @@ -260,10 +262,43 @@ impl PublishArgs { } else { None }; + + // If the input is a directory, build a WIT package from it into a temp + // file first. _tmp is held until the publish completes so the file + // isn't deleted out from under us. + let metadata = tokio::fs::metadata(&self.path) + .await + .with_context(|| format!("Failed to stat {:?}", self.path))?; + let (publish_path, _tmp) = if metadata.is_dir() { + let mut lock_file = LockFile::load(false).await?; + let prev_lock_ref = (lock_file.version, lock_file.packages.clone()); + let (build_ref, _, bytes) = + wit::build_wit_dir(&self.path, client.clone(), &mut lock_file).await?; + let tmp = tempfile::Builder::new() + .prefix(&build_ref.to_string()) + .suffix(".wasm") + .tempfile() + .context("Failed to create temporary file for built WIT package")?; + // There is no way to check if we are in a git repository unlike `cargo publish --allow-dirty` so + // check against previous values. + if lock_file != prev_lock_ref { + return Err(anyhow::anyhow!( + "wkg.lock would be updated during publish, aborting" + )) + .context("Run `wkg wit fetch` before attempting to publish"); + } + tokio::fs::write(tmp.path(), &bytes) + .await + .context("Failed to write built WIT package to temp file")?; + (tmp.path().to_path_buf(), Some(tmp)) + } else { + (self.path.clone(), None) + }; + let (package, version) = client .client()? .publish_release_file( - &self.file, + &publish_path, PublishOpts { package, registry: self.registry_args.registry, diff --git a/crates/wkg/src/wit.rs b/crates/wkg/src/wit.rs index 36e2895..3c2462e 100644 --- a/crates/wkg/src/wit.rs +++ b/crates/wkg/src/wit.rs @@ -3,6 +3,8 @@ use std::path::{Path, PathBuf}; use anyhow::Context; use clap::{Args, Subcommand}; +use wasm_pkg_client::caching::{CachingClient, FileCache}; +use wasm_pkg_common::package::{PackageRef, Version}; use wasm_pkg_core::{ lock::LockFile, wit::{self, OutputType}, @@ -86,12 +88,9 @@ pub struct UpdateArgs { impl BuildArgs { pub async fn run(self) -> anyhow::Result<()> { - check_dir(&self.dir).await?; let client = self.common.get_client().await?; - let wkg_config = wasm_pkg_core::config::Config::load().await?; let mut lock_file = LockFile::load(false).await?; - let (pkg_ref, version, bytes) = - wit::build_package(&wkg_config, self.dir, &mut lock_file, client).await?; + let (pkg_ref, version, bytes) = build_wit_dir(&self.dir, client, &mut lock_file).await?; let output_path = if let Some(path) = self.output { path } else { @@ -111,6 +110,19 @@ impl BuildArgs { } } +/// Build a WIT package from a directory, returning the resolved package ref, optional +/// version, and the encoded component bytes. +pub async fn build_wit_dir( + dir: impl AsRef, + client: CachingClient, + mut lock_file: &mut LockFile, +) -> anyhow::Result<(PackageRef, Option, Vec)> { + check_dir(&dir).await?; + let wkg_config = wasm_pkg_core::config::Config::load().await?; + let result = wit::build_package(&wkg_config, dir, &mut lock_file, client).await?; + Ok(result) +} + impl FetchArgs { pub async fn run(self) -> anyhow::Result<()> { check_dir(&self.dir).await?; @@ -154,5 +166,9 @@ impl UpdateArgs { } async fn check_dir(dir: impl AsRef) -> anyhow::Result<()> { - tokio::fs::metadata(dir).await.context("Unable to read wit directory. This command should be run from the parent directory of the wit directory or a directory can be overridden with the --wit-dir argument").map(|_|()) + let dir = dir.as_ref(); + tokio::fs::metadata(dir).await + .with_context(|| format!("unable to read wit directory: {}", dir.display())) + .context("This command should be run from the parent directory of the wit directory or a directory can be overridden with the --wit-dir argument") + .map(|_|()) } From d314aed3689a1f403c2dab91f967396f8aa21168 Mon Sep 17 00:00:00 2001 From: Mikhail Katychev Date: Tue, 2 Jun 2026 14:37:54 -0500 Subject: [PATCH 02/61] added logging for tempfile --- crates/wkg/src/main.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/crates/wkg/src/main.rs b/crates/wkg/src/main.rs index 6ad01fa..0d1d0a0 100644 --- a/crates/wkg/src/main.rs +++ b/crates/wkg/src/main.rs @@ -290,7 +290,10 @@ impl PublishArgs { tokio::fs::write(tmp.path(), &bytes) .await .context("Failed to write built WIT package to temp file")?; - (tmp.path().to_path_buf(), Some(tmp)) + let tmp_pkg_path = tmp.path().to_path_buf(); + tracing::debug!(tmp_pkg_path = %tmp_pkg_path.display(), "Wrote temporary WIT package file"); + + (tmp_pkg_path, Some(tmp)) } else { (self.path.clone(), None) }; From 63e24622efae59eb798210236661632c616c2e42 Mon Sep 17 00:00:00 2001 From: Mikhail Katychev Date: Tue, 2 Jun 2026 14:41:17 -0500 Subject: [PATCH 03/61] dont use metadata for path checking --- crates/wkg/src/main.rs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/crates/wkg/src/main.rs b/crates/wkg/src/main.rs index 0d1d0a0..e35f3d5 100644 --- a/crates/wkg/src/main.rs +++ b/crates/wkg/src/main.rs @@ -266,10 +266,7 @@ impl PublishArgs { // If the input is a directory, build a WIT package from it into a temp // file first. _tmp is held until the publish completes so the file // isn't deleted out from under us. - let metadata = tokio::fs::metadata(&self.path) - .await - .with_context(|| format!("Failed to stat {:?}", self.path))?; - let (publish_path, _tmp) = if metadata.is_dir() { + let (publish_path, _tmp) = if self.path.is_dir() { let mut lock_file = LockFile::load(false).await?; let prev_lock_ref = (lock_file.version, lock_file.packages.clone()); let (build_ref, _, bytes) = From a84022459b8c3e71f4d8d1304eca71a7c3d55277 Mon Sep 17 00:00:00 2001 From: Mikhail Katychev Date: Tue, 2 Jun 2026 14:42:25 -0500 Subject: [PATCH 04/61] move lock check up --- crates/wkg/src/main.rs | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/crates/wkg/src/main.rs b/crates/wkg/src/main.rs index e35f3d5..efb3282 100644 --- a/crates/wkg/src/main.rs +++ b/crates/wkg/src/main.rs @@ -271,11 +271,6 @@ impl PublishArgs { let prev_lock_ref = (lock_file.version, lock_file.packages.clone()); let (build_ref, _, bytes) = wit::build_wit_dir(&self.path, client.clone(), &mut lock_file).await?; - let tmp = tempfile::Builder::new() - .prefix(&build_ref.to_string()) - .suffix(".wasm") - .tempfile() - .context("Failed to create temporary file for built WIT package")?; // There is no way to check if we are in a git repository unlike `cargo publish --allow-dirty` so // check against previous values. if lock_file != prev_lock_ref { @@ -284,6 +279,12 @@ impl PublishArgs { )) .context("Run `wkg wit fetch` before attempting to publish"); } + + let tmp = tempfile::Builder::new() + .prefix(&build_ref.to_string()) + .suffix(".wasm") + .tempfile() + .context("Failed to create temporary file for built WIT package")?; tokio::fs::write(tmp.path(), &bytes) .await .context("Failed to write built WIT package to temp file")?; From 66cb7d63d3432d585f76c01ff9b9a7eddb8002ad Mon Sep 17 00:00:00 2001 From: Mikhail Katychev Date: Tue, 2 Jun 2026 14:50:01 -0500 Subject: [PATCH 05/61] clippy --- crates/wkg/src/wit.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/wkg/src/wit.rs b/crates/wkg/src/wit.rs index 3c2462e..90878cf 100644 --- a/crates/wkg/src/wit.rs +++ b/crates/wkg/src/wit.rs @@ -119,7 +119,7 @@ pub async fn build_wit_dir( ) -> anyhow::Result<(PackageRef, Option, Vec)> { check_dir(&dir).await?; let wkg_config = wasm_pkg_core::config::Config::load().await?; - let result = wit::build_package(&wkg_config, dir, &mut lock_file, client).await?; + let result = wit::build_package(&wkg_config, dir, lock_file, client).await?; Ok(result) } From f0a896fdcfa6ca740d2a5fcbb34730b31893a9be Mon Sep 17 00:00:00 2001 From: Mikhail Katychev Date: Wed, 10 Jun 2026 12:33:13 -0500 Subject: [PATCH 06/61] initial packaging commit --- Cargo.lock | 53 +++++++++---- Cargo.toml | 12 +++ crates/wasm-pkg-client/Cargo.toml | 1 + crates/wasm-pkg-client/src/lib.rs | 18 ++++- crates/wasm-pkg-client/src/publisher.rs | 99 +++++++++++++++++++++++++ crates/wasm-pkg-common/Cargo.toml | 1 + crates/wasm-pkg-common/src/registry.rs | 7 ++ crates/wasm-pkg-core/Cargo.toml | 1 + crates/wasm-pkg-core/src/resolver.rs | 31 ++++++++ crates/wkg/src/main.rs | 6 +- 10 files changed, 207 insertions(+), 22 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6a89846..9071b68 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -466,9 +466,9 @@ dependencies = [ [[package]] name = "bitflags" -version = "2.11.1" +version = "2.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4512299f36f043ab09a583e57bceb5a5aab7a73db1805848e8fef3c9e8c78b3" +checksum = "84d7ced0ae9557296835c32bf1b1e02b44c746701f898460fb000d7eaa84f00a" dependencies = [ "serde_core", ] @@ -652,9 +652,9 @@ checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" [[package]] name = "chrono" -version = "0.4.44" +version = "0.4.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c673075a2e0e5f4a1dde27ce9dee1ea4558c7ffe648f576438a20ca1d2acc4b0" +checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" dependencies = [ "iana-time-zone", "js-sys", @@ -1361,6 +1361,12 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" +[[package]] +name = "fixedbitset" +version = "0.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" + [[package]] name = "flate2" version = "1.1.9" @@ -2297,9 +2303,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.30" +version = "0.4.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "616ec5685824bcc94416c6d4a7a446eea774a31efd7062c8480ba6fd06d7a6e5" +checksum = "953f07c43838f8e6f9758cab68bf5bed85465e7587ebe0b823f1bcd81978ad3a" [[package]] name = "logos" @@ -2867,10 +2873,22 @@ version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4c5cc86750666a3ed20bdaf5ca2a0344f9c67674cae0515bec2da16fbaa47db" dependencies = [ - "fixedbitset", + "fixedbitset 0.4.2", "indexmap 2.14.0", ] +[[package]] +name = "petgraph" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8701b58ea97060d5e5b155d383a69952a60943f0e6dfe30b04c287beb0b27455" +dependencies = [ + "fixedbitset 0.5.7", + "hashbrown 0.15.5", + "indexmap 2.14.0", + "serde", +] + [[package]] name = "pin-project" version = "1.1.13" @@ -3059,7 +3077,7 @@ dependencies = [ "log", "multimap", "once_cell", - "petgraph", + "petgraph 0.6.5", "prettyplease", "prost 0.12.6", "prost-types 0.12.6", @@ -3161,7 +3179,7 @@ dependencies = [ "anstyle", "config", "directories", - "petgraph", + "petgraph 0.6.5", "serde", "serde-value", "tint", @@ -3921,9 +3939,9 @@ dependencies = [ [[package]] name = "serde_with" -version = "3.20.0" +version = "3.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e72c1c2cb7b223fafb600a619537a871c2818583d619401b785e7c0b746ccde2" +checksum = "76a5c54c7310e7b8b9577c286d7e399ddd876c3e12b3ed917a8aabc4b96e9e8c" dependencies = [ "base64 0.22.1", "bs58", @@ -3941,9 +3959,9 @@ dependencies = [ [[package]] name = "serde_with_macros" -version = "3.20.0" +version = "3.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b90c488738ecb4fb0262f41f43bc40efc5868d9fb744319ddf5f5317f417bfac" +checksum = "84d57bc0c8b9a17920c178daa6bb924850d54a9c97ab45194bb8c17ad66bb660" dependencies = [ "darling 0.23.0", "proc-macro2", @@ -5088,7 +5106,7 @@ dependencies = [ "im-rc", "indexmap 2.14.0", "log", - "petgraph", + "petgraph 0.6.5", "serde", "serde_derive", "serde_yaml", @@ -5160,6 +5178,7 @@ dependencies = [ "futures-util", "oci-client", "oci-wasm", + "petgraph 0.8.3", "rcgen", "reqwest 0.12.28", "secrecy", @@ -5192,6 +5211,7 @@ dependencies = [ "etcetera 0.11.0", "futures-util", "http", + "petgraph 0.8.3", "semver", "serde", "serde_json", @@ -5210,6 +5230,7 @@ dependencies = [ "futures-util", "indexmap 2.14.0", "libc", + "petgraph 0.8.3", "rstest", "semver", "serde", @@ -5891,9 +5912,9 @@ dependencies = [ [[package]] name = "yoke" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "abe8c5fda708d9ca3df187cae8bfb9ceda00dd96231bed36e445a1a48e66f9ca" +checksum = "709fe23a0424b6a435d82152b1bd3fdfb0833487d5fa90d05d42762a9891fef5" dependencies = [ "stable_deref_trait", "yoke-derive", diff --git a/Cargo.toml b/Cargo.toml index 7c4bd80..18134e1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -21,6 +21,7 @@ oci-client = { version = "0.16", default-features = false, features = [ oci-wasm = { version = "0.4", default-features = false, features = [ "rustls-tls", ] } +petgraph = "0.8.3" rcgen = "0.14.8" semver = "1.0.23" serde = { version = "1.0", features = ["derive"] } @@ -43,3 +44,14 @@ wasm-pkg-core = { version = "0.15.1", path = "crates/wasm-pkg-core" } wasm-metadata = "0.244" wit-component = "0.244" wit-parser = "0.244" +async-trait = "0.1.77" +clap = "4.5" +http = "1.1.0" +indexmap = "2.5" +reqwest = { version = "0.12.0", default-features = false } +rstest = "0.23" +secrecy = "0.8" +url = "2.5.0" +warg-client = "0.9.2" +warg-crypto = "0.9.2" +warg-protocol = "0.9.2" diff --git a/crates/wasm-pkg-client/Cargo.toml b/crates/wasm-pkg-client/Cargo.toml index 3fa5f82..b8efdca 100644 --- a/crates/wasm-pkg-client/Cargo.toml +++ b/crates/wasm-pkg-client/Cargo.toml @@ -47,6 +47,7 @@ wasm-metadata = { workspace = true } warg-protocol = "0.9.2" wasm-pkg-common = { workspace = true, features = ["registry-config"] } wit-component = { workspace = true } +petgraph = "0.8.3" [dev-dependencies] rcgen = { workspace = true } diff --git a/crates/wasm-pkg-client/src/lib.rs b/crates/wasm-pkg-client/src/lib.rs index b093b88..1f62e20 100644 --- a/crates/wasm-pkg-client/src/lib.rs +++ b/crates/wasm-pkg-client/src/lib.rs @@ -1,5 +1,3 @@ -//! Wasm Package Client -//! //! [`Client`] implements a unified interface for loading package content from //! multiple kinds of package registries. //! @@ -144,6 +142,20 @@ impl Client { source.stream_content(package, release).await } + pub async fn publish_release_files( + &self, + files: &[impl AsRef], + additional_options: PublishOpts, + ) -> Result<(PackageRef, Version), Error> { + let data = tokio::fs::OpenOptions::new() + .read(true) + .open(files[0].as_ref()) + .await?; + + self.publish_release_data(Box::pin(data), additional_options) + .await + } + /// Publishes the given file as a package release. The package name and version will be read /// from the component if not given as part of `additional_options`. Returns the package name /// and version of the published release. @@ -158,7 +170,7 @@ impl Client { .await } - /// Publishes the given reader as a package release. TThe package name and version will be read + /// Publishes the given reader as a package release. The package name and version will be read /// from the component if not given as part of `additional_options`. Returns the package name /// and version of the published release. pub async fn publish_release_data( diff --git a/crates/wasm-pkg-client/src/publisher.rs b/crates/wasm-pkg-client/src/publisher.rs index 2d5e3d4..52e30aa 100644 --- a/crates/wasm-pkg-client/src/publisher.rs +++ b/crates/wasm-pkg-client/src/publisher.rs @@ -1,3 +1,11 @@ +use std::collections::{BTreeSet, HashMap}; + +use petgraph::{ + acyclic::Acyclic, + graph::{DiGraph, NodeIndex}, +}; +use wasm_pkg_common::registry::{DependencyGraph, DependencyOf}; + use crate::{PackageRef, PublishingSource, Version}; #[async_trait::async_trait] @@ -13,3 +21,94 @@ pub trait PackagePublisher: Send + Sync { dry_run: bool, ) -> Result<(), crate::Error>; } + +/// State for tracking dependencies during upload. +struct PublishPlan { + /// Graph of publishable packages where the edges are `(dependency -DependencyOf->) dependent)` + dependents: DependencyGraph, + /// Mapping [`PackageRef`]s to the respective index inside the dependency graph. + // TODO look at using cargo's `InternedString` type for `PackageRef`: + // https://docs.rs/cargo/latest/cargo/util/interning/struct.InternedString.html + indices: HashMap, +} + +impl PublishPlan { + /// Given a package dependency graph, creates a `PublishPlan` for tracking state. + fn new(graph: &DependencyGraph) -> Self { + let mut dependents = graph.clone().into_inner(); + dependents.reverse(); + // graph was already found to be acyclic + let dependents = DependencyGraph::try_from(dependents).unwrap(); + + let indices: HashMap<_, _> = dependents + .nodes_iter() + .map(|id| (dependents[id].clone(), id)) + .collect(); + + Self { + dependents, + indices, + } + } + + fn iter<'a>(&'a self) -> impl Iterator + 'a { + self.indices.iter().map(|(pkg, _)| pkg) + } + + fn is_empty(&self) -> bool { + self.indices.is_empty() + } + + fn len(&self) -> usize { + self.indices.len() + } + + /// Returns the set of packages that are ready for publishing (i.e. have no outstanding dependencies). + /// + /// These will not be returned in future calls. + fn take_ready(&mut self) -> BTreeSet { + self.dependents + .nodes_iter() + // there are no dependents on `self.dendents[id]` + .filter(|id| self.dependents.neighbors(*id).count() == 0) + .map(|id| { + let pkg = &self.dependents[id]; + self.indices.remove(&pkg); + pkg.clone() + }) + .collect() + } + + /// Packages confirmed to be available in the registry, potentially allowing additional + /// packages to be "ready". + fn mark_confirmed(&mut self, published: impl IntoIterator) { + for pkg in published { + let id = self + .indices + .remove(&pkg) + .expect("PackageRef has no associated index"); + self.dependents + .remove_node(id) + .expect("index has no associated PackageRef"); + } + } +} + +/// Format a collection of packages as a list +/// +/// e.g. "foo:a@0.1.0, bar:b@0.2.0, and baz:c@0.3.0". +/// +/// Note: the final separator (e.g. "and" in the previous example) can be chosen. +fn package_list(pkgs: impl IntoIterator, final_sep: &str) -> String { + let mut names: Vec<_> = pkgs.into_iter().map(|pkg| pkg.to_string()).collect(); + names.sort(); + + match &names[..] { + [] => String::new(), + [a] => a.clone(), + [a, b] => format!("{a} {final_sep} {b}"), + [names @ .., last] => { + format!("{}, {final_sep} {last}", names.join(", ")) + } + } +} diff --git a/crates/wasm-pkg-common/Cargo.toml b/crates/wasm-pkg-common/Cargo.toml index 104515a..9f7d0c4 100644 --- a/crates/wasm-pkg-common/Cargo.toml +++ b/crates/wasm-pkg-common/Cargo.toml @@ -31,6 +31,7 @@ tokio = { workspace = true, optional = true, features = ["fs"] } toml = { workspace = true, optional = true } thiserror = { workspace = true } tracing.workspace = true +petgraph.workspace = true [dev-dependencies] tokio = { workspace = true, features = ["macros", "rt"] } diff --git a/crates/wasm-pkg-common/src/registry.rs b/crates/wasm-pkg-common/src/registry.rs index 2fd42dc..0b89bb9 100644 --- a/crates/wasm-pkg-common/src/registry.rs +++ b/crates/wasm-pkg-common/src/registry.rs @@ -1,4 +1,5 @@ use http::uri::Authority; +use petgraph::{acyclic::Acyclic, graph::DiGraph}; use serde::{Deserialize, Serialize}; use crate::Error; @@ -55,3 +56,9 @@ impl TryFrom for Registry { Ok(Self(value.try_into()?)) } } + +/// Represents a directed edge in a package dependnecy graph. +#[derive(Clone, Debug)] +pub struct DependencyOf; + +pub type DependencyGraph = Acyclic>; diff --git a/crates/wasm-pkg-core/Cargo.toml b/crates/wasm-pkg-core/Cargo.toml index a43e021..d0b9fe2 100644 --- a/crates/wasm-pkg-core/Cargo.toml +++ b/crates/wasm-pkg-core/Cargo.toml @@ -22,6 +22,7 @@ wasm-pkg-common = { workspace = true } wasm-pkg-client = { workspace = true } wit-component = { workspace = true } wit-parser = { workspace = true } +petgraph = { workspace = true } [target.'cfg(unix)'.dependencies.libc] version = "0.2" diff --git a/crates/wasm-pkg-core/src/resolver.rs b/crates/wasm-pkg-core/src/resolver.rs index 6ffd112..f0a298b 100644 --- a/crates/wasm-pkg-core/src/resolver.rs +++ b/crates/wasm-pkg-core/src/resolver.rs @@ -12,6 +12,11 @@ use std::{ use anyhow::{bail, Context, Result}; use futures_util::TryStreamExt; use indexmap::{IndexMap, IndexSet}; +use petgraph::{ + acyclic::{Acyclic, AcyclicEdgeError}, + graph::DiGraph, + Graph, +}; use semver::{Comparator, Op, Version, VersionReq}; use tokio::io::{AsyncRead, AsyncReadExt}; use wasm_pkg_client::{ @@ -163,6 +168,32 @@ pub struct LocalResolution { pub path: PathBuf, } +pub struct LocalDependencies { + pub packages: HashMap, + pub graph: DiGraph, +} + +impl LocalDependencies { + pub fn sort(&self) -> Result> { + // sort our packages topologically + let acyclic_graph = Acyclic::try_from(self.graph.clone()).map_err(|e| { + anyhow::anyhow!( + "detected cyclical dependencies with package: {}", + self.graph[e.node_id()] + ) + })?; + + Ok(acyclic_graph + .nodes_iter() + .map(|id| self.graph[id].clone()) + .collect()) + } + + pub fn has_no_dependencies(&self) -> bool { + self.graph.raw_edges().is_empty() + } +} + /// Represents a resolution of a dependency. #[derive(Debug, Clone)] #[allow(clippy::large_enum_variant)] diff --git a/crates/wkg/src/main.rs b/crates/wkg/src/main.rs index 7272062..b7347a4 100644 --- a/crates/wkg/src/main.rs +++ b/crates/wkg/src/main.rs @@ -228,7 +228,7 @@ struct GetArgs { #[derive(Args, Debug)] struct PublishArgs { /// The file to publish - file: PathBuf, + files: Vec, #[command(flatten)] registry_args: RegistryArgs, @@ -262,8 +262,8 @@ impl PublishArgs { }; let (package, version) = client .client()? - .publish_release_file( - &self.file, + .publish_release_files( + &self.files, PublishOpts { package, registry: self.registry_args.registry, From a6ec15ebb244668151149fa74ae870bed80da533 Mon Sep 17 00:00:00 2001 From: Mikhail Katychev Date: Wed, 10 Jun 2026 17:08:39 -0500 Subject: [PATCH 07/61] added overlay client --- crates/wasm-pkg-client/src/local.rs | 2 +- crates/wasm-pkg-client/src/overlay/mod.rs | 86 +++++++++++++++++++++++ crates/wasm-pkg-common/src/config.rs | 9 +-- crates/wasm-pkg-core/src/resolver.rs | 1 + 4 files changed, 91 insertions(+), 7 deletions(-) create mode 100644 crates/wasm-pkg-client/src/overlay/mod.rs diff --git a/crates/wasm-pkg-client/src/local.rs b/crates/wasm-pkg-client/src/local.rs index 28a84bc..1f08ae6 100644 --- a/crates/wasm-pkg-client/src/local.rs +++ b/crates/wasm-pkg-client/src/local.rs @@ -30,7 +30,7 @@ pub struct LocalConfig { } pub(crate) struct LocalBackend { - root: PathBuf, + pub(crate) root: PathBuf, } impl LocalBackend { diff --git a/crates/wasm-pkg-client/src/overlay/mod.rs b/crates/wasm-pkg-client/src/overlay/mod.rs new file mode 100644 index 0000000..81ea716 --- /dev/null +++ b/crates/wasm-pkg-client/src/overlay/mod.rs @@ -0,0 +1,86 @@ +use async_trait::async_trait; +use tempfile::TempDir; +use wasm_pkg_common::Error; + +use crate::{loader::PackageLoader, local::LocalBackend, publisher::PackagePublisher, InnerClient}; + +pub(crate) struct OverlayBackend { + local: LocalBackend, + remote: InnerClient, + _handle: TempDir, +} + +impl OverlayBackend { + fn new(remote: InnerClient) -> Result { + let handle = TempDir::new()?; + let root = handle.path().to_owned(); + let local = LocalBackend { root }; + Ok(Self { + local, + remote, + _handle: handle, + }) + } +} + +#[async_trait] +impl PackageLoader for OverlayBackend { + async fn list_all_versions(&self, package: &PackageRef) -> Result, Error> { + let mut versions = self.local.list_all_versions(package).await?; + let mut remote_versions = self.remote.list_all_versions(package).await?; + versions.append(&mut remote_versions); + versions.sort(); + versions.dedup(); + Ok(versions) + } + + async fn get_release(&self, package: &PackageRef, version: &Version) -> Result { + if let Ok(release) = self.local.get_release(package, version).await { + return Ok(release); + } + tracing::debug!(%package, %version, method = "get_release", "OverlayBackend falling back to remote"); + self.remote.get_release(package, version).await + } + + async fn stream_content_unvalidated( + &self, + package: &PackageRef, + content: &Release, + ) -> Result { + if let Ok(stream) = self + .local + .stream_content_unvalidated(package, content) + .await + { + return Ok(stream); + } + tracing::debug!(%package, %version, method = "stream_content_unvalidated", "OverlayBackend falling back to remote"); + + self.local + .stream_content_unvalidated(package, content) + .await + } +} + +#[async_trait::async_trait] +impl PackagePublisher for LocalBackend { + async fn publish( + &self, + package: &PackageRef, + version: &Version, + mut data: PublishingSource, + dry_run: bool, + ) -> Result<(), Error> { + self.local + .publish(&package, &version, data, additional_options.dry_run) + .await?; + if dry_run { + return Ok(()); + } + self.remote + .publish(&package, &version, data, additional_options.dry_run) + .await?; + + Ok(()) + } +} diff --git a/crates/wasm-pkg-common/src/config.rs b/crates/wasm-pkg-common/src/config.rs index a20da34..e59ae96 100644 --- a/crates/wasm-pkg-common/src/config.rs +++ b/crates/wasm-pkg-common/src/config.rs @@ -302,13 +302,10 @@ impl RegistryConfig { pub fn default_backend(&self) -> Option<&str> { match self.default_backend.as_deref() { Some(ty) => Some(ty), - None => { - if self.backend_configs.len() == 1 { - self.backend_configs.keys().next().map(|ty| ty.as_str()) - } else { - None - } + _ if self.backend_configs.len() == 1 => { + self.backend_configs.keys().next().map(|ty| ty.as_str()) } + None => None, } } diff --git a/crates/wasm-pkg-core/src/resolver.rs b/crates/wasm-pkg-core/src/resolver.rs index f0a298b..87e0b79 100644 --- a/crates/wasm-pkg-core/src/resolver.rs +++ b/crates/wasm-pkg-core/src/resolver.rs @@ -23,6 +23,7 @@ use wasm_pkg_client::{ caching::{CachingClient, FileCache}, Client, Config, ContentDigest, Error as WasmPkgError, PackageRef, Release, VersionInfo, }; +use wasm_pkg_common::registry::DependencyOf; use wit_component::DecodedWasm; use wit_parser::{PackageId, PackageName, Resolve, UnresolvedPackageGroup, WorldId}; From a7084a652a28ca1c56f5bf2adfbba78904e08927 Mon Sep 17 00:00:00 2001 From: Mikhail Katychev Date: Thu, 11 Jun 2026 13:42:33 -0500 Subject: [PATCH 08/61] breaking up methods --- crates/wasm-pkg-client/Cargo.toml | 2 +- crates/wasm-pkg-client/src/lib.rs | 268 ++++++++++++++-------- crates/wasm-pkg-client/src/metadata.rs | 1 + crates/wasm-pkg-client/src/overlay/mod.rs | 23 +- crates/wasm-pkg-common/src/config.rs | 8 + crates/wasm-pkg-common/src/metadata.rs | 8 +- 6 files changed, 207 insertions(+), 103 deletions(-) diff --git a/crates/wasm-pkg-client/Cargo.toml b/crates/wasm-pkg-client/Cargo.toml index b8efdca..d10fc69 100644 --- a/crates/wasm-pkg-client/Cargo.toml +++ b/crates/wasm-pkg-client/Cargo.toml @@ -48,8 +48,8 @@ warg-protocol = "0.9.2" wasm-pkg-common = { workspace = true, features = ["registry-config"] } wit-component = { workspace = true } petgraph = "0.8.3" +tempfile = { workspace = true } [dev-dependencies] rcgen = { workspace = true } -tempfile = { workspace = true } testcontainers = { workspace = true } diff --git a/crates/wasm-pkg-client/src/lib.rs b/crates/wasm-pkg-client/src/lib.rs index 1f62e20..63b050c 100644 --- a/crates/wasm-pkg-client/src/lib.rs +++ b/crates/wasm-pkg-client/src/lib.rs @@ -29,6 +29,7 @@ mod loader; pub mod local; pub mod metadata; pub mod oci; +pub mod overlay; mod publisher; mod release; pub mod warg; @@ -44,6 +45,7 @@ use publisher::PackagePublisher; use tokio::io::AsyncSeekExt; use tokio::sync::RwLock; use tokio_util::io::SyncIoBridge; +use wasm_pkg_common::config::RegistryConfig; pub use wasm_pkg_common::{ config::{Config, CustomConfig, RegistryMapping}, digest::ContentDigest, @@ -203,111 +205,189 @@ impl Client { .map(|_| (package, version)) } + fn resolve_registry( + &self, + package: &PackageRef, + registry_override: Option, + ) -> Result { + if let Some(registry) = registry_override { + return Ok(registry); + } + self.config + .resolve_registry(package) + .cloned() + .ok_or_else(|| Error::NoRegistryForNamespace(package.namespace().clone())) + } + + fn resolve_backend( + &self, + registry: &Registry, + registry_meta: RegistryMetadata, + ) -> Result { + let registry_config = self + .config + .registry_config(®istry) + .cloned() + .unwrap_or_default(); + + let backend_type = match registry_config.default_backend() { + // If the local config specifies a backend type, use it + Some(backend_type) => Some(backend_type), + None => { + // If the registry metadata indicates a preferred protocol, use it + let preferred_protocol = registry_meta.preferred_protocol(); + // ...except registry metadata cannot force a local backend + if preferred_protocol == Some("local") { + return Err(Error::InvalidRegistryMetadata(anyhow!( + "registry metadata with 'local' protocol not allowed" + ))); + } + preferred_protocol + } + } + // Otherwise use the default backend + .unwrap_or("oci"); + + todo!(); + } + + fn resolve_metadata( + &self, + package: &PackageRef, + registry_config: &RegistryConfig, + registry_override: bool, + registry: &Registry, + ) -> &RegistryMetadata { + if let Some(metadata) = self + .config + .package_registry_override(package) + .and_then(|mapping| mapping.metadata()) + { + return metadata; + } + + let _ = self + .config + .namespace_registry(package.namespace()) + .and_then(|mapping| { + // If the overridden registry matches the registry we are trying to resolve, we + // should use the metadata, otherwise we'll need to fetch the metadata from the + // registry + match (mapping, registry_override) { + (RegistryMapping::Custom(custom), true) if custom.registry == *registry => { + Some(custom.metadata.clone()) + } + (RegistryMapping::Custom(custom), false) => Some(custom.metadata.clone()), + _ => None, + } + }); + todo!(); + } + + async fn has_registry(&self, registry: &Registry) -> bool { + let sources = self.sources.read().await; + sources.contains_key(registry) + } + async fn resolve_source( &self, package: &PackageRef, registry_override: Option, ) -> Result, Error> { let is_override = registry_override.is_some(); - let registry = if let Some(registry) = registry_override { - registry + let registry = self.resolve_registry(package, registry_override)?; + + if let Some(source) = self.sources.read().await.get(®istry) { + return Ok(source.clone()); + } + + let registry_config = self + .config + .registry_config(®istry) + .cloned() + .unwrap_or_default(); + + // Skip fetching metadata for "local" source + let should_fetch_meta = registry_config.default_backend() != Some("local"); + let maybe_metadata = self + .config + .package_registry_override(package) + .and_then(|mapping| match mapping { + RegistryMapping::Custom(custom) => Some(custom.metadata.clone()), + _ => None, + }) + .or_else(|| { + self.config + .namespace_registry(package.namespace()) + .and_then(|meta| { + // If the overridden registry matches the registry we are trying to resolve, we + // should use the metadata, otherwise we'll need to fetch the metadata from the + // registry + match (meta, is_override) { + (RegistryMapping::Custom(custom), true) + if custom.registry == registry => + { + Some(custom.metadata.clone()) + } + (RegistryMapping::Custom(custom), false) => { + Some(custom.metadata.clone()) + } + _ => None, + } + }) + }); + + let registry_meta = if let Some(meta) = maybe_metadata { + meta + } else if should_fetch_meta { + RegistryMetadata::fetch_or_default(®istry).await } else { - self.config - .resolve_registry(package) - .ok_or_else(|| Error::NoRegistryForNamespace(package.namespace().clone()))? - .to_owned() - }; - let has_key = { - let sources = self.sources.read().await; - sources.contains_key(®istry) + RegistryMetadata::default() }; - if !has_key { - let registry_config = self - .config - .registry_config(®istry) - .cloned() - .unwrap_or_default(); - - // Skip fetching metadata for "local" source - let should_fetch_meta = registry_config.default_backend() != Some("local"); - let maybe_metadata = self - .config - .package_registry_override(package) - .and_then(|mapping| match mapping { - RegistryMapping::Custom(custom) => Some(custom.metadata.clone()), - _ => None, - }) - .or_else(|| { - self.config - .namespace_registry(package.namespace()) - .and_then(|meta| { - // If the overridden registry matches the registry we are trying to resolve, we - // should use the metadata, otherwise we'll need to fetch the metadata from the - // registry - match (meta, is_override) { - (RegistryMapping::Custom(custom), true) - if custom.registry == registry => - { - Some(custom.metadata.clone()) - } - (RegistryMapping::Custom(custom), false) => { - Some(custom.metadata.clone()) - } - _ => None, - } - }) - }); - - let registry_meta = if let Some(meta) = maybe_metadata { - meta - } else if should_fetch_meta { - RegistryMetadata::fetch_or_default(®istry).await - } else { - RegistryMetadata::default() - }; - - // Resolve backend type - let backend_type = match registry_config.default_backend() { - // If the local config specifies a backend type, use it - Some(backend_type) => Some(backend_type), - None => { - // If the registry metadata indicates a preferred protocol, use it - let preferred_protocol = registry_meta.preferred_protocol(); - // ...except registry metadata cannot force a local backend - if preferred_protocol == Some("local") { - return Err(Error::InvalidRegistryMetadata(anyhow!( - "registry metadata with 'local' protocol not allowed" - ))); - } - preferred_protocol - } - } - // Otherwise use the default backend - .unwrap_or("oci"); - tracing::debug!(?backend_type, "Resolved backend type"); - - let source: InnerClient = match backend_type { - "local" => Box::new(LocalBackend::new(registry_config)?), - "oci" => Box::new(OciBackend::new( - ®istry, - ®istry_config, - ®istry_meta, - )?), - "warg" => { - Box::new(WargBackend::new(®istry, ®istry_config, ®istry_meta).await?) - } - other => { - return Err(Error::InvalidConfig(anyhow!( - "unknown backend type {other:?}" + + // Resolve backend type + let backend_type = match registry_config.default_backend() { + // If the local config specifies a backend type, use it + Some(backend_type) => Some(backend_type), + None => { + // If the registry metadata indicates a preferred protocol, use it + let preferred_protocol = registry_meta.preferred_protocol(); + // ...except registry metadata cannot force a local backend + if preferred_protocol == Some("local") { + return Err(Error::InvalidRegistryMetadata(anyhow!( + "registry metadata with 'local' protocol not allowed" ))); } - }; - self.sources - .write() - .await - .insert(registry.clone(), Arc::new(source)); + preferred_protocol + } } - Ok(self.sources.read().await.get(®istry).unwrap().clone()) + // Otherwise use the default backend + .unwrap_or("oci"); + tracing::debug!(?backend_type, "Resolved backend type"); + + let source: InnerClient = match backend_type { + "local" => Box::new(LocalBackend::new(registry_config)?), + "oci" => Box::new(OciBackend::new( + ®istry, + ®istry_config, + ®istry_meta, + )?), + "warg" => { + Box::new(WargBackend::new(®istry, ®istry_config, ®istry_meta).await?) + } + other => { + return Err(Error::InvalidConfig(anyhow!( + "unknown backend type {other:?}" + ))); + } + }; + let source = Arc::new(source); + self.sources + .write() + .await + .insert(registry.clone(), source.clone()); + + Ok(source) } } diff --git a/crates/wasm-pkg-client/src/metadata.rs b/crates/wasm-pkg-client/src/metadata.rs index 27827c3..d88d66d 100644 --- a/crates/wasm-pkg-client/src/metadata.rs +++ b/crates/wasm-pkg-client/src/metadata.rs @@ -37,6 +37,7 @@ impl RegistryMetadataExt for RegistryMetadata { } async fn fetch(registry: &Registry) -> Result, Error> { + // TODO use `core::net::Ipv4Addr::is_loopback()` let scheme = if registry.host() == "localhost" || registry.host() == "127.0.0.1" { "http" } else { diff --git a/crates/wasm-pkg-client/src/overlay/mod.rs b/crates/wasm-pkg-client/src/overlay/mod.rs index 81ea716..d99edcf 100644 --- a/crates/wasm-pkg-client/src/overlay/mod.rs +++ b/crates/wasm-pkg-client/src/overlay/mod.rs @@ -1,8 +1,16 @@ +use std::io::Cursor; + use async_trait::async_trait; use tempfile::TempDir; -use wasm_pkg_common::Error; +use wasm_pkg_common::{ + package::{PackageRef, Version}, + Error, +}; -use crate::{loader::PackageLoader, local::LocalBackend, publisher::PackagePublisher, InnerClient}; +use crate::{ + loader::PackageLoader, local::LocalBackend, publisher::PackagePublisher, ContentStream, + InnerClient, PublishingSource, Release, VersionInfo, +}; pub(crate) struct OverlayBackend { local: LocalBackend, @@ -54,7 +62,7 @@ impl PackageLoader for OverlayBackend { { return Ok(stream); } - tracing::debug!(%package, %version, method = "stream_content_unvalidated", "OverlayBackend falling back to remote"); + tracing::debug!(%package, version = %content.version, method = "stream_content_unvalidated", "OverlayBackend falling back to remote"); self.local .stream_content_unvalidated(package, content) @@ -63,7 +71,7 @@ impl PackageLoader for OverlayBackend { } #[async_trait::async_trait] -impl PackagePublisher for LocalBackend { +impl PackagePublisher for OverlayBackend { async fn publish( &self, package: &PackageRef, @@ -71,14 +79,17 @@ impl PackagePublisher for LocalBackend { mut data: PublishingSource, dry_run: bool, ) -> Result<(), Error> { + let mut local_data = Box::pin(Cursor::new(Vec::new())); + tokio::io::copy(&mut data, &mut local_data).await?; + self.local - .publish(&package, &version, data, additional_options.dry_run) + .publish(&package, &version, local_data.clone(), dry_run) .await?; if dry_run { return Ok(()); } self.remote - .publish(&package, &version, data, additional_options.dry_run) + .publish(&package, &version, local_data, dry_run) .await?; Ok(()) diff --git a/crates/wasm-pkg-common/src/config.rs b/crates/wasm-pkg-common/src/config.rs index e59ae96..7f46533 100644 --- a/crates/wasm-pkg-common/src/config.rs +++ b/crates/wasm-pkg-common/src/config.rs @@ -50,6 +50,14 @@ impl RegistryMapping { RegistryMapping::Custom(custom) => &custom.registry, } } + + /// returns the inner [`RegistryMetadata`] if `Self` holds a [`CustomConfig`] + pub fn metadata(&self) -> Option<&RegistryMetadata> { + if let Self::Custom(config) = self { + return Some(&config.metadata); + } + None + } } /// Custom registry configuration diff --git a/crates/wasm-pkg-common/src/metadata.rs b/crates/wasm-pkg-common/src/metadata.rs index c3fd663..82c36f9 100644 --- a/crates/wasm-pkg-common/src/metadata.rs +++ b/crates/wasm-pkg-common/src/metadata.rs @@ -36,8 +36,12 @@ pub struct RegistryMetadata { warg_url: Option, } -const OCI_PROTOCOL: &str = "oci"; -const WARG_PROTOCOL: &str = "warg"; +/// OCI registry try +pub const OCI_PROTOCOL: &str = "oci"; +/// Warg registry key +pub const WARG_PROTOCOL: &str = "warg"; +/// Local filesystem key +pub const LOCAL_FS: &str = "local"; impl RegistryMetadata { /// Returns the registry's preferred protocol. From 4e244e8a1930767f032995779f745807708430ab Mon Sep 17 00:00:00 2001 From: Mikhail Katychev Date: Mon, 15 Jun 2026 13:40:25 -0500 Subject: [PATCH 09/61] added multi client support for overlay --- crates/wasm-pkg-client/src/lib.rs | 83 ++--------------------- crates/wasm-pkg-client/src/overlay/mod.rs | 27 +++++--- crates/wasm-pkg-common/src/config.rs | 16 +++-- crates/wasm-pkg-common/src/metadata.rs | 2 +- crates/wasm-pkg-common/src/package.rs | 2 +- 5 files changed, 36 insertions(+), 94 deletions(-) diff --git a/crates/wasm-pkg-client/src/lib.rs b/crates/wasm-pkg-client/src/lib.rs index 63b050c..6dfe74b 100644 --- a/crates/wasm-pkg-client/src/lib.rs +++ b/crates/wasm-pkg-client/src/lib.rs @@ -46,6 +46,7 @@ use tokio::io::AsyncSeekExt; use tokio::sync::RwLock; use tokio_util::io::SyncIoBridge; use wasm_pkg_common::config::RegistryConfig; +use wasm_pkg_common::metadata::{LOCAL_PROTOCOL, OCI_PROTOCOL, WARG_PROTOCOL}; pub use wasm_pkg_common::{ config::{Config, CustomConfig, RegistryMapping}, digest::ContentDigest, @@ -219,76 +220,6 @@ impl Client { .ok_or_else(|| Error::NoRegistryForNamespace(package.namespace().clone())) } - fn resolve_backend( - &self, - registry: &Registry, - registry_meta: RegistryMetadata, - ) -> Result { - let registry_config = self - .config - .registry_config(®istry) - .cloned() - .unwrap_or_default(); - - let backend_type = match registry_config.default_backend() { - // If the local config specifies a backend type, use it - Some(backend_type) => Some(backend_type), - None => { - // If the registry metadata indicates a preferred protocol, use it - let preferred_protocol = registry_meta.preferred_protocol(); - // ...except registry metadata cannot force a local backend - if preferred_protocol == Some("local") { - return Err(Error::InvalidRegistryMetadata(anyhow!( - "registry metadata with 'local' protocol not allowed" - ))); - } - preferred_protocol - } - } - // Otherwise use the default backend - .unwrap_or("oci"); - - todo!(); - } - - fn resolve_metadata( - &self, - package: &PackageRef, - registry_config: &RegistryConfig, - registry_override: bool, - registry: &Registry, - ) -> &RegistryMetadata { - if let Some(metadata) = self - .config - .package_registry_override(package) - .and_then(|mapping| mapping.metadata()) - { - return metadata; - } - - let _ = self - .config - .namespace_registry(package.namespace()) - .and_then(|mapping| { - // If the overridden registry matches the registry we are trying to resolve, we - // should use the metadata, otherwise we'll need to fetch the metadata from the - // registry - match (mapping, registry_override) { - (RegistryMapping::Custom(custom), true) if custom.registry == *registry => { - Some(custom.metadata.clone()) - } - (RegistryMapping::Custom(custom), false) => Some(custom.metadata.clone()), - _ => None, - } - }); - todo!(); - } - - async fn has_registry(&self, registry: &Registry) -> bool { - let sources = self.sources.read().await; - sources.contains_key(registry) - } - async fn resolve_source( &self, package: &PackageRef, @@ -308,7 +239,7 @@ impl Client { .unwrap_or_default(); // Skip fetching metadata for "local" source - let should_fetch_meta = registry_config.default_backend() != Some("local"); + let should_fetch_meta = registry_config.default_backend() != LOCAL_PROTOCOL.into(); let maybe_metadata = self .config .package_registry_override(package) @@ -353,7 +284,7 @@ impl Client { // If the registry metadata indicates a preferred protocol, use it let preferred_protocol = registry_meta.preferred_protocol(); // ...except registry metadata cannot force a local backend - if preferred_protocol == Some("local") { + if preferred_protocol == Some(LOCAL_PROTOCOL) { return Err(Error::InvalidRegistryMetadata(anyhow!( "registry metadata with 'local' protocol not allowed" ))); @@ -362,17 +293,17 @@ impl Client { } } // Otherwise use the default backend - .unwrap_or("oci"); + .unwrap_or(OCI_PROTOCOL); tracing::debug!(?backend_type, "Resolved backend type"); let source: InnerClient = match backend_type { - "local" => Box::new(LocalBackend::new(registry_config)?), - "oci" => Box::new(OciBackend::new( + LOCAL_PROTOCOL => Box::new(LocalBackend::new(registry_config)?), + OCI_PROTOCOL => Box::new(OciBackend::new( ®istry, ®istry_config, ®istry_meta, )?), - "warg" => { + WARG_PROTOCOL => { Box::new(WargBackend::new(®istry, ®istry_config, ®istry_meta).await?) } other => { diff --git a/crates/wasm-pkg-client/src/overlay/mod.rs b/crates/wasm-pkg-client/src/overlay/mod.rs index d99edcf..89731cc 100644 --- a/crates/wasm-pkg-client/src/overlay/mod.rs +++ b/crates/wasm-pkg-client/src/overlay/mod.rs @@ -1,4 +1,4 @@ -use std::io::Cursor; +use std::{collections::HashMap, io::Cursor}; use async_trait::async_trait; use tempfile::TempDir; @@ -14,28 +14,34 @@ use crate::{ pub(crate) struct OverlayBackend { local: LocalBackend, - remote: InnerClient, + remotes: HashMap, _handle: TempDir, } impl OverlayBackend { - fn new(remote: InnerClient) -> Result { + fn new(remotes: HashMap) -> Result { let handle = TempDir::new()?; let root = handle.path().to_owned(); let local = LocalBackend { root }; Ok(Self { local, - remote, + remotes, _handle: handle, }) } + + fn remote(&self, package: &PackageRef) -> Result<&InnerClient, Error> { + self.remotes + .get(package) + .ok_or_else(|| Error::InvalidPackageRef(package.to_string())) + } } #[async_trait] impl PackageLoader for OverlayBackend { async fn list_all_versions(&self, package: &PackageRef) -> Result, Error> { let mut versions = self.local.list_all_versions(package).await?; - let mut remote_versions = self.remote.list_all_versions(package).await?; + let mut remote_versions = self.remote(package)?.list_all_versions(package).await?; versions.append(&mut remote_versions); versions.sort(); versions.dedup(); @@ -47,7 +53,7 @@ impl PackageLoader for OverlayBackend { return Ok(release); } tracing::debug!(%package, %version, method = "get_release", "OverlayBackend falling back to remote"); - self.remote.get_release(package, version).await + self.remote(package)?.get_release(package, version).await } async fn stream_content_unvalidated( @@ -82,13 +88,14 @@ impl PackagePublisher for OverlayBackend { let mut local_data = Box::pin(Cursor::new(Vec::new())); tokio::io::copy(&mut data, &mut local_data).await?; - self.local - .publish(&package, &version, local_data.clone(), dry_run) - .await?; if dry_run { + self.local + .publish(&package, &version, local_data.clone(), dry_run) + .await?; return Ok(()); } - self.remote + + self.remote(package)? .publish(&package, &version, local_data, dry_run) .await?; diff --git a/crates/wasm-pkg-common/src/config.rs b/crates/wasm-pkg-common/src/config.rs index 7f46533..7edca19 100644 --- a/crates/wasm-pkg-common/src/config.rs +++ b/crates/wasm-pkg-common/src/config.rs @@ -204,12 +204,7 @@ impl Config { let namespace = package.namespace(); // look in `self.package_registry_overrides ` // then in `self.namespace_registries` - if let Some(reg) = self - .package_registry_overrides - .get(package) - .or_else(|| self.namespace_registries.get(namespace)) - .map(|pkg| pkg.registry()) - { + if let Some(reg) = self.resolve_mapping(package).map(|ns| ns.registry()) { return Some(reg); } else if let Some(reg) = self.default_registry.as_ref() { return Some(reg); @@ -218,6 +213,15 @@ impl Config { self.fallback_namespace_registries.get(namespace) } + pub fn resolve_mapping(&self, package: &PackageRef) -> Option<&RegistryMapping> { + let namespace = package.namespace(); + // look in `self.package_registry_overrides ` + // then in `self.namespace_registries` + self.package_registry_overrides + .get(package) + .or_else(|| self.namespace_registries.get(namespace)) + } + /// Returns the default registry. pub fn default_registry(&self) -> Option<&Registry> { self.default_registry.as_ref() diff --git a/crates/wasm-pkg-common/src/metadata.rs b/crates/wasm-pkg-common/src/metadata.rs index 82c36f9..dda8332 100644 --- a/crates/wasm-pkg-common/src/metadata.rs +++ b/crates/wasm-pkg-common/src/metadata.rs @@ -41,7 +41,7 @@ pub const OCI_PROTOCOL: &str = "oci"; /// Warg registry key pub const WARG_PROTOCOL: &str = "warg"; /// Local filesystem key -pub const LOCAL_FS: &str = "local"; +pub const LOCAL_PROTOCOL: &str = "local"; impl RegistryMetadata { /// Returns the registry's preferred protocol. diff --git a/crates/wasm-pkg-common/src/package.rs b/crates/wasm-pkg-common/src/package.rs index 8360c97..7d971e5 100644 --- a/crates/wasm-pkg-common/src/package.rs +++ b/crates/wasm-pkg-common/src/package.rs @@ -23,7 +23,7 @@ impl PackageRef { } /// Returns the namespace of the package. - pub fn namespace(&self) -> &Label { + pub const fn namespace(&self) -> &Label { &self.namespace } From 8e5ab5ff1c84de80e1c73d1a84d6f38b3ccde5f8 Mon Sep 17 00:00:00 2001 From: Mikhail Katychev Date: Mon, 15 Jun 2026 15:08:01 -0500 Subject: [PATCH 10/61] clean up publish source --- crates/wasm-pkg-client/src/lib.rs | 9 ++++----- crates/wasm-pkg-client/src/overlay/mod.rs | 20 ++++++-------------- 2 files changed, 10 insertions(+), 19 deletions(-) diff --git a/crates/wasm-pkg-client/src/lib.rs b/crates/wasm-pkg-client/src/lib.rs index 6dfe74b..0d31113 100644 --- a/crates/wasm-pkg-client/src/lib.rs +++ b/crates/wasm-pkg-client/src/lib.rs @@ -238,8 +238,6 @@ impl Client { .cloned() .unwrap_or_default(); - // Skip fetching metadata for "local" source - let should_fetch_meta = registry_config.default_backend() != LOCAL_PROTOCOL.into(); let maybe_metadata = self .config .package_registry_override(package) @@ -270,10 +268,11 @@ impl Client { let registry_meta = if let Some(meta) = maybe_metadata { meta - } else if should_fetch_meta { - RegistryMetadata::fetch_or_default(®istry).await - } else { + } else if registry_config.default_backend() == LOCAL_PROTOCOL.into() { + // Skip fetching metadata for "local" source RegistryMetadata::default() + } else { + RegistryMetadata::fetch_or_default(®istry).await }; // Resolve backend type diff --git a/crates/wasm-pkg-client/src/overlay/mod.rs b/crates/wasm-pkg-client/src/overlay/mod.rs index 89731cc..b6067ae 100644 --- a/crates/wasm-pkg-client/src/overlay/mod.rs +++ b/crates/wasm-pkg-client/src/overlay/mod.rs @@ -82,23 +82,15 @@ impl PackagePublisher for OverlayBackend { &self, package: &PackageRef, version: &Version, - mut data: PublishingSource, + data: PublishingSource, dry_run: bool, ) -> Result<(), Error> { - let mut local_data = Box::pin(Cursor::new(Vec::new())); - tokio::io::copy(&mut data, &mut local_data).await?; - if dry_run { - self.local - .publish(&package, &version, local_data.clone(), dry_run) - .await?; - return Ok(()); + self.local.publish(&package, &version, data, dry_run).await + } else { + self.remote(package)? + .publish(&package, &version, data, dry_run) + .await } - - self.remote(package)? - .publish(&package, &version, local_data, dry_run) - .await?; - - Ok(()) } } From 4e50668e1658a694af97673fd5648a47a08b38e1 Mon Sep 17 00:00:00 2001 From: Mikhail Katychev Date: Tue, 16 Jun 2026 11:41:41 -0500 Subject: [PATCH 11/61] add path to errors --- crates/wasm-pkg-client/src/local.rs | 31 ++++++++++++++++++++++------ crates/wasm-pkg-common/src/config.rs | 10 +++++++++ crates/wasm-pkg-common/src/lib.rs | 2 +- 3 files changed, 36 insertions(+), 7 deletions(-) diff --git a/crates/wasm-pkg-client/src/local.rs b/crates/wasm-pkg-client/src/local.rs index 28a84bc..2aa2436 100644 --- a/crates/wasm-pkg-client/src/local.rs +++ b/crates/wasm-pkg-client/src/local.rs @@ -2,7 +2,10 @@ //! //! Each package release is a file: `///.wasm` -use std::path::{Path, PathBuf}; +use std::{ + io, + path::{Path, PathBuf}, +}; use anyhow::anyhow; use async_trait::async_trait; @@ -33,6 +36,11 @@ pub(crate) struct LocalBackend { root: PathBuf, } +fn registry_path_context(err: io::Error, path: &Path) -> Error { + let err = anyhow::Error::new(err).context(format!("path: {}", path.display())); + Error::RegistryError(err) +} + impl LocalBackend { pub fn new(registry_config: RegistryConfig) -> Result { let config = registry_config @@ -60,7 +68,10 @@ impl PackageLoader for LocalBackend { let mut versions = vec![]; let package_dir = self.package_dir(package); tracing::debug!(?package_dir, "Reading versions from path"); - let mut entries = tokio::fs::read_dir(package_dir).await?; + let mut entries = tokio::fs::read_dir(&package_dir) + .await + .map_err(|e| registry_path_context(e, &package_dir))?; + tracing::debug!("READ IT"); while let Some(entry) = entries.next_entry().await? { let path = entry.path(); if path.extension() != Some("wasm".as_ref()) { @@ -86,7 +97,9 @@ impl PackageLoader for LocalBackend { async fn get_release(&self, package: &PackageRef, version: &Version) -> Result { let path = self.version_path(package, version); tracing::debug!(path = %path.display(), "Reading content from path"); - let content_digest = sha256_from_file(path).await?; + let content_digest = sha256_from_file(&path) + .await + .map_err(|e| registry_path_context(e, &path))?; Ok(Release { version: version.clone(), content_digest, @@ -100,7 +113,9 @@ impl PackageLoader for LocalBackend { ) -> Result { let path = self.version_path(package, &content.version); tracing::debug!("Streaming content from {path:?}"); - let file = tokio::fs::File::open(path).await?; + let file = tokio::fs::File::open(&path) + .await + .map_err(|e| registry_path_context(e, &path))?; Ok(ReaderStream::new(file).map_err(Into::into).boxed()) } } @@ -116,12 +131,16 @@ impl PackagePublisher for LocalBackend { ) -> Result<(), Error> { let package_dir = self.package_dir(package); // Ensure the package directory exists. - tokio::fs::create_dir_all(package_dir).await?; + tokio::fs::create_dir_all(&package_dir) + .await + .map_err(|e| registry_path_context(e, &package_dir))?; let path = self.version_path(package, version); if dry_run { return Ok(()); } - let mut out = tokio::fs::File::create(path).await?; + let mut out = tokio::fs::File::create(&path) + .await + .map_err(|e| registry_path_context(e, &path))?; tokio::io::copy(&mut data, &mut out) .await .map_err(Error::IoError) diff --git a/crates/wasm-pkg-common/src/config.rs b/crates/wasm-pkg-common/src/config.rs index a20da34..05f6e31 100644 --- a/crates/wasm-pkg-common/src/config.rs +++ b/crates/wasm-pkg-common/src/config.rs @@ -136,6 +136,16 @@ impl Config { .map(|strat| strat.config_dir().join("wasm-pkg").join("config.toml")) } + // take relative paths (such as those provided by [`LocalConfig`]'s `root` field) and make them + // absolute + fn normalize_paths(mut self, config_file: &Path) -> Result { + for (_, registry_config) in self.registry_configs.iter_mut() { + if let Some(local_config) = registry_config.backend_config::("local")? {} + + if let Some(path) = backend_configs.get_mut("local").map(|v| v.get_mut("root")) {} + } + } + /// Reads config from a TOML file at the given path. pub async fn from_file(path: impl AsRef) -> Result { let contents = tokio::fs::read_to_string(path) diff --git a/crates/wasm-pkg-common/src/lib.rs b/crates/wasm-pkg-common/src/lib.rs index e8759bd..4ea1b5b 100644 --- a/crates/wasm-pkg-common/src/lib.rs +++ b/crates/wasm-pkg-common/src/lib.rs @@ -45,7 +45,7 @@ pub enum Error { NoRegistryForNamespace(Label), #[error("Package not found")] PackageNotFound, - #[error("registry error: {0}")] + #[error("registry error")] RegistryError(#[source] anyhow::Error), #[error("registry metadata error: {0:#}")] RegistryMetadataError(#[source] anyhow::Error), From 19edc4f18c81812053d0833ffe243db6ff16312f Mon Sep 17 00:00:00 2001 From: Mikhail Katychev Date: Tue, 16 Jun 2026 15:30:12 -0500 Subject: [PATCH 12/61] reflow formatting --- crates/wasm-pkg-client/src/local.rs | 2 +- crates/wasm-pkg-common/src/config.rs | 11 +---------- crates/wasm-pkg-core/src/resolver.rs | 14 +++++++++++--- 3 files changed, 13 insertions(+), 14 deletions(-) diff --git a/crates/wasm-pkg-client/src/local.rs b/crates/wasm-pkg-client/src/local.rs index 2aa2436..6302027 100644 --- a/crates/wasm-pkg-client/src/local.rs +++ b/crates/wasm-pkg-client/src/local.rs @@ -71,7 +71,6 @@ impl PackageLoader for LocalBackend { let mut entries = tokio::fs::read_dir(&package_dir) .await .map_err(|e| registry_path_context(e, &package_dir))?; - tracing::debug!("READ IT"); while let Some(entry) = entries.next_entry().await? { let path = entry.path(); if path.extension() != Some("wasm".as_ref()) { @@ -141,6 +140,7 @@ impl PackagePublisher for LocalBackend { let mut out = tokio::fs::File::create(&path) .await .map_err(|e| registry_path_context(e, &path))?; + println!("writing to {}", path.display()); tokio::io::copy(&mut data, &mut out) .await .map_err(Error::IoError) diff --git a/crates/wasm-pkg-common/src/config.rs b/crates/wasm-pkg-common/src/config.rs index 05f6e31..5f36887 100644 --- a/crates/wasm-pkg-common/src/config.rs +++ b/crates/wasm-pkg-common/src/config.rs @@ -4,6 +4,7 @@ use std::{ path::{Path, PathBuf}, }; +use ::toml::Value; use serde::{Deserialize, Serialize}; use crate::{ @@ -136,16 +137,6 @@ impl Config { .map(|strat| strat.config_dir().join("wasm-pkg").join("config.toml")) } - // take relative paths (such as those provided by [`LocalConfig`]'s `root` field) and make them - // absolute - fn normalize_paths(mut self, config_file: &Path) -> Result { - for (_, registry_config) in self.registry_configs.iter_mut() { - if let Some(local_config) = registry_config.backend_config::("local")? {} - - if let Some(path) = backend_configs.get_mut("local").map(|v| v.get_mut("root")) {} - } - } - /// Reads config from a TOML file at the given path. pub async fn from_file(path: impl AsRef) -> Result { let contents = tokio::fs::read_to_string(path) diff --git a/crates/wasm-pkg-core/src/resolver.rs b/crates/wasm-pkg-core/src/resolver.rs index 6ffd112..51a41eb 100644 --- a/crates/wasm-pkg-core/src/resolver.rs +++ b/crates/wasm-pkg-core/src/resolver.rs @@ -556,10 +556,18 @@ impl<'a> DependencyResolver<'a> { // the version requirement; this can happen when packages are yanked. If we did // find an exact match, return the digest for comparison after fetching the // release - find_latest_release(versions, &exact_req).map(|v| (&v.version, Some(digest))).or_else(|| find_latest_release(versions, &dependency.version).map(|v| (&v.version, None))) - } + find_latest_release(versions, &exact_req) + .map(|v| (&v.version, Some(digest))) + .or_else(|| find_latest_release(versions, &dependency.version).map(|v| (&v.version, None))) + } None => find_latest_release(versions, &dependency.version).map(|v| (&v.version, None)), - }.with_context(|| format!("component registry package `{name}` has no release matching version requirement `{version}`", name = dependency.package, version = dependency.version))? + }.with_context(|| + format!( + "component registry package `{name}` has no release matching version requirement `{version}`", + name = dependency.package, + version = dependency.version + ) + )? }; // We need to clone a handle to the client because we mutably borrow self above. Might From 4c2c2cf27bc3c5a19b083b1846dfb6a7e2cd9799 Mon Sep 17 00:00:00 2001 From: Mikhail Katychev Date: Wed, 17 Jun 2026 12:12:13 -0500 Subject: [PATCH 13/61] added back dry run fix --- crates/wasm-pkg-client/src/local.rs | 2 +- crates/wasm-pkg-common/src/config.rs | 1 - crates/wkg/src/main.rs | 9 +++++++-- crates/wkg/src/wit.rs | 2 +- 4 files changed, 9 insertions(+), 5 deletions(-) diff --git a/crates/wasm-pkg-client/src/local.rs b/crates/wasm-pkg-client/src/local.rs index 6302027..acc8440 100644 --- a/crates/wasm-pkg-client/src/local.rs +++ b/crates/wasm-pkg-client/src/local.rs @@ -140,7 +140,7 @@ impl PackagePublisher for LocalBackend { let mut out = tokio::fs::File::create(&path) .await .map_err(|e| registry_path_context(e, &path))?; - println!("writing to {}", path.display()); + tracing::info!("publishing to {}", path.display()); tokio::io::copy(&mut data, &mut out) .await .map_err(Error::IoError) diff --git a/crates/wasm-pkg-common/src/config.rs b/crates/wasm-pkg-common/src/config.rs index 5f36887..a20da34 100644 --- a/crates/wasm-pkg-common/src/config.rs +++ b/crates/wasm-pkg-common/src/config.rs @@ -4,7 +4,6 @@ use std::{ path::{Path, PathBuf}, }; -use ::toml::Value; use serde::{Deserialize, Serialize}; use crate::{ diff --git a/crates/wkg/src/main.rs b/crates/wkg/src/main.rs index efb3282..d4e83d9 100644 --- a/crates/wkg/src/main.rs +++ b/crates/wkg/src/main.rs @@ -273,11 +273,16 @@ impl PublishArgs { wit::build_wit_dir(&self.path, client.clone(), &mut lock_file).await?; // There is no way to check if we are in a git repository unlike `cargo publish --allow-dirty` so // check against previous values. - if lock_file != prev_lock_ref { + if lock_file != prev_lock_ref && !self.dry_run { return Err(anyhow::anyhow!( "wkg.lock would be updated during publish, aborting" )) - .context("Run `wkg wit fetch` before attempting to publish"); + .with_context(|| { + format!( + "Run `wkg wit build {}` before attempting to publish", + self.path.to_string_lossy() + ) + }); } let tmp = tempfile::Builder::new() diff --git a/crates/wkg/src/wit.rs b/crates/wkg/src/wit.rs index 90878cf..f6d891d 100644 --- a/crates/wkg/src/wit.rs +++ b/crates/wkg/src/wit.rs @@ -115,7 +115,7 @@ impl BuildArgs { pub async fn build_wit_dir( dir: impl AsRef, client: CachingClient, - mut lock_file: &mut LockFile, + lock_file: &mut LockFile, ) -> anyhow::Result<(PackageRef, Option, Vec)> { check_dir(&dir).await?; let wkg_config = wasm_pkg_core::config::Config::load().await?; From db51fe6c29abd733673ce91240f30ab9120007b6 Mon Sep 17 00:00:00 2001 From: Mikhail Katychev Date: Wed, 17 Jun 2026 16:30:04 -0500 Subject: [PATCH 14/61] cargo update --- Cargo.lock | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6a89846..14a1032 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -466,9 +466,9 @@ dependencies = [ [[package]] name = "bitflags" -version = "2.11.1" +version = "2.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4512299f36f043ab09a583e57bceb5a5aab7a73db1805848e8fef3c9e8c78b3" +checksum = "84d7ced0ae9557296835c32bf1b1e02b44c746701f898460fb000d7eaa84f00a" dependencies = [ "serde_core", ] @@ -652,9 +652,9 @@ checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" [[package]] name = "chrono" -version = "0.4.44" +version = "0.4.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c673075a2e0e5f4a1dde27ce9dee1ea4558c7ffe648f576438a20ca1d2acc4b0" +checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" dependencies = [ "iana-time-zone", "js-sys", @@ -2297,9 +2297,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.30" +version = "0.4.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "616ec5685824bcc94416c6d4a7a446eea774a31efd7062c8480ba6fd06d7a6e5" +checksum = "953f07c43838f8e6f9758cab68bf5bed85465e7587ebe0b823f1bcd81978ad3a" [[package]] name = "logos" @@ -3921,9 +3921,9 @@ dependencies = [ [[package]] name = "serde_with" -version = "3.20.0" +version = "3.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e72c1c2cb7b223fafb600a619537a871c2818583d619401b785e7c0b746ccde2" +checksum = "76a5c54c7310e7b8b9577c286d7e399ddd876c3e12b3ed917a8aabc4b96e9e8c" dependencies = [ "base64 0.22.1", "bs58", @@ -3941,9 +3941,9 @@ dependencies = [ [[package]] name = "serde_with_macros" -version = "3.20.0" +version = "3.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b90c488738ecb4fb0262f41f43bc40efc5868d9fb744319ddf5f5317f417bfac" +checksum = "84d57bc0c8b9a17920c178daa6bb924850d54a9c97ab45194bb8c17ad66bb660" dependencies = [ "darling 0.23.0", "proc-macro2", @@ -5891,9 +5891,9 @@ dependencies = [ [[package]] name = "yoke" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "abe8c5fda708d9ca3df187cae8bfb9ceda00dd96231bed36e445a1a48e66f9ca" +checksum = "709fe23a0424b6a435d82152b1bd3fdfb0833487d5fa90d05d42762a9891fef5" dependencies = [ "stable_deref_trait", "yoke-derive", From f8498fba01f55b4d47e1c332833ae64bac8ed6ab Mon Sep 17 00:00:00 2001 From: Mikhail Katychev Date: Thu, 18 Jun 2026 12:15:01 -0500 Subject: [PATCH 15/61] set lockfile to be readonly --- crates/wkg/src/main.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/wkg/src/main.rs b/crates/wkg/src/main.rs index d4e83d9..ba70553 100644 --- a/crates/wkg/src/main.rs +++ b/crates/wkg/src/main.rs @@ -267,7 +267,7 @@ impl PublishArgs { // file first. _tmp is held until the publish completes so the file // isn't deleted out from under us. let (publish_path, _tmp) = if self.path.is_dir() { - let mut lock_file = LockFile::load(false).await?; + let mut lock_file = LockFile::load(true).await?; let prev_lock_ref = (lock_file.version, lock_file.packages.clone()); let (build_ref, _, bytes) = wit::build_wit_dir(&self.path, client.clone(), &mut lock_file).await?; From 8e1c617b2fa8b95176b7ca58a0ab159b469f002f Mon Sep 17 00:00:00 2001 From: Mikhail Katychev Date: Thu, 18 Jun 2026 16:14:48 -0500 Subject: [PATCH 16/61] Apply suggestion from @ricochet Co-authored-by: Bailey Hayes --- crates/wasm-pkg-common/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/wasm-pkg-common/src/lib.rs b/crates/wasm-pkg-common/src/lib.rs index 4ea1b5b..a6d401f 100644 --- a/crates/wasm-pkg-common/src/lib.rs +++ b/crates/wasm-pkg-common/src/lib.rs @@ -45,7 +45,7 @@ pub enum Error { NoRegistryForNamespace(Label), #[error("Package not found")] PackageNotFound, - #[error("registry error")] + #[error("registry error: {0:#}")] RegistryError(#[source] anyhow::Error), #[error("registry metadata error: {0:#}")] RegistryMetadataError(#[source] anyhow::Error), From 80dd587d4158cb3eb15fc0f42825214199015d82 Mon Sep 17 00:00:00 2001 From: Mikhail Katychev Date: Thu, 18 Jun 2026 16:17:15 -0500 Subject: [PATCH 17/61] Apply suggestions from code review Co-authored-by: Bailey Hayes --- crates/wasm-pkg-core/src/lock.rs | 3 +++ crates/wkg/src/main.rs | 6 +++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/crates/wasm-pkg-core/src/lock.rs b/crates/wasm-pkg-core/src/lock.rs index 7a864ee..f27d5c5 100644 --- a/crates/wasm-pkg-core/src/lock.rs +++ b/crates/wasm-pkg-core/src/lock.rs @@ -48,6 +48,9 @@ impl PartialEq for LockFile { self.packages == other.packages && self.version == other.version } } +/// Compares a [`LockFile`] against a `(version, packages)` snapshot. This is used to +/// detect whether a build would mutate the lock file (e.g. before publishing) without +/// having to hold a second [`LockFile`]. impl PartialEq<(u64, BTreeSet)> for LockFile { fn eq(&self, other: &(u64, BTreeSet)) -> bool { self.packages == other.1 && self.version == other.0 diff --git a/crates/wkg/src/main.rs b/crates/wkg/src/main.rs index ba70553..08b687b 100644 --- a/crates/wkg/src/main.rs +++ b/crates/wkg/src/main.rs @@ -285,8 +285,12 @@ impl PublishArgs { }); } + // Sanitize the package ref for use as a filename prefix: `namespace:name` + // contains characters (`:`, `/`) that are invalid in filenames on some + // platforms (notably Windows). + let prefix: String = build_ref.to_string().replace([':', '/'], "_"); let tmp = tempfile::Builder::new() - .prefix(&build_ref.to_string()) + .prefix(&prefix) .suffix(".wasm") .tempfile() .context("Failed to create temporary file for built WIT package")?; From 20d1923fb36e620928ac58e415b2bf96d770145b Mon Sep 17 00:00:00 2001 From: Mikhail Katychev Date: Thu, 18 Jun 2026 16:51:31 -0500 Subject: [PATCH 18/61] initial graph input data generation --- crates/wasm-pkg-common/src/package.rs | 19 ++++++++++ crates/wasm-pkg-core/src/wit.rs | 50 ++++++++++++++++++++++++--- crates/wkg/src/main.rs | 26 +++++++------- 3 files changed, 77 insertions(+), 18 deletions(-) diff --git a/crates/wasm-pkg-common/src/package.rs b/crates/wasm-pkg-common/src/package.rs index 7d971e5..f4c921b 100644 --- a/crates/wasm-pkg-common/src/package.rs +++ b/crates/wasm-pkg-common/src/package.rs @@ -68,6 +68,15 @@ impl FromStr for PackageRef { s.to_string().try_into() } } +impl std::fmt::Display for PackageSpec { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + if let Some(version) = &self.version { + write!(f, "{}@{}", self.package, version) + } else { + write!(f, "{}", self.package) + } + } +} /// A package spec combines a [`PackageRef`] with an optional version. #[derive(Clone, Debug)] @@ -76,6 +85,16 @@ pub struct PackageSpec { pub version: Option, } +impl PackageSpec { + pub fn package(&self) -> PackageRef { + self.package.clone() + } + + pub fn version(&self) -> Option { + self.version.clone() + } +} + impl FromStr for PackageSpec { type Err = Error; diff --git a/crates/wasm-pkg-core/src/wit.rs b/crates/wasm-pkg-core/src/wit.rs index 38bd70d..57dd6c8 100644 --- a/crates/wasm-pkg-core/src/wit.rs +++ b/crates/wasm-pkg-core/src/wit.rs @@ -1,14 +1,23 @@ //! Functions for building WIT packages and fetching their dependencies. -use std::{collections::HashSet, path::Path, str::FromStr}; +use std::{ + collections::{HashMap, HashSet}, + path::Path, + str::FromStr, +}; use anyhow::{Context, Result}; +use petgraph::{data::Build, graph::NodeIndex}; use semver::{Version, VersionReq}; use wasm_metadata::{AddMetadata, AddMetadataField}; use wasm_pkg_client::{ caching::{CachingClient, FileCache}, PackageRef, }; +use wasm_pkg_common::{ + package::PackageSpec, + registry::{DependencyGraph, DependencyOf}, +}; use wit_component::WitPrinter; use wit_parser::{PackageId, PackageName, Resolve}; @@ -134,11 +143,11 @@ pub async fn fetch_dependencies( /// for resolving dependencies. pub fn get_packages( path: impl AsRef, -) -> Result<(PackageRef, HashSet<(PackageRef, VersionReq)>)> { +) -> Result<(PackageSpec, HashSet<(PackageRef, VersionReq)>)> { let group = wit_parser::UnresolvedPackageGroup::parse_path(path).context("Couldn't parse package")?; - let name = PackageRef::new( + let package = PackageRef::new( group .main .name @@ -152,6 +161,10 @@ pub fn get_packages( .parse() .context("Invalid name found in package")?, ); + let package = PackageSpec { + package, + version: group.main.name.version.clone(), + }; // Get all package refs from the main package and then from any nested packages let packages: HashSet<(PackageRef, VersionReq)> = @@ -164,7 +177,34 @@ pub fn get_packages( ) .collect(); - Ok((name, packages)) + Ok((package, packages)) +} + +pub fn get_local_dependencies( + paths: Vec>, +) -> Result<(DependencyGraph, HashMap)> { + let pkg_trees = paths + .into_iter() + .map(|p| get_packages(p)) + .collect::, _>>()?; + let mut graph = DependencyGraph::new(); + let mut indices = HashMap::new(); + // establish all nodes + for (spec, _) in &pkg_trees { + let id = graph.add_node(spec.package()); + if indices.insert(spec.package(), id).is_some() { + anyhow::bail!("duplicate references to package detected: {spec}"); + } + } + for (spec, deps) in pkg_trees { + // TODO handle version matching for dependencies + for (dep, _version) in deps { + if let Some(&dep_id) = indices.get(&dep) { + graph.add_edge(dep_id, indices[&spec.package], DependencyOf); + } + } + } + Ok((graph, indices)) } /// Builds a list of resolved dependencies loaded from the component or path containing the WIT. @@ -211,7 +251,7 @@ pub async fn resolve_dependencies( .with_context(|| format!("unable to add dependency {dep}"))?; } } - let (_name, packages) = get_packages(path)?; + let (_spec, packages) = get_packages(path)?; resolver.add_packages(packages).await?; resolver.resolve().await } diff --git a/crates/wkg/src/main.rs b/crates/wkg/src/main.rs index b7347a4..810f2ea 100644 --- a/crates/wkg/src/main.rs +++ b/crates/wkg/src/main.rs @@ -227,8 +227,8 @@ struct GetArgs { #[derive(Args, Debug)] struct PublishArgs { - /// The file to publish - files: Vec, + /// The directories and files to publish + paths: Vec, #[command(flatten)] registry_args: RegistryArgs, @@ -250,22 +250,22 @@ impl PublishArgs { pub async fn run(self) -> anyhow::Result<()> { let client = self.common.get_client().await?; - let package = if let Some(package) = self.package { - Some(( - package.package, - package.version.ok_or_else(|| { - anyhow::anyhow!("version is required when manually overriding the package ID") - })?, - )) - } else { - None + let spec = match self.package { + Some(PackageSpec { + package, + version: Some(v), + }) => Some((package, v)), + Some(PackageSpec { version: None, .. }) => { + anyhow::bail!("version is required when manually overriding the package ID"); + } + None => None, }; let (package, version) = client .client()? .publish_release_files( - &self.files, + &self.paths, PublishOpts { - package, + package: spec, registry: self.registry_args.registry, dry_run: self.dry_run, }, From 6b156ad41736021116ad2a27bccc384a05e1f0c4 Mon Sep 17 00:00:00 2001 From: Mikhail Katychev Date: Fri, 19 Jun 2026 09:51:11 -0500 Subject: [PATCH 19/61] moved PublishPlan to core --- crates/wasm-pkg-client/src/lib.rs | 2 +- crates/wasm-pkg-client/src/publisher.rs | 91 ----------------------- crates/wasm-pkg-core/src/resolver.rs | 98 ++++++++++++++++++++++++- crates/wasm-pkg-core/src/wit.rs | 2 +- 4 files changed, 96 insertions(+), 97 deletions(-) diff --git a/crates/wasm-pkg-client/src/lib.rs b/crates/wasm-pkg-client/src/lib.rs index 0d31113..31d26b1 100644 --- a/crates/wasm-pkg-client/src/lib.rs +++ b/crates/wasm-pkg-client/src/lib.rs @@ -41,7 +41,7 @@ use std::{collections::HashMap, pin::Pin}; use anyhow::anyhow; use bytes::Bytes; use futures_util::Stream; -use publisher::PackagePublisher; +pub use publisher::PackagePublisher; use tokio::io::AsyncSeekExt; use tokio::sync::RwLock; use tokio_util::io::SyncIoBridge; diff --git a/crates/wasm-pkg-client/src/publisher.rs b/crates/wasm-pkg-client/src/publisher.rs index 52e30aa..ad26b3c 100644 --- a/crates/wasm-pkg-client/src/publisher.rs +++ b/crates/wasm-pkg-client/src/publisher.rs @@ -21,94 +21,3 @@ pub trait PackagePublisher: Send + Sync { dry_run: bool, ) -> Result<(), crate::Error>; } - -/// State for tracking dependencies during upload. -struct PublishPlan { - /// Graph of publishable packages where the edges are `(dependency -DependencyOf->) dependent)` - dependents: DependencyGraph, - /// Mapping [`PackageRef`]s to the respective index inside the dependency graph. - // TODO look at using cargo's `InternedString` type for `PackageRef`: - // https://docs.rs/cargo/latest/cargo/util/interning/struct.InternedString.html - indices: HashMap, -} - -impl PublishPlan { - /// Given a package dependency graph, creates a `PublishPlan` for tracking state. - fn new(graph: &DependencyGraph) -> Self { - let mut dependents = graph.clone().into_inner(); - dependents.reverse(); - // graph was already found to be acyclic - let dependents = DependencyGraph::try_from(dependents).unwrap(); - - let indices: HashMap<_, _> = dependents - .nodes_iter() - .map(|id| (dependents[id].clone(), id)) - .collect(); - - Self { - dependents, - indices, - } - } - - fn iter<'a>(&'a self) -> impl Iterator + 'a { - self.indices.iter().map(|(pkg, _)| pkg) - } - - fn is_empty(&self) -> bool { - self.indices.is_empty() - } - - fn len(&self) -> usize { - self.indices.len() - } - - /// Returns the set of packages that are ready for publishing (i.e. have no outstanding dependencies). - /// - /// These will not be returned in future calls. - fn take_ready(&mut self) -> BTreeSet { - self.dependents - .nodes_iter() - // there are no dependents on `self.dendents[id]` - .filter(|id| self.dependents.neighbors(*id).count() == 0) - .map(|id| { - let pkg = &self.dependents[id]; - self.indices.remove(&pkg); - pkg.clone() - }) - .collect() - } - - /// Packages confirmed to be available in the registry, potentially allowing additional - /// packages to be "ready". - fn mark_confirmed(&mut self, published: impl IntoIterator) { - for pkg in published { - let id = self - .indices - .remove(&pkg) - .expect("PackageRef has no associated index"); - self.dependents - .remove_node(id) - .expect("index has no associated PackageRef"); - } - } -} - -/// Format a collection of packages as a list -/// -/// e.g. "foo:a@0.1.0, bar:b@0.2.0, and baz:c@0.3.0". -/// -/// Note: the final separator (e.g. "and" in the previous example) can be chosen. -fn package_list(pkgs: impl IntoIterator, final_sep: &str) -> String { - let mut names: Vec<_> = pkgs.into_iter().map(|pkg| pkg.to_string()).collect(); - names.sort(); - - match &names[..] { - [] => String::new(), - [a] => a.clone(), - [a, b] => format!("{a} {final_sep} {b}"), - [names @ .., last] => { - format!("{}, {final_sep} {last}", names.join(", ")) - } - } -} diff --git a/crates/wasm-pkg-core/src/resolver.rs b/crates/wasm-pkg-core/src/resolver.rs index 87e0b79..36ab548 100644 --- a/crates/wasm-pkg-core/src/resolver.rs +++ b/crates/wasm-pkg-core/src/resolver.rs @@ -2,7 +2,7 @@ // NOTE(thomastaylor312): This is copied and adapted from the `cargo-component` crate: https://github.com/bytecodealliance/cargo-component/blob/f0be1c7d9917aa97e9102e69e3b838dae38d624b/crates/core/src/registry.rs use std::{ - collections::{hash_map, HashMap, HashSet}, + collections::{hash_map, BTreeSet, HashMap, HashSet}, fmt::Debug, ops::{Deref, DerefMut}, path::{Path, PathBuf}, @@ -14,7 +14,7 @@ use futures_util::TryStreamExt; use indexmap::{IndexMap, IndexSet}; use petgraph::{ acyclic::{Acyclic, AcyclicEdgeError}, - graph::DiGraph, + graph::{DiGraph, NodeIndex}, Graph, }; use semver::{Comparator, Op, Version, VersionReq}; @@ -23,11 +23,14 @@ use wasm_pkg_client::{ caching::{CachingClient, FileCache}, Client, Config, ContentDigest, Error as WasmPkgError, PackageRef, Release, VersionInfo, }; -use wasm_pkg_common::registry::DependencyOf; +use wasm_pkg_common::registry::{DependencyGraph, DependencyOf}; use wit_component::DecodedWasm; use wit_parser::{PackageId, PackageName, Resolve, UnresolvedPackageGroup, WorldId}; -use crate::{lock::LockFile, wit::get_packages}; +use crate::{ + lock::LockFile, + wit::{get_local_dependencies, get_packages}, +}; /// The name of the default registry. pub const DEFAULT_REGISTRY_NAME: &str = "default"; @@ -858,3 +861,90 @@ fn visit<'a>( Ok(()) } + +/// State for tracking dependencies during upload. +pub struct PublishPlan { + /// Graph of publishable packages where the edges are `(dependency -DependencyOf->) dependent)` + dependents: DependencyGraph, + /// Mapping [`PackageRef`]s to the respective index inside the dependency graph. + // TODO look at using cargo's `InternedString` type for `PackageRef`: + // https://docs.rs/cargo/latest/cargo/util/interning/struct.InternedString.html + indices: HashMap, +} + +impl PublishPlan { + /// Generate [`Self`] from a list of WIT package paths (files or directories). + fn from_paths(paths: &[impl AsRef]) -> Result { + let (graph, indices) = get_local_dependencies(paths)?; + let mut dependents = graph.into_inner(); + dependents.reverse(); + // graph was already found to be acyclic + let dependents = DependencyGraph::try_from(dependents).unwrap(); + + Ok(Self { + dependents, + indices, + }) + } + + fn iter<'a>(&'a self) -> impl Iterator + 'a { + self.indices.iter().map(|(pkg, _)| pkg) + } + + fn is_empty(&self) -> bool { + self.indices.is_empty() + } + + fn len(&self) -> usize { + self.indices.len() + } + + /// Returns the set of packages that are ready for publishing (i.e. have no outstanding dependencies). + /// + /// These will not be returned in future calls. + fn take_ready(&mut self) -> BTreeSet { + self.dependents + .nodes_iter() + // there are no dependents on `self.dendents[id]` + .filter(|id| self.dependents.neighbors(*id).count() == 0) + .map(|id| { + let pkg = &self.dependents[id]; + self.indices.remove(&pkg); + pkg.clone() + }) + .collect() + } + + /// Packages confirmed to be available in the registry, potentially allowing additional + /// packages to be "ready". + fn mark_confirmed(&mut self, published: impl IntoIterator) { + for pkg in published { + let id = self + .indices + .remove(&pkg) + .expect("PackageRef has no associated index"); + self.dependents + .remove_node(id) + .expect("index has no associated PackageRef"); + } + } +} + +/// Format a collection of packages as a list +/// +/// e.g. "foo:a@0.1.0, bar:b@0.2.0, and baz:c@0.3.0". +/// +/// Note: the final separator (e.g. "and" in the previous example) can be chosen. +fn package_list(pkgs: impl IntoIterator, final_sep: &str) -> String { + let mut names: Vec<_> = pkgs.into_iter().map(|pkg| pkg.to_string()).collect(); + names.sort(); + + match &names[..] { + [] => String::new(), + [a] => a.clone(), + [a, b] => format!("{a} {final_sep} {b}"), + [names @ .., last] => { + format!("{}, {final_sep} {last}", names.join(", ")) + } + } +} diff --git a/crates/wasm-pkg-core/src/wit.rs b/crates/wasm-pkg-core/src/wit.rs index 57dd6c8..21f7669 100644 --- a/crates/wasm-pkg-core/src/wit.rs +++ b/crates/wasm-pkg-core/src/wit.rs @@ -181,7 +181,7 @@ pub fn get_packages( } pub fn get_local_dependencies( - paths: Vec>, + paths: &[impl AsRef], ) -> Result<(DependencyGraph, HashMap)> { let pkg_trees = paths .into_iter() From 6ef21e2047a1270701e3519481258ff0f018aede Mon Sep 17 00:00:00 2001 From: Mikhail Katychev Date: Fri, 19 Jun 2026 11:16:00 -0500 Subject: [PATCH 20/61] moved tempfile method into separate function --- crates/wkg/src/main.rs | 39 ++++++++++++++------------------------- crates/wkg/src/wit.rs | 21 +++++++++++++++++++++ 2 files changed, 35 insertions(+), 25 deletions(-) diff --git a/crates/wkg/src/main.rs b/crates/wkg/src/main.rs index 08b687b..0475884 100644 --- a/crates/wkg/src/main.rs +++ b/crates/wkg/src/main.rs @@ -24,6 +24,8 @@ mod wit; use oci::OciCommands; use wit::WitCommands; +use crate::wit::temp_wit_file; + #[derive(Parser, Debug)] #[command(version)] struct Cli { @@ -252,15 +254,15 @@ impl PublishArgs { pub async fn run(self) -> anyhow::Result<()> { let client = self.common.get_client().await?; - let package = if let Some(package) = self.package { - Some(( - package.package, - package.version.ok_or_else(|| { - anyhow::anyhow!("version is required when manually overriding the package ID") - })?, - )) - } else { - None + let package = match self.package { + Some(PackageSpec { + package, + version: Some(v), + }) => Some((package, v)), + Some(PackageSpec { version: None, .. }) => { + anyhow::bail!("version is required when manually overriding the package ID"); + } + None => None, }; // If the input is a directory, build a WIT package from it into a temp @@ -269,7 +271,7 @@ impl PublishArgs { let (publish_path, _tmp) = if self.path.is_dir() { let mut lock_file = LockFile::load(true).await?; let prev_lock_ref = (lock_file.version, lock_file.packages.clone()); - let (build_ref, _, bytes) = + let (pkg_ref, _, bytes) = wit::build_wit_dir(&self.path, client.clone(), &mut lock_file).await?; // There is no way to check if we are in a git repository unlike `cargo publish --allow-dirty` so // check against previous values. @@ -285,22 +287,9 @@ impl PublishArgs { }); } - // Sanitize the package ref for use as a filename prefix: `namespace:name` - // contains characters (`:`, `/`) that are invalid in filenames on some - // platforms (notably Windows). - let prefix: String = build_ref.to_string().replace([':', '/'], "_"); - let tmp = tempfile::Builder::new() - .prefix(&prefix) - .suffix(".wasm") - .tempfile() - .context("Failed to create temporary file for built WIT package")?; - tokio::fs::write(tmp.path(), &bytes) - .await - .context("Failed to write built WIT package to temp file")?; - let tmp_pkg_path = tmp.path().to_path_buf(); - tracing::debug!(tmp_pkg_path = %tmp_pkg_path.display(), "Wrote temporary WIT package file"); + let tmp = temp_wit_file(&pkg_ref, &bytes).await?; - (tmp_pkg_path, Some(tmp)) + (tmp.path().to_path_buf(), Some(tmp)) } else { (self.path.clone(), None) }; diff --git a/crates/wkg/src/wit.rs b/crates/wkg/src/wit.rs index f6d891d..9d9e4ce 100644 --- a/crates/wkg/src/wit.rs +++ b/crates/wkg/src/wit.rs @@ -3,6 +3,7 @@ use std::path::{Path, PathBuf}; use anyhow::Context; use clap::{Args, Subcommand}; +use tempfile::NamedTempFile; use wasm_pkg_client::caching::{CachingClient, FileCache}; use wasm_pkg_common::package::{PackageRef, Version}; use wasm_pkg_core::{ @@ -123,6 +124,26 @@ pub async fn build_wit_dir( Ok(result) } +pub async fn temp_wit_file(package: &PackageRef, bytes: &[u8]) -> anyhow::Result { + // Sanitize the package ref for use as a filename prefix: `namespace:name` + // contains characters (`:`, `/`) that are invalid in filenames on some + // platforms (notably Windows). + let prefix: String = package.to_string().replace([':', '/'], "_"); + let tmp_handle = tempfile::Builder::new() + .prefix(&prefix) + .suffix(".wasm") + .tempfile() + .context("Failed to create temporary file for built WIT package") + .with_context(|| format!("package: {package}"))?; + tokio::fs::write(tmp_handle.path(), &bytes) + .await + .context("Failed to write built WIT package to temp file") + .with_context(|| format!("package: {package}"))?; + + tracing::debug!(tmp_pkg_path = %tmp_handle.path().display(), "Wrote temporary WIT package file"); + Ok(tmp_handle) +} + impl FetchArgs { pub async fn run(self) -> anyhow::Result<()> { check_dir(&self.dir).await?; From dca7d50177407eae763ecd3d80826583ad948911 Mon Sep 17 00:00:00 2001 From: Mikhail Katychev Date: Fri, 19 Jun 2026 11:44:22 -0500 Subject: [PATCH 21/61] added multi pkg match arm --- Cargo.toml | 13 ++++++++++++ crates/wasm-pkg-client/src/lib.rs | 1 - crates/wasm-pkg-client/src/overlay/mod.rs | 2 +- crates/wasm-pkg-client/src/publisher.rs | 6 ------ crates/wasm-pkg-common/src/registry.rs | 2 +- crates/wasm-pkg-core/src/resolver.rs | 17 ++++++++------- crates/wkg/src/main.rs | 25 ++++++++++++++++++----- 7 files changed, 43 insertions(+), 23 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 18134e1..8354838 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -55,3 +55,16 @@ url = "2.5.0" warg-client = "0.9.2" warg-crypto = "0.9.2" warg-protocol = "0.9.2" + +# https://github.com/crate-ci/typos/blob/master/docs/reference.md +[workspace.metadata.typos.default] +extend-ignore-re = [ + "\\d\\w{4,}\\d", +] +extend-ignore-words-re = [ + '^[a-zA-Z]{1,3}$' # ignore words up to length 3 +] + +[workspace.metadata.typos.default.extend-words] +# strategy shorthand +strat = "strat" diff --git a/crates/wasm-pkg-client/src/lib.rs b/crates/wasm-pkg-client/src/lib.rs index 31d26b1..b409e9f 100644 --- a/crates/wasm-pkg-client/src/lib.rs +++ b/crates/wasm-pkg-client/src/lib.rs @@ -45,7 +45,6 @@ pub use publisher::PackagePublisher; use tokio::io::AsyncSeekExt; use tokio::sync::RwLock; use tokio_util::io::SyncIoBridge; -use wasm_pkg_common::config::RegistryConfig; use wasm_pkg_common::metadata::{LOCAL_PROTOCOL, OCI_PROTOCOL, WARG_PROTOCOL}; pub use wasm_pkg_common::{ config::{Config, CustomConfig, RegistryMapping}, diff --git a/crates/wasm-pkg-client/src/overlay/mod.rs b/crates/wasm-pkg-client/src/overlay/mod.rs index b6067ae..86e0f6a 100644 --- a/crates/wasm-pkg-client/src/overlay/mod.rs +++ b/crates/wasm-pkg-client/src/overlay/mod.rs @@ -1,4 +1,4 @@ -use std::{collections::HashMap, io::Cursor}; +use std::collections::HashMap; use async_trait::async_trait; use tempfile::TempDir; diff --git a/crates/wasm-pkg-client/src/publisher.rs b/crates/wasm-pkg-client/src/publisher.rs index ad26b3c..7bc395e 100644 --- a/crates/wasm-pkg-client/src/publisher.rs +++ b/crates/wasm-pkg-client/src/publisher.rs @@ -1,10 +1,4 @@ -use std::collections::{BTreeSet, HashMap}; -use petgraph::{ - acyclic::Acyclic, - graph::{DiGraph, NodeIndex}, -}; -use wasm_pkg_common::registry::{DependencyGraph, DependencyOf}; use crate::{PackageRef, PublishingSource, Version}; diff --git a/crates/wasm-pkg-common/src/registry.rs b/crates/wasm-pkg-common/src/registry.rs index 0b89bb9..6b65d33 100644 --- a/crates/wasm-pkg-common/src/registry.rs +++ b/crates/wasm-pkg-common/src/registry.rs @@ -57,7 +57,7 @@ impl TryFrom for Registry { } } -/// Represents a directed edge in a package dependnecy graph. +/// Represents a directed edge in a package dependency graph. #[derive(Clone, Debug)] pub struct DependencyOf; diff --git a/crates/wasm-pkg-core/src/resolver.rs b/crates/wasm-pkg-core/src/resolver.rs index 8bbb9ed..ea52efd 100644 --- a/crates/wasm-pkg-core/src/resolver.rs +++ b/crates/wasm-pkg-core/src/resolver.rs @@ -13,9 +13,8 @@ use anyhow::{bail, Context, Result}; use futures_util::TryStreamExt; use indexmap::{IndexMap, IndexSet}; use petgraph::{ - acyclic::{Acyclic, AcyclicEdgeError}, + acyclic::Acyclic, graph::{DiGraph, NodeIndex}, - Graph, }; use semver::{Comparator, Op, Version, VersionReq}; use tokio::io::{AsyncRead, AsyncReadExt}; @@ -882,7 +881,7 @@ pub struct PublishPlan { impl PublishPlan { /// Generate [`Self`] from a list of WIT package paths (files or directories). - fn from_paths(paths: &[impl AsRef]) -> Result { + pub fn from_paths(paths: &[impl AsRef]) -> Result { let (graph, indices) = get_local_dependencies(paths)?; let mut dependents = graph.into_inner(); dependents.reverse(); @@ -895,22 +894,22 @@ impl PublishPlan { }) } - fn iter<'a>(&'a self) -> impl Iterator + 'a { + pub fn iter<'a>(&'a self) -> impl Iterator + 'a { self.indices.iter().map(|(pkg, _)| pkg) } - fn is_empty(&self) -> bool { + pub fn is_empty(&self) -> bool { self.indices.is_empty() } - fn len(&self) -> usize { + pub fn len(&self) -> usize { self.indices.len() } /// Returns the set of packages that are ready for publishing (i.e. have no outstanding dependencies). /// /// These will not be returned in future calls. - fn take_ready(&mut self) -> BTreeSet { + pub fn take_ready(&mut self) -> BTreeSet { self.dependents .nodes_iter() // there are no dependents on `self.dendents[id]` @@ -925,7 +924,7 @@ impl PublishPlan { /// Packages confirmed to be available in the registry, potentially allowing additional /// packages to be "ready". - fn mark_confirmed(&mut self, published: impl IntoIterator) { + pub fn mark_confirmed(&mut self, published: impl IntoIterator) { for pkg in published { let id = self .indices @@ -943,7 +942,7 @@ impl PublishPlan { /// e.g. "foo:a@0.1.0, bar:b@0.2.0, and baz:c@0.3.0". /// /// Note: the final separator (e.g. "and" in the previous example) can be chosen. -fn package_list(pkgs: impl IntoIterator, final_sep: &str) -> String { +pub fn package_list(pkgs: impl IntoIterator, final_sep: &str) -> String { let mut names: Vec<_> = pkgs.into_iter().map(|pkg| pkg.to_string()).collect(); names.sort(); diff --git a/crates/wkg/src/main.rs b/crates/wkg/src/main.rs index 6252c33..e37eedf 100644 --- a/crates/wkg/src/main.rs +++ b/crates/wkg/src/main.rs @@ -15,7 +15,7 @@ use wasm_pkg_common::{ package::PackageSpec, registry::Registry, }; -use wasm_pkg_core::lock::LockFile; +use wasm_pkg_core::{lock::LockFile, resolver::PublishPlan}; use wit_component::DecodedWasm; mod oci; @@ -255,6 +255,9 @@ impl PublishArgs { let client = self.common.get_client().await?; let package = match self.package { + Some(_) if self.paths.len() > 2 => { + anyhow::bail!("`--package` is currently unsupported when providing more than one path argument"); + } Some(PackageSpec { package, version: Some(v), @@ -265,14 +268,26 @@ impl PublishArgs { None => None, }; + let path = match &self.paths[..] { + [path] => path, + paths => { + let plan = PublishPlan::from_paths(paths)?; + + for pkg in plan.iter() { + println!("{pkg}"); + } + todo!(); + } + }; + // If the input is a directory, build a WIT package from it into a temp // file first. _tmp is held until the publish completes so the file // isn't deleted out from under us. - let (publish_path, _tmp) = if self.path.is_dir() { + let (publish_path, _tmp) = if path.is_dir() { let mut lock_file = LockFile::load(true).await?; let prev_lock_ref = (lock_file.version, lock_file.packages.clone()); let (pkg_ref, _, bytes) = - wit::build_wit_dir(&self.path, client.clone(), &mut lock_file).await?; + wit::build_wit_dir(&path.clone(), client.clone(), &mut lock_file).await?; // There is no way to check if we are in a git repository unlike `cargo publish --allow-dirty` so // check against previous values. if lock_file != prev_lock_ref && !self.dry_run { @@ -282,7 +297,7 @@ impl PublishArgs { .with_context(|| { format!( "Run `wkg wit build {}` before attempting to publish", - self.path.to_string_lossy() + path.to_string_lossy() ) }); } @@ -291,7 +306,7 @@ impl PublishArgs { (tmp.path().to_path_buf(), Some(tmp)) } else { - (self.path.clone(), None) + (path.clone(), None) }; let (package, version) = client From 3e84ee95aba77ddfc6350f3da0f82c0e6599b13d Mon Sep 17 00:00:00 2001 From: Mikhail Katychev Date: Fri, 19 Jun 2026 15:32:17 -0500 Subject: [PATCH 22/61] interim commit --- crates/wasm-pkg-client/src/caching/mod.rs | 2 +- crates/wasm-pkg-client/src/local.rs | 15 ++++- crates/wasm-pkg-client/src/overlay/mod.rs | 16 +++-- crates/wasm-pkg-common/src/package.rs | 2 +- crates/wasm-pkg-common/src/registry.rs | 6 +- crates/wasm-pkg-core/src/resolver.rs | 79 ++++++++++++----------- crates/wasm-pkg-core/src/wit.rs | 45 +++++++++---- crates/wkg/src/main.rs | 52 ++++++++++++--- 8 files changed, 141 insertions(+), 76 deletions(-) diff --git a/crates/wasm-pkg-client/src/caching/mod.rs b/crates/wasm-pkg-client/src/caching/mod.rs index 59c0d93..51406c6 100644 --- a/crates/wasm-pkg-client/src/caching/mod.rs +++ b/crates/wasm-pkg-client/src/caching/mod.rs @@ -46,7 +46,7 @@ pub trait Cache { /// A client that caches response data using the given cache implementation. Can be used without an /// underlying client to be used as a read-only cache. pub struct CachingClient { - client: Option, + pub client: Option, cache: Arc, } diff --git a/crates/wasm-pkg-client/src/local.rs b/crates/wasm-pkg-client/src/local.rs index d733b8f..faa9379 100644 --- a/crates/wasm-pkg-client/src/local.rs +++ b/crates/wasm-pkg-client/src/local.rs @@ -10,8 +10,9 @@ use std::{ use anyhow::anyhow; use async_trait::async_trait; use futures_util::{StreamExt, TryStreamExt}; -use serde::Deserialize; +use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256}; +use tempfile::TempDir; use tokio_util::io::ReaderStream; use wasm_pkg_common::{ config::RegistryConfig, @@ -27,12 +28,13 @@ use crate::{ ContentStream, PublishingSource, }; -#[derive(Clone, Debug, Deserialize)] +#[derive(Clone, Debug, Deserialize, Serialize)] pub struct LocalConfig { pub root: PathBuf, } -pub(crate) struct LocalBackend { +#[derive(Clone)] +pub struct LocalBackend { pub(crate) root: PathBuf, } @@ -60,6 +62,13 @@ impl LocalBackend { fn version_path(&self, package: &PackageRef, version: &Version) -> PathBuf { self.package_dir(package).join(format!("{version}.wasm")) } + + pub fn temp_dir() -> Result<(Self, TempDir), Error> { + let handle = TempDir::new()?; + let root = handle.path().to_owned(); + tracing::debug!(registry_dir=%root.display(), "created temporary directory"); + Ok((Self { root }, handle)) + } } #[async_trait] diff --git a/crates/wasm-pkg-client/src/overlay/mod.rs b/crates/wasm-pkg-client/src/overlay/mod.rs index 86e0f6a..8f594cc 100644 --- a/crates/wasm-pkg-client/src/overlay/mod.rs +++ b/crates/wasm-pkg-client/src/overlay/mod.rs @@ -20,9 +20,7 @@ pub(crate) struct OverlayBackend { impl OverlayBackend { fn new(remotes: HashMap) -> Result { - let handle = TempDir::new()?; - let root = handle.path().to_owned(); - let local = LocalBackend { root }; + let (local, handle) = LocalBackend::temp_dir()?; Ok(Self { local, remotes, @@ -41,10 +39,14 @@ impl OverlayBackend { impl PackageLoader for OverlayBackend { async fn list_all_versions(&self, package: &PackageRef) -> Result, Error> { let mut versions = self.local.list_all_versions(package).await?; - let mut remote_versions = self.remote(package)?.list_all_versions(package).await?; - versions.append(&mut remote_versions); - versions.sort(); - versions.dedup(); + + if let Some(remote) = self.remotes.get(package) { + let mut remote_versions = remote.list_all_versions(package).await?; + versions.append(&mut remote_versions); + versions.sort(); + versions.dedup(); + } + Ok(versions) } diff --git a/crates/wasm-pkg-common/src/package.rs b/crates/wasm-pkg-common/src/package.rs index f4c921b..26c7e99 100644 --- a/crates/wasm-pkg-common/src/package.rs +++ b/crates/wasm-pkg-common/src/package.rs @@ -79,7 +79,7 @@ impl std::fmt::Display for PackageSpec { } /// A package spec combines a [`PackageRef`] with an optional version. -#[derive(Clone, Debug)] +#[derive(Clone, Debug, Eq, PartialEq, PartialOrd, Ord)] pub struct PackageSpec { pub package: PackageRef, pub version: Option, diff --git a/crates/wasm-pkg-common/src/registry.rs b/crates/wasm-pkg-common/src/registry.rs index 6b65d33..d059f6d 100644 --- a/crates/wasm-pkg-common/src/registry.rs +++ b/crates/wasm-pkg-common/src/registry.rs @@ -57,8 +57,4 @@ impl TryFrom for Registry { } } -/// Represents a directed edge in a package dependency graph. -#[derive(Clone, Debug)] -pub struct DependencyOf; - -pub type DependencyGraph = Acyclic>; +pub type DependencyGraph = Acyclic>; diff --git a/crates/wasm-pkg-core/src/resolver.rs b/crates/wasm-pkg-core/src/resolver.rs index ea52efd..63fa8f3 100644 --- a/crates/wasm-pkg-core/src/resolver.rs +++ b/crates/wasm-pkg-core/src/resolver.rs @@ -15,14 +15,16 @@ use indexmap::{IndexMap, IndexSet}; use petgraph::{ acyclic::Acyclic, graph::{DiGraph, NodeIndex}, + Direction, }; use semver::{Comparator, Op, Version, VersionReq}; use tokio::io::{AsyncRead, AsyncReadExt}; +use tracing::Level; use wasm_pkg_client::{ caching::{CachingClient, FileCache}, Client, Config, ContentDigest, Error as WasmPkgError, PackageRef, Release, VersionInfo, }; -use wasm_pkg_common::registry::{DependencyGraph, DependencyOf}; +use wasm_pkg_common::{package::PackageSpec, registry::DependencyGraph}; use wit_component::DecodedWasm; use wit_parser::{PackageId, PackageName, Resolve, UnresolvedPackageGroup, WorldId}; @@ -171,32 +173,6 @@ pub struct LocalResolution { pub path: PathBuf, } -pub struct LocalDependencies { - pub packages: HashMap, - pub graph: DiGraph, -} - -impl LocalDependencies { - pub fn sort(&self) -> Result> { - // sort our packages topologically - let acyclic_graph = Acyclic::try_from(self.graph.clone()).map_err(|e| { - anyhow::anyhow!( - "detected cyclical dependencies with package: {}", - self.graph[e.node_id()] - ) - })?; - - Ok(acyclic_graph - .nodes_iter() - .map(|id| self.graph[id].clone()) - .collect()) - } - - pub fn has_no_dependencies(&self) -> bool { - self.graph.raw_edges().is_empty() - } -} - /// Represents a resolution of a dependency. #[derive(Debug, Clone)] #[allow(clippy::large_enum_variant)] @@ -872,11 +848,11 @@ fn visit<'a>( /// State for tracking dependencies during upload. pub struct PublishPlan { /// Graph of publishable packages where the edges are `(dependency -DependencyOf->) dependent)` - dependents: DependencyGraph, + dependents: DependencyGraph, /// Mapping [`PackageRef`]s to the respective index inside the dependency graph. // TODO look at using cargo's `InternedString` type for `PackageRef`: // https://docs.rs/cargo/latest/cargo/util/interning/struct.InternedString.html - indices: HashMap, + indices: HashMap, } impl PublishPlan { @@ -894,8 +870,32 @@ impl PublishPlan { }) } - pub fn iter<'a>(&'a self) -> impl Iterator + 'a { - self.indices.iter().map(|(pkg, _)| pkg) + pub fn iter<'a>(&'a self) -> impl Iterator + 'a { + self.dependents.nodes_iter().map(|id| &self.dependents[id]) + } + + pub fn iter_edges<'a>(&'a self) -> impl Iterator + 'a { + use petgraph::visit::IntoNeighborsDirected; + self.dependents.nodes_iter().map(|id| { + let dep = &self.dependents[id]; + tracing::warn!("ITER {dep}"); + let mut neighbors = self + .dependents + .neighbors_directed(id, Direction::Outgoing) + .peekable(); + if neighbors.peek().is_none() { + tracing::debug!("{dep} has no dependents"); + } + + if tracing::enabled!(Level::DEBUG) { + while let Some(id) = neighbors.next() { + let pkg = &self.dependents[id]; + tracing::debug!("{dep} -(DependencyOF)-> {pkg}"); + } + } + + dep + }) } pub fn is_empty(&self) -> bool { @@ -909,30 +909,35 @@ impl PublishPlan { /// Returns the set of packages that are ready for publishing (i.e. have no outstanding dependencies). /// /// These will not be returned in future calls. - pub fn take_ready(&mut self) -> BTreeSet { + pub fn take_ready(&mut self) -> BTreeSet { self.dependents .nodes_iter() // there are no dependents on `self.dendents[id]` .filter(|id| self.dependents.neighbors(*id).count() == 0) .map(|id| { let pkg = &self.dependents[id]; - self.indices.remove(&pkg); + self.indices.remove(&pkg.package); pkg.clone() }) .collect() } + /// + pub fn get_path(&self, pkg: &PackageRef) -> Option<&Path> { + self.indices.get(pkg).map(|(_, p)| p.as_ref()) + } + /// Packages confirmed to be available in the registry, potentially allowing additional /// packages to be "ready". pub fn mark_confirmed(&mut self, published: impl IntoIterator) { for pkg in published { - let id = self + let (id, _) = self .indices .remove(&pkg) - .expect("PackageRef has no associated index"); + .expect("PackageSpec has no associated index"); self.dependents .remove_node(id) - .expect("index has no associated PackageRef"); + .expect("index has no associated PackageSpec"); } } } @@ -942,7 +947,7 @@ impl PublishPlan { /// e.g. "foo:a@0.1.0, bar:b@0.2.0, and baz:c@0.3.0". /// /// Note: the final separator (e.g. "and" in the previous example) can be chosen. -pub fn package_list(pkgs: impl IntoIterator, final_sep: &str) -> String { +pub fn package_list(pkgs: impl IntoIterator, final_sep: &str) -> String { let mut names: Vec<_> = pkgs.into_iter().map(|pkg| pkg.to_string()).collect(); names.sort(); diff --git a/crates/wasm-pkg-core/src/wit.rs b/crates/wasm-pkg-core/src/wit.rs index 21f7669..d60fc02 100644 --- a/crates/wasm-pkg-core/src/wit.rs +++ b/crates/wasm-pkg-core/src/wit.rs @@ -2,22 +2,19 @@ use std::{ collections::{HashMap, HashSet}, - path::Path, + path::{Path, PathBuf}, str::FromStr, }; use anyhow::{Context, Result}; -use petgraph::{data::Build, graph::NodeIndex}; +use petgraph::{data::Build, graph::NodeIndex, Direction}; use semver::{Version, VersionReq}; use wasm_metadata::{AddMetadata, AddMetadataField}; use wasm_pkg_client::{ caching::{CachingClient, FileCache}, PackageRef, }; -use wasm_pkg_common::{ - package::PackageSpec, - registry::{DependencyGraph, DependencyOf}, -}; +use wasm_pkg_common::{package::PackageSpec, registry::DependencyGraph}; use wit_component::WitPrinter; use wit_parser::{PackageId, PackageName, Resolve}; @@ -182,25 +179,45 @@ pub fn get_packages( pub fn get_local_dependencies( paths: &[impl AsRef], -) -> Result<(DependencyGraph, HashMap)> { +) -> Result<( + DependencyGraph, + HashMap, +)> { let pkg_trees = paths .into_iter() - .map(|p| get_packages(p)) + .map(|path| get_packages(path).map(|(pkg, deps)| ((pkg, path), deps))) .collect::, _>>()?; let mut graph = DependencyGraph::new(); let mut indices = HashMap::new(); // establish all nodes - for (spec, _) in &pkg_trees { - let id = graph.add_node(spec.package()); - if indices.insert(spec.package(), id).is_some() { + for ((spec, path), _) in &pkg_trees { + let id = graph.add_node(spec.clone()); + if indices + .insert(spec.package(), (id, path.as_ref().to_owned())) + .is_some() + { anyhow::bail!("duplicate references to package detected: {spec}"); } } - for (spec, deps) in pkg_trees { + for ((spec, _), deps) in pkg_trees { // TODO handle version matching for dependencies for (dep, _version) in deps { - if let Some(&dep_id) = indices.get(&dep) { - graph.add_edge(dep_id, indices[&spec.package], DependencyOf); + if let Some(&(dep, _)) = indices.get(&dep) { + let (pkg, _) = indices[&spec.package]; + // // dep -(DependencyOF)-> pkg + // graph.try_update_edge(dep, pkg, Direction::Outgoing); + // // pkg -(DependsOn)-> dep + graph + .try_update_edge(pkg, dep, Direction::Outgoing) + .map_err(|e| match e { + petgraph::acyclic::AcyclicEdgeError::Cycle(cycle) => { + anyhow::anyhow!("cyclical depndency detected") + .context(format!("package {}", graph[cycle.node_id()])) + .context(format!("package {}", graph[pkg])) + } + petgraph::acyclic::AcyclicEdgeError::SelfLoop => todo!(), + petgraph::acyclic::AcyclicEdgeError::InvalidEdge => todo!(), + })?; } } } diff --git a/crates/wkg/src/main.rs b/crates/wkg/src/main.rs index e37eedf..71a2935 100644 --- a/crates/wkg/src/main.rs +++ b/crates/wkg/src/main.rs @@ -7,11 +7,11 @@ use tokio::io::AsyncWriteExt; use tracing::level_filters::LevelFilter; use wasm_pkg_client::{ caching::{CachingClient, FileCache}, - Client, PublishOpts, + Client, PackagePublisher, PublishOpts, }; use wasm_pkg_common::{ self, - config::{Config, RegistryMapping}, + config::{Config, RegistryConfig, RegistryMapping}, package::PackageSpec, registry::Registry, }; @@ -40,7 +40,7 @@ struct RegistryArgs { registry: Option, } -#[derive(Args, Debug)] +#[derive(Args, Debug, Default)] struct Common { /// The path to the configuration file. #[arg(long = "config", value_name = "CONFIG", env = "WKG_CONFIG_FILE")] @@ -252,7 +252,7 @@ struct PublishArgs { impl PublishArgs { pub async fn run(self) -> anyhow::Result<()> { - let client = self.common.get_client().await?; + let mut client = self.common.get_client().await?; let package = match self.package { Some(_) if self.paths.len() > 2 => { @@ -271,12 +271,48 @@ impl PublishArgs { let path = match &self.paths[..] { [path] => path, paths => { + let (overlay, tmp_dir) = wasm_pkg_client::local::LocalBackend::temp_dir()?; let plan = PublishPlan::from_paths(paths)?; - - for pkg in plan.iter() { - println!("{pkg}"); + let mut lock_file = LockFile::load(true).await?; + for spec in plan.iter_edges() { + let mut reg_config = RegistryConfig::default(); + reg_config.set_default_backend(Some("local".to_owned())); + reg_config.set_backend_config( + "local", + wasm_pkg_client::local::LocalConfig { + root: tmp_dir.path().to_path_buf(), + }, + ); + let client = client.client.as_mut().unwrap(); + let path = plan.get_path(&spec.package).unwrap(); + let (publish_path, _tmp) = if path.is_dir() { + let _prev_lock_ref = (lock_file.version, lock_file.packages.clone()); + let (pkg_ref, _, bytes) = + wit::build_wit_dir(&path, client.clone(), &mut lock_file).await?; + let tmp = temp_wit_file(&pkg_ref, &bytes).await?; + + (tmp.path().to_path_buf(), Some(tmp)) + } else { + (path.to_owned(), None) + }; + + let data = tokio::fs::OpenOptions::new() + .read(true) + .open(publish_path) + .await?; + + overlay + .publish( + &spec.package, + spec.version.as_ref().unwrap(), + Box::pin(data), + false, + ) + .await + .unwrap(); } - todo!(); + return Ok(()); + // todo!(); } }; From bf240f73bacd429fe3d24ec7fa318bb886d6b5d1 Mon Sep 17 00:00:00 2001 From: Mikhail Katychev Date: Fri, 19 Jun 2026 15:44:19 -0500 Subject: [PATCH 23/61] successful package publishing to temp --- crates/wasm-pkg-core/src/resolver.rs | 8 ++++--- crates/wkg/src/main.rs | 35 +++++++++++++++++++++------- 2 files changed, 31 insertions(+), 12 deletions(-) diff --git a/crates/wasm-pkg-core/src/resolver.rs b/crates/wasm-pkg-core/src/resolver.rs index 63fa8f3..7bf12ec 100644 --- a/crates/wasm-pkg-core/src/resolver.rs +++ b/crates/wasm-pkg-core/src/resolver.rs @@ -884,13 +884,15 @@ impl PublishPlan { .neighbors_directed(id, Direction::Outgoing) .peekable(); if neighbors.peek().is_none() { - tracing::debug!("{dep} has no dependents"); + // tracing::debug!("{dep} has no dependents"); + println!("{dep} has no dependents"); } - if tracing::enabled!(Level::DEBUG) { + // if tracing::enabled!(Level::DEBUG) { + if true { while let Some(id) = neighbors.next() { let pkg = &self.dependents[id]; - tracing::debug!("{dep} -(DependencyOF)-> {pkg}"); + println!("{dep} -(DependencyOF)-> {pkg}"); } } diff --git a/crates/wkg/src/main.rs b/crates/wkg/src/main.rs index 71a2935..785e219 100644 --- a/crates/wkg/src/main.rs +++ b/crates/wkg/src/main.rs @@ -272,18 +272,35 @@ impl PublishArgs { [path] => path, paths => { let (overlay, tmp_dir) = wasm_pkg_client::local::LocalBackend::temp_dir()?; + let local_backend = wasm_pkg_client::local::LocalConfig { + root: tmp_dir.path().to_path_buf(), + }; + let mut reg_config = RegistryConfig::default(); + reg_config.set_default_backend(Some("local".to_owned())); + reg_config.set_backend_config("local", &local_backend)?; + let plan = PublishPlan::from_paths(paths)?; let mut lock_file = LockFile::load(true).await?; - for spec in plan.iter_edges() { - let mut reg_config = RegistryConfig::default(); - reg_config.set_default_backend(Some("local".to_owned())); - reg_config.set_backend_config( - "local", - wasm_pkg_client::local::LocalConfig { - root: tmp_dir.path().to_path_buf(), - }, + + // Route every package in the plan to the local overlay registry + // backed by `reg_config`, so the client used in `build_wit_dir` + // resolves these packages against the local overlay instead of + // an upstream remote. + let mut config = self.common.load_config().await?; + let local_registry: Registry = "local".parse()?; + config + .get_or_insert_registry_config_mut(&local_registry) + .merge(reg_config); + for spec in plan.iter() { + config.set_package_registry_override( + spec.package.clone(), + RegistryMapping::Registry(local_registry.clone()), ); - let client = client.client.as_mut().unwrap(); + } + let cache = self.common.load_cache().await?; + let client = CachingClient::new(Some(Client::new(config)), cache); + + for spec in plan.iter_edges() { let path = plan.get_path(&spec.package).unwrap(); let (publish_path, _tmp) = if path.is_dir() { let _prev_lock_ref = (lock_file.version, lock_file.packages.clone()); From 1968b61d8d8137b068d338e269876a9a2abde05d Mon Sep 17 00:00:00 2001 From: Mikhail Katychev Date: Fri, 19 Jun 2026 15:52:15 -0500 Subject: [PATCH 24/61] added expect unused --- crates/wasm-pkg-client/src/overlay/mod.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/wasm-pkg-client/src/overlay/mod.rs b/crates/wasm-pkg-client/src/overlay/mod.rs index 8f594cc..1bcc6d1 100644 --- a/crates/wasm-pkg-client/src/overlay/mod.rs +++ b/crates/wasm-pkg-client/src/overlay/mod.rs @@ -18,6 +18,7 @@ pub(crate) struct OverlayBackend { _handle: TempDir, } +#[expect(unused)] impl OverlayBackend { fn new(remotes: HashMap) -> Result { let (local, handle) = LocalBackend::temp_dir()?; From b1db797d9df0323fc7f289bbcb14f20f5f91db10 Mon Sep 17 00:00:00 2001 From: Mikhail Katychev Date: Mon, 22 Jun 2026 09:38:39 -0500 Subject: [PATCH 25/61] added overlay setyp --- crates/wasm-pkg-client/src/caching/file.rs | 1 + crates/wasm-pkg-client/src/caching/mod.rs | 23 ++++++---- crates/wasm-pkg-client/src/lib.rs | 16 +++++++ crates/wasm-pkg-common/src/config.rs | 10 ++++ crates/wasm-pkg-core/src/resolver.rs | 53 ++++++++++++++++++++++ crates/wkg/src/main.rs | 37 +++++++++------ 6 files changed, 115 insertions(+), 25 deletions(-) diff --git a/crates/wasm-pkg-client/src/caching/file.rs b/crates/wasm-pkg-client/src/caching/file.rs index 7c65332..e54a1e8 100644 --- a/crates/wasm-pkg-client/src/caching/file.rs +++ b/crates/wasm-pkg-client/src/caching/file.rs @@ -16,6 +16,7 @@ use crate::{ContentStream, Release}; use super::Cache; +#[derive(Clone)] pub struct FileCache { root: PathBuf, } diff --git a/crates/wasm-pkg-client/src/caching/mod.rs b/crates/wasm-pkg-client/src/caching/mod.rs index 51406c6..c7df52d 100644 --- a/crates/wasm-pkg-client/src/caching/mod.rs +++ b/crates/wasm-pkg-client/src/caching/mod.rs @@ -4,10 +4,11 @@ use std::sync::Arc; use wasm_pkg_common::{ digest::ContentDigest, package::{PackageRef, Version}, + registry::Registry, Error, }; -use crate::{Client, ContentStream, Release, VersionInfo}; +use crate::{Client, ContentStream, InnerClient, Release, VersionInfo}; mod file; @@ -45,20 +46,12 @@ pub trait Cache { /// A client that caches response data using the given cache implementation. Can be used without an /// underlying client to be used as a read-only cache. +#[derive(Clone)] pub struct CachingClient { pub client: Option, cache: Arc, } -impl Clone for CachingClient { - fn clone(&self) -> Self { - Self { - client: self.client.clone(), - cache: self.cache.clone(), - } - } -} - impl CachingClient { /// Creates a new caching client from the given client and cache implementation. If no client is /// given, the client will be in offline or read-only mode, meaning it will only be able to return @@ -70,6 +63,16 @@ impl CachingClient { } } + async fn with_overlays( + mut self, + overlays: impl IntoIterator)>, + ) -> Self { + if let Some(client) = &mut self.client { + client.overlays = overlays.into_iter().collect(); + } + self + } + /// Returns whether or not the client is in read-only mode. pub fn is_readonly(&self) -> bool { self.client.is_none() diff --git a/crates/wasm-pkg-client/src/lib.rs b/crates/wasm-pkg-client/src/lib.rs index b409e9f..1267c94 100644 --- a/crates/wasm-pkg-client/src/lib.rs +++ b/crates/wasm-pkg-client/src/lib.rs @@ -95,6 +95,8 @@ pub struct PublishOpts { pub struct Client { config: Arc, sources: Arc>, + /// Mapping of sources to local registries that will be overlaid on them. + overlays: RegistrySources, } impl Client { @@ -103,6 +105,20 @@ impl Client { Self { config: Arc::new(config), sources: Default::default(), + overlays: Default::default(), + } + } + + /// Like [`Self::new`] but includes sources from source + /// replacement configurations. + pub fn new_with_overlays( + config: Config, + overlays: impl IntoIterator)>, + ) -> Self { + Self { + config: Arc::new(config), + sources: Default::default(), + overlays: overlays.into_iter().collect(), } } diff --git a/crates/wasm-pkg-common/src/config.rs b/crates/wasm-pkg-common/src/config.rs index 7edca19..cd17e17 100644 --- a/crates/wasm-pkg-common/src/config.rs +++ b/crates/wasm-pkg-common/src/config.rs @@ -290,6 +290,16 @@ pub struct RegistryConfig { } impl RegistryConfig { + pub fn with_default_backend( + mut self, + default_backend: &str, + backend_config: T, + ) -> Result { + self.default_backend = Some(default_backend.to_string()); + self.set_backend_config("local", backend_config)?; + Ok(self) + } + /// Merges the given other config into this one. pub fn merge(&mut self, other: Self) { let Self { diff --git a/crates/wasm-pkg-core/src/resolver.rs b/crates/wasm-pkg-core/src/resolver.rs index 7bf12ec..cc02538 100644 --- a/crates/wasm-pkg-core/src/resolver.rs +++ b/crates/wasm-pkg-core/src/resolver.rs @@ -944,6 +944,59 @@ impl PublishPlan { } } +#[cfg(test)] +mod tests { + use std::collections::HashSet; + use std::path::PathBuf; + + use super::PublishPlan; + use wasm_pkg_client::PackageRef; + + fn transitive_local_paths() -> Vec { + let fixtures = + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/transitive-local"); + ["example-a", "example-b", "example-c"] + .into_iter() + .map(|name| fixtures.join(name).join("wit")) + .collect() + } + + #[test] + fn publish_plan_from_paths_collects_all_packages() { + let paths = transitive_local_paths(); + let plan = PublishPlan::from_paths(&paths).expect("plan should build"); + + assert!(!plan.is_empty()); + assert_eq!(plan.len(), 3); + + for name in ["example-a:foo", "example-b:bar", "example-c:baz"] { + let pkg: PackageRef = name.parse().expect("valid package ref"); + assert!( + plan.get_path(&pkg).is_some(), + "expected `{name}` to have an associated path in the plan", + ); + } + } + + #[test] + fn publish_plan_iter_yields_each_package_once() { + let paths = transitive_local_paths(); + let plan = PublishPlan::from_paths(&paths).expect("plan should build"); + + let seen: Vec = plan.iter().map(|spec| spec.package.clone()).collect(); + assert_eq!(seen.len(), 3, "iter should yield one entry per package"); + + let unique: HashSet<_> = seen.iter().cloned().collect(); + assert_eq!(unique.len(), seen.len(), "iter must not yield duplicates"); + + let expected: HashSet = ["example-a:foo", "example-b:bar", "example-c:baz"] + .into_iter() + .map(|s| s.parse().unwrap()) + .collect(); + assert_eq!(unique, expected); + } +} + /// Format a collection of packages as a list /// /// e.g. "foo:a@0.1.0, bar:b@0.2.0, and baz:c@0.3.0". diff --git a/crates/wkg/src/main.rs b/crates/wkg/src/main.rs index 785e219..51e480d 100644 --- a/crates/wkg/src/main.rs +++ b/crates/wkg/src/main.rs @@ -7,6 +7,7 @@ use tokio::io::AsyncWriteExt; use tracing::level_filters::LevelFilter; use wasm_pkg_client::{ caching::{CachingClient, FileCache}, + local::LocalConfig, Client, PackagePublisher, PublishOpts, }; use wasm_pkg_common::{ @@ -252,8 +253,6 @@ struct PublishArgs { impl PublishArgs { pub async fn run(self) -> anyhow::Result<()> { - let mut client = self.common.get_client().await?; - let package = match self.package { Some(_) if self.paths.len() > 2 => { anyhow::bail!("`--package` is currently unsupported when providing more than one path argument"); @@ -271,35 +270,41 @@ impl PublishArgs { let path = match &self.paths[..] { [path] => path, paths => { - let (overlay, tmp_dir) = wasm_pkg_client::local::LocalBackend::temp_dir()?; - let local_backend = wasm_pkg_client::local::LocalConfig { - root: tmp_dir.path().to_path_buf(), - }; - let mut reg_config = RegistryConfig::default(); - reg_config.set_default_backend(Some("local".to_owned())); - reg_config.set_backend_config("local", &local_backend)?; - - let plan = PublishPlan::from_paths(paths)?; - let mut lock_file = LockFile::load(true).await?; + let mut config = self.common.load_config().await?; + let cache = self.common.load_cache().await?; + let (overlay, tmp_dir) = wasm_pkg_client::local::LocalBackend::temp_dir()?; + let reg_config = RegistryConfig::default().with_default_backend( + "local", + LocalConfig { + root: tmp_dir.path().to_path_buf(), + }, + )?; // Route every package in the plan to the local overlay registry // backed by `reg_config`, so the client used in `build_wit_dir` // resolves these packages against the local overlay instead of // an upstream remote. - let mut config = self.common.load_config().await?; - let local_registry: Registry = "local".parse()?; + let local_registry: Registry = "tmp_local_publish".parse()?; config .get_or_insert_registry_config_mut(&local_registry) .merge(reg_config); + + let plan = PublishPlan::from_paths(paths)?; + + // TODO(mkatychev): Add support for `PackageLoader::get_release` to handle + // querying on a per package, namespace, and registry level + // to handle cargo style overlays. + // see this reference of `cargo::core::Dependency` usage for local overlays in Cargo: + // https://github.com/rust-lang/cargo/blob/d6900d00af2644ea1c0068c5694d9dbe11a3ab39/src/cargo/sources/overlay.rs#L47 for spec in plan.iter() { config.set_package_registry_override( spec.package.clone(), RegistryMapping::Registry(local_registry.clone()), ); } - let cache = self.common.load_cache().await?; let client = CachingClient::new(Some(Client::new(config)), cache); + let mut lock_file = LockFile::load(true).await?; for spec in plan.iter_edges() { let path = plan.get_path(&spec.package).unwrap(); let (publish_path, _tmp) = if path.is_dir() { @@ -333,6 +338,8 @@ impl PublishArgs { } }; + let client = self.common.get_client().await?; + // If the input is a directory, build a WIT package from it into a temp // file first. _tmp is held until the publish completes so the file // isn't deleted out from under us. From 4a00db31b61234e3b19116cd9b51557b11b2f545 Mon Sep 17 00:00:00 2001 From: Mikhail Katychev Date: Mon, 22 Jun 2026 11:33:09 -0500 Subject: [PATCH 26/61] removed client pub --- crates/wasm-pkg-client/src/caching/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/wasm-pkg-client/src/caching/mod.rs b/crates/wasm-pkg-client/src/caching/mod.rs index c7df52d..586f658 100644 --- a/crates/wasm-pkg-client/src/caching/mod.rs +++ b/crates/wasm-pkg-client/src/caching/mod.rs @@ -48,7 +48,7 @@ pub trait Cache { /// underlying client to be used as a read-only cache. #[derive(Clone)] pub struct CachingClient { - pub client: Option, + client: Option, cache: Arc, } From 9de6af4ed5fad54b71f7ccb8b8e77b953e0910c5 Mon Sep 17 00:00:00 2001 From: Mikhail Katychev Date: Mon, 22 Jun 2026 12:40:16 -0500 Subject: [PATCH 27/61] remove overlay --- crates/wasm-pkg-client/src/caching/mod.rs | 10 ---------- crates/wasm-pkg-client/src/lib.rs | 13 ------------- 2 files changed, 23 deletions(-) diff --git a/crates/wasm-pkg-client/src/caching/mod.rs b/crates/wasm-pkg-client/src/caching/mod.rs index 586f658..9287425 100644 --- a/crates/wasm-pkg-client/src/caching/mod.rs +++ b/crates/wasm-pkg-client/src/caching/mod.rs @@ -63,16 +63,6 @@ impl CachingClient { } } - async fn with_overlays( - mut self, - overlays: impl IntoIterator)>, - ) -> Self { - if let Some(client) = &mut self.client { - client.overlays = overlays.into_iter().collect(); - } - self - } - /// Returns whether or not the client is in read-only mode. pub fn is_readonly(&self) -> bool { self.client.is_none() diff --git a/crates/wasm-pkg-client/src/lib.rs b/crates/wasm-pkg-client/src/lib.rs index 1267c94..521e17a 100644 --- a/crates/wasm-pkg-client/src/lib.rs +++ b/crates/wasm-pkg-client/src/lib.rs @@ -109,19 +109,6 @@ impl Client { } } - /// Like [`Self::new`] but includes sources from source - /// replacement configurations. - pub fn new_with_overlays( - config: Config, - overlays: impl IntoIterator)>, - ) -> Self { - Self { - config: Arc::new(config), - sources: Default::default(), - overlays: overlays.into_iter().collect(), - } - } - /// Returns a reference to the configuration this client was initialized with. pub fn config(&self) -> &Config { &self.config From 42c9cbd4ca3eafc0a041e70c9a0233608e5aed96 Mon Sep 17 00:00:00 2001 From: Mikhail Katychev Date: Tue, 23 Jun 2026 12:04:10 -0500 Subject: [PATCH 28/61] rename overlay --- Cargo.lock | 29 ++++++++++++++++++++++++---- crates/wasm-pkg-client/src/local.rs | 5 ----- crates/wasm-pkg-core/src/resolver.rs | 23 ++++++++++------------ crates/wkg/src/main.rs | 10 +++++----- 4 files changed, 40 insertions(+), 27 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f577fdf..7afdf95 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1360,6 +1360,12 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" +[[package]] +name = "fixedbitset" +version = "0.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" + [[package]] name = "flate2" version = "1.1.9" @@ -2862,8 +2868,20 @@ version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4c5cc86750666a3ed20bdaf5ca2a0344f9c67674cae0515bec2da16fbaa47db" dependencies = [ - "fixedbitset", + "fixedbitset 0.4.2", + "indexmap 2.14.0", +] + +[[package]] +name = "petgraph" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8701b58ea97060d5e5b155d383a69952a60943f0e6dfe30b04c287beb0b27455" +dependencies = [ + "fixedbitset 0.5.7", + "hashbrown 0.15.5", "indexmap 2.14.0", + "serde", ] [[package]] @@ -3032,7 +3050,7 @@ dependencies = [ "log", "multimap", "once_cell", - "petgraph", + "petgraph 0.6.5", "prettyplease", "prost 0.12.6", "prost-types 0.12.6", @@ -3134,7 +3152,7 @@ dependencies = [ "anstyle", "config", "directories", - "petgraph", + "petgraph 0.6.5", "serde", "serde-value", "tint", @@ -5051,7 +5069,7 @@ dependencies = [ "im-rc", "indexmap 2.14.0", "log", - "petgraph", + "petgraph 0.6.5", "serde", "serde_derive", "serde_yaml", @@ -5123,6 +5141,7 @@ dependencies = [ "futures-util", "oci-client", "oci-wasm", + "petgraph 0.8.3", "rcgen", "reqwest 0.12.28", "secrecy", @@ -5155,6 +5174,7 @@ dependencies = [ "etcetera 0.11.0", "futures-util", "http", + "petgraph 0.8.3", "semver", "serde", "serde_json", @@ -5173,6 +5193,7 @@ dependencies = [ "futures-util", "indexmap 2.14.0", "libc", + "petgraph 0.8.3", "rstest", "semver", "serde", diff --git a/crates/wasm-pkg-client/src/local.rs b/crates/wasm-pkg-client/src/local.rs index 3fe559e..faa9379 100644 --- a/crates/wasm-pkg-client/src/local.rs +++ b/crates/wasm-pkg-client/src/local.rs @@ -43,11 +43,6 @@ fn registry_path_context(err: io::Error, path: &Path) -> Error { Error::RegistryError(err) } -fn registry_path_context(err: io::Error, path: &Path) -> Error { - let err = anyhow::Error::new(err).context(format!("path: {}", path.display())); - Error::RegistryError(err) -} - impl LocalBackend { pub fn new(registry_config: RegistryConfig) -> Result { let config = registry_config diff --git a/crates/wasm-pkg-core/src/resolver.rs b/crates/wasm-pkg-core/src/resolver.rs index cc02538..050c58b 100644 --- a/crates/wasm-pkg-core/src/resolver.rs +++ b/crates/wasm-pkg-core/src/resolver.rs @@ -871,29 +871,26 @@ impl PublishPlan { } pub fn iter<'a>(&'a self) -> impl Iterator + 'a { - self.dependents.nodes_iter().map(|id| &self.dependents[id]) - } - - pub fn iter_edges<'a>(&'a self) -> impl Iterator + 'a { - use petgraph::visit::IntoNeighborsDirected; self.dependents.nodes_iter().map(|id| { let dep = &self.dependents[id]; - tracing::warn!("ITER {dep}"); + if !tracing::enabled!(Level::DEBUG) { + return dep; + } + + // initial dependency graph visualization let mut neighbors = self .dependents .neighbors_directed(id, Direction::Outgoing) .peekable(); + if neighbors.peek().is_none() { // tracing::debug!("{dep} has no dependents"); println!("{dep} has no dependents"); } - // if tracing::enabled!(Level::DEBUG) { - if true { - while let Some(id) = neighbors.next() { - let pkg = &self.dependents[id]; - println!("{dep} -(DependencyOF)-> {pkg}"); - } + while let Some(id) = neighbors.next() { + let pkg = &self.dependents[id]; + println!("{dep} -(DependencyOF)-> {pkg}"); } dep @@ -924,7 +921,7 @@ impl PublishPlan { .collect() } - /// + /// NOTE pub fn get_path(&self, pkg: &PackageRef) -> Option<&Path> { self.indices.get(pkg).map(|(_, p)| p.as_ref()) } diff --git a/crates/wkg/src/main.rs b/crates/wkg/src/main.rs index 51e480d..77869aa 100644 --- a/crates/wkg/src/main.rs +++ b/crates/wkg/src/main.rs @@ -270,7 +270,7 @@ impl PublishArgs { let path = match &self.paths[..] { [path] => path, paths => { - let mut config = self.common.load_config().await?; + let mut overlay_config = self.common.load_config().await?; let cache = self.common.load_cache().await?; let (overlay, tmp_dir) = wasm_pkg_client::local::LocalBackend::temp_dir()?; @@ -285,7 +285,7 @@ impl PublishArgs { // resolves these packages against the local overlay instead of // an upstream remote. let local_registry: Registry = "tmp_local_publish".parse()?; - config + overlay_config .get_or_insert_registry_config_mut(&local_registry) .merge(reg_config); @@ -297,15 +297,15 @@ impl PublishArgs { // see this reference of `cargo::core::Dependency` usage for local overlays in Cargo: // https://github.com/rust-lang/cargo/blob/d6900d00af2644ea1c0068c5694d9dbe11a3ab39/src/cargo/sources/overlay.rs#L47 for spec in plan.iter() { - config.set_package_registry_override( + overlay_config.set_package_registry_override( spec.package.clone(), RegistryMapping::Registry(local_registry.clone()), ); } - let client = CachingClient::new(Some(Client::new(config)), cache); + let client = CachingClient::new(Some(Client::new(overlay_config)), cache); let mut lock_file = LockFile::load(true).await?; - for spec in plan.iter_edges() { + for spec in plan.iter() { let path = plan.get_path(&spec.package).unwrap(); let (publish_path, _tmp) = if path.is_dir() { let _prev_lock_ref = (lock_file.version, lock_file.packages.clone()); From bc8db16bb48a878a9b87d417efa57491a794333e Mon Sep 17 00:00:00 2001 From: Mikhail Katychev Date: Tue, 23 Jun 2026 15:07:49 -0500 Subject: [PATCH 29/61] added take_ready test case --- Cargo.lock | 1 + crates/wasm-pkg-common/src/package.rs | 6 + crates/wasm-pkg-core/Cargo.toml | 1 + crates/wasm-pkg-core/src/resolver.rs | 147 ++++++++++-------- crates/wasm-pkg-core/src/wit.rs | 8 +- .../transitive-local/example-d/wit/world.wit | 5 + 6 files changed, 99 insertions(+), 69 deletions(-) create mode 100644 crates/wasm-pkg-core/tests/fixtures/transitive-local/example-d/wit/world.wit diff --git a/Cargo.lock b/Cargo.lock index 7afdf95..a0ce0f2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5191,6 +5191,7 @@ version = "0.15.1" dependencies = [ "anyhow", "futures-util", + "glob", "indexmap 2.14.0", "libc", "petgraph 0.8.3", diff --git a/crates/wasm-pkg-common/src/package.rs b/crates/wasm-pkg-common/src/package.rs index 26c7e99..0f12bf5 100644 --- a/crates/wasm-pkg-common/src/package.rs +++ b/crates/wasm-pkg-common/src/package.rs @@ -95,6 +95,12 @@ impl PackageSpec { } } +impl PartialEq for PackageSpec { + fn eq(&self, other: &str) -> bool { + &format!("{self}") == other + } +} + impl FromStr for PackageSpec { type Err = Error; diff --git a/crates/wasm-pkg-core/Cargo.toml b/crates/wasm-pkg-core/Cargo.toml index d0b9fe2..1e8818b 100644 --- a/crates/wasm-pkg-core/Cargo.toml +++ b/crates/wasm-pkg-core/Cargo.toml @@ -43,3 +43,4 @@ features = [ tempfile = { workspace = true } sha2 = { workspace = true } rstest = "0.23" +glob = "0.3.3" diff --git a/crates/wasm-pkg-core/src/resolver.rs b/crates/wasm-pkg-core/src/resolver.rs index 050c58b..5de1e04 100644 --- a/crates/wasm-pkg-core/src/resolver.rs +++ b/crates/wasm-pkg-core/src/resolver.rs @@ -885,12 +885,20 @@ impl PublishPlan { if neighbors.peek().is_none() { // tracing::debug!("{dep} has no dependents"); - println!("{dep} has no dependents"); + println!("[{dep} has no dependents]"); + } else { + println!("[{dep}]"); } while let Some(id) = neighbors.next() { let pkg = &self.dependents[id]; - println!("{dep} -(DependencyOF)-> {pkg}"); + let separator = if neighbors.peek().is_some() { + "├─" + } else { + "╰─" + }; + + println!("{separator}(DependencyOf)─▶ {pkg}"); } dep @@ -912,12 +920,13 @@ impl PublishPlan { self.dependents .nodes_iter() // there are no dependents on `self.dendents[id]` - .filter(|id| self.dependents.neighbors(*id).count() == 0) - .map(|id| { - let pkg = &self.dependents[id]; - self.indices.remove(&pkg.package); - pkg.clone() + .filter(|id| { + self.dependents + .neighbors_directed(*id, Direction::Incoming) + .count() + == 0 }) + .map(|id| self.dependents[id].clone()) .collect() } @@ -928,15 +937,37 @@ impl PublishPlan { /// Packages confirmed to be available in the registry, potentially allowing additional /// packages to be "ready". - pub fn mark_confirmed(&mut self, published: impl IntoIterator) { - for pkg in published { + pub fn mark_confirmed(&mut self, published: impl IntoIterator) { + for spec in published { let (id, _) = self .indices - .remove(&pkg) + .remove(&spec.package) .expect("PackageSpec has no associated index"); - self.dependents - .remove_node(id) - .expect("index has no associated PackageSpec"); + // NOTE: nodes without edges will return None here + self.dependents.remove_node(id); + } + } +} + +/// Format a collection of packages as a list +/// +/// e.g. "foo:a@0.1.0, bar:b@0.2.0, and baz:c@0.3.0". +/// +/// Note: the final separator (e.g. "and" in the previous example) can be chosen. +pub fn package_list<'a>( + pkgs: impl IntoIterator, + final_sep: Option<&str>, +) -> String { + let final_sep = final_sep.unwrap_or("and"); + let mut names: Vec<_> = pkgs.into_iter().map(|pkg| pkg.to_string()).collect(); + names.sort(); + + match &names[..] { + [] => String::new(), + [a] => a.clone(), + [a, b] => format!("{a} {final_sep} {b}"), + [names @ .., last] => { + format!("{}, {final_sep} {last}", names.join(", ")) } } } @@ -946,69 +977,57 @@ mod tests { use std::collections::HashSet; use std::path::PathBuf; - use super::PublishPlan; + use super::*; + use glob::glob; use wasm_pkg_client::PackageRef; fn transitive_local_paths() -> Vec { - let fixtures = - PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/transitive-local"); - ["example-a", "example-b", "example-c"] + let fixtures = PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("tests/fixtures/transitive-local/*/wit/"); + glob(fixtures.to_str().unwrap()) + .unwrap() .into_iter() - .map(|name| fixtures.join(name).join("wit")) + .map(|p| p.expect("glob path error")) .collect() } #[test] - fn publish_plan_from_paths_collects_all_packages() { + fn publish_plan_iter() { let paths = transitive_local_paths(); - let plan = PublishPlan::from_paths(&paths).expect("plan should build"); - - assert!(!plan.is_empty()); - assert_eq!(plan.len(), 3); - - for name in ["example-a:foo", "example-b:bar", "example-c:baz"] { - let pkg: PackageRef = name.parse().expect("valid package ref"); - assert!( - plan.get_path(&pkg).is_some(), - "expected `{name}` to have an associated path in the plan", - ); - } + let plan = PublishPlan::from_paths(&paths).unwrap(); + assert_eq!( + plan.iter().count(), + 4, + "unexpected package count\npackages found: {}", + package_list(plan.iter(), None) + ); } #[test] - fn publish_plan_iter_yields_each_package_once() { + fn publish_plan_chunks() { let paths = transitive_local_paths(); - let plan = PublishPlan::from_paths(&paths).expect("plan should build"); - - let seen: Vec = plan.iter().map(|spec| spec.package.clone()).collect(); - assert_eq!(seen.len(), 3, "iter should yield one entry per package"); - - let unique: HashSet<_> = seen.iter().cloned().collect(); - assert_eq!(unique.len(), seen.len(), "iter must not yield duplicates"); - - let expected: HashSet = ["example-a:foo", "example-b:bar", "example-c:baz"] - .into_iter() - .map(|s| s.parse().unwrap()) - .collect(); - assert_eq!(unique, expected); - } -} - -/// Format a collection of packages as a list -/// -/// e.g. "foo:a@0.1.0, bar:b@0.2.0, and baz:c@0.3.0". -/// -/// Note: the final separator (e.g. "and" in the previous example) can be chosen. -pub fn package_list(pkgs: impl IntoIterator, final_sep: &str) -> String { - let mut names: Vec<_> = pkgs.into_iter().map(|pkg| pkg.to_string()).collect(); - names.sort(); - - match &names[..] { - [] => String::new(), - [a] => a.clone(), - [a, b] => format!("{a} {final_sep} {b}"), - [names @ .., last] => { - format!("{}, {final_sep} {last}", names.join(", ")) - } + let mut plan = PublishPlan::from_paths(&paths).unwrap(); + let mut ready_for_publish = plan.take_ready(); + assert_eq!( + ready_for_publish.iter().collect::>(), + ["example-c:baz@0.1.0", "example-d:foo@0.1.0"], + ); + plan.mark_confirmed(ready_for_publish.into_iter()); + + ready_for_publish = plan.take_ready(); + assert_eq!( + ready_for_publish.iter().collect::>(), + ["example-b:bar@0.1.0"], + ); + plan.mark_confirmed(ready_for_publish.into_iter()); + + ready_for_publish = plan.take_ready(); + assert_eq!( + ready_for_publish.iter().collect::>(), + ["example-a:foo@0.1.0"], + ); + plan.mark_confirmed(ready_for_publish.into_iter()); + + assert!(plan.is_empty()); } } diff --git a/crates/wasm-pkg-core/src/wit.rs b/crates/wasm-pkg-core/src/wit.rs index d60fc02..23591f4 100644 --- a/crates/wasm-pkg-core/src/wit.rs +++ b/crates/wasm-pkg-core/src/wit.rs @@ -204,14 +204,12 @@ pub fn get_local_dependencies( for (dep, _version) in deps { if let Some(&(dep, _)) = indices.get(&dep) { let (pkg, _) = indices[&spec.package]; - // // dep -(DependencyOF)-> pkg - // graph.try_update_edge(dep, pkg, Direction::Outgoing); - // // pkg -(DependsOn)-> dep + // // pkg <=DependsOn= dep graph - .try_update_edge(pkg, dep, Direction::Outgoing) + .try_update_edge(pkg, dep, Direction::Incoming) .map_err(|e| match e { petgraph::acyclic::AcyclicEdgeError::Cycle(cycle) => { - anyhow::anyhow!("cyclical depndency detected") + anyhow::anyhow!("cyclical dependency detected") .context(format!("package {}", graph[cycle.node_id()])) .context(format!("package {}", graph[pkg])) } diff --git a/crates/wasm-pkg-core/tests/fixtures/transitive-local/example-d/wit/world.wit b/crates/wasm-pkg-core/tests/fixtures/transitive-local/example-d/wit/world.wit new file mode 100644 index 0000000..427a9a5 --- /dev/null +++ b/crates/wasm-pkg-core/tests/fixtures/transitive-local/example-d/wit/world.wit @@ -0,0 +1,5 @@ +package example-d:foo@0.1.0; + +interface bar { + type baz = string; +} From cadcb33cbd5fa564ceced7f3361c85188e6f2686 Mon Sep 17 00:00:00 2001 From: Mikhail Katychev Date: Wed, 24 Jun 2026 15:32:43 -0500 Subject: [PATCH 30/61] handled secondary publishing flow --- crates/wasm-pkg-client/src/lib.rs | 25 +++--- crates/wasm-pkg-client/src/local.rs | 18 +++-- crates/wasm-pkg-core/src/resolver.rs | 67 +++++++-------- crates/wkg/src/main.rs | 117 ++++++++++++++++----------- 4 files changed, 125 insertions(+), 102 deletions(-) diff --git a/crates/wasm-pkg-client/src/lib.rs b/crates/wasm-pkg-client/src/lib.rs index 521e17a..147bc76 100644 --- a/crates/wasm-pkg-client/src/lib.rs +++ b/crates/wasm-pkg-client/src/lib.rs @@ -41,7 +41,6 @@ use std::{collections::HashMap, pin::Pin}; use anyhow::anyhow; use bytes::Bytes; use futures_util::Stream; -pub use publisher::PackagePublisher; use tokio::io::AsyncSeekExt; use tokio::sync::RwLock; use tokio_util::io::SyncIoBridge; @@ -57,7 +56,8 @@ pub use wasm_pkg_common::{ use wit_component::DecodedWasm; use crate::metadata::RegistryMetadataExt; -use crate::{loader::PackageLoader, local::LocalBackend, oci::OciBackend, warg::WargBackend}; +pub use crate::{loader::PackageLoader, local::LocalBackend, publisher::PackagePublisher}; +use crate::{oci::OciBackend, warg::WargBackend}; pub use release::{Release, VersionInfo}; @@ -67,8 +67,14 @@ pub type ContentStream = Pin> + Send /// An alias for a PublishingSource (generally a file) pub type PublishingSource = Pin>; +// Convenience method to convert a byte array into a [`PublishingSource`] +pub fn source_from_slice<'a>(data: &'a [u8]) -> Pin> { + Box::pin(std::io::Cursor::new(data)) +} + /// A supertrait combining tokio's AsyncRead and AsyncSeek. pub trait ReaderSeeker: tokio::io::AsyncRead + tokio::io::AsyncSeek {} + impl ReaderSeeker for T where T: tokio::io::AsyncRead + tokio::io::AsyncSeek {} trait LoaderPublisher: PackageLoader + PackagePublisher {} @@ -147,20 +153,6 @@ impl Client { source.stream_content(package, release).await } - pub async fn publish_release_files( - &self, - files: &[impl AsRef], - additional_options: PublishOpts, - ) -> Result<(PackageRef, Version), Error> { - let data = tokio::fs::OpenOptions::new() - .read(true) - .open(files[0].as_ref()) - .await?; - - self.publish_release_data(Box::pin(data), additional_options) - .await - } - /// Publishes the given file as a package release. The package name and version will be read /// from the component if not given as part of `additional_options`. Returns the package name /// and version of the published release. @@ -229,6 +221,7 @@ impl Client { ) -> Result, Error> { let is_override = registry_override.is_some(); let registry = self.resolve_registry(package, registry_override)?; + tracing::debug!(?registry, "resolved registry"); if let Some(source) = self.sources.read().await.get(®istry) { return Ok(source.clone()); diff --git a/crates/wasm-pkg-client/src/local.rs b/crates/wasm-pkg-client/src/local.rs index faa9379..6f5a26d 100644 --- a/crates/wasm-pkg-client/src/local.rs +++ b/crates/wasm-pkg-client/src/local.rs @@ -33,8 +33,17 @@ pub struct LocalConfig { pub root: PathBuf, } +impl LocalConfig { + pub fn temp_dir() -> Result<(Self, TempDir), Error> { + let handle = TempDir::new()?; + let root = handle.path().to_owned(); + tracing::debug!(registry_dir=%root.display(), "created temporary directory"); + Ok((Self { root }, handle)) + } +} + #[derive(Clone)] -pub struct LocalBackend { +struct LocalBackend { pub(crate) root: PathBuf, } @@ -62,13 +71,6 @@ impl LocalBackend { fn version_path(&self, package: &PackageRef, version: &Version) -> PathBuf { self.package_dir(package).join(format!("{version}.wasm")) } - - pub fn temp_dir() -> Result<(Self, TempDir), Error> { - let handle = TempDir::new()?; - let root = handle.path().to_owned(); - tracing::debug!(registry_dir=%root.display(), "created temporary directory"); - Ok((Self { root }, handle)) - } } #[async_trait] diff --git a/crates/wasm-pkg-core/src/resolver.rs b/crates/wasm-pkg-core/src/resolver.rs index 5de1e04..42ed229 100644 --- a/crates/wasm-pkg-core/src/resolver.rs +++ b/crates/wasm-pkg-core/src/resolver.rs @@ -871,38 +871,9 @@ impl PublishPlan { } pub fn iter<'a>(&'a self) -> impl Iterator + 'a { - self.dependents.nodes_iter().map(|id| { - let dep = &self.dependents[id]; - if !tracing::enabled!(Level::DEBUG) { - return dep; - } - - // initial dependency graph visualization - let mut neighbors = self - .dependents - .neighbors_directed(id, Direction::Outgoing) - .peekable(); - - if neighbors.peek().is_none() { - // tracing::debug!("{dep} has no dependents"); - println!("[{dep} has no dependents]"); - } else { - println!("[{dep}]"); - } - - while let Some(id) = neighbors.next() { - let pkg = &self.dependents[id]; - let separator = if neighbors.peek().is_some() { - "├─" - } else { - "╰─" - }; - - println!("{separator}(DependencyOf)─▶ {pkg}"); - } - - dep - }) + self.dependents + .nodes_iter() + .map(|id| &(self.dependents[id])) } pub fn is_empty(&self) -> bool { @@ -949,6 +920,38 @@ impl PublishPlan { } } +impl std::fmt::Display for PublishPlan { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + for id in self.dependents.nodes_iter() { + let dep = &self.dependents[id]; + // initial dependency graph visualization + let mut neighbors = self + .dependents + .neighbors_directed(id, Direction::Outgoing) + .peekable(); + + if neighbors.peek().is_none() { + // tracing::debug!("{dep} has no dependents"); + writeln!(f, "[{dep} has no dependents]")?; + } else { + writeln!(f, "[{dep}]")?; + } + + while let Some(id) = neighbors.next() { + let pkg = &self.dependents[id]; + let separator = if neighbors.peek().is_some() { + "├─" + } else { + "╰─" + }; + + writeln!(f, "{separator}(DependencyOf)─▶ {pkg}")?; + } + } + Ok(()) + } +} + /// Format a collection of packages as a list /// /// e.g. "foo:a@0.1.0, bar:b@0.2.0, and baz:c@0.3.0". diff --git a/crates/wkg/src/main.rs b/crates/wkg/src/main.rs index 77869aa..07e6455 100644 --- a/crates/wkg/src/main.rs +++ b/crates/wkg/src/main.rs @@ -1,14 +1,19 @@ -use std::{io::Seek, path::PathBuf}; +use std::{ + io::{Cursor, Seek}, + path::PathBuf, + pin::Pin, +}; use anyhow::{ensure, Context}; use clap::{Args, Parser, Subcommand, ValueEnum}; use futures_util::TryStreamExt; -use tokio::io::AsyncWriteExt; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; use tracing::level_filters::LevelFilter; use wasm_pkg_client::{ caching::{CachingClient, FileCache}, local::LocalConfig, - Client, PackagePublisher, PublishOpts, + source_from_slice, Client, PackageLoader, PackagePublisher, PublishOpts, PublishingSource, + ReaderSeeker, }; use wasm_pkg_common::{ self, @@ -253,27 +258,14 @@ struct PublishArgs { impl PublishArgs { pub async fn run(self) -> anyhow::Result<()> { - let package = match self.package { - Some(_) if self.paths.len() > 2 => { - anyhow::bail!("`--package` is currently unsupported when providing more than one path argument"); - } - Some(PackageSpec { - package, - version: Some(v), - }) => Some((package, v)), - Some(PackageSpec { version: None, .. }) => { - anyhow::bail!("version is required when manually overriding the package ID"); - } - None => None, - }; - + let opts = self.opts()?; let path = match &self.paths[..] { [path] => path, paths => { let mut overlay_config = self.common.load_config().await?; let cache = self.common.load_cache().await?; - let (overlay, tmp_dir) = wasm_pkg_client::local::LocalBackend::temp_dir()?; + let (backend, tmp_dir) = LocalConfig::temp_dir()?; let reg_config = RegistryConfig::default().with_default_backend( "local", LocalConfig { @@ -290,12 +282,14 @@ impl PublishArgs { .merge(reg_config); let plan = PublishPlan::from_paths(paths)?; + println!("{plan}"); // TODO(mkatychev): Add support for `PackageLoader::get_release` to handle // querying on a per package, namespace, and registry level // to handle cargo style overlays. // see this reference of `cargo::core::Dependency` usage for local overlays in Cargo: // https://github.com/rust-lang/cargo/blob/d6900d00af2644ea1c0068c5694d9dbe11a3ab39/src/cargo/sources/overlay.rs#L47 + // this is still needed for `wit::build_wit_dir` for spec in plan.iter() { overlay_config.set_package_registry_override( spec.package.clone(), @@ -304,37 +298,54 @@ impl PublishArgs { } let client = CachingClient::new(Some(Client::new(overlay_config)), cache); - let mut lock_file = LockFile::load(true).await?; + let mut lock_file = LockFile::load(false).await?; + // these are packages that have been successfully pushed to our "tmp_local_publish" + let mut validated_packages = Vec::new(); for spec in plan.iter() { let path = plan.get_path(&spec.package).unwrap(); - let (publish_path, _tmp) = if path.is_dir() { + let data = if path.is_dir() { let _prev_lock_ref = (lock_file.version, lock_file.packages.clone()); - let (pkg_ref, _, bytes) = + let (_pkg_ref, _version, bytes) = wit::build_wit_dir(&path, client.clone(), &mut lock_file).await?; - let tmp = temp_wit_file(&pkg_ref, &bytes).await?; - - (tmp.path().to_path_buf(), Some(tmp)) + bytes } else { - (path.to_owned(), None) + let mut file = tokio::fs::OpenOptions::new().read(true).open(path).await?; + let mut buf = Vec::new(); + file.read(&mut buf).await?; + buf }; - let data = tokio::fs::OpenOptions::new() - .read(true) - .open(publish_path) + let source = Box::pin(Cursor::new(data.clone())); + client + .client()? + .publish_release_data( + source, + PublishOpts { + package: None, + registry: Some(local_registry.clone()), + // we want to publish to "tmp_local_publish" regardless of flags passed in + dry_run: false, + }, + ) .await?; - overlay - .publish( - &spec.package, - spec.version.as_ref().unwrap(), - Box::pin(data), - false, - ) - .await - .unwrap(); + validated_packages.push(data); + } + + let client = self.common.get_client().await?; + for data in validated_packages { + let source = Box::pin(Cursor::new(data)); + let (package, version) = client + .client()? + .publish_release_data(source, opts.clone()) + .await?; + if self.dry_run { + println!("Aborting publish due to dry run: {}@{}", package, version); + } else { + println!("Published {}@{}", package, version); + } } return Ok(()); - // todo!(); } }; @@ -371,14 +382,7 @@ impl PublishArgs { let (package, version) = client .client()? - .publish_release_file( - &publish_path, - PublishOpts { - package, - registry: self.registry_args.registry, - dry_run: self.dry_run, - }, - ) + .publish_release_file(&publish_path, opts) .await?; if self.dry_run { println!("Aborting publish due to dry run: {}@{}", package, version); @@ -387,6 +391,27 @@ impl PublishArgs { } Ok(()) } + + fn opts(&self) -> anyhow::Result { + let package = match self.package.clone() { + Some(_) if self.paths.len() > 2 => { + anyhow::bail!("`--package` is currently unsupported when providing more than one path argument"); + } + Some(PackageSpec { + package, + version: Some(v), + }) => Some((package, v)), + Some(PackageSpec { version: None, .. }) => { + anyhow::bail!("version is required when manually overriding the package ID"); + } + None => None, + }; + Ok(PublishOpts { + package, + registry: self.registry_args.registry.clone(), + dry_run: self.dry_run, + }) + } } #[derive(ValueEnum, Clone, Debug, PartialEq)] From 25fee68162ca20d99be63df91f425a254d0fc8e9 Mon Sep 17 00:00:00 2001 From: Mikhail Katychev Date: Wed, 24 Jun 2026 15:42:15 -0500 Subject: [PATCH 31/61] cleaned up local backend visiblity --- crates/wasm-pkg-client/src/lib.rs | 4 +- crates/wasm-pkg-client/src/local.rs | 2 +- crates/wasm-pkg-client/src/overlay/mod.rs | 99 ----------------------- crates/wkg/src/main.rs | 10 +-- 4 files changed, 6 insertions(+), 109 deletions(-) delete mode 100644 crates/wasm-pkg-client/src/overlay/mod.rs diff --git a/crates/wasm-pkg-client/src/lib.rs b/crates/wasm-pkg-client/src/lib.rs index 147bc76..e19aa26 100644 --- a/crates/wasm-pkg-client/src/lib.rs +++ b/crates/wasm-pkg-client/src/lib.rs @@ -29,7 +29,6 @@ mod loader; pub mod local; pub mod metadata; pub mod oci; -pub mod overlay; mod publisher; mod release; pub mod warg; @@ -55,8 +54,9 @@ pub use wasm_pkg_common::{ }; use wit_component::DecodedWasm; +use crate::local::LocalBackend; use crate::metadata::RegistryMetadataExt; -pub use crate::{loader::PackageLoader, local::LocalBackend, publisher::PackagePublisher}; +pub use crate::{loader::PackageLoader, publisher::PackagePublisher}; use crate::{oci::OciBackend, warg::WargBackend}; pub use release::{Release, VersionInfo}; diff --git a/crates/wasm-pkg-client/src/local.rs b/crates/wasm-pkg-client/src/local.rs index 6f5a26d..6ed03cf 100644 --- a/crates/wasm-pkg-client/src/local.rs +++ b/crates/wasm-pkg-client/src/local.rs @@ -43,7 +43,7 @@ impl LocalConfig { } #[derive(Clone)] -struct LocalBackend { +pub(crate) struct LocalBackend { pub(crate) root: PathBuf, } diff --git a/crates/wasm-pkg-client/src/overlay/mod.rs b/crates/wasm-pkg-client/src/overlay/mod.rs deleted file mode 100644 index 1bcc6d1..0000000 --- a/crates/wasm-pkg-client/src/overlay/mod.rs +++ /dev/null @@ -1,99 +0,0 @@ -use std::collections::HashMap; - -use async_trait::async_trait; -use tempfile::TempDir; -use wasm_pkg_common::{ - package::{PackageRef, Version}, - Error, -}; - -use crate::{ - loader::PackageLoader, local::LocalBackend, publisher::PackagePublisher, ContentStream, - InnerClient, PublishingSource, Release, VersionInfo, -}; - -pub(crate) struct OverlayBackend { - local: LocalBackend, - remotes: HashMap, - _handle: TempDir, -} - -#[expect(unused)] -impl OverlayBackend { - fn new(remotes: HashMap) -> Result { - let (local, handle) = LocalBackend::temp_dir()?; - Ok(Self { - local, - remotes, - _handle: handle, - }) - } - - fn remote(&self, package: &PackageRef) -> Result<&InnerClient, Error> { - self.remotes - .get(package) - .ok_or_else(|| Error::InvalidPackageRef(package.to_string())) - } -} - -#[async_trait] -impl PackageLoader for OverlayBackend { - async fn list_all_versions(&self, package: &PackageRef) -> Result, Error> { - let mut versions = self.local.list_all_versions(package).await?; - - if let Some(remote) = self.remotes.get(package) { - let mut remote_versions = remote.list_all_versions(package).await?; - versions.append(&mut remote_versions); - versions.sort(); - versions.dedup(); - } - - Ok(versions) - } - - async fn get_release(&self, package: &PackageRef, version: &Version) -> Result { - if let Ok(release) = self.local.get_release(package, version).await { - return Ok(release); - } - tracing::debug!(%package, %version, method = "get_release", "OverlayBackend falling back to remote"); - self.remote(package)?.get_release(package, version).await - } - - async fn stream_content_unvalidated( - &self, - package: &PackageRef, - content: &Release, - ) -> Result { - if let Ok(stream) = self - .local - .stream_content_unvalidated(package, content) - .await - { - return Ok(stream); - } - tracing::debug!(%package, version = %content.version, method = "stream_content_unvalidated", "OverlayBackend falling back to remote"); - - self.local - .stream_content_unvalidated(package, content) - .await - } -} - -#[async_trait::async_trait] -impl PackagePublisher for OverlayBackend { - async fn publish( - &self, - package: &PackageRef, - version: &Version, - data: PublishingSource, - dry_run: bool, - ) -> Result<(), Error> { - if dry_run { - self.local.publish(&package, &version, data, dry_run).await - } else { - self.remote(package)? - .publish(&package, &version, data, dry_run) - .await - } - } -} diff --git a/crates/wkg/src/main.rs b/crates/wkg/src/main.rs index 07e6455..86ddf94 100644 --- a/crates/wkg/src/main.rs +++ b/crates/wkg/src/main.rs @@ -265,13 +265,9 @@ impl PublishArgs { let mut overlay_config = self.common.load_config().await?; let cache = self.common.load_cache().await?; - let (backend, tmp_dir) = LocalConfig::temp_dir()?; - let reg_config = RegistryConfig::default().with_default_backend( - "local", - LocalConfig { - root: tmp_dir.path().to_path_buf(), - }, - )?; + let (local_config, _tmp_dir_handle) = LocalConfig::temp_dir()?; + let reg_config = + RegistryConfig::default().with_default_backend("local", local_config)?; // Route every package in the plan to the local overlay registry // backed by `reg_config`, so the client used in `build_wit_dir` // resolves these packages against the local overlay instead of From cf2cf6c4efccf3f7dc432a5de70f5bbb43ce780b Mon Sep 17 00:00:00 2001 From: Mikhail Katychev Date: Wed, 24 Jun 2026 15:48:19 -0500 Subject: [PATCH 32/61] used LOCAL_PROTOCOL const where possible --- crates/wasm-pkg-client/src/caching/mod.rs | 3 +-- crates/wasm-pkg-client/src/lib.rs | 3 --- crates/wasm-pkg-client/src/local.rs | 3 ++- crates/wasm-pkg-common/src/config.rs | 8 ++++++-- crates/wasm-pkg-core/src/resolver.rs | 8 +++----- crates/wkg/src/main.rs | 7 +++---- 6 files changed, 15 insertions(+), 17 deletions(-) diff --git a/crates/wasm-pkg-client/src/caching/mod.rs b/crates/wasm-pkg-client/src/caching/mod.rs index 9287425..e90a161 100644 --- a/crates/wasm-pkg-client/src/caching/mod.rs +++ b/crates/wasm-pkg-client/src/caching/mod.rs @@ -4,11 +4,10 @@ use std::sync::Arc; use wasm_pkg_common::{ digest::ContentDigest, package::{PackageRef, Version}, - registry::Registry, Error, }; -use crate::{Client, ContentStream, InnerClient, Release, VersionInfo}; +use crate::{Client, ContentStream, Release, VersionInfo}; mod file; diff --git a/crates/wasm-pkg-client/src/lib.rs b/crates/wasm-pkg-client/src/lib.rs index e19aa26..7795c04 100644 --- a/crates/wasm-pkg-client/src/lib.rs +++ b/crates/wasm-pkg-client/src/lib.rs @@ -101,8 +101,6 @@ pub struct PublishOpts { pub struct Client { config: Arc, sources: Arc>, - /// Mapping of sources to local registries that will be overlaid on them. - overlays: RegistrySources, } impl Client { @@ -111,7 +109,6 @@ impl Client { Self { config: Arc::new(config), sources: Default::default(), - overlays: Default::default(), } } diff --git a/crates/wasm-pkg-client/src/local.rs b/crates/wasm-pkg-client/src/local.rs index 6ed03cf..cac05d2 100644 --- a/crates/wasm-pkg-client/src/local.rs +++ b/crates/wasm-pkg-client/src/local.rs @@ -17,6 +17,7 @@ use tokio_util::io::ReaderStream; use wasm_pkg_common::{ config::RegistryConfig, digest::ContentDigest, + metadata::LOCAL_PROTOCOL, package::{PackageRef, Version}, Error, }; @@ -55,7 +56,7 @@ fn registry_path_context(err: io::Error, path: &Path) -> Error { impl LocalBackend { pub fn new(registry_config: RegistryConfig) -> Result { let config = registry_config - .backend_config::("local")? + .backend_config::(LOCAL_PROTOCOL)? .ok_or_else(|| { Error::InvalidConfig(anyhow!("'local' backend requires configuration")) })?; diff --git a/crates/wasm-pkg-common/src/config.rs b/crates/wasm-pkg-common/src/config.rs index cd17e17..652eaae 100644 --- a/crates/wasm-pkg-common/src/config.rs +++ b/crates/wasm-pkg-common/src/config.rs @@ -7,7 +7,11 @@ use std::{ use serde::{Deserialize, Serialize}; use crate::{ - label::Label, metadata::RegistryMetadata, package::PackageRef, registry::Registry, Error, + label::Label, + metadata::{RegistryMetadata, LOCAL_PROTOCOL}, + package::PackageRef, + registry::Registry, + Error, }; mod toml; @@ -296,7 +300,7 @@ impl RegistryConfig { backend_config: T, ) -> Result { self.default_backend = Some(default_backend.to_string()); - self.set_backend_config("local", backend_config)?; + self.set_backend_config(LOCAL_PROTOCOL, backend_config)?; Ok(self) } diff --git a/crates/wasm-pkg-core/src/resolver.rs b/crates/wasm-pkg-core/src/resolver.rs index 42ed229..aa085b9 100644 --- a/crates/wasm-pkg-core/src/resolver.rs +++ b/crates/wasm-pkg-core/src/resolver.rs @@ -13,13 +13,11 @@ use anyhow::{bail, Context, Result}; use futures_util::TryStreamExt; use indexmap::{IndexMap, IndexSet}; use petgraph::{ - acyclic::Acyclic, - graph::{DiGraph, NodeIndex}, + graph::NodeIndex, Direction, }; use semver::{Comparator, Op, Version, VersionReq}; use tokio::io::{AsyncRead, AsyncReadExt}; -use tracing::Level; use wasm_pkg_client::{ caching::{CachingClient, FileCache}, Client, Config, ContentDigest, Error as WasmPkgError, PackageRef, Release, VersionInfo, @@ -977,12 +975,12 @@ pub fn package_list<'a>( #[cfg(test)] mod tests { - use std::collections::HashSet; + use std::path::PathBuf; use super::*; use glob::glob; - use wasm_pkg_client::PackageRef; + fn transitive_local_paths() -> Vec { let fixtures = PathBuf::from(env!("CARGO_MANIFEST_DIR")) diff --git a/crates/wkg/src/main.rs b/crates/wkg/src/main.rs index 86ddf94..b533758 100644 --- a/crates/wkg/src/main.rs +++ b/crates/wkg/src/main.rs @@ -1,7 +1,6 @@ use std::{ io::{Cursor, Seek}, path::PathBuf, - pin::Pin, }; use anyhow::{ensure, Context}; @@ -12,12 +11,12 @@ use tracing::level_filters::LevelFilter; use wasm_pkg_client::{ caching::{CachingClient, FileCache}, local::LocalConfig, - source_from_slice, Client, PackageLoader, PackagePublisher, PublishOpts, PublishingSource, - ReaderSeeker, + Client, PackageLoader, PublishOpts, }; use wasm_pkg_common::{ self, config::{Config, RegistryConfig, RegistryMapping}, + metadata::LOCAL_PROTOCOL, package::PackageSpec, registry::Registry, }; @@ -267,7 +266,7 @@ impl PublishArgs { let (local_config, _tmp_dir_handle) = LocalConfig::temp_dir()?; let reg_config = - RegistryConfig::default().with_default_backend("local", local_config)?; + RegistryConfig::default().with_default_backend(LOCAL_PROTOCOL, local_config)?; // Route every package in the plan to the local overlay registry // backed by `reg_config`, so the client used in `build_wit_dir` // resolves these packages against the local overlay instead of From 635171dc7e8ddc2acb1a3928be1e9e8497a69af2 Mon Sep 17 00:00:00 2001 From: Mikhail Katychev Date: Wed, 24 Jun 2026 15:58:29 -0500 Subject: [PATCH 33/61] cleanup even more --- crates/wasm-pkg-client/src/lib.rs | 5 ----- crates/wasm-pkg-client/src/local.rs | 2 ++ crates/wasm-pkg-common/src/package.rs | 2 ++ crates/wkg/src/main.rs | 2 +- 4 files changed, 5 insertions(+), 6 deletions(-) diff --git a/crates/wasm-pkg-client/src/lib.rs b/crates/wasm-pkg-client/src/lib.rs index 7795c04..492a13f 100644 --- a/crates/wasm-pkg-client/src/lib.rs +++ b/crates/wasm-pkg-client/src/lib.rs @@ -67,11 +67,6 @@ pub type ContentStream = Pin> + Send /// An alias for a PublishingSource (generally a file) pub type PublishingSource = Pin>; -// Convenience method to convert a byte array into a [`PublishingSource`] -pub fn source_from_slice<'a>(data: &'a [u8]) -> Pin> { - Box::pin(std::io::Cursor::new(data)) -} - /// A supertrait combining tokio's AsyncRead and AsyncSeek. pub trait ReaderSeeker: tokio::io::AsyncRead + tokio::io::AsyncSeek {} diff --git a/crates/wasm-pkg-client/src/local.rs b/crates/wasm-pkg-client/src/local.rs index cac05d2..4862a4f 100644 --- a/crates/wasm-pkg-client/src/local.rs +++ b/crates/wasm-pkg-client/src/local.rs @@ -35,6 +35,8 @@ pub struct LocalConfig { } impl LocalConfig { + // creates a [`Self`] using a generated temporary directory. Upon dropping the [`TempDir`] handle, + // the temporary directory will be deleted. pub fn temp_dir() -> Result<(Self, TempDir), Error> { let handle = TempDir::new()?; let root = handle.path().to_owned(); diff --git a/crates/wasm-pkg-common/src/package.rs b/crates/wasm-pkg-common/src/package.rs index 0f12bf5..adb08cc 100644 --- a/crates/wasm-pkg-common/src/package.rs +++ b/crates/wasm-pkg-common/src/package.rs @@ -86,10 +86,12 @@ pub struct PackageSpec { } impl PackageSpec { + // Returns a copy of the associated package reference. pub fn package(&self) -> PackageRef { self.package.clone() } + // Returns a copy of the attached semver [`Version`] object if it exists. pub fn version(&self) -> Option { self.version.clone() } diff --git a/crates/wkg/src/main.rs b/crates/wkg/src/main.rs index b533758..fab2366 100644 --- a/crates/wkg/src/main.rs +++ b/crates/wkg/src/main.rs @@ -11,7 +11,7 @@ use tracing::level_filters::LevelFilter; use wasm_pkg_client::{ caching::{CachingClient, FileCache}, local::LocalConfig, - Client, PackageLoader, PublishOpts, + Client, PublishOpts, }; use wasm_pkg_common::{ self, From 35a75761e87d0273b440ea3460334219fafd2696 Mon Sep 17 00:00:00 2001 From: Mikhail Katychev Date: Wed, 24 Jun 2026 16:10:56 -0500 Subject: [PATCH 34/61] cleaned up dependencies --- Cargo.toml | 11 ----------- crates/wasm-pkg-common/src/config.rs | 9 ++++----- 2 files changed, 4 insertions(+), 16 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 8354838..a73991f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -44,17 +44,6 @@ wasm-pkg-core = { version = "0.15.1", path = "crates/wasm-pkg-core" } wasm-metadata = "0.244" wit-component = "0.244" wit-parser = "0.244" -async-trait = "0.1.77" -clap = "4.5" -http = "1.1.0" -indexmap = "2.5" -reqwest = { version = "0.12.0", default-features = false } -rstest = "0.23" -secrecy = "0.8" -url = "2.5.0" -warg-client = "0.9.2" -warg-crypto = "0.9.2" -warg-protocol = "0.9.2" # https://github.com/crate-ci/typos/blob/master/docs/reference.md [workspace.metadata.typos.default] diff --git a/crates/wasm-pkg-common/src/config.rs b/crates/wasm-pkg-common/src/config.rs index 652eaae..cefd9ef 100644 --- a/crates/wasm-pkg-common/src/config.rs +++ b/crates/wasm-pkg-common/src/config.rs @@ -206,8 +206,6 @@ impl Config { /// - Hard-coded fallbacks for certain well-known namespaces pub fn resolve_registry(&self, package: &PackageRef) -> Option<&Registry> { let namespace = package.namespace(); - // look in `self.package_registry_overrides ` - // then in `self.namespace_registries` if let Some(reg) = self.resolve_mapping(package).map(|ns| ns.registry()) { return Some(reg); } else if let Some(reg) = self.default_registry.as_ref() { @@ -219,7 +217,7 @@ impl Config { pub fn resolve_mapping(&self, package: &PackageRef) -> Option<&RegistryMapping> { let namespace = package.namespace(); - // look in `self.package_registry_overrides ` + // look in `self.package_registry_overrides` // then in `self.namespace_registries` self.package_registry_overrides .get(package) @@ -294,12 +292,13 @@ pub struct RegistryConfig { } impl RegistryConfig { + // Created a [`Self`] a default local backend associated with the provided `backend_name` label. pub fn with_default_backend( mut self, - default_backend: &str, + backend_name: &str, backend_config: T, ) -> Result { - self.default_backend = Some(default_backend.to_string()); + self.default_backend = Some(backend_name.to_string()); self.set_backend_config(LOCAL_PROTOCOL, backend_config)?; Ok(self) } From 3dbff6ed5e4e1d2565a7d837a36914abff74bebb Mon Sep 17 00:00:00 2001 From: Mikhail Katychev Date: Wed, 24 Jun 2026 16:18:39 -0500 Subject: [PATCH 35/61] cargo clippy fixes --- crates/wasm-pkg-client/src/lib.rs | 2 ++ crates/wasm-pkg-common/src/package.rs | 2 +- crates/wasm-pkg-common/src/registry.rs | 1 + crates/wasm-pkg-core/src/resolver.rs | 19 ++++++++----------- crates/wasm-pkg-core/src/wit.rs | 13 +++++-------- crates/wkg/src/main.rs | 2 +- 6 files changed, 18 insertions(+), 21 deletions(-) diff --git a/crates/wasm-pkg-client/src/lib.rs b/crates/wasm-pkg-client/src/lib.rs index 492a13f..0f50364 100644 --- a/crates/wasm-pkg-client/src/lib.rs +++ b/crates/wasm-pkg-client/src/lib.rs @@ -1,3 +1,5 @@ +//! Wasm Package Client +//! //! [`Client`] implements a unified interface for loading package content from //! multiple kinds of package registries. //! diff --git a/crates/wasm-pkg-common/src/package.rs b/crates/wasm-pkg-common/src/package.rs index adb08cc..8d09f9c 100644 --- a/crates/wasm-pkg-common/src/package.rs +++ b/crates/wasm-pkg-common/src/package.rs @@ -99,7 +99,7 @@ impl PackageSpec { impl PartialEq for PackageSpec { fn eq(&self, other: &str) -> bool { - &format!("{self}") == other + format!("{self}") == other } } diff --git a/crates/wasm-pkg-common/src/registry.rs b/crates/wasm-pkg-common/src/registry.rs index d059f6d..5147e2f 100644 --- a/crates/wasm-pkg-common/src/registry.rs +++ b/crates/wasm-pkg-common/src/registry.rs @@ -57,4 +57,5 @@ impl TryFrom for Registry { } } +/// Graph of publishable packages with the [`petgraph::Direction`] edges describing the dependency direction. pub type DependencyGraph = Acyclic>; diff --git a/crates/wasm-pkg-core/src/resolver.rs b/crates/wasm-pkg-core/src/resolver.rs index aa085b9..c4091ac 100644 --- a/crates/wasm-pkg-core/src/resolver.rs +++ b/crates/wasm-pkg-core/src/resolver.rs @@ -12,10 +12,7 @@ use std::{ use anyhow::{bail, Context, Result}; use futures_util::TryStreamExt; use indexmap::{IndexMap, IndexSet}; -use petgraph::{ - graph::NodeIndex, - Direction, -}; +use petgraph::{graph::NodeIndex, Direction}; use semver::{Comparator, Op, Version, VersionReq}; use tokio::io::{AsyncRead, AsyncReadExt}; use wasm_pkg_client::{ @@ -843,11 +840,13 @@ fn visit<'a>( Ok(()) } +/// Mapping of [`PackageRef`]s to the respective index inside the dependency graph. +pub type LocalPackageIndex = HashMap; + /// State for tracking dependencies during upload. pub struct PublishPlan { /// Graph of publishable packages where the edges are `(dependency -DependencyOf->) dependent)` dependents: DependencyGraph, - /// Mapping [`PackageRef`]s to the respective index inside the dependency graph. // TODO look at using cargo's `InternedString` type for `PackageRef`: // https://docs.rs/cargo/latest/cargo/util/interning/struct.InternedString.html indices: HashMap, @@ -975,19 +974,17 @@ pub fn package_list<'a>( #[cfg(test)] mod tests { - + use std::path::PathBuf; use super::*; use glob::glob; - fn transitive_local_paths() -> Vec { let fixtures = PathBuf::from(env!("CARGO_MANIFEST_DIR")) .join("tests/fixtures/transitive-local/*/wit/"); glob(fixtures.to_str().unwrap()) .unwrap() - .into_iter() .map(|p| p.expect("glob path error")) .collect() } @@ -1013,21 +1010,21 @@ mod tests { ready_for_publish.iter().collect::>(), ["example-c:baz@0.1.0", "example-d:foo@0.1.0"], ); - plan.mark_confirmed(ready_for_publish.into_iter()); + plan.mark_confirmed(ready_for_publish); ready_for_publish = plan.take_ready(); assert_eq!( ready_for_publish.iter().collect::>(), ["example-b:bar@0.1.0"], ); - plan.mark_confirmed(ready_for_publish.into_iter()); + plan.mark_confirmed(ready_for_publish); ready_for_publish = plan.take_ready(); assert_eq!( ready_for_publish.iter().collect::>(), ["example-a:foo@0.1.0"], ); - plan.mark_confirmed(ready_for_publish.into_iter()); + plan.mark_confirmed(ready_for_publish); assert!(plan.is_empty()); } diff --git a/crates/wasm-pkg-core/src/wit.rs b/crates/wasm-pkg-core/src/wit.rs index 23591f4..3c09185 100644 --- a/crates/wasm-pkg-core/src/wit.rs +++ b/crates/wasm-pkg-core/src/wit.rs @@ -2,12 +2,12 @@ use std::{ collections::{HashMap, HashSet}, - path::{Path, PathBuf}, + path::Path, str::FromStr, }; use anyhow::{Context, Result}; -use petgraph::{data::Build, graph::NodeIndex, Direction}; +use petgraph::{data::Build, Direction}; use semver::{Version, VersionReq}; use wasm_metadata::{AddMetadata, AddMetadataField}; use wasm_pkg_client::{ @@ -23,7 +23,7 @@ use crate::{ lock::LockFile, resolver::{ DecodedDependency, Dependency, DependencyResolution, DependencyResolutionMap, - DependencyResolver, LocalResolution, RegistryPackage, + DependencyResolver, LocalPackageIndex, LocalResolution, RegistryPackage, }, }; @@ -179,12 +179,9 @@ pub fn get_packages( pub fn get_local_dependencies( paths: &[impl AsRef], -) -> Result<( - DependencyGraph, - HashMap, -)> { +) -> Result<(DependencyGraph, LocalPackageIndex)> { let pkg_trees = paths - .into_iter() + .iter() .map(|path| get_packages(path).map(|(pkg, deps)| ((pkg, path), deps))) .collect::, _>>()?; let mut graph = DependencyGraph::new(); diff --git a/crates/wkg/src/main.rs b/crates/wkg/src/main.rs index fab2366..d11da78 100644 --- a/crates/wkg/src/main.rs +++ b/crates/wkg/src/main.rs @@ -306,7 +306,7 @@ impl PublishArgs { } else { let mut file = tokio::fs::OpenOptions::new().read(true).open(path).await?; let mut buf = Vec::new(); - file.read(&mut buf).await?; + file.read_exact(&mut buf).await?; buf }; From 9a4669aca7550c584c0c8fef3a9e4666a054adc9 Mon Sep 17 00:00:00 2001 From: Mikhail Katychev Date: Wed, 24 Jun 2026 16:18:45 -0500 Subject: [PATCH 36/61] fmt --- crates/wasm-pkg-client/src/publisher.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/crates/wasm-pkg-client/src/publisher.rs b/crates/wasm-pkg-client/src/publisher.rs index 7bc395e..2d5e3d4 100644 --- a/crates/wasm-pkg-client/src/publisher.rs +++ b/crates/wasm-pkg-client/src/publisher.rs @@ -1,5 +1,3 @@ - - use crate::{PackageRef, PublishingSource, Version}; #[async_trait::async_trait] From 16a923d5c0b387673286b5bef5e7357a2c7f80e8 Mon Sep 17 00:00:00 2001 From: Mikhail Katychev Date: Wed, 24 Jun 2026 16:35:03 -0500 Subject: [PATCH 37/61] add anstream logic --- crates/wasm-pkg-client/src/local.rs | 2 +- crates/wasm-pkg-core/src/resolver.rs | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/crates/wasm-pkg-client/src/local.rs b/crates/wasm-pkg-client/src/local.rs index 4862a4f..ce16d40 100644 --- a/crates/wasm-pkg-client/src/local.rs +++ b/crates/wasm-pkg-client/src/local.rs @@ -154,7 +154,7 @@ impl PackagePublisher for LocalBackend { let mut out = tokio::fs::File::create(&path) .await .map_err(|e| registry_path_context(e, &path))?; - tracing::info!("publishing to {}", path.display()); + tracing::debug!("publishing to {}", path.display()); tokio::io::copy(&mut data, &mut out) .await .map_err(Error::IoError) diff --git a/crates/wasm-pkg-core/src/resolver.rs b/crates/wasm-pkg-core/src/resolver.rs index c4091ac..47e9c11 100644 --- a/crates/wasm-pkg-core/src/resolver.rs +++ b/crates/wasm-pkg-core/src/resolver.rs @@ -919,6 +919,7 @@ impl PublishPlan { impl std::fmt::Display for PublishPlan { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + // TODO(mkatychev): handle with anstyle and anstream, passing in `anstream::AutoStream` for colour choice for id in self.dependents.nodes_iter() { let dep = &self.dependents[id]; // initial dependency graph visualization @@ -942,7 +943,7 @@ impl std::fmt::Display for PublishPlan { "╰─" }; - writeln!(f, "{separator}(DependencyOf)─▶ {pkg}")?; + writeln!(f, "{separator}─▶ {pkg}")?; } } Ok(()) From 0f569bb10298f91dfa42b207e08deeb38a31c777 Mon Sep 17 00:00:00 2001 From: Mikhail Katychev Date: Wed, 24 Jun 2026 16:46:32 -0500 Subject: [PATCH 38/61] fixup doc comments --- Cargo.lock | 1 + crates/wasm-pkg-client/Cargo.toml | 1 + crates/wasm-pkg-client/src/local.rs | 4 ++-- crates/wasm-pkg-common/src/config.rs | 2 +- crates/wasm-pkg-core/src/resolver.rs | 4 ++-- crates/wasm-pkg-core/src/wit.rs | 6 ++++++ 6 files changed, 13 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a0ce0f2..6a6caf9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5162,6 +5162,7 @@ dependencies = [ "warg-protocol", "wasm-metadata", "wasm-pkg-common", + "wasm-pkg-core", "wit-component", ] diff --git a/crates/wasm-pkg-client/Cargo.toml b/crates/wasm-pkg-client/Cargo.toml index d10fc69..e86a123 100644 --- a/crates/wasm-pkg-client/Cargo.toml +++ b/crates/wasm-pkg-client/Cargo.toml @@ -53,3 +53,4 @@ tempfile = { workspace = true } [dev-dependencies] rcgen = { workspace = true } testcontainers = { workspace = true } +wasm-pkg-core = { workspace = true } diff --git a/crates/wasm-pkg-client/src/local.rs b/crates/wasm-pkg-client/src/local.rs index ce16d40..2ef3cfd 100644 --- a/crates/wasm-pkg-client/src/local.rs +++ b/crates/wasm-pkg-client/src/local.rs @@ -35,8 +35,8 @@ pub struct LocalConfig { } impl LocalConfig { - // creates a [`Self`] using a generated temporary directory. Upon dropping the [`TempDir`] handle, - // the temporary directory will be deleted. + /// Creates a [`Self`] using a generated temporary directory. Upon dropping the [`TempDir`] handle, + /// the temporary directory will be deleted. pub fn temp_dir() -> Result<(Self, TempDir), Error> { let handle = TempDir::new()?; let root = handle.path().to_owned(); diff --git a/crates/wasm-pkg-common/src/config.rs b/crates/wasm-pkg-common/src/config.rs index cefd9ef..94c53ca 100644 --- a/crates/wasm-pkg-common/src/config.rs +++ b/crates/wasm-pkg-common/src/config.rs @@ -55,7 +55,7 @@ impl RegistryMapping { } } - /// returns the inner [`RegistryMetadata`] if `Self` holds a [`CustomConfig`] + /// Returns the inner [`RegistryMetadata`] if `Self` holds a [`CustomConfig`]. pub fn metadata(&self) -> Option<&RegistryMetadata> { if let Self::Custom(config) = self { return Some(&config.metadata); diff --git a/crates/wasm-pkg-core/src/resolver.rs b/crates/wasm-pkg-core/src/resolver.rs index 47e9c11..0e2e691 100644 --- a/crates/wasm-pkg-core/src/resolver.rs +++ b/crates/wasm-pkg-core/src/resolver.rs @@ -849,7 +849,7 @@ pub struct PublishPlan { dependents: DependencyGraph, // TODO look at using cargo's `InternedString` type for `PackageRef`: // https://docs.rs/cargo/latest/cargo/util/interning/struct.InternedString.html - indices: HashMap, + indices: LocalPackageIndex, } impl PublishPlan { @@ -898,7 +898,7 @@ impl PublishPlan { .collect() } - /// NOTE + /// Return the path associated with a local package. pub fn get_path(&self, pkg: &PackageRef) -> Option<&Path> { self.indices.get(pkg).map(|(_, p)| p.as_ref()) } diff --git a/crates/wasm-pkg-core/src/wit.rs b/crates/wasm-pkg-core/src/wit.rs index 3c09185..fb1ec93 100644 --- a/crates/wasm-pkg-core/src/wit.rs +++ b/crates/wasm-pkg-core/src/wit.rs @@ -177,6 +177,12 @@ pub fn get_packages( Ok((package, packages)) } +/// Build an acyclic [`DependencyGraph`] for the provided WIT package paths alongside a [`LocalPackageIndex`]. +/// +/// # Errors +/// +/// The function will return an error if there are duplicate references to the same package or if +/// there are cyclical dependencies between packages. pub fn get_local_dependencies( paths: &[impl AsRef], ) -> Result<(DependencyGraph, LocalPackageIndex)> { From 864d69318b1b96867eb228db286d075d8353e413 Mon Sep 17 00:00:00 2001 From: Mikhail Katychev Date: Wed, 24 Jun 2026 16:57:05 -0500 Subject: [PATCH 39/61] revert directory use --- Cargo.lock | 1 - crates/wasm-pkg-client/Cargo.toml | 1 - 2 files changed, 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6a6caf9..a0ce0f2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5162,7 +5162,6 @@ dependencies = [ "warg-protocol", "wasm-metadata", "wasm-pkg-common", - "wasm-pkg-core", "wit-component", ] diff --git a/crates/wasm-pkg-client/Cargo.toml b/crates/wasm-pkg-client/Cargo.toml index e86a123..d10fc69 100644 --- a/crates/wasm-pkg-client/Cargo.toml +++ b/crates/wasm-pkg-client/Cargo.toml @@ -53,4 +53,3 @@ tempfile = { workspace = true } [dev-dependencies] rcgen = { workspace = true } testcontainers = { workspace = true } -wasm-pkg-core = { workspace = true } From 40d2dd3f84e0fb46661feb965d2269b2080fb90a Mon Sep 17 00:00:00 2001 From: Mikhail Katychev Date: Wed, 24 Jun 2026 17:44:04 -0500 Subject: [PATCH 40/61] added e2e test --- crates/wkg/tests/common.rs | 6 ++-- crates/wkg/tests/e2e.rs | 67 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 71 insertions(+), 2 deletions(-) diff --git a/crates/wkg/tests/common.rs b/crates/wkg/tests/common.rs index 8ae6b49..ef222d2 100644 --- a/crates/wkg/tests/common.rs +++ b/crates/wkg/tests/common.rs @@ -135,8 +135,10 @@ pub async fn load_fixture(fixture: &str) -> Fixture { } } -#[allow(dead_code)] -async fn copy_dir(source: impl AsRef, destination: impl AsRef) -> anyhow::Result<()> { +pub async fn copy_dir( + source: impl AsRef, + destination: impl AsRef, +) -> anyhow::Result<()> { tokio::fs::create_dir_all(&destination).await?; let mut entries = tokio::fs::read_dir(source).await?; while let Some(entry) = entries.next_entry().await? { diff --git a/crates/wkg/tests/e2e.rs b/crates/wkg/tests/e2e.rs index 658027e..14c1bb8 100644 --- a/crates/wkg/tests/e2e.rs +++ b/crates/wkg/tests/e2e.rs @@ -1,3 +1,7 @@ +use wasm_pkg_client::{Version, VersionInfo}; + +use crate::common::copy_dir; + mod common; #[cfg(feature = "docker-tests")] @@ -85,6 +89,69 @@ async fn build_and_publish_with_metadata() { ); } +#[cfg(feature = "docker-tests")] +#[tokio::test] +async fn publish_multiple_transitive_local_packages() { + use std::path::PathBuf; + + let (config, registry, _container) = common::start_registry().await; + let namespaces = ["example-a", "example-b", "example-c", "example-d"]; + + // copy the transitive-local fixtures from wasm-pkg-core into a temp dir + let temp_dir = tempfile::tempdir().expect("Failed to create tempdir"); + let src_root = PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("..") + .join("wasm-pkg-core") + .join("tests") + .join("fixtures") + .join("transitive-local"); + let fixture_root = temp_dir.path().join("transitive-local"); + copy_dir(&src_root, &fixture_root).await.unwrap(); + + let mut mapped = config.clone(); + for ns in namespaces { + mapped = common::map_namespace(&mapped, ns, ®istry); + } + let config_path = temp_dir.path().join("config.toml"); + mapped.to_file(&config_path).await.expect("write config"); + + // pass all fixture dirs to a single `wkg publish` invocation + // TODO use glob suchas in `wasm_pkgs_core::resolver::tests::transitive_local_paths` + let dirs = namespaces.map(|name| fixture_root.join(name).join("wit")); + let mut publish = tokio::process::Command::new(env!("CARGO_BIN_EXE_wkg")); + publish + .current_dir(temp_dir.path()) + .env("WKG_CACHE_DIR", temp_dir.path().join("cache")) + .env("WKG_CONFIG_FILE", &config_path) + .arg("publish"); + for dir in &dirs { + publish.arg(dir); + } + let status = publish.status().await.expect("spawn wkg publish"); + assert!(status.success(), "wkg publish should succeed"); + + let client = wasm_pkg_client::Client::new(mapped); + + let expected_version = "0.1.0".parse::().unwrap(); + for name in [ + "example-a:foo", + "example-b:bar", + "example-c:baz", + "example-d:foo", + ] { + let pkg = name.parse().unwrap(); + let versions = client + .list_all_versions(&pkg) + .await + .unwrap_or_else(|e| panic!("list versions for {name}: {e:#}")); + std::assert_matches!( + &versions[..], + [VersionInfo { version, .. }] if version == &expected_version, + "{name} should have exactly one published version", + ); + } +} + #[tokio::test] pub async fn check() { let fixture = common::load_fixture("wasi-http").await; From 6466ce50aa2bce09e0ccd226adb3820ffb0c11fc Mon Sep 17 00:00:00 2001 From: Mikhail Katychev Date: Wed, 24 Jun 2026 18:04:31 -0500 Subject: [PATCH 41/61] added PublishPlan::take_ready usage --- crates/wasm-pkg-core/src/resolver.rs | 7 +++- crates/wkg/src/main.rs | 48 +++++++++++++++++----------- 2 files changed, 35 insertions(+), 20 deletions(-) diff --git a/crates/wasm-pkg-core/src/resolver.rs b/crates/wasm-pkg-core/src/resolver.rs index 0e2e691..000050e 100644 --- a/crates/wasm-pkg-core/src/resolver.rs +++ b/crates/wasm-pkg-core/src/resolver.rs @@ -884,7 +884,7 @@ impl PublishPlan { /// Returns the set of packages that are ready for publishing (i.e. have no outstanding dependencies). /// /// These will not be returned in future calls. - pub fn take_ready(&mut self) -> BTreeSet { + pub fn take_ready(&self) -> BTreeSet { self.dependents .nodes_iter() // there are no dependents on `self.dendents[id]` @@ -903,6 +903,11 @@ impl PublishPlan { self.indices.get(pkg).map(|(_, p)| p.as_ref()) } + /// Return the [`NodeIndex`] associated with a local package. + pub fn get_node_index(&self, pkg: &PackageRef) -> Option { + self.indices.get(pkg).map(|(id, _)| *id) + } + /// Packages confirmed to be available in the registry, potentially allowing additional /// packages to be "ready". pub fn mark_confirmed(&mut self, published: impl IntoIterator) { diff --git a/crates/wkg/src/main.rs b/crates/wkg/src/main.rs index d11da78..f4cb13d 100644 --- a/crates/wkg/src/main.rs +++ b/crates/wkg/src/main.rs @@ -1,9 +1,10 @@ use std::{ + collections::HashMap, io::{Cursor, Seek}, path::PathBuf, }; -use anyhow::{ensure, Context}; +use anyhow::{anyhow, ensure, Context}; use clap::{Args, Parser, Subcommand, ValueEnum}; use futures_util::TryStreamExt; use tokio::io::{AsyncReadExt, AsyncWriteExt}; @@ -129,8 +130,7 @@ impl ConfigArgs { let path = if let Some(path) = self.common.config { path } else { - Config::global_config_path() - .ok_or(anyhow::anyhow!("global config path not available"))? + Config::global_config_path().ok_or(anyhow!("global config path not available"))? }; // Check if the parent directory exists, if not create it @@ -152,7 +152,7 @@ impl ConfigArgs { } if self.edit { - let editor = std::env::var("EDITOR").or(Err(anyhow::anyhow!( + let editor = std::env::var("EDITOR").or(Err(anyhow!( "failed to read `$EDITOR` environment variable" )))?; @@ -175,7 +175,7 @@ impl ConfigArgs { let mut config = match tokio::fs::read_to_string(&path).await { Ok(contents) => Config::from_toml(&contents)?, Err(err) if err.kind() == std::io::ErrorKind::NotFound => Config::default(), - Err(err) => return Err(anyhow::anyhow!("error reading config file: {0}", err)), + Err(err) => return Err(anyhow!("error reading config file: {0}", err)), }; if let Some(default) = self.default_registry { @@ -276,7 +276,7 @@ impl PublishArgs { .get_or_insert_registry_config_mut(&local_registry) .merge(reg_config); - let plan = PublishPlan::from_paths(paths)?; + let mut plan = PublishPlan::from_paths(paths)?; println!("{plan}"); // TODO(mkatychev): Add support for `PackageLoader::get_release` to handle @@ -295,7 +295,7 @@ impl PublishArgs { let mut lock_file = LockFile::load(false).await?; // these are packages that have been successfully pushed to our "tmp_local_publish" - let mut validated_packages = Vec::new(); + let mut validated_packages = HashMap::new(); for spec in plan.iter() { let path = plan.get_path(&spec.package).unwrap(); let data = if path.is_dir() { @@ -324,21 +324,31 @@ impl PublishArgs { ) .await?; - validated_packages.push(data); + let id = plan + .get_node_index(&spec.package) + .expect("missing node index"); + validated_packages.insert(id, data); } let client = self.common.get_client().await?; - for data in validated_packages { - let source = Box::pin(Cursor::new(data)); - let (package, version) = client - .client()? - .publish_release_data(source, opts.clone()) - .await?; - if self.dry_run { - println!("Aborting publish due to dry run: {}@{}", package, version); - } else { - println!("Published {}@{}", package, version); + while !plan.is_empty() { + let ready_for_publish = plan.take_ready(); + for spec in &ready_for_publish { + let id = plan + .get_node_index(&spec.package) + .expect("missing node index"); + let source = Box::pin(Cursor::new(validated_packages[&id].clone())); + let (package, version) = client + .client()? + .publish_release_data(source, opts.clone()) + .await?; + if self.dry_run { + println!("Aborting publish due to dry run: {}@{}", package, version); + } else { + println!("Published {}@{}", package, version); + } } + plan.mark_confirmed(ready_for_publish); } return Ok(()); } @@ -357,7 +367,7 @@ impl PublishArgs { // There is no way to check if we are in a git repository unlike `cargo publish --allow-dirty` so // check against previous values. if lock_file != prev_lock_ref && !self.dry_run { - return Err(anyhow::anyhow!( + return Err(anyhow!( "wkg.lock would be updated during publish, aborting" )) .with_context(|| { From b68c1f1c501b110f021bd0110be68fc8186fe4d6 Mon Sep 17 00:00:00 2001 From: Mikhail Katychev Date: Wed, 24 Jun 2026 18:07:33 -0500 Subject: [PATCH 42/61] added take_ready comments --- crates/wkg/src/main.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/crates/wkg/src/main.rs b/crates/wkg/src/main.rs index f4cb13d..7e79564 100644 --- a/crates/wkg/src/main.rs +++ b/crates/wkg/src/main.rs @@ -296,6 +296,9 @@ impl PublishArgs { let mut lock_file = LockFile::load(false).await?; // these are packages that have been successfully pushed to our "tmp_local_publish" let mut validated_packages = HashMap::new(); + + // 1. Publish our packages to "tmp_local_publish" ensuring all dependencies are + // resolved by the local backend for spec in plan.iter() { let path = plan.get_path(&spec.package).unwrap(); let data = if path.is_dir() { @@ -331,6 +334,8 @@ impl PublishArgs { } let client = self.common.get_client().await?; + // 2. Publish our packages in "waves" to the actual registries ensuring all + // possible dependency free pacakges are published in the same group while !plan.is_empty() { let ready_for_publish = plan.take_ready(); for spec in &ready_for_publish { From 6b2178ba45c6c6d976e6945325611ad7a13f6aef Mon Sep 17 00:00:00 2001 From: Mikhail Katychev Date: Wed, 24 Jun 2026 22:14:26 -0500 Subject: [PATCH 43/61] Update crates/wkg/src/main.rs Co-authored-by: Victor Adossi <123968127+vados-cosmonic@users.noreply.github.com> --- crates/wkg/src/main.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/wkg/src/main.rs b/crates/wkg/src/main.rs index 7e79564..9c588ab 100644 --- a/crates/wkg/src/main.rs +++ b/crates/wkg/src/main.rs @@ -378,7 +378,7 @@ impl PublishArgs { .with_context(|| { format!( "Run `wkg wit build {}` before attempting to publish", - path.to_string_lossy() + path.display() ) }); } From 73467f58686967d3e78899ee26517b2ed67310a0 Mon Sep 17 00:00:00 2001 From: Mikhail Katychev Date: Wed, 24 Jun 2026 22:18:13 -0500 Subject: [PATCH 44/61] Update crates/wkg/src/main.rs Co-authored-by: Victor Adossi <123968127+vados-cosmonic@users.noreply.github.com> --- crates/wkg/src/main.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/wkg/src/main.rs b/crates/wkg/src/main.rs index 9c588ab..f9202fc 100644 --- a/crates/wkg/src/main.rs +++ b/crates/wkg/src/main.rs @@ -402,7 +402,7 @@ impl PublishArgs { Ok(()) } - fn opts(&self) -> anyhow::Result { + fn publish_opts(&self) -> anyhow::Result { let package = match self.package.clone() { Some(_) if self.paths.len() > 2 => { anyhow::bail!("`--package` is currently unsupported when providing more than one path argument"); From b7cb19a6f37b51955af873015229955538f18c12 Mon Sep 17 00:00:00 2001 From: Mikhail Katychev Date: Wed, 24 Jun 2026 22:19:56 -0500 Subject: [PATCH 45/61] brought rest in line of publish_opts --- crates/wkg/src/main.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/wkg/src/main.rs b/crates/wkg/src/main.rs index f9202fc..95a71ba 100644 --- a/crates/wkg/src/main.rs +++ b/crates/wkg/src/main.rs @@ -257,7 +257,7 @@ struct PublishArgs { impl PublishArgs { pub async fn run(self) -> anyhow::Result<()> { - let opts = self.opts()?; + let publish_opts = self.publish_opts()?; let path = match &self.paths[..] { [path] => path, paths => { @@ -345,7 +345,7 @@ impl PublishArgs { let source = Box::pin(Cursor::new(validated_packages[&id].clone())); let (package, version) = client .client()? - .publish_release_data(source, opts.clone()) + .publish_release_data(source, publish_opts.clone()) .await?; if self.dry_run { println!("Aborting publish due to dry run: {}@{}", package, version); @@ -392,7 +392,7 @@ impl PublishArgs { let (package, version) = client .client()? - .publish_release_file(&publish_path, opts) + .publish_release_file(&publish_path, publish_opts) .await?; if self.dry_run { println!("Aborting publish due to dry run: {}@{}", package, version); From ce353f6d02339f8b848a1c4667099581a6f2e93b Mon Sep 17 00:00:00 2001 From: Mikhail Katychev Date: Wed, 24 Jun 2026 23:07:06 -0500 Subject: [PATCH 46/61] typos --- crates/wasm-pkg-core/src/resolver.rs | 2 +- crates/wkg/src/main.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/wasm-pkg-core/src/resolver.rs b/crates/wasm-pkg-core/src/resolver.rs index 000050e..411e61b 100644 --- a/crates/wasm-pkg-core/src/resolver.rs +++ b/crates/wasm-pkg-core/src/resolver.rs @@ -887,7 +887,7 @@ impl PublishPlan { pub fn take_ready(&self) -> BTreeSet { self.dependents .nodes_iter() - // there are no dependents on `self.dendents[id]` + // there are no dependents on `self.dependents[id]` .filter(|id| { self.dependents .neighbors_directed(*id, Direction::Incoming) diff --git a/crates/wkg/src/main.rs b/crates/wkg/src/main.rs index 95a71ba..f931cd2 100644 --- a/crates/wkg/src/main.rs +++ b/crates/wkg/src/main.rs @@ -335,7 +335,7 @@ impl PublishArgs { let client = self.common.get_client().await?; // 2. Publish our packages in "waves" to the actual registries ensuring all - // possible dependency free pacakges are published in the same group + // possible dependency free packages are published in the same group while !plan.is_empty() { let ready_for_publish = plan.take_ready(); for spec in &ready_for_publish { From da54db88692968ac543d784672ded01bf4a57a65 Mon Sep 17 00:00:00 2001 From: Mikhail Katychev Date: Thu, 25 Jun 2026 10:58:36 -0500 Subject: [PATCH 47/61] moved DependencyGraph into core with pub(crate) --- Cargo.lock | 1 - crates/wasm-pkg-common/Cargo.toml | 1 - crates/wasm-pkg-common/src/registry.rs | 4 ---- crates/wasm-pkg-core/src/resolver.rs | 11 +++++++++-- crates/wasm-pkg-core/src/wit.rs | 7 ++++--- crates/wkg/tests/e2e.rs | 6 +----- 6 files changed, 14 insertions(+), 16 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a0ce0f2..29fc3be 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5174,7 +5174,6 @@ dependencies = [ "etcetera 0.11.0", "futures-util", "http", - "petgraph 0.8.3", "semver", "serde", "serde_json", diff --git a/crates/wasm-pkg-common/Cargo.toml b/crates/wasm-pkg-common/Cargo.toml index 9f7d0c4..104515a 100644 --- a/crates/wasm-pkg-common/Cargo.toml +++ b/crates/wasm-pkg-common/Cargo.toml @@ -31,7 +31,6 @@ tokio = { workspace = true, optional = true, features = ["fs"] } toml = { workspace = true, optional = true } thiserror = { workspace = true } tracing.workspace = true -petgraph.workspace = true [dev-dependencies] tokio = { workspace = true, features = ["macros", "rt"] } diff --git a/crates/wasm-pkg-common/src/registry.rs b/crates/wasm-pkg-common/src/registry.rs index 5147e2f..2fd42dc 100644 --- a/crates/wasm-pkg-common/src/registry.rs +++ b/crates/wasm-pkg-common/src/registry.rs @@ -1,5 +1,4 @@ use http::uri::Authority; -use petgraph::{acyclic::Acyclic, graph::DiGraph}; use serde::{Deserialize, Serialize}; use crate::Error; @@ -56,6 +55,3 @@ impl TryFrom for Registry { Ok(Self(value.try_into()?)) } } - -/// Graph of publishable packages with the [`petgraph::Direction`] edges describing the dependency direction. -pub type DependencyGraph = Acyclic>; diff --git a/crates/wasm-pkg-core/src/resolver.rs b/crates/wasm-pkg-core/src/resolver.rs index 411e61b..6b82a2d 100644 --- a/crates/wasm-pkg-core/src/resolver.rs +++ b/crates/wasm-pkg-core/src/resolver.rs @@ -12,14 +12,18 @@ use std::{ use anyhow::{bail, Context, Result}; use futures_util::TryStreamExt; use indexmap::{IndexMap, IndexSet}; -use petgraph::{graph::NodeIndex, Direction}; +use petgraph::{ + acyclic::Acyclic, + graph::{DiGraph, NodeIndex}, + Direction, +}; use semver::{Comparator, Op, Version, VersionReq}; use tokio::io::{AsyncRead, AsyncReadExt}; use wasm_pkg_client::{ caching::{CachingClient, FileCache}, Client, Config, ContentDigest, Error as WasmPkgError, PackageRef, Release, VersionInfo, }; -use wasm_pkg_common::{package::PackageSpec, registry::DependencyGraph}; +use wasm_pkg_common::package::PackageSpec; use wit_component::DecodedWasm; use wit_parser::{PackageId, PackageName, Resolve, UnresolvedPackageGroup, WorldId}; @@ -840,6 +844,9 @@ fn visit<'a>( Ok(()) } +/// Graph of publishable packages with the [`petgraph::Direction`] edges describing the dependency direction. +pub(crate) type DependencyGraph = Acyclic>; + /// Mapping of [`PackageRef`]s to the respective index inside the dependency graph. pub type LocalPackageIndex = HashMap; diff --git a/crates/wasm-pkg-core/src/wit.rs b/crates/wasm-pkg-core/src/wit.rs index fb1ec93..a236223 100644 --- a/crates/wasm-pkg-core/src/wit.rs +++ b/crates/wasm-pkg-core/src/wit.rs @@ -14,7 +14,7 @@ use wasm_pkg_client::{ caching::{CachingClient, FileCache}, PackageRef, }; -use wasm_pkg_common::{package::PackageSpec, registry::DependencyGraph}; +use wasm_pkg_common::package::PackageSpec; use wit_component::WitPrinter; use wit_parser::{PackageId, PackageName, Resolve}; @@ -22,8 +22,9 @@ use crate::{ config::Config, lock::LockFile, resolver::{ - DecodedDependency, Dependency, DependencyResolution, DependencyResolutionMap, - DependencyResolver, LocalPackageIndex, LocalResolution, RegistryPackage, + DecodedDependency, Dependency, DependencyGraph, DependencyResolution, + DependencyResolutionMap, DependencyResolver, LocalPackageIndex, LocalResolution, + RegistryPackage, }, }; diff --git a/crates/wkg/tests/e2e.rs b/crates/wkg/tests/e2e.rs index 14c1bb8..9b83757 100644 --- a/crates/wkg/tests/e2e.rs +++ b/crates/wkg/tests/e2e.rs @@ -100,11 +100,7 @@ async fn publish_multiple_transitive_local_packages() { // copy the transitive-local fixtures from wasm-pkg-core into a temp dir let temp_dir = tempfile::tempdir().expect("Failed to create tempdir"); let src_root = PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("..") - .join("wasm-pkg-core") - .join("tests") - .join("fixtures") - .join("transitive-local"); + .join("../wasm-pkg-core/tests/fixtures/transitive-local"); let fixture_root = temp_dir.path().join("transitive-local"); copy_dir(&src_root, &fixture_root).await.unwrap(); From c26b6bfdc351769427b21c12e06a9fc72779acdc Mon Sep 17 00:00:00 2001 From: Mikhail Katychev Date: Thu, 25 Jun 2026 11:08:53 -0500 Subject: [PATCH 48/61] removed PackageRef methods --- crates/wasm-pkg-client/src/local.rs | 2 +- crates/wasm-pkg-common/src/package.rs | 12 ------------ crates/wasm-pkg-core/src/wit.rs | 2 +- 3 files changed, 2 insertions(+), 14 deletions(-) diff --git a/crates/wasm-pkg-client/src/local.rs b/crates/wasm-pkg-client/src/local.rs index 2ef3cfd..1da08a2 100644 --- a/crates/wasm-pkg-client/src/local.rs +++ b/crates/wasm-pkg-client/src/local.rs @@ -154,7 +154,7 @@ impl PackagePublisher for LocalBackend { let mut out = tokio::fs::File::create(&path) .await .map_err(|e| registry_path_context(e, &path))?; - tracing::debug!("publishing to {}", path.display()); + tracing::info!("publishing to {}", path.display()); tokio::io::copy(&mut data, &mut out) .await .map_err(Error::IoError) diff --git a/crates/wasm-pkg-common/src/package.rs b/crates/wasm-pkg-common/src/package.rs index 8d09f9c..456ba54 100644 --- a/crates/wasm-pkg-common/src/package.rs +++ b/crates/wasm-pkg-common/src/package.rs @@ -85,18 +85,6 @@ pub struct PackageSpec { pub version: Option, } -impl PackageSpec { - // Returns a copy of the associated package reference. - pub fn package(&self) -> PackageRef { - self.package.clone() - } - - // Returns a copy of the attached semver [`Version`] object if it exists. - pub fn version(&self) -> Option { - self.version.clone() - } -} - impl PartialEq for PackageSpec { fn eq(&self, other: &str) -> bool { format!("{self}") == other diff --git a/crates/wasm-pkg-core/src/wit.rs b/crates/wasm-pkg-core/src/wit.rs index a236223..9c1e7ec 100644 --- a/crates/wasm-pkg-core/src/wit.rs +++ b/crates/wasm-pkg-core/src/wit.rs @@ -197,7 +197,7 @@ pub fn get_local_dependencies( for ((spec, path), _) in &pkg_trees { let id = graph.add_node(spec.clone()); if indices - .insert(spec.package(), (id, path.as_ref().to_owned())) + .insert(spec.package.clone(), (id, path.as_ref().to_owned())) .is_some() { anyhow::bail!("duplicate references to package detected: {spec}"); From 8555d1bbee9ac20290b5516437f35346add95be0 Mon Sep 17 00:00:00 2001 From: Mikhail Katychev Date: Thu, 25 Jun 2026 11:34:01 -0500 Subject: [PATCH 49/61] improved messaging for invalid dependency edges during try_update_edge --- crates/wasm-pkg-core/src/wit.rs | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/crates/wasm-pkg-core/src/wit.rs b/crates/wasm-pkg-core/src/wit.rs index 9c1e7ec..882a060 100644 --- a/crates/wasm-pkg-core/src/wit.rs +++ b/crates/wasm-pkg-core/src/wit.rs @@ -207,18 +207,25 @@ pub fn get_local_dependencies( // TODO handle version matching for dependencies for (dep, _version) in deps { if let Some(&(dep, _)) = indices.get(&dep) { - let (pkg, _) = indices[&spec.package]; + let pkg = &spec.package; + let (id, _) = indices[pkg]; // // pkg <=DependsOn= dep graph - .try_update_edge(pkg, dep, Direction::Incoming) - .map_err(|e| match e { - petgraph::acyclic::AcyclicEdgeError::Cycle(cycle) => { - anyhow::anyhow!("cyclical dependency detected") - .context(format!("package {}", graph[cycle.node_id()])) - .context(format!("package {}", graph[pkg])) + .try_update_edge(id, dep, Direction::Incoming) + .map_err(|e| { + match e { + petgraph::acyclic::AcyclicEdgeError::Cycle(cycle) => { + anyhow::anyhow!("cyclical dependency detected") + .context(format!("other package: {}", graph[cycle.node_id()])) + } + petgraph::acyclic::AcyclicEdgeError::SelfLoop => { + anyhow::anyhow!("Package is declaring self as a dependency.") + } + petgraph::acyclic::AcyclicEdgeError::InvalidEdge => anyhow::anyhow!( + "Could not successfully add the edge to the underlying graph." + ), } - petgraph::acyclic::AcyclicEdgeError::SelfLoop => todo!(), - petgraph::acyclic::AcyclicEdgeError::InvalidEdge => todo!(), + .context(format!("package: {pkg}")) })?; } } From d5a625000970276ea16de7594a92b101c03f8fe4 Mon Sep 17 00:00:00 2001 From: Mikhail Katychev Date: Thu, 25 Jun 2026 15:17:50 -0500 Subject: [PATCH 50/61] added setup for nested publishing and better errors --- crates/wasm-pkg-client/src/lib.rs | 11 +++-- crates/wasm-pkg-core/src/config.rs | 16 +++++++ crates/wasm-pkg-core/src/resolver.rs | 42 ++++++++++++++++--- crates/wasm-pkg-core/src/wit.rs | 12 +++++- .../example-c/wit/nested/nested.wit | 7 ++++ .../transitive-local/example-c/wit/world.wit | 4 +- 6 files changed, 79 insertions(+), 13 deletions(-) create mode 100644 crates/wasm-pkg-core/tests/fixtures/transitive-local/example-c/wit/nested/nested.wit diff --git a/crates/wasm-pkg-client/src/lib.rs b/crates/wasm-pkg-client/src/lib.rs index 0f50364..3c78cfe 100644 --- a/crates/wasm-pkg-client/src/lib.rs +++ b/crates/wasm-pkg-client/src/lib.rs @@ -39,7 +39,7 @@ use std::path::Path; use std::sync::Arc; use std::{collections::HashMap, pin::Pin}; -use anyhow::anyhow; +use anyhow::{anyhow, Context}; use bytes::Bytes; use futures_util::Stream; use tokio::io::AsyncSeekExt; @@ -353,8 +353,9 @@ fn resolve_package( })?; let version = version.ok_or_else(|| { - crate::Error::InvalidComponent(anyhow::anyhow!( - "component package version not found in the Wasm binary\n\ + crate::Error::InvalidComponent( + anyhow::anyhow!( + "component package version not found in the Wasm binary\n\ \n\ The Wasm file was built without a version in the WIT `package` statement.\n\ Add a version to the `package` statement in your .wit file, e.g.:\n\ @@ -364,7 +365,9 @@ fn resolve_package( Alternatively, specify the package and version explicitly with the --package flag:\n\ \n\ \twkg publish --package :@" - )) + ) + .context(format!("package: {package}")), + ) })?; Ok((data.into_inner(), package, version)) } diff --git a/crates/wasm-pkg-core/src/config.rs b/crates/wasm-pkg-core/src/config.rs index bfc8ac3..d4e86ca 100644 --- a/crates/wasm-pkg-core/src/config.rs +++ b/crates/wasm-pkg-core/src/config.rs @@ -58,6 +58,22 @@ impl Config { .await .context("unable to write config to path") } + + /// Returns a matching override name and value for the input path + pub(crate) fn has_override(&self, path: impl AsRef) -> bool { + let path = path.as_ref().canonicalize().ok(); + self.overrides + .iter() + .map(|map| map.iter()) + .flatten() + .filter(|(_, o)| { + dbg!(o.path.as_deref()); + dbg!(path.as_ref()); + o.path.as_ref().map(|p| p.canonicalize().ok()).flatten() == path + }) + .next() + .is_some() + } } #[derive(Debug, Clone, Serialize, Deserialize, Default, PartialEq, Eq)] diff --git a/crates/wasm-pkg-core/src/resolver.rs b/crates/wasm-pkg-core/src/resolver.rs index 6b82a2d..9fb61f8 100644 --- a/crates/wasm-pkg-core/src/resolver.rs +++ b/crates/wasm-pkg-core/src/resolver.rs @@ -863,7 +863,25 @@ impl PublishPlan { /// Generate [`Self`] from a list of WIT package paths (files or directories). pub fn from_paths(paths: &[impl AsRef]) -> Result { let (graph, indices) = get_local_dependencies(paths)?; + { + // collection of local packages that have no version + let missing_version = graph + .nodes_iter() + .map(|f| graph[f].clone()) + .filter(|pkg| pkg.version.is_none()) + .collect::>(); + if !missing_version.is_empty() { + return Err(anyhow::anyhow!( + "Unable to publish packages without a version specified" + ) + .context(format!( + "packages: {}", + package_list(missing_version.iter(), None) + ))); + } + } let mut dependents = graph.into_inner(); + dependents.reverse(); // graph was already found to be acyclic let dependents = DependencyGraph::try_from(dependents).unwrap(); @@ -994,12 +1012,17 @@ mod tests { use glob::glob; fn transitive_local_paths() -> Vec { - let fixtures = PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("tests/fixtures/transitive-local/*/wit/"); - glob(fixtures.to_str().unwrap()) + let fixtures_root = + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/transitive-local"); + // pass all fixture dirs to a single `wkg publish` invocation + let mut paths: Vec = glob(fixtures_root.join("**/*.wit").to_str().unwrap()) .unwrap() .map(|p| p.expect("glob path error")) - .collect() + .map(|p| p.parent().unwrap().to_path_buf()) + .collect(); + paths.sort(); + paths.dedup(); + paths } #[test] @@ -1008,7 +1031,7 @@ mod tests { let plan = PublishPlan::from_paths(&paths).unwrap(); assert_eq!( plan.iter().count(), - 4, + 5, "unexpected package count\npackages found: {}", package_list(plan.iter(), None) ); @@ -1021,7 +1044,14 @@ mod tests { let mut ready_for_publish = plan.take_ready(); assert_eq!( ready_for_publish.iter().collect::>(), - ["example-c:baz@0.1.0", "example-d:foo@0.1.0"], + ["example-c:nested@0.1.0", "example-d:foo@0.1.0",], + ); + plan.mark_confirmed(ready_for_publish); + + ready_for_publish = plan.take_ready(); + assert_eq!( + ready_for_publish.iter().collect::>(), + ["example-c:baz@0.1.0"], ); plan.mark_confirmed(ready_for_publish); diff --git a/crates/wasm-pkg-core/src/wit.rs b/crates/wasm-pkg-core/src/wit.rs index 882a060..717968d 100644 --- a/crates/wasm-pkg-core/src/wit.rs +++ b/crates/wasm-pkg-core/src/wit.rs @@ -61,12 +61,20 @@ pub async fn build_package( ) -> Result<(PackageRef, Option, Vec)> { let dependencies = resolve_dependencies(config, &wit_dir, Some(lock_file), client) .await - .with_context(|| format!("wit_dir: {}", wit_dir.as_ref().display())) .context("Unable to resolve dependencies")?; lock_file.update_dependencies(&dependencies); - let (resolve, pkg_id) = dependencies.generate_resolve(wit_dir).await?; + let (resolve, pkg_id) = dependencies + .generate_resolve(wit_dir.as_ref()) + .await + .map_err(|e| { + if config.has_override(wit_dir.as_ref()) { + e.context(format!("hint: override present for WIT directory")) + } else { + e + } + })?; let bytes = wit_component::encode(&resolve, pkg_id)?; let pkg = &resolve.packages[pkg_id]; diff --git a/crates/wasm-pkg-core/tests/fixtures/transitive-local/example-c/wit/nested/nested.wit b/crates/wasm-pkg-core/tests/fixtures/transitive-local/example-c/wit/nested/nested.wit new file mode 100644 index 0000000..667aff4 --- /dev/null +++ b/crates/wasm-pkg-core/tests/fixtures/transitive-local/example-c/wit/nested/nested.wit @@ -0,0 +1,7 @@ +package example-c:nested@0.1.0; + +interface nested { + record meta { + label: string, + } +} diff --git a/crates/wasm-pkg-core/tests/fixtures/transitive-local/example-c/wit/world.wit b/crates/wasm-pkg-core/tests/fixtures/transitive-local/example-c/wit/world.wit index 98db971..15655fb 100644 --- a/crates/wasm-pkg-core/tests/fixtures/transitive-local/example-c/wit/world.wit +++ b/crates/wasm-pkg-core/tests/fixtures/transitive-local/example-c/wit/world.wit @@ -1,12 +1,14 @@ package example-c:baz@0.1.0; interface baz { + use example-c:nested/nested@0.1.0.{meta}; + record requestresponse { baz: list, + meta: meta, } } world handler { - // No dependencies to import export baz; } From a85e1e365f251caddf32f521bb2729efde397730 Mon Sep 17 00:00:00 2001 From: Mikhail Katychev Date: Thu, 25 Jun 2026 15:18:00 -0500 Subject: [PATCH 51/61] added e2e --- crates/wkg/src/wit.rs | 4 +++- crates/wkg/tests/e2e.rs | 8 +++++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/crates/wkg/src/wit.rs b/crates/wkg/src/wit.rs index 9d9e4ce..1c62ba7 100644 --- a/crates/wkg/src/wit.rs +++ b/crates/wkg/src/wit.rs @@ -120,7 +120,9 @@ pub async fn build_wit_dir( ) -> anyhow::Result<(PackageRef, Option, Vec)> { check_dir(&dir).await?; let wkg_config = wasm_pkg_core::config::Config::load().await?; - let result = wit::build_package(&wkg_config, dir, lock_file, client).await?; + let result = wit::build_package(&wkg_config, dir.as_ref(), lock_file, client) + .await + .with_context(|| format!("failed to build WIT directory `{}`", dir.as_ref().display()))?; Ok(result) } diff --git a/crates/wkg/tests/e2e.rs b/crates/wkg/tests/e2e.rs index 9b83757..cbe7dca 100644 --- a/crates/wkg/tests/e2e.rs +++ b/crates/wkg/tests/e2e.rs @@ -112,8 +112,13 @@ async fn publish_multiple_transitive_local_packages() { mapped.to_file(&config_path).await.expect("write config"); // pass all fixture dirs to a single `wkg publish` invocation + let mut dirs: Vec = namespaces + .iter() + .map(|name| fixture_root.join(name).join("wit")) + .collect(); // TODO use glob suchas in `wasm_pkgs_core::resolver::tests::transitive_local_paths` - let dirs = namespaces.map(|name| fixture_root.join(name).join("wit")); + dirs.push(fixture_root.join("example-c/wit/nested")); + let mut publish = tokio::process::Command::new(env!("CARGO_BIN_EXE_wkg")); publish .current_dir(temp_dir.path()) @@ -133,6 +138,7 @@ async fn publish_multiple_transitive_local_packages() { "example-a:foo", "example-b:bar", "example-c:baz", + "example-c:nested", "example-d:foo", ] { let pkg = name.parse().unwrap(); From 31f7944db6560421ebc81215591b45bc088f0469 Mon Sep 17 00:00:00 2001 From: Mikhail Katychev Date: Thu, 25 Jun 2026 15:38:56 -0500 Subject: [PATCH 52/61] moved to StabeDiGraph for DependencyGraph to avoid invalidating NodeIndex keys --- crates/wasm-pkg-client/src/lib.rs | 2 +- crates/wasm-pkg-core/src/config.rs | 6 +----- crates/wasm-pkg-core/src/resolver.rs | 6 ++++-- crates/wasm-pkg-core/tests/fetch.rs | 22 ++++++++++++++-------- 4 files changed, 20 insertions(+), 16 deletions(-) diff --git a/crates/wasm-pkg-client/src/lib.rs b/crates/wasm-pkg-client/src/lib.rs index 3c78cfe..3d1c85c 100644 --- a/crates/wasm-pkg-client/src/lib.rs +++ b/crates/wasm-pkg-client/src/lib.rs @@ -39,7 +39,7 @@ use std::path::Path; use std::sync::Arc; use std::{collections::HashMap, pin::Pin}; -use anyhow::{anyhow, Context}; +use anyhow::anyhow; use bytes::Bytes; use futures_util::Stream; use tokio::io::AsyncSeekExt; diff --git a/crates/wasm-pkg-core/src/config.rs b/crates/wasm-pkg-core/src/config.rs index d4e86ca..d309ac9 100644 --- a/crates/wasm-pkg-core/src/config.rs +++ b/crates/wasm-pkg-core/src/config.rs @@ -66,11 +66,7 @@ impl Config { .iter() .map(|map| map.iter()) .flatten() - .filter(|(_, o)| { - dbg!(o.path.as_deref()); - dbg!(path.as_ref()); - o.path.as_ref().map(|p| p.canonicalize().ok()).flatten() == path - }) + .filter(|(_, o)| o.path.as_ref().map(|p| p.canonicalize().ok()).flatten() == path) .next() .is_some() } diff --git a/crates/wasm-pkg-core/src/resolver.rs b/crates/wasm-pkg-core/src/resolver.rs index 9fb61f8..b490452 100644 --- a/crates/wasm-pkg-core/src/resolver.rs +++ b/crates/wasm-pkg-core/src/resolver.rs @@ -14,7 +14,8 @@ use futures_util::TryStreamExt; use indexmap::{IndexMap, IndexSet}; use petgraph::{ acyclic::Acyclic, - graph::{DiGraph, NodeIndex}, + graph::NodeIndex, + stable_graph::StableDiGraph, Direction, }; use semver::{Comparator, Op, Version, VersionReq}; @@ -845,7 +846,7 @@ fn visit<'a>( } /// Graph of publishable packages with the [`petgraph::Direction`] edges describing the dependency direction. -pub(crate) type DependencyGraph = Acyclic>; +pub(crate) type DependencyGraph = Acyclic>; /// Mapping of [`PackageRef`]s to the respective index inside the dependency graph. pub type LocalPackageIndex = HashMap; @@ -910,6 +911,7 @@ impl PublishPlan { /// /// These will not be returned in future calls. pub fn take_ready(&self) -> BTreeSet { + // dbg!(&self.dependents); self.dependents .nodes_iter() // there are no dependents on `self.dependents[id]` diff --git a/crates/wasm-pkg-core/tests/fetch.rs b/crates/wasm-pkg-core/tests/fetch.rs index 2b8fe1b..25a093f 100644 --- a/crates/wasm-pkg-core/tests/fetch.rs +++ b/crates/wasm-pkg-core/tests/fetch.rs @@ -90,6 +90,7 @@ async fn test_transitive_local(#[values(OutputType::Wasm, OutputType::Wit)] outp // [overrides] // "example-b:bar" = { "path" = "../example-b/wit" } // "example-c:baz" = { "path" = "../example-c/wit" } + // "example-c:nested" = { "path" = "../example-c/wit/nested" } // ``` let config = Config { overrides: Some(HashMap::from([ @@ -107,18 +108,22 @@ async fn test_transitive_local(#[values(OutputType::Wasm, OutputType::Wit)] outp version: None, }, ), + ( + "example-c:nested".to_string(), + Override { + path: Some(fixture_path.join("example-c").join("wit/nested")), + version: None, + }, + ), ])), ..Default::default() }; let (_temp_cache, client) = common::get_client().await.unwrap(); - assert!( - // If overrides didn't properly resolve, this will fail - wit::fetch_dependencies(&config, project_path.join("wit"), &mut lock, client, output) - .await - .is_ok(), - "Should be able to fetch the dependencies" - ); + // If overrides didn't properly resolve, this will fail + wit::fetch_dependencies(&config, project_path.join("wit"), &mut lock, client, output) + .await + .unwrap_or_else(|e| panic!("Should be able to fetch the dependencies: {e:#}")); // Ensure that the deps directory contains the correct dependencies let mut deps_dir = tokio::fs::read_dir(project_path.join("wit").join("deps")) @@ -128,9 +133,10 @@ async fn test_transitive_local(#[values(OutputType::Wasm, OutputType::Wit)] outp while let Ok(Some(entry)) = deps_dir.next_entry().await { deps.push(entry.file_name().to_string_lossy().to_string()); } - assert_eq!(deps.len(), 2); + assert_eq!(deps.len(), 3); assert!(deps.contains(&"example-b-bar-0.1.0".to_string())); assert!(deps.contains(&"example-c-baz-0.1.0".to_string())); + assert!(deps.contains(&"example-c-nested-0.1.0".to_string())); // All dependencies are local, so the lock file should be empty assert_eq!( From 465b4aa7d7d1a6fc8aeec641bfa091609d2ba706 Mon Sep 17 00:00:00 2001 From: Mikhail Katychev Date: Thu, 25 Jun 2026 16:21:37 -0500 Subject: [PATCH 53/61] added tempdir handle to LocalBackend --- crates/wasm-pkg-client/src/local.rs | 21 +++++++++++++++------ crates/wkg/src/main.rs | 4 ++-- 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/crates/wasm-pkg-client/src/local.rs b/crates/wasm-pkg-client/src/local.rs index 1da08a2..aaf570e 100644 --- a/crates/wasm-pkg-client/src/local.rs +++ b/crates/wasm-pkg-client/src/local.rs @@ -5,6 +5,7 @@ use std::{ io, path::{Path, PathBuf}, + sync::Arc, }; use anyhow::anyhow; @@ -32,16 +33,24 @@ use crate::{ #[derive(Clone, Debug, Deserialize, Serialize)] pub struct LocalConfig { pub root: PathBuf, + // NOTE: set by [`Self::temp_dir`] so callers don't need to hold a separate + // `TempDir` handle. + #[serde(skip)] + #[doc(hidden)] + _temp_handle: Arc>, } impl LocalConfig { - /// Creates a [`Self`] using a generated temporary directory. Upon dropping the [`TempDir`] handle, - /// the temporary directory will be deleted. - pub fn temp_dir() -> Result<(Self, TempDir), Error> { + /// Creates a [`Self`] with a new temporary directory. + /// The returned config owns the directory and removes the config upon drop. + pub fn temp_dir() -> Result { let handle = TempDir::new()?; - let root = handle.path().to_owned(); - tracing::debug!(registry_dir=%root.display(), "created temporary directory"); - Ok((Self { root }, handle)) + let root = handle.path().to_path_buf(); + tracing::debug!(registry_dir = %root.display(), "created temporary directory"); + Ok(Self { + root, + _temp_handle: Arc::new(Some(handle)), + }) } } diff --git a/crates/wkg/src/main.rs b/crates/wkg/src/main.rs index f931cd2..3678bfa 100644 --- a/crates/wkg/src/main.rs +++ b/crates/wkg/src/main.rs @@ -264,9 +264,9 @@ impl PublishArgs { let mut overlay_config = self.common.load_config().await?; let cache = self.common.load_cache().await?; - let (local_config, _tmp_dir_handle) = LocalConfig::temp_dir()?; + let local_config = LocalConfig::temp_dir()?; let reg_config = - RegistryConfig::default().with_default_backend(LOCAL_PROTOCOL, local_config)?; + RegistryConfig::default().with_default_backend(LOCAL_PROTOCOL, &local_config)?; // Route every package in the plan to the local overlay registry // backed by `reg_config`, so the client used in `build_wit_dir` // resolves these packages against the local overlay instead of From e95b33654f44f6310987605404a924ed0f432218 Mon Sep 17 00:00:00 2001 From: Mikhail Katychev Date: Thu, 25 Jun 2026 16:22:09 -0500 Subject: [PATCH 54/61] clippy --- crates/wasm-pkg-core/src/config.rs | 6 ++---- crates/wasm-pkg-core/src/resolver.rs | 7 +------ crates/wasm-pkg-core/src/wit.rs | 2 +- crates/wkg/src/main.rs | 4 ++-- 4 files changed, 6 insertions(+), 13 deletions(-) diff --git a/crates/wasm-pkg-core/src/config.rs b/crates/wasm-pkg-core/src/config.rs index d309ac9..ef183df 100644 --- a/crates/wasm-pkg-core/src/config.rs +++ b/crates/wasm-pkg-core/src/config.rs @@ -64,10 +64,8 @@ impl Config { let path = path.as_ref().canonicalize().ok(); self.overrides .iter() - .map(|map| map.iter()) - .flatten() - .filter(|(_, o)| o.path.as_ref().map(|p| p.canonicalize().ok()).flatten() == path) - .next() + .flat_map(|map| map.iter()) + .find(|(_, o)| o.path.as_ref().and_then(|p| p.canonicalize().ok()) == path) .is_some() } } diff --git a/crates/wasm-pkg-core/src/resolver.rs b/crates/wasm-pkg-core/src/resolver.rs index b490452..593f594 100644 --- a/crates/wasm-pkg-core/src/resolver.rs +++ b/crates/wasm-pkg-core/src/resolver.rs @@ -12,12 +12,7 @@ use std::{ use anyhow::{bail, Context, Result}; use futures_util::TryStreamExt; use indexmap::{IndexMap, IndexSet}; -use petgraph::{ - acyclic::Acyclic, - graph::NodeIndex, - stable_graph::StableDiGraph, - Direction, -}; +use petgraph::{acyclic::Acyclic, graph::NodeIndex, stable_graph::StableDiGraph, Direction}; use semver::{Comparator, Op, Version, VersionReq}; use tokio::io::{AsyncRead, AsyncReadExt}; use wasm_pkg_client::{ diff --git a/crates/wasm-pkg-core/src/wit.rs b/crates/wasm-pkg-core/src/wit.rs index 717968d..bcfd442 100644 --- a/crates/wasm-pkg-core/src/wit.rs +++ b/crates/wasm-pkg-core/src/wit.rs @@ -70,7 +70,7 @@ pub async fn build_package( .await .map_err(|e| { if config.has_override(wit_dir.as_ref()) { - e.context(format!("hint: override present for WIT directory")) + e.context("hint: override present for WIT directory".to_string()) } else { e } diff --git a/crates/wkg/src/main.rs b/crates/wkg/src/main.rs index 3678bfa..de67121 100644 --- a/crates/wkg/src/main.rs +++ b/crates/wkg/src/main.rs @@ -265,8 +265,8 @@ impl PublishArgs { let cache = self.common.load_cache().await?; let local_config = LocalConfig::temp_dir()?; - let reg_config = - RegistryConfig::default().with_default_backend(LOCAL_PROTOCOL, &local_config)?; + let reg_config = RegistryConfig::default() + .with_default_backend(LOCAL_PROTOCOL, &local_config)?; // Route every package in the plan to the local overlay registry // backed by `reg_config`, so the client used in `build_wit_dir` // resolves these packages against the local overlay instead of From ff084ea9622135a206dd7592750d772849b2eeba Mon Sep 17 00:00:00 2001 From: Mikhail Katychev Date: Thu, 25 Jun 2026 16:23:51 -0500 Subject: [PATCH 55/61] doc comment --- crates/wasm-pkg-client/src/local.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/crates/wasm-pkg-client/src/local.rs b/crates/wasm-pkg-client/src/local.rs index aaf570e..4de9b46 100644 --- a/crates/wasm-pkg-client/src/local.rs +++ b/crates/wasm-pkg-client/src/local.rs @@ -33,8 +33,7 @@ use crate::{ #[derive(Clone, Debug, Deserialize, Serialize)] pub struct LocalConfig { pub root: PathBuf, - // NOTE: set by [`Self::temp_dir`] so callers don't need to hold a separate - // `TempDir` handle. + // NOTE: set by [`Self::temp_dir`] to avoid holding onto a separate `TempDir` handle. #[serde(skip)] #[doc(hidden)] _temp_handle: Arc>, From 9634898f481f3b4a6fb9c42b6fd25fc79086775c Mon Sep 17 00:00:00 2001 From: Mikhail Katychev Date: Thu, 25 Jun 2026 16:28:00 -0500 Subject: [PATCH 56/61] to_string in PackageSpec::eq impl --- crates/wasm-pkg-common/src/package.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/wasm-pkg-common/src/package.rs b/crates/wasm-pkg-common/src/package.rs index 456ba54..0a3d047 100644 --- a/crates/wasm-pkg-common/src/package.rs +++ b/crates/wasm-pkg-common/src/package.rs @@ -87,7 +87,7 @@ pub struct PackageSpec { impl PartialEq for PackageSpec { fn eq(&self, other: &str) -> bool { - format!("{self}") == other + self.to_string() == other } } From 634808073a3eefbb99518fd9210a3c737313daa7 Mon Sep 17 00:00:00 2001 From: Mikhail Katychev Date: Thu, 25 Jun 2026 16:32:59 -0500 Subject: [PATCH 57/61] bumped minor version --- Cargo.lock | 68 +++++++++++++++++++++++++++--------------------------- Cargo.toml | 8 +++---- 2 files changed, 38 insertions(+), 38 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 29fc3be..9d3578c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -107,9 +107,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.102" +version = "1.0.103" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" +checksum = "2a4385e2e34eb35d6b3efe798b9eb88096925d87726c0798709bf56d9ed84af3" [[package]] name = "arraydeque" @@ -2149,9 +2149,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.102" +version = "0.3.103" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03d04c30968dffe80775bd4d7fb676131cd04a1fb46d2686dbffbaec2d9dfd31" +checksum = "53b44bfcdb3f8d5837a46dae1ca9660a837176eee74a28b229bc626816589102" dependencies = [ "cfg-if", "futures-util", @@ -2298,9 +2298,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.32" +version = "0.4.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "953f07c43838f8e6f9758cab68bf5bed85465e7587ebe0b823f1bcd81978ad3a" +checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" [[package]] name = "logos" @@ -3160,9 +3160,9 @@ dependencies = [ [[package]] name = "quinn" -version = "0.11.9" +version = "0.11.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9e20a958963c291dc322d98411f541009df2ced7b5a4f2bd52337638cfccf20" +checksum = "0c1a41e437b6bbd489372cd4971de128e85c855f56c57f283d20ff016cf7c0a8" dependencies = [ "bytes", "cfg_aliases", @@ -3180,9 +3180,9 @@ dependencies = [ [[package]] name = "quinn-proto" -version = "0.11.14" +version = "0.11.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "434b42fec591c96ef50e21e886936e66d3cc3f737104fdb9b737c40ffb94c098" +checksum = "4fcb935c5bec503c2f0e306bdd3e58bb9029dcb14fa8d9ac76e3a5256ac0763e" dependencies = [ "aws-lc-rs", "bytes", @@ -3216,9 +3216,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.45" +version = "1.0.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" +checksum = "dfbc457d0c7a0759a614551b11a6409e5951f6c7537be1f1b7682b9ae9230368" dependencies = [ "proc-macro2", ] @@ -3592,9 +3592,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.40" +version = "0.23.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef86cd5876211988985292b91c96a8f2d298df24e75989a43a3c73f2d4d8168b" +checksum = "6b92b125634d9b795e7beca796cc790df15a7fb38323bf3196fda83292d06b1f" dependencies = [ "aws-lc-rs", "log", @@ -4324,9 +4324,9 @@ dependencies = [ [[package]] name = "time" -version = "0.3.49" +version = "0.3.51" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "711a53c2d47bbd818258c498c8dbfe186a2526c631495cfe7e078567f86b8469" +checksum = "85c17d80feb7334b40c484e45ed1a5273dfd8bfda537c3be2e74a06a6686f327" dependencies = [ "deranged", "num-conv", @@ -4344,9 +4344,9 @@ checksum = "9e1c906769ad99c88eaa54e728060edef082f8e358ff32030cb7c7d315e81109" [[package]] name = "time-macros" -version = "0.2.29" +version = "0.2.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "71c652a3727a9cbb9a02f707f530b618ce00d0ccd762009c8c23bd191df3c17d" +checksum = "dcef1a61bdb119096e153208ec5cbec23944ce8bca13be5c7f60c634f7403935" dependencies = [ "num-conv", "time-core", @@ -5005,9 +5005,9 @@ dependencies = [ [[package]] name = "wasm-bindgen" -version = "0.2.125" +version = "0.2.126" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ddb3f79143bced6de84270411622a2699cee572fc0875aeaf1e7867cf9fca1a" +checksum = "4b067c0c11094aef6b7a801c1e34a26affafdf3d051dba08456b868789aaf9a4" dependencies = [ "cfg-if", "once_cell", @@ -5018,9 +5018,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.75" +version = "0.4.76" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "503b14d284f2c8dac03b819967e155ea753f573586193b2b2c95990cb5d69280" +checksum = "c62df1340f32221cb9c54d6a27b030e3dba64361d4a95bed55f9aacb44da291d" dependencies = [ "js-sys", "wasm-bindgen", @@ -5028,9 +5028,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.125" +version = "0.2.126" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e21a184b13fb19e157296e2c46056aec9092264fab83e4ba59e68c61b323c3d" +checksum = "167ce5e579f6bcf889c4f7175a8a5a585de84e8ff93976ce393efa5f2837aab1" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -5038,9 +5038,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.125" +version = "0.2.126" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fecefd9c35bd935a20fc3fc344b5f29138961e4f47fb03297d88f2587afb5ebd" +checksum = "f3997c7839262f4ef12cf90b818d6340c18e80f263f1a94bf157d0ec4420380e" dependencies = [ "bumpalo", "proc-macro2", @@ -5051,9 +5051,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.125" +version = "0.2.126" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23939e44bb9a5d7576fa2b563dc2e136628f1224e88a8deed09e04858b77871f" +checksum = "dc1b4cb0cc549fcf58d7dfc081778139b3d283a081644e833e84682ad71cea24" dependencies = [ "unicode-ident", ] @@ -5130,7 +5130,7 @@ dependencies = [ [[package]] name = "wasm-pkg-client" -version = "0.15.1" +version = "0.16.0" dependencies = [ "anyhow", "async-trait", @@ -5167,7 +5167,7 @@ dependencies = [ [[package]] name = "wasm-pkg-common" -version = "0.15.1" +version = "0.16.0" dependencies = [ "anyhow", "bytes", @@ -5186,7 +5186,7 @@ dependencies = [ [[package]] name = "wasm-pkg-core" -version = "0.15.1" +version = "0.16.0" dependencies = [ "anyhow", "futures-util", @@ -5305,9 +5305,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.102" +version = "0.3.103" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6430a72df5eb332242960fe84b3002a241163998241eb596d4f739b9757061d" +checksum = "8622dcb61c0bcc9fffa6938bed81210af2da9a7e4a1a834b2e37a59b6dfb6141" dependencies = [ "js-sys", "wasm-bindgen", @@ -5736,7 +5736,7 @@ dependencies = [ [[package]] name = "wkg" -version = "0.15.1" +version = "0.16.0" dependencies = [ "anyhow", "base64 0.22.1", diff --git a/Cargo.toml b/Cargo.toml index a73991f..03eef6f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,7 +4,7 @@ resolver = "2" [workspace.package] edition = "2021" -version = "0.15.1" +version = "0.16.0" authors = ["The Wasmtime Project Developers"] license = "Apache-2.0 WITH LLVM-exception" @@ -38,9 +38,9 @@ tracing-subscriber = { version = "0.3.20", default-features = false, features = "fmt", "env-filter", ] } -wasm-pkg-common = { version = "0.15.1", path = "crates/wasm-pkg-common" } -wasm-pkg-client = { version = "0.15.1", path = "crates/wasm-pkg-client" } -wasm-pkg-core = { version = "0.15.1", path = "crates/wasm-pkg-core" } +wasm-pkg-common = { version = "0.16.0", path = "crates/wasm-pkg-common" } +wasm-pkg-client = { version = "0.16.0", path = "crates/wasm-pkg-client" } +wasm-pkg-core = { version = "0.16.0", path = "crates/wasm-pkg-core" } wasm-metadata = "0.244" wit-component = "0.244" wit-parser = "0.244" From 16cb6a45c7eb1590dae0a20a9921edb6680f4ea8 Mon Sep 17 00:00:00 2001 From: Mikhail Katychev Date: Fri, 26 Jun 2026 22:03:36 -0500 Subject: [PATCH 58/61] fix(logging): move println macro calls to eprintln to avoid populating stdout --- crates/wasm-pkg-core/src/wit.rs | 2 +- crates/wkg/src/main.rs | 26 +++++++++++++------------- crates/wkg/src/wit.rs | 2 +- 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/crates/wasm-pkg-core/src/wit.rs b/crates/wasm-pkg-core/src/wit.rs index bcfd442..6fbec3c 100644 --- a/crates/wasm-pkg-core/src/wit.rs +++ b/crates/wasm-pkg-core/src/wit.rs @@ -192,7 +192,7 @@ pub fn get_packages( /// /// The function will return an error if there are duplicate references to the same package or if /// there are cyclical dependencies between packages. -pub fn get_local_dependencies( +pub(crate) fn get_local_dependencies( paths: &[impl AsRef], ) -> Result<(DependencyGraph, LocalPackageIndex)> { let pkg_trees = paths diff --git a/crates/wkg/src/main.rs b/crates/wkg/src/main.rs index de67121..bb01307 100644 --- a/crates/wkg/src/main.rs +++ b/crates/wkg/src/main.rs @@ -184,14 +184,14 @@ impl ConfigArgs { // write config file config.to_file(&path).await?; - println!("Updated config file: {path}", path = path.display()); + eprintln!("Updated config file: {path}", path = path.display()); } // print config if let Some(registry) = config.default_registry() { - println!("Default registry: {}", registry); + eprintln!("Default registry: {}", registry); } else { - println!("Default registry is not set"); + eprintln!("Default registry is not set"); } Ok(()) @@ -277,7 +277,7 @@ impl PublishArgs { .merge(reg_config); let mut plan = PublishPlan::from_paths(paths)?; - println!("{plan}"); + eprintln!("{plan}"); // TODO(mkatychev): Add support for `PackageLoader::get_release` to handle // querying on a per package, namespace, and registry level @@ -348,9 +348,9 @@ impl PublishArgs { .publish_release_data(source, publish_opts.clone()) .await?; if self.dry_run { - println!("Aborting publish due to dry run: {}@{}", package, version); + eprintln!("Aborting publish due to dry run: {}@{}", package, version); } else { - println!("Published {}@{}", package, version); + eprintln!("Published {}@{}", package, version); } } plan.mark_confirmed(ready_for_publish); @@ -395,9 +395,9 @@ impl PublishArgs { .publish_release_file(&publish_path, publish_opts) .await?; if self.dry_run { - println!("Aborting publish due to dry run: {}@{}", package, version); + eprintln!("Aborting publish due to dry run: {}@{}", package, version); } else { - println!("Published {}@{}", package, version); + eprintln!("Published {}@{}", package, version); } Ok(()) } @@ -449,7 +449,7 @@ impl GetArgs { let version = match version { Some(ver) => ver, None => { - println!("No version specified; fetching version list..."); + eprintln!("No version specified; fetching version list..."); let versions = client.list_all_versions(&package).await?; tracing::trace!(?versions, "Fetched version list"); versions @@ -460,7 +460,7 @@ impl GetArgs { } }; - println!("Getting {package}@{version}..."); + eprintln!("Getting {package}@{version}..."); let release = client .get_release(&package, &version) .await @@ -501,7 +501,7 @@ impl GetArgs { "wasm" => Format::Wasm, "wit" => Format::Wit, _ => { - println!( + eprintln!( "Couldn't infer output format from file name {:?}", self.output.file_name().unwrap_or_default() ); @@ -528,7 +528,7 @@ impl GetArgs { if format == Format::Wit { return Err(err); } - println!("Failed to detect package content type: {err:#}"); + eprintln!("Failed to detect package content type: {err:#}"); None } } @@ -574,7 +574,7 @@ impl GetArgs { .persist(&output_path) .with_context(|| format!("Failed to persist WASM to {output_path:?}"))? } - println!("Wrote '{}'", output_path.display()); + eprintln!("Wrote '{}'", output_path.display()); } Ok(()) } diff --git a/crates/wkg/src/wit.rs b/crates/wkg/src/wit.rs index 1c62ba7..18d5d37 100644 --- a/crates/wkg/src/wit.rs +++ b/crates/wkg/src/wit.rs @@ -106,7 +106,7 @@ impl BuildArgs { tokio::fs::write(&output_path, bytes).await?; // Now write out the lock file since everything else succeeded lock_file.write().await?; - println!("WIT package written to {}", output_path.display()); + eprintln!("WIT package written to {}", output_path.display()); Ok(()) } } From 778b1cc6ba025ec2b9f4b976b760029edc9d8213 Mon Sep 17 00:00:00 2001 From: Mikhail Katychev Date: Fri, 26 Jun 2026 22:16:02 -0500 Subject: [PATCH 59/61] fix(vis): moved functions with needless pub to pub(crate) --- crates/wasm-pkg-common/src/config.rs | 10 +--------- crates/wasm-pkg-core/src/resolver.rs | 5 ++--- 2 files changed, 3 insertions(+), 12 deletions(-) diff --git a/crates/wasm-pkg-common/src/config.rs b/crates/wasm-pkg-common/src/config.rs index 94c53ca..809d194 100644 --- a/crates/wasm-pkg-common/src/config.rs +++ b/crates/wasm-pkg-common/src/config.rs @@ -54,14 +54,6 @@ impl RegistryMapping { RegistryMapping::Custom(custom) => &custom.registry, } } - - /// Returns the inner [`RegistryMetadata`] if `Self` holds a [`CustomConfig`]. - pub fn metadata(&self) -> Option<&RegistryMetadata> { - if let Self::Custom(config) = self { - return Some(&config.metadata); - } - None - } } /// Custom registry configuration @@ -215,7 +207,7 @@ impl Config { self.fallback_namespace_registries.get(namespace) } - pub fn resolve_mapping(&self, package: &PackageRef) -> Option<&RegistryMapping> { + fn resolve_mapping(&self, package: &PackageRef) -> Option<&RegistryMapping> { let namespace = package.namespace(); // look in `self.package_registry_overrides` // then in `self.namespace_registries` diff --git a/crates/wasm-pkg-core/src/resolver.rs b/crates/wasm-pkg-core/src/resolver.rs index 593f594..af6f2f2 100644 --- a/crates/wasm-pkg-core/src/resolver.rs +++ b/crates/wasm-pkg-core/src/resolver.rs @@ -841,7 +841,7 @@ fn visit<'a>( } /// Graph of publishable packages with the [`petgraph::Direction`] edges describing the dependency direction. -pub(crate) type DependencyGraph = Acyclic>; +pub type DependencyGraph = Acyclic>; /// Mapping of [`PackageRef`]s to the respective index inside the dependency graph. pub type LocalPackageIndex = HashMap; @@ -906,7 +906,6 @@ impl PublishPlan { /// /// These will not be returned in future calls. pub fn take_ready(&self) -> BTreeSet { - // dbg!(&self.dependents); self.dependents .nodes_iter() // there are no dependents on `self.dependents[id]` @@ -982,7 +981,7 @@ impl std::fmt::Display for PublishPlan { /// e.g. "foo:a@0.1.0, bar:b@0.2.0, and baz:c@0.3.0". /// /// Note: the final separator (e.g. "and" in the previous example) can be chosen. -pub fn package_list<'a>( +fn package_list<'a>( pkgs: impl IntoIterator, final_sep: Option<&str>, ) -> String { From 8540821523d848a4e0d7438b94634611adfcf9b9 Mon Sep 17 00:00:00 2001 From: Mikhail Katychev Date: Fri, 26 Jun 2026 22:22:53 -0500 Subject: [PATCH 60/61] fix: clippy shenanigans --- crates/wasm-pkg-common/src/package.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/crates/wasm-pkg-common/src/package.rs b/crates/wasm-pkg-common/src/package.rs index 0a3d047..00a816e 100644 --- a/crates/wasm-pkg-common/src/package.rs +++ b/crates/wasm-pkg-common/src/package.rs @@ -87,7 +87,8 @@ pub struct PackageSpec { impl PartialEq for PackageSpec { fn eq(&self, other: &str) -> bool { - self.to_string() == other + // clippy --fix will create a recursive callsite here if `self.to_string()` is used instead + format!("{self}") == other } } From 161bfba321c13a767c86e5150273acd287713923 Mon Sep 17 00:00:00 2001 From: Mikhail Katychev Date: Fri, 26 Jun 2026 22:25:05 -0500 Subject: [PATCH 61/61] feat(ci): iimplement concurrency lock for duplicate test runs in older commits --- .github/workflows/ci.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b40098b..1171e85 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,5 +1,11 @@ on: [push, pull_request] +# https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#concurrency +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + + name: CI jobs: @@ -40,4 +46,4 @@ jobs: - name: Test wasm32-wasip1 build for wasm-pkg-common if: matrix.test_wasm_build - run: cargo build -p wasm-pkg-common --target wasm32-wasip1 \ No newline at end of file + run: cargo build -p wasm-pkg-common --target wasm32-wasip1