diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e656c8c..efde891 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -38,6 +38,9 @@ jobs: - name: Check no-std run: make check-wasm + - name: Install pvq-program-metadata-gen + run: cargo install --path pvq-program-metadata-gen + - name: Clippy run: make clippy @@ -65,6 +68,9 @@ jobs: - name: Install polkatool run: make polkatool + - name: Install pvq-program-metadata-gen + run: cargo install --path pvq-program-metadata-gen + - name: Build guests run: make guests diff --git a/Cargo.lock b/Cargo.lock index b0235a9..1cf0c1d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2790,6 +2790,7 @@ dependencies = [ "pvq-extension-procedural", "pvq-primitives", "scale-info", + "serde", "tracing", "tracing-subscriber", ] @@ -2843,7 +2844,22 @@ dependencies = [ "polkavm-derive 0.21.0", "pvq-program-procedural", "scale-info", - "trybuild", +] + +[[package]] +name = "pvq-program-metadata-gen" +version = "0.1.0" +dependencies = [ + "clap", + "parity-scale-codec", + "proc-macro2", + "quote", + "scale-info", + "syn 2.0.100", + "tempfile", + "toml", + "tracing", + "tracing-subscriber", ] [[package]] @@ -2877,6 +2893,8 @@ dependencies = [ "pvq-extension-fungibles", "pvq-primitives", "scale-info", + "serde", + "serde_json", "sp-core", "tracing", "tracing-subscriber", @@ -4179,6 +4197,7 @@ version = "0.8.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cd87a5cdd6ffab733b2f74bc4fd7ee5fff6634124999ac278c35fc78c6120148" dependencies = [ + "indexmap", "serde", "serde_spanned", "toml_datetime", diff --git a/Cargo.toml b/Cargo.toml index e3ba60e..d156cd9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,6 +11,7 @@ members = [ "poc/runtime", "pvq-program", + "pvq-program-metadata-gen", "pvq-executor", "pvq-extension-core", "pvq-extension-fungibles", @@ -32,6 +33,7 @@ opt-level = 3 [workspace.dependencies] # local pvq-program = { path = "pvq-program", default-features = false } +pvq-program-metadata-gen = { path = "pvq-program-metadata-gen" } pvq-executor = { path = "pvq-executor", default-features = false } pvq-extension-core = { path = "pvq-extension-core", default-features = false } pvq-extension-fungibles = { path = "pvq-extension-fungibles", default-features = false } @@ -68,13 +70,18 @@ parity-scale-codec = { version = "3.6.12", default-features = false, features = ] } scale-info = { version = "2.11.3", default-features = false, features = [ "derive", + "serde", ] } tracing = { version = "0.1.40", default-features = false } +serde = { version = "1.0.215", default-features = false, features = ["derive"] } +serde_json = { version = "1.0.110", default-features = false } # std clap = { version = "4.5.4", features = ["derive"] } env_logger = { version = "0.11.3" } tracing-subscriber = { version = "0.3.18", features = ["env-filter"] } +tempfile = { version = "3.9.0" } +toml = { version = "0.8", features = ["preserve_order"] } fortuples = "0.9" diff --git a/Makefile b/Makefile index 7cd1d83..90dca01 100644 --- a/Makefile +++ b/Makefile @@ -23,6 +23,9 @@ tools: polkatool chain-spec-builder polkatool: cargo install --path vendor/polkavm/tools/polkatool +pvq-program-metadata-gen: + cargo install --path pvq-program-metadata-gen + chain-spec-builder: cargo install --path vendor/polkadot-sdk/substrate/bin/utils/chain-spec-builder @@ -37,7 +40,7 @@ check: check-wasm SKIP_WASM_BUILD= cargo check cd pvq-program/examples; cargo check -clippy: +clippy: pvq-program-metadata-gen SKIP_WASM_BUILD= cargo clippy -- -D warnings cd guest-examples; cargo clippy --all diff --git a/guest-examples/Cargo.lock b/guest-examples/Cargo.lock index 2093557..d56ca0e 100644 --- a/guest-examples/Cargo.lock +++ b/guest-examples/Cargo.lock @@ -14,6 +14,12 @@ version = "1.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7575182f7272186991736b70173b0ea045398f984bf5ebbb3804736ce1330c9d" +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + [[package]] name = "const_format" version = "0.2.34" @@ -34,6 +40,26 @@ dependencies = [ "unicode-xid", ] +[[package]] +name = "derive_more" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a9b99b9cbbe49445b21764dc0625032a89b145a2642e67603e1c936f5458d05" +dependencies = [ + "derive_more-impl", +] + +[[package]] +name = "derive_more-impl" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7330aeadfbe296029522e6c40f315320aba36fc43a5b3632f3795348f3bd22" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "equivalent" version = "1.0.2" @@ -44,6 +70,7 @@ checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" name = "guest-sum-balance" version = "0.1.0" dependencies = [ + "cfg-if", "parity-scale-codec", "polkavm-derive", "pvq-program", @@ -200,7 +227,10 @@ dependencies = [ name = "pvq-program" version = "0.1.0" dependencies = [ + "parity-scale-codec", + "polkavm-derive", "pvq-program-procedural", + "scale-info", ] [[package]] @@ -228,6 +258,51 @@ version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eded382c5f5f786b989652c49544c4877d9f015cc22e145a5ea8ea66c2921cd2" +[[package]] +name = "scale-info" +version = "2.11.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "346a3b32eba2640d17a9cb5927056b08f3de90f65b72fe09402c2ad07d684d0b" +dependencies = [ + "cfg-if", + "derive_more", + "parity-scale-codec", + "scale-info-derive", + "serde", +] + +[[package]] +name = "scale-info-derive" +version = "2.11.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6630024bf739e2179b91fb424b28898baf819414262c5d376677dbff1fe7ebf" +dependencies = [ + "proc-macro-crate", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde" +version = "1.0.219" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f0e2c6ed6606019b4e29e69dbaba95b11854410e5347d525002456dbbb786b6" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.219" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b0276cf7f2c73365f7157c8123c21cd9a50fbbd844757af28ca1f5925fc2a00" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "syn" version = "2.0.100" diff --git a/guest-examples/Cargo.toml b/guest-examples/Cargo.toml index 32d03e6..30ff9b9 100644 --- a/guest-examples/Cargo.toml +++ b/guest-examples/Cargo.toml @@ -14,4 +14,6 @@ parity-scale-codec = { version = "3", default-features = false, features = [ "derive", ] } pvq-program = { path = "../pvq-program", default-features = false } +pvq-program-metadata-gen = { path = "../pvq-program-metadata-gen" } polkavm-derive = { path = "../vendor/polkavm/crates/polkavm-derive" } +cfg-if = "1.0" diff --git a/guest-examples/rust-toolchain.toml b/guest-examples/rust-toolchain.toml index 303a7f4..764468d 100644 --- a/guest-examples/rust-toolchain.toml +++ b/guest-examples/rust-toolchain.toml @@ -1,3 +1,3 @@ [toolchain] -channel = "nightly-2024-11-01" +channel = "nightly-2024-11-19" components = ["rust-src"] diff --git a/guest-examples/sum-balance/Cargo.toml b/guest-examples/sum-balance/Cargo.toml index 1239970..96997b9 100644 --- a/guest-examples/sum-balance/Cargo.toml +++ b/guest-examples/sum-balance/Cargo.toml @@ -8,3 +8,8 @@ publish = false parity-scale-codec = { workspace = true } polkavm-derive = { workspace = true } pvq-program = { workspace = true } +cfg-if = { workspace = true } + +[features] +option_version_1 = [] +option_version_2 = [] diff --git a/guest-examples/sum-balance/build.rs b/guest-examples/sum-balance/build.rs new file mode 100644 index 0000000..8622dfd --- /dev/null +++ b/guest-examples/sum-balance/build.rs @@ -0,0 +1,25 @@ +use std::env; +use std::path::PathBuf; +use std::process::Command; + +fn main() { + // Tell Cargo to rerun this build script if the source file changes + println!("cargo:rerun-if-changed=src/main.rs"); + let current_dir = env::current_dir().expect("Failed to get current directory"); + // Determine the output directory for the metadata + let output_dir = PathBuf::from(env::var("OUT_DIR").expect("OUT_DIR is not set")); + + // Build and run the command + let status = Command::new("pvq-program-metadata-gen") + .arg("--crate-path") + .arg(¤t_dir) + .arg("--output-dir") + .arg(&output_dir) + .env("RUST_LOG", "info") + .status() + .expect("Failed to execute pvq-program-metadata-gen"); + + if !status.success() { + panic!("Failed to generate program metadata"); + } +} diff --git a/guest-examples/sum-balance/src/main.rs b/guest-examples/sum-balance/src/main.rs index 771a33f..764cf32 100644 --- a/guest-examples/sum-balance/src/main.rs +++ b/guest-examples/sum-balance/src/main.rs @@ -3,9 +3,22 @@ #[pvq_program::program] mod sum_balance { - type AccountId = [u8; 32]; - type AssetId = u32; - type Balance = u64; + + cfg_if::cfg_if! { + if #[cfg(feature = "option_version_1")] { + type AccountId = [u8; 64]; + type AssetId = u64; + type Balance = u128; + } else if #[cfg(feature = "option_version_2")] { + type AccountId = [u8; 32]; + type AssetId = u32; + type Balance = u64; + } else { + type AccountId = [u8; 32]; + type AssetId = u32; + type Balance = u64; + } + } #[program::extension_fn(extension_id = 4071833530116166512u64, fn_index = 1)] fn balance(asset: AssetId, who: AccountId) -> Balance {} diff --git a/pvq-extension/Cargo.toml b/pvq-extension/Cargo.toml index f441f89..c1ee690 100644 --- a/pvq-extension/Cargo.toml +++ b/pvq-extension/Cargo.toml @@ -16,6 +16,7 @@ pvq-extension-procedural = { path = "procedural" } parity-scale-codec = { workspace = true } scale-info = { workspace = true } pvq-primitives = { workspace = true } +serde = { workspace = true } [dev-dependencies] tracing-subscriber = { workspace = true } @@ -29,4 +30,5 @@ std = [ "parity-scale-codec/std", "scale-info/std", "tracing/std", + "serde/std", ] diff --git a/pvq-extension/procedural/src/extensions_impl/expand/metadata.rs b/pvq-extension/procedural/src/extensions_impl/expand/metadata.rs index 4fa8f00..1dfcef8 100644 --- a/pvq-extension/procedural/src/extensions_impl/expand/metadata.rs +++ b/pvq-extension/procedural/src/extensions_impl/expand/metadata.rs @@ -5,6 +5,8 @@ use quote::quote; /// generate the `metadata` function in the #[extensions_impl] module pub fn expand_metadata(def: &Def) -> TokenStream2 { let pvq_extension = &def.pvq_extension; + let scale_info = &def.scale_info; + let mut extension_id_call_list = Vec::new(); let mut extension_metadata_call_list = Vec::new(); for impl_ in &def.extension_impls { @@ -14,30 +16,22 @@ pub fn expand_metadata(def: &Def) -> TokenStream2 { // Replace trait_path with a call to the metadata function with the impl struct as generic parameter let impl_struct_ident = &def.impl_struct.ident; + let extension_id_call = quote!( + #trait_path extension_id() + ); // Create a method call expression instead of a path let method_call = quote!( #trait_path metadata::<#impl_struct_ident>() ); + extension_id_call_list.push(extension_id_call); extension_metadata_call_list.push(method_call); } - // let query_metadata_by_extension_id = quote! { - // impl #pvq_extension::ExtensionImplMetadata for #extension_impl_name { - // fn extension_metadata(extension_id: #pvq_extension::ExtensionIdTy) -> #pvq_extension::metadata::ExtensionMetadata { - // let extension_metadata = match extension_id { - // #(#extension_ids => #extension_metadata_list,)* - // _ => panic!("Unknown extension id"), - // }; - // extension_metadata - // } - // } - // }; - let metadata = quote! { pub fn metadata() -> #pvq_extension::metadata::Metadata { #pvq_extension::metadata::Metadata::new( - scale_info::prelude::vec![ #( #extension_metadata_call_list, )* ], + #scale_info::prelude::collections::BTreeMap::from([ #( (#extension_id_call_list, #extension_metadata_call_list), )* ]), ) } }; diff --git a/pvq-extension/procedural/src/extensions_impl/parse/mod.rs b/pvq-extension/procedural/src/extensions_impl/parse/mod.rs index d45c4a7..e3006db 100644 --- a/pvq-extension/procedural/src/extensions_impl/parse/mod.rs +++ b/pvq-extension/procedural/src/extensions_impl/parse/mod.rs @@ -15,11 +15,13 @@ pub struct Def { pub impl_struct: impl_struct::ImplStruct, pub extension_impls: Vec, pub pvq_extension: syn::Path, + pub scale_info: syn::Path, } impl Def { pub fn try_from(mut item: syn::ItemMod) -> syn::Result { let pvq_extension = generate_crate_access("pvq-extension")?; + let scale_info = generate_crate_access("scale-info")?; let item_span = item.span(); let items = &mut item .content @@ -58,6 +60,7 @@ impl Def { .ok_or_else(|| syn::Error::new(item_span, "Missing `#[extensions_impl::impl_struct]`"))?, extension_impls, pvq_extension, + scale_info, }) } } diff --git a/pvq-extension/src/metadata.rs b/pvq-extension/src/metadata.rs index 3d0e78c..323f253 100644 --- a/pvq-extension/src/metadata.rs +++ b/pvq-extension/src/metadata.rs @@ -8,20 +8,25 @@ pub trait ExtensionImplMetadata { use parity_scale_codec::Encode; use scale_info::{ form::{Form, MetaForm, PortableForm}, + prelude::collections::BTreeMap, prelude::vec::Vec, IntoPortable, PortableRegistry, Registry, }; +use serde::Serialize; /// Metadata of extensions -#[derive(Clone, PartialEq, Eq, Encode, Debug)] +#[derive(Clone, PartialEq, Eq, Encode, Debug, Serialize)] pub struct Metadata { pub types: PortableRegistry, - pub extensions: Vec>, + pub extensions: BTreeMap>, } impl Metadata { - pub fn new(extensions: Vec) -> Self { + pub fn new(extensions: BTreeMap) -> Self { let mut registry = Registry::new(); - let extensions = registry.map_into_portable(extensions); + let extensions = extensions + .into_iter() + .map(|(id, metadata)| (id, metadata.into_portable(&mut registry))) + .collect(); Self { types: registry.into(), extensions, @@ -47,6 +52,19 @@ impl IntoPortable for ExtensionMetadata { } } +impl Serialize for ExtensionMetadata { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + use serde::ser::SerializeStruct; + let mut state = serializer.serialize_struct("ExtensionMetadata", 2)?; + state.serialize_field("name", &self.name)?; + state.serialize_field("functions", &self.functions)?; + state.end() + } +} + /// Metadata of a runtime function. #[derive(Clone, PartialEq, Eq, Encode, Debug)] pub struct FunctionMetadata { @@ -70,6 +88,20 @@ impl IntoPortable for FunctionMetadata { } } +impl Serialize for FunctionMetadata { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + use serde::ser::SerializeStruct; + let mut state = serializer.serialize_struct("FunctionMetadata", 3)?; + state.serialize_field("name", &self.name)?; + state.serialize_field("inputs", &self.inputs)?; + state.serialize_field("output", &self.output)?; + state.end() + } +} + /// Metadata of a runtime method parameter. #[derive(Clone, PartialEq, Eq, Encode, Debug)] pub struct FunctionParamMetadata { @@ -89,3 +121,16 @@ impl IntoPortable for FunctionParamMetadata { } } } + +impl Serialize for FunctionParamMetadata { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + use serde::ser::SerializeStruct; + let mut state = serializer.serialize_struct("FunctionParamMetadata", 2)?; + state.serialize_field("name", &self.name)?; + state.serialize_field("ty", &self.ty)?; + state.end() + } +} diff --git a/pvq-program-metadata-gen/Cargo.toml b/pvq-program-metadata-gen/Cargo.toml new file mode 100644 index 0000000..5f3ead5 --- /dev/null +++ b/pvq-program-metadata-gen/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "pvq-program-metadata-gen" +description = "PVQ program metadata generation" +authors.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +version.workspace = true + +[dependencies] +quote = { workspace = true } +syn = { workspace = true } +proc-macro2 = { workspace = true } +clap = { workspace = true } +parity-scale-codec = { workspace = true } +scale-info = { workspace = true } +toml = { workspace = true } +tempfile = { workspace = true } +tracing = { workspace = true } +tracing-subscriber = { workspace = true } diff --git a/pvq-program-metadata-gen/README.md b/pvq-program-metadata-gen/README.md new file mode 100644 index 0000000..adb8cf2 --- /dev/null +++ b/pvq-program-metadata-gen/README.md @@ -0,0 +1,67 @@ +# PVQ Program Metadata Generator + +A command-line tool for generating metadata for PVQ programs. This tool extracts metadata from your PVQ program source code, allowing the UI to know about your program's metadata. + +## Installation + +You can install the tool globally using Cargo: + +```bash +cargo install --path /path/to/pvq-program-metadata-gen +``` + +## Usage + +The basic usage is as follows: + +```bash +pvq-program-metadata-gen --crate-path /path/to/your/crate --output-dir /path/to/output/dir +``` + +### Arguments + +- `--crate-path, -c`: Path to the crate directory containing a PVQ program +- `--output-dir, -o`: Output directory for the metadata file, typically `OUT_DIR` specified in your `build.rs` file + +## Integration with Build Scripts + +You can integrate this tool into your crate's `build.rs` file: + +```rust +use std::env; +use std::path::PathBuf; +use std::process::Command; + +fn main() { + // Tell Cargo to rerun this build script if the source file changes + let current_dir = env::current_dir().expect("Failed to get current directory"); + // Determine the output directory for the metadata + let output_dir = PathBuf::from(env::var("OUT_DIR").expect("OUT_DIR is not set")); + + // Build and run the command + let status = Command::new("pvq-program-metadata-gen") + .arg("--crate-path") + .arg(¤t_dir) + .arg("--output-dir") + .arg(&output_dir) + .env("RUST_LOG", "info") + .status() + .expect("Failed to execute pvq-program-metadata-gen"); + + if !status.success() { + panic!("Failed to generate program metadata"); + } +} + +``` + +## How It Works + +The tool: + +1. Reads the source code of your PVQ program +2. Generates metadata generation code +3. Creates a temporary crate that store the metadata generation code +4. Compiles and runs the temporary crate using the same conditions as your original crate + +The metadata includes information about function names, parameter types, and return types, allowing the UI to know about your program's metadata. diff --git a/pvq-program-metadata-gen/src/bin/pvq-program-metadata-gen.rs b/pvq-program-metadata-gen/src/bin/pvq-program-metadata-gen.rs new file mode 100644 index 0000000..f7e642c --- /dev/null +++ b/pvq-program-metadata-gen/src/bin/pvq-program-metadata-gen.rs @@ -0,0 +1,81 @@ +use clap::Parser; +use std::fs; +use std::path::PathBuf; +use std::process::Command; +use tracing::{debug, info}; + +#[derive(Parser, Debug)] +#[command(author, version, about = "PVQ Program Metadata Generator")] +struct Args { + /// Path to the crate directory containing a PVQ program + #[arg(short, long)] + crate_path: PathBuf, + + #[arg(short, long)] + output_dir: PathBuf, +} + +fn main() { + // Initialize tracing + tracing_subscriber::fmt::init(); + + // Parse command line arguments + let args = Args::parse(); + + // Logging arguments + info!("Generating metadata for program at: {}", args.crate_path.display()); + info!("Output dir: {}", args.output_dir.display()); + + // Create a temp crate for the metadata generation + let temp_dir = tempfile::tempdir().expect("Failed to create temp directory"); + let temp_crate_path = temp_dir.path(); + fs::create_dir_all(temp_crate_path).expect("Failed to create `temp_crate` directory"); + info!("Temp crate path: {}", temp_crate_path.display()); + + // Read the program source + let source = fs::read_to_string(args.crate_path.join("src/main.rs")) + .expect("Failed to read pvq program source file, expected `src/main.rs`"); + + // Generate the metadata generator source codes + let metadata_gen_src = + pvq_program_metadata_gen::metadata_gen_src(&source, args.output_dir.to_string_lossy().as_ref()) + .expect("Failed to generate metadata generator source code"); + debug!("Metadata generator source code: {}", metadata_gen_src); + + // Create src directory and write main.rs + fs::create_dir_all(temp_crate_path.join("src")).expect("Failed to create `temp_crate/src directory"); + fs::write(temp_crate_path.join("src/main.rs"), metadata_gen_src.to_string()) + .expect("Failed to write metadata generator source code"); + + // Extract features section from the original manifest + let original_manifest_content = + std::fs::read_to_string(args.crate_path.join("Cargo.toml")).expect("Failed to read original Cargo.toml"); + let optional_features = pvq_program_metadata_gen::extract_features(&original_manifest_content) + .expect("Failed to extract features section from the original Cargo.toml"); + debug!("Features section: {:?}", optional_features); + + // Create Cargo.toml with features from the original crate + let manifest = pvq_program_metadata_gen::create_manifest(optional_features.as_ref()); + debug!("Manifest: {}", manifest); + std::fs::write(temp_crate_path.join("Cargo.toml"), manifest).expect("Failed to write Cargo.toml"); + + // Compile and run the metadata generator in one step + let mut cargo_cmd = Command::new("cargo"); + cargo_cmd.current_dir(temp_crate_path).args(["run"]); + + // Add active features to the cargo command + let active_features = pvq_program_metadata_gen::get_active_features(optional_features.as_ref()) + .expect("Failed to get active features"); + debug!("Active features: {:?}", active_features); + for feature in active_features { + cargo_cmd.arg("--features").arg(feature); + } + + // Compile and run the metadata generator in one step + info!("Compiling and running metadata generator..."); + let status = cargo_cmd.status().expect("Failed to run metadata generator"); + if !status.success() { + panic!("Failed to generate metadata"); + } + info!("Metadata generation successful!"); +} diff --git a/pvq-program-metadata-gen/src/features.rs b/pvq-program-metadata-gen/src/features.rs new file mode 100644 index 0000000..2f008e5 --- /dev/null +++ b/pvq-program-metadata-gen/src/features.rs @@ -0,0 +1,40 @@ +use std::collections::HashSet; +pub fn get_active_features(optional_features: Option<&toml::Table>) -> Result, String> { + let features_env = std::env::vars() + .filter(|(var, _)| var.starts_with("CARGO_FEATURE_")) + .map(|(var, _)| var) + .collect::>(); + if features_env.is_empty() { + Ok(vec![]) + } else { + let features = optional_features + .as_ref() + .ok_or_else(|| "Some features are set, but there is no features section in the manifest".to_string())?; + Ok(features + .keys() + .filter(|feature| { + features_env.contains(&format!("CARGO_FEATURE_{}", feature.to_uppercase().replace("-", "_"))) + }) + .map(|feature| feature.to_string()) + .collect()) + } +} + +/// Extracts features from the original crate's Cargo.toml +pub fn extract_features(original_manifest_content: &str) -> Result, String> { + match toml::from_str::(original_manifest_content) { + Ok(manifest) => { + // Extract features section if it exists + if let Some(features) = manifest.get("features") { + if let toml::Value::Table(features) = features { + Ok(Some(features.clone())) + } else { + Err("features section is not a table".to_string()) + } + } else { + Ok(None) + } + } + Err(e) => Err(e.to_string()), + } +} diff --git a/pvq-program-metadata-gen/src/helper.rs b/pvq-program-metadata-gen/src/helper.rs new file mode 100644 index 0000000..920c223 --- /dev/null +++ b/pvq-program-metadata-gen/src/helper.rs @@ -0,0 +1,79 @@ +pub trait MutItemAttrs { + fn mut_item_attrs(&mut self) -> Option<&mut Vec>; +} +/// Take the first item attribute (e.g. attribute like `#[pvq..]`) and decode it to `Attr` +pub(crate) fn take_first_program_attr(item: &mut impl MutItemAttrs) -> syn::Result> { + let Some(attrs) = item.mut_item_attrs() else { + return Ok(None); + }; + + let Some(index) = attrs.iter().position(|attr| { + attr.path() + .segments + .first() + .is_some_and(|segment| segment.ident == "program") + }) else { + return Ok(None); + }; + + let pvq_attr = attrs.remove(index); + Ok(Some(pvq_attr)) +} +impl MutItemAttrs for syn::Item { + fn mut_item_attrs(&mut self) -> Option<&mut Vec> { + match self { + Self::Const(item) => Some(item.attrs.as_mut()), + Self::Enum(item) => Some(item.attrs.as_mut()), + Self::ExternCrate(item) => Some(item.attrs.as_mut()), + Self::Fn(item) => Some(item.attrs.as_mut()), + Self::ForeignMod(item) => Some(item.attrs.as_mut()), + Self::Impl(item) => Some(item.attrs.as_mut()), + Self::Macro(item) => Some(item.attrs.as_mut()), + Self::Mod(item) => Some(item.attrs.as_mut()), + Self::Static(item) => Some(item.attrs.as_mut()), + Self::Struct(item) => Some(item.attrs.as_mut()), + Self::Trait(item) => Some(item.attrs.as_mut()), + Self::TraitAlias(item) => Some(item.attrs.as_mut()), + Self::Type(item) => Some(item.attrs.as_mut()), + Self::Union(item) => Some(item.attrs.as_mut()), + Self::Use(item) => Some(item.attrs.as_mut()), + _ => None, + } + } +} + +impl MutItemAttrs for syn::TraitItem { + fn mut_item_attrs(&mut self) -> Option<&mut Vec> { + match self { + Self::Const(item) => Some(item.attrs.as_mut()), + Self::Fn(item) => Some(item.attrs.as_mut()), + Self::Type(item) => Some(item.attrs.as_mut()), + Self::Macro(item) => Some(item.attrs.as_mut()), + _ => None, + } + } +} + +impl MutItemAttrs for Vec { + fn mut_item_attrs(&mut self) -> Option<&mut Vec> { + Some(self) + } +} + +impl MutItemAttrs for syn::ItemMod { + fn mut_item_attrs(&mut self) -> Option<&mut Vec> { + Some(&mut self.attrs) + } +} + +impl MutItemAttrs for syn::ImplItemFn { + fn mut_item_attrs(&mut self) -> Option<&mut Vec> { + Some(&mut self.attrs) + } +} + +impl MutItemAttrs for syn::ItemType { + fn mut_item_attrs(&mut self) -> Option<&mut Vec> { + Some(&mut self.attrs) + } +} diff --git a/pvq-program-metadata-gen/src/lib.rs b/pvq-program-metadata-gen/src/lib.rs new file mode 100644 index 0000000..82961fe --- /dev/null +++ b/pvq-program-metadata-gen/src/lib.rs @@ -0,0 +1,9 @@ +pub type ExtensionId = u64; +pub type FnIndex = u8; +mod features; +mod helper; +pub use features::{extract_features, get_active_features}; +mod manifest; +pub use manifest::create_manifest; +mod metadata_gen; +pub use metadata_gen::metadata_gen_src; diff --git a/pvq-program-metadata-gen/src/manifest.rs b/pvq-program-metadata-gen/src/manifest.rs new file mode 100644 index 0000000..480b1df --- /dev/null +++ b/pvq-program-metadata-gen/src/manifest.rs @@ -0,0 +1,23 @@ +/// Creates a Cargo.toml file for the metadata generator +pub fn create_manifest(features: Option<&toml::Table>) -> String { + // Create a basic Cargo.toml for the temp crate + format!( + r#"[package] +name = "metadata_gen" +version = "0.1.0" +edition = "2021" + +[dependencies] +scale-info = {{ version = "2.0.0", features = ["derive","serde"] }} +parity-scale-codec = {{ version = "3.0.0", features = ["derive"] }} +serde = {{ version = "1", features = ["derive" ] }} +serde_json = "1" +cfg-if = "1.0" +{0} +"#, + features.map_or_else(String::new, |features| format!( + "\n[features]\n{}", + toml::to_string(&features).expect("Should be checked in parsing") + )) + ) +} diff --git a/pvq-program-metadata-gen/src/metadata_gen.rs b/pvq-program-metadata-gen/src/metadata_gen.rs new file mode 100644 index 0000000..02837d7 --- /dev/null +++ b/pvq-program-metadata-gen/src/metadata_gen.rs @@ -0,0 +1,289 @@ +use quote::{quote, ToTokens}; +use syn::spanned::Spanned; + +type ExtensionId = u64; +type FnIndex = u8; + +pub fn metadata_gen_src(source: &str, output_dir: &str) -> syn::Result { + // Parse the source code + let mut syntax = syn::parse_file(source)?; + + // Find the program module + // Find the index of the program module + let program_mod_idx = syntax + .items + .iter() + .position(|item| matches!(item, syn::Item::Mod(m) if m.attrs.iter().any(|attr|attr.path().segments.last().is_some_and(|last|last.ident == "program")))) + .ok_or(syn::Error::new( + proc_macro2::Span::call_site(), + "No program module found", + ))?; + + // Remove the program module from syntax.items + let mut program_mod = match syntax.items.remove(program_mod_idx) { + syn::Item::Mod(m) => m, + _ => unreachable!("We already checked this is a module"), + }; + + // Remove the program attr + program_mod.attrs.clear(); + let program_mod_items = &mut program_mod.content.as_mut().expect("This is checked before").1; + + // Find entrypoint and extension functions + let mut entrypoint_metadata = None; + let mut extension_fns_metadata = Vec::new(); + + for i in (0..program_mod_items.len()).rev() { + let item = &mut program_mod_items[i]; + if let Some(attr) = crate::helper::take_first_program_attr(item)? { + if let Some(last_segment) = attr.path().segments.last() { + if last_segment.ident == "extension_fn" { + let mut extension_id = None; + let mut fn_index = None; + attr.parse_nested_meta(|meta| { + if meta.path.is_ident("extension_id") { + let value = meta.value()?; + extension_id = Some(value.parse::()?.base10_parse::()?); + } else if meta.path.is_ident("fn_index") { + let value = meta.value()?; + fn_index = Some(value.parse::()?.base10_parse::()?); + } else { + return Err(syn::Error::new( + meta.path.span(), + "Invalid attribute meta, expected `extension_id` or `fn_index`", + )); + } + Ok(()) + })?; + let removed_item = program_mod_items.remove(i); + if extension_id.is_none() || fn_index.is_none() { + return Err(syn::Error::new( + attr.span(), + "Extension ID and function index are required", + )); + } + let extension_id = + extension_id.ok_or_else(|| syn::Error::new(attr.span(), "Extension ID is required"))?; + let fn_index = + fn_index.ok_or_else(|| syn::Error::new(attr.span(), "Function index is required"))?; + let extension_fn_metadata = generate_extension_fn_metadata(removed_item, extension_id, fn_index)?; + extension_fns_metadata.push(extension_fn_metadata); + } else if last_segment.ident == "entrypoint" { + if entrypoint_metadata.is_some() { + return Err(syn::Error::new(attr.span(), "Multiple entrypoint functions found")); + } + let removed_item = program_mod_items.remove(i); + entrypoint_metadata = Some(generate_entrypoint_metadata(removed_item)?); + } else { + return Err(syn::Error::new( + attr.span(), + "Invalid attribute, expected `#[program::extension_fn]` or `#[program::entrypoint]`", + )); + } + } + } + } + + let entrypoint_metadata = entrypoint_metadata + .ok_or_else(|| syn::Error::new(proc_macro2::Span::call_site(), "No entrypoint function found"))?; + + let metadata_defs = metadata_defs(); + let import_packages = import_packages(); + + let new_items = quote! { + #(#program_mod_items)* + #import_packages + #metadata_defs + fn main() { + let extension_fns = vec![ #( #extension_fns_metadata, )* ]; + let entrypoint = #entrypoint_metadata; + let metadata = Metadata::new(extension_fns, entrypoint); + // Serialize to both formats + let encoded = parity_scale_codec::Encode::encode(&metadata); + let json = serde_json::to_string(&metadata).expect("Failed to serialize metadata to JSON"); + + let bin_path = Path::new(#output_dir).join("metadata.bin"); + let json_path = Path::new(#output_dir).join("metadata.json"); + + // Write the binary format + std::fs::write(bin_path, &encoded).expect("Failed to write binary metadata"); + + // Write the JSON format + std::fs::write(json_path, json).expect("Failed to write JSON metadata"); + } + }; + + // Remove #![no_main] and #![no_std] attributes if present + syntax.attrs.retain(|attr| { + if let Some(segment) = attr.path().segments.last() { + let ident = &segment.ident; + !(ident == "no_main" || ident == "no_std") + } else { + true + } + }); + + syntax.items.push(syn::Item::Verbatim(new_items)); + + Ok(syntax.into_token_stream()) +} + +fn generate_extension_fn_metadata( + f: syn::Item, + extension_id: ExtensionId, + fn_index: FnIndex, +) -> syn::Result { + if let syn::Item::Fn(f) = f { + let fn_name = f.sig.ident.to_string(); + let mut inputs = Vec::new(); + for input in &f.sig.inputs { + if let syn::FnArg::Typed(syn::PatType { pat, ty, .. }) = input { + if let syn::Pat::Ident(pat_ident) = &**pat { + let name = pat_ident.ident.to_string(); + inputs.push(quote!( + FunctionParamMetadata { + name: #name, + ty: scale_info::meta_type::<#ty>(), + } + )); + } else { + return Err(syn::Error::new(input.span(), "Expected a typed argument")); + } + } else { + return Err(syn::Error::new(input.span(), "Expected a typed argument")); + } + } + let output = match &f.sig.output { + syn::ReturnType::Default => quote!(scale_info::meta_type::<()>()), + syn::ReturnType::Type(_, ty) => { + quote!(scale_info::meta_type::<#ty>()) + } + }; + Ok(quote! { + (#extension_id, #fn_index, FunctionMetadata { + name: #fn_name, + inputs: vec![#(#inputs,)*], + output: #output + }) + }) + } else { + Err(syn::Error::new(f.span(), "Expected a function")) + } +} +fn generate_entrypoint_metadata(f: syn::Item) -> syn::Result { + if let syn::Item::Fn(f) = f { + let name = f.sig.ident.to_string(); + let mut inputs = Vec::new(); + for input in &f.sig.inputs { + if let syn::FnArg::Typed(syn::PatType { pat, ty, .. }) = input { + if let syn::Pat::Ident(pat_ident) = &**pat { + let name = pat_ident.ident.to_string(); + inputs.push(quote!( + FunctionParamMetadata { + name: #name, + ty: scale_info::meta_type::<#ty>(), + } + )); + } else { + return Err(syn::Error::new(input.span(), "Expected a typed argument")); + } + } else { + return Err(syn::Error::new(input.span(), "Expected a typed argument")); + } + } + let output = match &f.sig.output { + syn::ReturnType::Default => quote!(scale_info::meta_type::<()>()), + syn::ReturnType::Type(_, ty) => { + quote!(scale_info::meta_type::<#ty>()) + } + }; + Ok(quote! { + FunctionMetadata { + name: #name, + inputs: vec![#(#inputs,)*], + output: #output + } + }) + } else { + Err(syn::Error::new(f.span(), "Expected a function")) + } +} + +fn import_packages() -> proc_macro2::TokenStream { + quote! { + extern crate alloc; + use std::path::Path; + use serde::Serialize; + use parity_scale_codec::Encode; + use scale_info::{ + form::{Form, MetaForm, PortableForm}, + prelude::vec::Vec, + IntoPortable, PortableRegistry, Registry, + }; + } +} +fn metadata_defs() -> proc_macro2::TokenStream { + quote! { + type ExtensionId = u64; + type FnIndex = u8; + /// Metadata of extensions + #[derive(Clone, PartialEq, Eq, Encode, Debug, Serialize)] + pub struct Metadata { + pub types: PortableRegistry, + pub extension_fns: Vec<(ExtensionId, FnIndex, FunctionMetadata)>, + pub entrypoint: FunctionMetadata, + } + + impl Metadata { + pub fn new(extension_fns: Vec<(ExtensionId, FnIndex, FunctionMetadata)>, entrypoint: FunctionMetadata) -> Self { + let mut registry = Registry::new(); + let extension_fns = extension_fns + .into_iter() + .map(|(id, index, metadata)| (id, index, metadata.into_portable(&mut registry))) + .collect(); + let entrypoint = entrypoint.into_portable(&mut registry); + Self { + types: registry.into(), + extension_fns, + entrypoint, + } + } + } + + #[derive(Clone, PartialEq, Eq, Encode, Debug, Serialize)] + pub struct FunctionMetadata { + pub name: T::String, + pub inputs: Vec>, + pub output: T::Type, + } + + impl IntoPortable for FunctionMetadata { + type Output = FunctionMetadata; + + fn into_portable(self, registry: &mut Registry) -> Self::Output { + FunctionMetadata { + name: self.name.into_portable(registry), + inputs: registry.map_into_portable(self.inputs), + output: registry.register_type(&self.output), + } + } + } + + #[derive(Clone, PartialEq, Eq, Encode, Debug, Serialize)] + pub struct FunctionParamMetadata { + pub name: T::String, + pub ty: T::Type, + } + + impl IntoPortable for FunctionParamMetadata { + type Output = FunctionParamMetadata; + + fn into_portable(self, registry: &mut Registry) -> Self::Output { + FunctionParamMetadata { + name: self.name.into_portable(registry), + ty: registry.register_type(&self.ty), + } + } + } + } +} diff --git a/pvq-program/Cargo.toml b/pvq-program/Cargo.toml index c2a5999..03e34c5 100644 --- a/pvq-program/Cargo.toml +++ b/pvq-program/Cargo.toml @@ -9,12 +9,9 @@ version.workspace = true [dependencies] pvq-program-procedural = { path = "procedural" } - -[dev-dependencies] parity-scale-codec = { workspace = true } scale-info = { workspace = true } polkavm-derive = { workspace = true } -trybuild = { workspace = true } [features] default = ["std"] diff --git a/pvq-program/procedural/src/program/parse/mod.rs b/pvq-program/procedural/src/program/parse/mod.rs index 0101cdd..187ef53 100644 --- a/pvq-program/procedural/src/program/parse/mod.rs +++ b/pvq-program/procedural/src/program/parse/mod.rs @@ -30,8 +30,7 @@ impl Def { let mut extension_fns = Vec::new(); let mut entrypoint = None; - let mut i = 0; - while i < items.len() { + for i in (0..items.len()).rev() { let item = &mut items[i]; if let Some(attr) = helper::take_first_program_attr(item)? { if let Some(last_segment) = attr.path().segments.last() { @@ -72,7 +71,6 @@ impl Def { } } } - i += 1; } let entrypoint = diff --git a/pvq-test-runner/Cargo.toml b/pvq-test-runner/Cargo.toml index a1a8f98..9d5c327 100644 --- a/pvq-test-runner/Cargo.toml +++ b/pvq-test-runner/Cargo.toml @@ -12,8 +12,10 @@ clap = { workspace = true } tracing = { workspace = true, features = ["std"] } tracing-subscriber = { workspace = true } parity-scale-codec = { workspace = true, features = ["std"] } -scale-info = { workspace = true, features = ["std"] } +scale-info = { workspace = true, features = ["std", "serde"] } sp-core = { workspace = true, features = ["std"] } +serde = { workspace = true } +serde_json = { workspace = true } pvq-executor = { workspace = true, features = ["std"] } pvq-extension = { workspace = true, features = ["std"] } diff --git a/pvq-test-runner/src/main.rs b/pvq-test-runner/src/bin/pvq-test-runner.rs similarity index 83% rename from pvq-test-runner/src/main.rs rename to pvq-test-runner/src/bin/pvq-test-runner.rs index 5f1d7b3..e8e4a0c 100644 --- a/pvq-test-runner/src/main.rs +++ b/pvq-test-runner/src/bin/pvq-test-runner.rs @@ -34,5 +34,9 @@ fn main() { let mut runner = TestRunner::new(); let res = runner.execute_program(&blob, &input_data).unwrap(); + let metadata = pvq_test_runner::extensions::metadata(); + let metadata_json = serde_json::to_string(&metadata).expect("Failed to serialize metadata"); + tracing::info!("Metadata: {}", metadata_json); + tracing::info!("Result: {:?}", res); }