diff --git a/Cargo.lock b/Cargo.lock index aa9d0aa..ee2306e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -134,6 +134,12 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "ascii" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d92bec98840b8f03a5ff5413de5293bfcd8bf96467cf5452609f939ec6f5de16" + [[package]] name = "atomic" version = "0.6.1" @@ -185,6 +191,12 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7" +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + [[package]] name = "bitflags" version = "2.10.0" @@ -237,7 +249,6 @@ dependencies = [ "chrono", "citationberg", "clap", - "directories", "duct", "eipw-lint", "eipw-preamble", @@ -252,15 +263,16 @@ dependencies = [ "iref", "lazy_static", "log", + "notify", "pulldown-cmark", "pulldown-cmark-to-cmark", "regex", "semver", "serde", "serde_json", - "sha3", "snafu", "tempfile", + "tiny_http", "tokio", "toml 0.9.11+spec-1.1.0", "toml_datetime 0.7.5+spec-1.1.0", @@ -341,6 +353,12 @@ dependencies = [ "windows-link", ] +[[package]] +name = "chunked_transfer" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e4de3bc4ea267985becf712dc6d9eed8b04c953b3fcfb339ebc87acd9804901" + [[package]] name = "ciborium" version = "0.2.2" @@ -467,6 +485,21 @@ dependencies = [ "libc", ] +[[package]] +name = "crossbeam-channel" +version = "0.5.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82b8f8f868b36967f9606790d1903570de9ceaf870a7bf9fbbd3016d636a2cb2" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" + [[package]] name = "crunchy" version = "0.2.4" @@ -533,22 +566,13 @@ dependencies = [ "crypto-common", ] -[[package]] -name = "directories" -version = "6.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16f5094c54661b38d03bd7e50df373292118db60b585c08a411c6d840017fe7d" -dependencies = [ - "dirs-sys 0.5.0", -] - [[package]] name = "dirs" version = "5.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "44c45a9d03d6676652bcb5e724c7e988de1acad23a711b5217ab9cbecbec2225" dependencies = [ - "dirs-sys 0.4.1", + "dirs-sys", ] [[package]] @@ -559,22 +583,10 @@ checksum = "520f05a5cbd335fae5a99ff7a6ab8627577660ee5cfd6a94a6a929b52ff0321c" dependencies = [ "libc", "option-ext", - "redox_users 0.4.6", + "redox_users", "windows-sys 0.48.0", ] -[[package]] -name = "dirs-sys" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e01a3366d27ee9890022452ee61b2b63a67e6f13f58900b651ff5665f0bb1fab" -dependencies = [ - "libc", - "option-ext", - "redox_users 0.5.2", - "windows-sys 0.61.2", -] - [[package]] name = "displaydoc" version = "0.2.5" @@ -805,6 +817,17 @@ dependencies = [ "version_check", ] +[[package]] +name = "filetime" +version = "0.2.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f98844151eee8917efc50bd9e8318cb963ae8b297431495d3f758616ea5c57db" +dependencies = [ + "cfg-if", + "libc", + "libredox", +] + [[package]] name = "find-msvc-tools" version = "0.1.9" @@ -853,6 +876,15 @@ dependencies = [ "num", ] +[[package]] +name = "fsevent-sys" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76ee7a02da4d231650c7cea31349b889be2f45ddb3ef3032d2ec8185f6313fd2" +dependencies = [ + "libc", +] + [[package]] name = "fslock" version = "0.2.1" @@ -936,7 +968,7 @@ version = "0.20.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7b88256088d75a56f8ecfa070513a775dd9107f6530ef14919dac831af9cfe2b" dependencies = [ - "bitflags", + "bitflags 2.10.0", "libc", "libgit2-sys", "log", @@ -1051,6 +1083,12 @@ dependencies = [ "match_token", ] +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + [[package]] name = "iana-time-zone" version = "0.1.65" @@ -1221,6 +1259,26 @@ dependencies = [ "rustversion", ] +[[package]] +name = "inotify" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8069d3ec154eb856955c1c0fbffefbf5f3c40a104ec912d4797314c1801abff" +dependencies = [ + "bitflags 1.3.2", + "inotify-sys", + "libc", +] + +[[package]] +name = "inotify-sys" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e05c02b5e89bff3b946cedeca278abc628fe811e604f027c45a8aa3cf793d0eb" +dependencies = [ + "libc", +] + [[package]] name = "iref" version = "3.2.2" @@ -1340,6 +1398,26 @@ dependencies = [ "cpufeatures", ] +[[package]] +name = "kqueue" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eac30106d7dce88daf4a3fcb4879ea939476d5074a9b7ddd0fb97fa4bed5596a" +dependencies = [ + "kqueue-sys", + "libc", +] + +[[package]] +name = "kqueue-sys" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed9625ffda8729b85e45cf04090035ac368927b8cebc34898e7c120f52e4838b" +dependencies = [ + "bitflags 1.3.2", + "libc", +] + [[package]] name = "lazy_static" version = "1.5.0" @@ -1372,8 +1450,9 @@ version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3d0b95e02c851351f877147b7deea7b1afb1df71b63aa5f8270716e0c5720616" dependencies = [ - "bitflags", + "bitflags 2.10.0", "libc", + "redox_syscall 0.7.4", ] [[package]] @@ -1501,6 +1580,18 @@ dependencies = [ "adler2", ] +[[package]] +name = "mio" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4a650543ca06a924e8b371db273b2756685faae30f8487da1b56505a8f78b0c" +dependencies = [ + "libc", + "log", + "wasi", + "windows-sys 0.48.0", +] + [[package]] name = "new_debug_unreachable" version = "1.0.6" @@ -1517,6 +1608,25 @@ dependencies = [ "minimal-lexical", ] +[[package]] +name = "notify" +version = "6.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6205bd8bb1e454ad2e27422015fb5e4f2bcc7e08fa8f27058670d208324a4d2d" +dependencies = [ + "bitflags 2.10.0", + "crossbeam-channel", + "filetime", + "fsevent-sys", + "inotify", + "kqueue", + "libc", + "log", + "mio", + "walkdir", + "windows-sys 0.48.0", +] + [[package]] name = "num" version = "0.4.3" @@ -1686,7 +1796,7 @@ checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" dependencies = [ "cfg-if", "libc", - "redox_syscall", + "redox_syscall 0.5.18", "smallvec", "windows-link", ] @@ -1846,7 +1956,7 @@ version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e8bbe1a966bd2f362681a44f6edce3c2310ac21e4d5067a6e7ec396297a6ea0" dependencies = [ - "bitflags", + "bitflags 2.10.0", "getopts", "memchr", "pulldown-cmark-escape", @@ -1920,29 +2030,27 @@ version = "0.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" dependencies = [ - "bitflags", + "bitflags 2.10.0", ] [[package]] -name = "redox_users" -version = "0.4.6" +name = "redox_syscall" +version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba009ff324d1fc1b900bd1fdb31564febe58a8ccc8a6fdbb93b543d33b13ca43" +checksum = "f450ad9c3b1da563fb6948a8e0fb0fb9269711c9c73d9ea1de5058c79c8d643a" dependencies = [ - "getrandom 0.2.17", - "libredox", - "thiserror 1.0.69", + "bitflags 2.10.0", ] [[package]] name = "redox_users" -version = "0.5.2" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4e608c6638b9c18977b00b475ac1f28d14e84b27d8d42f70e0bf1e3dec127ac" +checksum = "ba009ff324d1fc1b900bd1fdb31564febe58a8ccc8a6fdbb93b543d33b13ca43" dependencies = [ "getrandom 0.2.17", "libredox", - "thiserror 2.0.18", + "thiserror 1.0.69", ] [[package]] @@ -2026,7 +2134,7 @@ version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" dependencies = [ - "bitflags", + "bitflags 2.10.0", "errno", "libc", "linux-raw-sys", @@ -2104,7 +2212,7 @@ version = "0.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fd568a4c9bb598e291a08244a5c1f5a8a6650bee243b5b0f8dbb3d9cc1d87fe8" dependencies = [ - "bitflags", + "bitflags 2.10.0", "cssparser", "derive_more", "fxhash", @@ -2514,6 +2622,18 @@ dependencies = [ "syn 2.0.114", ] +[[package]] +name = "tiny_http" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "389915df6413a2e74fb181895f933386023c71110878cd0825588928e64cdc82" +dependencies = [ + "ascii", + "chunked_transfer", + "httpdate", + "log", +] + [[package]] name = "tinystr" version = "0.8.2" diff --git a/Cargo.toml b/Cargo.toml index 4529c21..4bbfb89 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -21,7 +21,6 @@ depends = "$auto, libc6, libssl3, zlib1g, libgcc-s1, libgit2-1.5, git" [dependencies] chrono = "0.4.42" clap = { version = "4.5.53", features = ["cargo", "derive"] } -directories = "6.0.0" duct = "1.1.1" eipw-lint = { version = "0.10.0", features = [ "tokio", "schema-version" ] } eipw-snippets = "0.2.0" @@ -34,14 +33,15 @@ indicatif = "0.18.3" indicatif-log-bridge = "0.2.3" lazy_static = "1.5.0" log = { version = "0.4.29", features = ["std"] } +notify = "6.1.1" pulldown-cmark = "0.13.0" pulldown-cmark-to-cmark = "22.0.0" regex = "1.12.2" semver = {version = "1.0.27", features = ["serde"] } serde = { version = "1.0.228", features = ["derive"] } serde_json = "1.0.148" -sha3 = "0.10.8" snafu = { version = "0.8.9", features = ["rust_1_81"] } +tiny_http = "0.12.0" tokio = { version = "1.48.0", features = ["fs", "rt", "macros"] } toml = "0.9.10" toml_datetime = { version = "0.7.5", features = ["serde"] } diff --git a/src/cache.rs b/src/cache.rs deleted file mode 100644 index 9c7eb2d..0000000 --- a/src/cache.rs +++ /dev/null @@ -1,84 +0,0 @@ -/* - * This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at https://mozilla.org/MPL/2.0/. - */ - -use std::{io::ErrorKind, path::PathBuf, sync::Arc}; - -use directories::ProjectDirs; -use fslock::LockFile; -use log::{debug, info}; -use sha3::{Digest, Sha3_256}; -use snafu::{Backtrace, IntoError, OptionExt, Report, ResultExt, Snafu}; - -#[derive(Debug, Snafu)] -pub enum Error { - #[snafu(display("unable to discover application directories (unset $HOME?)"))] - Directories { backtrace: Backtrace }, - #[snafu(display("unable to access the path `{}`", path.to_string_lossy()))] - Fs { - path: PathBuf, - backtrace: Backtrace, - source: std::io::Error, - }, -} - -#[derive(Debug)] -struct Inner { - _lock: LockFile, - dir: PathBuf, -} - -#[derive(Debug, Clone)] -pub struct Cache(Arc); - -impl Cache { - pub fn open() -> Result { - debug!("opening local file cache"); - - let dirs = - ProjectDirs::from("org.ethereum", "eips", "eips-build").context(DirectoriesSnafu)?; - let cache_path = dirs.cache_dir(); - if let Err(e) = std::fs::create_dir_all(cache_path) { - debug!( - "got while creating cache directory: {}", - Report::from_error(e) - ); - } - - let lock_path = cache_path.join(".lock"); - let mut lock = LockFile::open(&lock_path).context(FsSnafu { path: &lock_path })?; - - let locked = lock - .try_lock_with_pid() - .context(FsSnafu { path: &lock_path })?; - - if !locked { - info!("waiting on cache directory..."); - lock.lock_with_pid().context(FsSnafu { path: &lock_path })?; - } - - Ok(Self(Arc::new(Inner { - _lock: lock, - dir: cache_path.into(), - }))) - } - - pub fn dir(&self, key: &str) -> Result { - let mut hasher = Sha3_256::new(); - hasher.update(key.as_bytes()); - let hash = hasher.finalize(); - let hash_text = format!("{:x}", hash); - let path = self.0.dir.join(hash_text); - - debug!("creating cache directory `{}`", path.to_string_lossy()); - match std::fs::create_dir(&path) { - Ok(()) => (), - Err(e) if e.kind() == ErrorKind::AlreadyExists => (), - Err(e) => return Err(FsSnafu { path }.into_error(e)), - } - - Ok(path) - } -} diff --git a/src/changed.rs b/src/changed.rs index 4472b59..07ad880 100644 --- a/src/changed.rs +++ b/src/changed.rs @@ -4,78 +4,46 @@ * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ -//! Changed-file command execution. +//! Changed-file command helpers. -use std::{ - ffi::OsStr, - path::{Path, PathBuf}, -}; +use std::path::Path; use snafu::{ResultExt, Whatever}; -use crate::{cli::ChangedFormat, config::RepositoryUse, git, layout::REPO_DIR}; - -pub(crate) fn is_proposal_path(mut p: PathBuf) -> bool { - // Only lint `content/00001.md` and `content/00001/index.md` files. - - // content/00000.md | content/00000/index.md - // ^^^^^^^^ | ^^^^^^^^ - match p.file_name() { - Some(n) if n == "index.md" => { - p.pop(); - } - Some(_) if p.extension().map(|x| x == "md").unwrap_or(false) => { - p.set_extension(""); - } - None | Some(_) => return false, - } - - // content/00000 - // ^^^^^ - match p.file_name().and_then(OsStr::to_str) { - None => return false, - Some(f) if f.parse::().is_err() => return false, - Some(_) => { - p.pop(); - } - } - - // content - // ^^^^^^^ - match p.file_name() { - Some(f) if f == "content" => { - p.pop(); - } - _ => return false, - } - - p == OsStr::new("") -} +use crate::{ + cli::ChangedFormat, execution::ResolvedExecution, git, layout::REPO_DIR, + proposal::is_proposal_path, +}; pub(crate) fn run( - root_path: &Path, + resolved: &ResolvedExecution, build_path: &Path, - repo_use: RepositoryUse, all: bool, format: &ChangedFormat, ) -> Result<(), Whatever> { let repo_path = build_path.join(REPO_DIR); - let both = git::Fresh::new(root_path, &repo_path, repo_use) - .whatever_context("initializing build repo")? - .clone_src() - .whatever_context("cloning source repo")? - .fetch_upstream() - .whatever_context("fetching upstream repo")?; + let both = git::Fresh::new( + &resolved.root_path, + &repo_path, + resolved.repository_use.clone(), + resolved.source_materialization, + ) + .whatever_context("initializing build repo")? + .clone_src() + .whatever_context("cloning source repo")? + .fetch_upstream() + .whatever_context("fetching upstream repo")?; let changed_files: Vec<_> = both .changed_files() .whatever_context("unable to list changed files")? .into_iter() - .filter(|p| all || is_proposal_path(p.into())) + .filter(|p| all || is_proposal_path(p)) .map(|p| repo_path.join(p)) .collect(); format.print(&changed_files, &repo_path); + Ok(()) } diff --git a/src/cli.rs b/src/cli.rs index ff93bbf..7186c55 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -9,8 +9,9 @@ use std::path::{Path, PathBuf}; use clap::{Parser, Subcommand}; +use url::Url; -use crate::{lint, print}; +use crate::{lint, print, proposal::ProposalNumber}; /// Build script for Ethereum EIPs and ERCs. #[derive(Parser, Debug)] @@ -20,11 +21,51 @@ pub(crate) struct Args { #[clap(short = 'C')] pub(crate) root: Option, + /// Use the configured remote sibling content repositories + #[clap(long)] + pub(crate) remote_siblings: bool, + + /// Write build artifacts under BUILD_ROOT instead of the default location + #[clap(long)] + pub(crate) build_root: Option, + #[clap(subcommand)] pub(crate) operation: Operation, } -#[derive(Debug, Subcommand)] +#[derive(Debug, Clone, Default, PartialEq, Eq, clap::Args)] +pub(crate) struct ServerCliArgs { + /// Host/interface for the local server to bind + #[arg(long)] + pub(crate) host: Option, + + /// Port for the local server to bind + #[arg(long)] + pub(crate) port: Option, +} + +#[derive(Debug, Clone, Default, PartialEq, Eq, clap::Args)] +pub(crate) struct BaseUrlCliArgs { + /// Override the rendered-site base URL for this command + #[arg(long, value_parser = clap::value_parser!(Url))] + pub(crate) base_url: Option, +} + +#[derive(Debug, Clone, Default, PartialEq, Eq, clap::Args)] +pub(crate) struct CleanCliArgs { + /// Ignore tracked working-tree changes in the active repo + #[arg(long)] + pub(crate) clean: bool, +} + +#[derive(Debug, Clone, Default, PartialEq, Eq, clap::Args)] +pub(crate) struct OnlyCliArgs { + /// Render only the selected proposal number(s) + #[arg(long, value_name = "NUMBER", value_parser = ProposalNumber::parse_cli_selector, num_args = 1..)] + pub(crate) only: Vec, +} + +#[derive(Debug, Clone, Subcommand)] pub(crate) enum Operation { /// Print various useful things, like available lints Print { @@ -35,13 +76,34 @@ pub(crate) enum Operation { /// Build the project and output HTML Build { #[command(flatten)] - eipw: lint::CmdArgs, + base_url: BaseUrlCliArgs, + + #[command(flatten)] + clean: CleanCliArgs, + + #[command(flatten)] + only: OnlyCliArgs, + }, + + /// Serve the existing built output without rebuilding it + Preview { + #[command(flatten)] + server: ServerCliArgs, }, /// Build the project and launch a web server to preview it Serve { #[command(flatten)] - eipw: lint::CmdArgs, + server: ServerCliArgs, + + #[command(flatten)] + base_url: BaseUrlCliArgs, + + #[command(flatten)] + clean: CleanCliArgs, + + #[command(flatten)] + only: OnlyCliArgs, }, /// Remove temporary and output files @@ -50,7 +112,7 @@ pub(crate) enum Operation { /// Analyze the repository and report errors, but don't build HTML files Check { #[command(flatten)] - eipw: lint::CmdArgs, + clean: CleanCliArgs, }, /// List files changed since the last commit common to both the local and upstream repositories @@ -61,6 +123,114 @@ pub(crate) enum Operation { #[clap(long, value_enum, default_value_t)] format: ChangedFormat, }, + + /// Run targeted editorial lint or check workflows + Editorial { + #[command(subcommand)] + command: EditorialCommand, + }, + + /// Create workspace config, docs, build root, and missing local repos + Init { + /// Workspace root directory + path: PathBuf, + + /// Also clone template for proposal-family scaffold work + #[arg(long)] + template: bool, + }, + + /// Check workspace layout, local repos, and required tools + Doctor, +} + + +#[derive(Debug, Subcommand, Clone)] +pub(crate) enum EditorialCommand { + Lint { #[command(flatten)] selectors: EditorialSelectorArgs, #[command(flatten)] eipw: lint::CmdArgs }, + Check { #[command(flatten)] selectors: EditorialSelectorArgs, #[command(flatten)] eipw: lint::CmdArgs }, +} + +#[derive(Debug, clap::Args, Clone)] +pub(crate) struct EditorialSelectorArgs { + #[arg(value_name = "TARGET")] + pub(crate) paths: Vec, + #[arg(long)] + pub(crate) batch: Option, + #[arg(long)] + pub(crate) working_tree: bool, + #[arg(long)] + pub(crate) against_upstream: bool, +} + +#[derive(Debug, Clone)] +pub(crate) enum RuntimeOperation { + Build, + Serve, + Clean, + Check, + Changed { all: bool, format: ChangedFormat }, + Preview, + Editorial { command: EditorialCommand }, +} + +impl Operation { + pub(crate) fn server_cli_args(&self) -> ServerCliArgs { + match self { + Self::Serve { server, .. } | Self::Preview { server } => server.clone(), + _ => ServerCliArgs::default(), + } + } + + pub(crate) fn base_url_cli_args(&self) -> BaseUrlCliArgs { + match self { + Self::Build { base_url, .. } | Self::Serve { base_url, .. } => base_url.clone(), + _ => BaseUrlCliArgs::default(), + } + } + + pub(crate) fn clean_cli_args(&self) -> CleanCliArgs { + match self { + Self::Build { clean, .. } | Self::Serve { clean, .. } | Self::Check { clean } => clean.clone(), + _ => CleanCliArgs::default(), + } + } + + pub(crate) fn only_cli_args(&self) -> Option<&OnlyCliArgs> { + match self { + Self::Build { only, .. } | Self::Serve { only, .. } => Some(only), + _ => None, + } + } + + pub(crate) fn is_plain_site_command(&self) -> bool { + matches!(self, Self::Build { .. } | Self::Serve { .. } | Self::Check { .. }) + } + + pub(crate) fn runtime_operation(&self) -> Option { + match self { + Self::Print { .. } | Self::Init { .. } | Self::Doctor => None, + Self::Build { .. } => Some(RuntimeOperation::Build), + Self::Serve { .. } => Some(RuntimeOperation::Serve), + Self::Preview { .. } => Some(RuntimeOperation::Preview), + Self::Clean => Some(RuntimeOperation::Clean), + Self::Check { .. } => Some(RuntimeOperation::Check), + Self::Changed { all, format } => Some(RuntimeOperation::Changed { all: *all, format: format.clone() }), + Self::Editorial { command } => Some(RuntimeOperation::Editorial { command: command.clone() }), + } + } + + pub(crate) fn is_editorial_command(&self) -> bool { + matches!(self, Self::Editorial { .. }) + } + + pub(crate) fn is_workspace_lifecycle_command(&self) -> bool { + matches!(self, Self::Init { .. } | Self::Doctor) + } + + pub(crate) fn is_print_command(&self) -> bool { + matches!(self, Self::Print { .. }) + } } #[derive(Debug, clap::ValueEnum, Clone, Default)] diff --git a/src/config.rs b/src/config.rs index bb01cef..a13df27 100644 --- a/src/config.rs +++ b/src/config.rs @@ -4,14 +4,28 @@ * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ -use std::{borrow::Borrow, collections::HashMap, path::PathBuf, str::FromStr}; +use std::{ + borrow::Borrow, + collections::HashMap, + fmt, + path::{Path, PathBuf}, + str::FromStr, +}; use regex::Regex; use serde::{Deserialize, Serialize}; use snafu::{Backtrace, IntoError, OptionExt, ResultExt, Snafu}; use url::Url; +use crate::proposal::ProposalNumber; + pub const MANIFEST_FILE: &str = "Build.toml"; +pub const LOCAL_CONFIG_FILE: &str = ".build-eips.toml"; +pub const DEFAULT_BUILD_ROOT_BASE: &str = ".local-build"; +pub const DEFAULT_THEME_DIR: &str = "theme"; +pub const DEFAULT_SERVER_HOST: &str = "127.0.0.1"; +pub const DEFAULT_SERVER_PORT: u16 = 1111; +pub const DEFAULT_SITE_BASE_URL: &str = "http://127.0.0.1:1111"; #[derive(Debug, Snafu)] #[non_exhaustive] @@ -24,7 +38,7 @@ pub enum Error { }, #[snafu(display( - "unable to parse repo manifest `{}`", + "unable to parse Build.toml `{}`", manifest_path.to_string_lossy() ))] Parse { @@ -35,7 +49,7 @@ pub enum Error { }, #[snafu(display( - "repo manifest `{}` is invalid: {}", + "Build.toml `{}` is invalid: {}", manifest_path.to_string_lossy(), source, ))] @@ -47,6 +61,30 @@ pub enum Error { }, } +#[derive(Debug, Snafu)] +pub enum WorkspaceError { + #[snafu(display("i/o error while accessing `{}`", path.to_string_lossy()))] + Fs { + path: PathBuf, + source: std::io::Error, + backtrace: Backtrace, + }, + + #[snafu( + context(name(WorkspaceParseSnafu)), + display( + "unable to parse workspace config `{}`", + config_path.to_string_lossy() + ) + )] + Parse { + config_path: PathBuf, + #[snafu(source(from(toml::de::Error, Box::new)))] + source: Box, + backtrace: Backtrace, + }, +} + #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] #[serde(deny_unknown_fields, rename_all = "kebab-case")] pub struct Theme { @@ -232,6 +270,258 @@ impl TryFrom for RepositoryUse { } } +/// Resolved manifest context for the active proposal repository. +#[derive(Debug, Clone)] +pub struct ActiveRepo { + /// Managed workspace title for the active repository. + pub title: String, + + /// Declared sibling repository titles. + pub sibling_ids: Vec, + + /// Normalized source selection for the active repository and its siblings. + pub repository_use: RepositoryUse, + + /// Theme metadata declared by the active repository manifest. + pub theme: Theme, +} + +impl ActiveRepo { + /// Load the active repository manifest from its working-tree checkout. + pub fn load(repo_root: &Path) -> Result { + let manifest = Manifest::load(repo_root.join(MANIFEST_FILE))?; + let manifest_path = manifest.manifest_path.clone(); + let theme = manifest.theme.clone(); + let repository_use = + RepositoryUse::try_from(manifest).context(InvalidSnafu { manifest_path })?; + let title = repository_use.title.clone(); + let mut sibling_ids = repository_use + .other_repos + .keys() + .cloned() + .collect::>(); + sibling_ids.sort(); + + Ok(Self { + title, + sibling_ids, + repository_use, + theme, + }) + } +} + +/// Workspace-local configuration loaded from `.build-eips.toml`. +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +#[serde(default, deny_unknown_fields)] +pub struct WorkspaceConfig { + /// Local server defaults for `build-eips serve` and `build-eips preview`. + #[serde(default)] + pub server: ServerSettings, + + /// Local rendered-site URL defaults for build and serve commands. + #[serde(default)] + pub site: SiteSettings, + + /// Local render filtering defaults. + #[serde(default)] + pub render: RenderSettings, +} + +impl WorkspaceConfig { + fn starter() -> Self { + Self { + server: ServerSettings::default(), + site: SiteSettings::starter(), + render: RenderSettings::default(), + } + } +} + +/// Local render filtering settings. +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(default, deny_unknown_fields)] +pub struct RenderSettings { + /// Proposal numbers to render for applicable local build and serve commands. + #[serde(default)] + pub only: Vec, +} + +/// Workspace-local bind address defaults for local server commands. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(default, deny_unknown_fields)] +pub struct ServerSettings { + /// Host or interface address used by `serve` and `preview`. + pub host: String, + + /// TCP port used by `serve` and `preview`. + pub port: u16, +} + +impl Default for ServerSettings { + fn default() -> Self { + Self { + host: DEFAULT_SERVER_HOST.to_owned(), + port: DEFAULT_SERVER_PORT, + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ServerBinding { + pub host: String, + pub port: u16, +} + +impl Default for ServerBinding { + fn default() -> Self { + ServerSettings::default().into() + } +} + +impl From for ServerBinding { + fn from(settings: ServerSettings) -> Self { + Self { + host: settings.host, + port: settings.port, + } + } +} + +impl From<&ServerSettings> for ServerBinding { + fn from(settings: &ServerSettings) -> Self { + settings.clone().into() + } +} + +impl fmt::Display for ServerBinding { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(formatter, "{}:{}", self.host, self.port) + } +} + +/// Workspace-local rendered-site URL defaults for build and serve commands. +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(default, deny_unknown_fields)] +pub struct SiteSettings { + /// Base URL written into rendered HTML, feeds, canonical links, and sitemaps. + #[serde(default)] + pub base_url: Option, +} + +impl SiteSettings { + fn starter() -> Self { + Self { + base_url: Some( + DEFAULT_SITE_BASE_URL + .parse() + .expect("default site base URL should parse"), + ), + } + } +} + +#[derive(Debug, Clone)] +pub struct LoadedWorkspaceConfig { + config_path: PathBuf, + workspace_root: PathBuf, + config: WorkspaceConfig, +} + +impl LoadedWorkspaceConfig { + pub fn from_path(path: &Path) -> Result { + let config_path = path.canonicalize().with_context(|_| FsSnafu { + path: path.to_path_buf(), + })?; + let contents = std::fs::read_to_string(&config_path).with_context(|_| FsSnafu { + path: config_path.clone(), + })?; + let config = + toml::from_str::(&contents).with_context(|_| WorkspaceParseSnafu { + config_path: config_path.clone(), + })?; + + let workspace_root = config_path + .parent() + .expect("workspace config should always have a parent") + .to_path_buf(); + + Ok(Self { + config_path, + workspace_root, + config, + }) + } + + pub fn discover(start: &Path) -> Result, WorkspaceError> { + match discover_path(start) { + Some(path) => Self::from_path(&path).map(Some), + None => Ok(None), + } + } + + pub fn config_path(&self) -> &Path { + &self.config_path + } + + pub fn workspace_root(&self) -> &Path { + &self.workspace_root + } + + pub fn workspace_build_root(&self, repo_name: &str) -> PathBuf { + self.workspace_root + .join(DEFAULT_BUILD_ROOT_BASE) + .join(repo_name) + } + + pub fn server_settings(&self) -> &ServerSettings { + &self.config.server + } + + pub fn site_settings(&self) -> &SiteSettings { + &self.config.site + } + + pub fn render_settings(&self) -> &RenderSettings { + &self.config.render + } + + pub fn local_theme_path(&self) -> PathBuf { + self.workspace_root.join(DEFAULT_THEME_DIR) + } + + pub fn local_repo_path(&self, repo_name: &str) -> PathBuf { + self.workspace_root.join(repo_name) + } +} + +pub fn discover_path(start: &Path) -> Option { + let mut current = Some(start); + + while let Some(candidate) = current { + let config_path = candidate.join(LOCAL_CONFIG_FILE); + match std::fs::File::open(&config_path) { + Ok(_) => return Some(config_path), + Err(error) + if matches!( + error.kind(), + std::io::ErrorKind::NotFound | std::io::ErrorKind::NotADirectory + ) => + { + current = candidate.parent(); + } + Err(_) => return Some(config_path), + } + } + + None +} + +pub fn default_workspace_config_text() -> String { + toml::to_string_pretty(&WorkspaceConfig::starter()) + .expect("workspace starter config should serialize") +} + #[cfg(test)] mod tests { use std::path::{Path, PathBuf}; @@ -270,9 +560,9 @@ mod tests { } #[test] - fn malformed_repo_manifest_reports_parse_error() { + fn malformed_build_manifest_reports_parse_error() { let repo = TestRepo::new(); - let manifest_path = repo.write_file(MANIFEST_FILE, "repo_id = ["); + let manifest_path = repo.write_file(MANIFEST_FILE, "name = ["); let error = Manifest::load(&manifest_path).unwrap_err(); @@ -280,7 +570,7 @@ mod tests { } #[test] - fn parses_repo_manifest() { + fn parses_build_manifest() { let repo = TestRepo::new(); let manifest_path = repo.write_file( MANIFEST_FILE, @@ -307,7 +597,7 @@ commit = "aaa" } #[test] - fn repo_manifest_rejects_unsafe_names() { + fn build_manifest_rejects_unsafe_names() { let repo = TestRepo::new(); let manifest_path = repo.write_file(MANIFEST_FILE, r#"name = "^^^^""#); @@ -321,7 +611,7 @@ commit = "aaa" } #[test] - fn repo_manifest_rejects_empty_names() { + fn build_manifest_rejects_empty_names() { let repo = TestRepo::new(); let manifest_path = repo.write_file(MANIFEST_FILE, r#"name = """#); @@ -335,7 +625,7 @@ commit = "aaa" } #[test] - fn repo_manifest_requires_self() { + fn build_manifest_requires_self() { let repo = TestRepo::new(); let manifest_path = repo.write_file( MANIFEST_FILE, @@ -357,3 +647,421 @@ commit = "aaa" assert!(reason.contains("this locations's name (`banana`) must appear in `locations`")); } } + +#[cfg(test)] +mod workspace_tests { + use std::path::{Path, PathBuf}; + + use tempfile::TempDir; + + use super::{ + default_workspace_config_text, discover_path, LoadedWorkspaceConfig, ServerBinding, + ServerSettings, WorkspaceError, DEFAULT_SERVER_HOST, DEFAULT_SERVER_PORT, + LOCAL_CONFIG_FILE, + }; + use crate::proposal::ProposalNumber; + + struct TestWorkspace { + tempdir: TempDir, + } + + impl TestWorkspace { + fn new() -> Self { + Self { + tempdir: TempDir::new().unwrap(), + } + } + + fn root(&self) -> &Path { + self.tempdir.path() + } + + fn path(&self, relative: impl AsRef) -> PathBuf { + self.root().join(relative) + } + + fn write_file(&self, relative: impl AsRef, contents: &str) -> PathBuf { + let path = self.path(relative); + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent).unwrap(); + } + std::fs::write(&path, contents).unwrap(); + path + } + + fn create_dir(&self, relative: impl AsRef) -> PathBuf { + let path = self.path(relative); + std::fs::create_dir_all(&path).unwrap(); + path + } + } + + #[test] + fn parses_default_workspace_config() { + let workspace = TestWorkspace::new(); + let config_path = workspace.write_file(LOCAL_CONFIG_FILE, &default_workspace_config_text()); + + let config = LoadedWorkspaceConfig::from_path(&config_path).unwrap(); + + assert_eq!(config.workspace_root(), workspace.root()); + assert_eq!(config.server_settings(), &ServerSettings::default()); + assert_eq!( + config.site_settings().base_url.as_ref().unwrap().as_str(), + "http://127.0.0.1:1111/" + ); + } + + #[test] + fn starter_workspace_config_roundtrips_stably() { + let original = default_workspace_config_text(); + let parsed = toml::from_str::(&original).unwrap(); + let reparsed = toml::to_string_pretty(&parsed).unwrap(); + + assert_eq!(reparsed, original); + assert!(!original.contains("build_root_base")); + assert!(original.contains("[server]")); + assert!(original.contains("host = \"127.0.0.1\"")); + assert!(original.contains("port = 1111")); + assert!(original.contains("[site]")); + assert!(original.contains("base_url = \"http://127.0.0.1:1111/\"")); + assert!(original.contains("[render]")); + assert!(original.contains("only = []")); + assert!(!original.contains("default_profile")); + assert!(!original.contains("[profiles")); + } + + #[test] + fn parses_workspace_config_server_settings() { + let workspace = TestWorkspace::new(); + let config_path = workspace.write_file( + LOCAL_CONFIG_FILE, + r#" +[server] +host = "0.0.0.0" +port = 8080 +"#, + ); + + let config = LoadedWorkspaceConfig::from_path(&config_path).unwrap(); + + assert_eq!( + config.server_settings(), + &ServerSettings { + host: "0.0.0.0".to_owned(), + port: 8080, + } + ); + } + + #[test] + fn missing_server_settings_use_default_binding() { + let workspace = TestWorkspace::new(); + let config_path = workspace.write_file( + LOCAL_CONFIG_FILE, + r#" +[site] +base_url = "http://localhost:4000" +"#, + ); + + let config = LoadedWorkspaceConfig::from_path(&config_path).unwrap(); + let binding = ServerBinding::from(config.server_settings()); + + assert_eq!(binding.host, DEFAULT_SERVER_HOST); + assert_eq!(binding.port, DEFAULT_SERVER_PORT); + assert_eq!(binding.to_string(), "127.0.0.1:1111"); + } + + #[test] + fn parses_workspace_config_site_settings() { + let workspace = TestWorkspace::new(); + let config_path = workspace.write_file( + LOCAL_CONFIG_FILE, + r#" +[site] +base_url = "http://localhost:4000" +"#, + ); + + let config = LoadedWorkspaceConfig::from_path(&config_path).unwrap(); + + assert_eq!( + config.site_settings().base_url.as_ref().unwrap().as_str(), + "http://localhost:4000/" + ); + } + + #[test] + fn invalid_workspace_config_site_base_url_errors() { + let workspace = TestWorkspace::new(); + let config_path = workspace.write_file( + LOCAL_CONFIG_FILE, + r#" +[site] +base_url = "not a url" +"#, + ); + let error = LoadedWorkspaceConfig::from_path(&config_path).unwrap_err(); + + assert!(error + .to_string() + .contains("unable to parse workspace config")); + } + + #[test] + fn missing_site_settings_preserve_no_base_url_override() { + let workspace = TestWorkspace::new(); + let config_path = workspace.write_file( + LOCAL_CONFIG_FILE, + r#" +[server] +host = "127.0.0.1" +port = 1111 +"#, + ); + + let config = LoadedWorkspaceConfig::from_path(&config_path).unwrap(); + + assert!(config.site_settings().base_url.is_none()); + } + + #[test] + fn minimal_workspace_config_parses() { + let workspace = TestWorkspace::new(); + let config_path = workspace.write_file( + LOCAL_CONFIG_FILE, + r#" +[server] +host = "127.0.0.1" +port = 1111 + +[site] +base_url = "http://127.0.0.1:1111" +"#, + ); + + let config = LoadedWorkspaceConfig::from_path(&config_path).unwrap(); + + assert_eq!(config.server_settings(), &ServerSettings::default()); + assert_eq!( + config.site_settings().base_url.as_ref().unwrap().as_str(), + "http://127.0.0.1:1111/" + ); + } + + #[test] + fn empty_workspace_config_uses_defaults() { + let workspace = TestWorkspace::new(); + let config_path = workspace.write_file(LOCAL_CONFIG_FILE, " \n"); + + let config = LoadedWorkspaceConfig::from_path(&config_path).unwrap(); + + assert_eq!(config.server_settings(), &ServerSettings::default()); + assert!(config.site_settings().base_url.is_none()); + assert!(config.render_settings().only.is_empty()); + } + + #[test] + fn parses_workspace_config_render_only_settings() { + let workspace = TestWorkspace::new(); + let config_path = workspace.write_file( + LOCAL_CONFIG_FILE, + r#" +[render] +only = [555, 678, 555] +"#, + ); + + let config = LoadedWorkspaceConfig::from_path(&config_path).unwrap(); + + assert_eq!( + config.render_settings().only, + vec![ + ProposalNumber::from_u32(555).unwrap(), + ProposalNumber::from_u32(678).unwrap(), + ProposalNumber::from_u32(555).unwrap(), + ] + ); + } + + #[test] + fn missing_render_missing_only_and_empty_only_disable_filtering() { + let cases = [ + ("missing render", ""), + ("missing only", "[render]\n"), + ("empty only", "[render]\nonly = []\n"), + ]; + + for (name, contents) in cases { + let workspace = TestWorkspace::new(); + let config_path = workspace.write_file(LOCAL_CONFIG_FILE, contents); + let config = LoadedWorkspaceConfig::from_path(&config_path).unwrap(); + + assert!( + config.render_settings().only.is_empty(), + "expected `{name}` to disable render filtering" + ); + } + } + + #[test] + fn workspace_config_render_only_rejects_non_positive_and_non_integer_values() { + let cases = [ + ("zero", "only = [0]"), + ("negative", "only = [-555]"), + ("quoted", "only = [\"555\"]"), + ("overflow", "only = [4294967296]"), + ]; + + for (name, contents) in cases { + let workspace = TestWorkspace::new(); + let config_path = + workspace.write_file(LOCAL_CONFIG_FILE, &format!("[render]\n{contents}\n")); + let error = LoadedWorkspaceConfig::from_path(&config_path).unwrap_err(); + + assert!( + matches!(error, WorkspaceError::Parse { .. }), + "expected `{name}` render only config to fail, got {error:?}" + ); + } + } + + #[test] + fn removed_workspace_config_fields_use_strict_parse_errors() { + let removed_theme_ref_field = concat!("co", "mmit"); + let cases = vec![ + ( + "build_root_base".to_owned(), + r#"build_root_base = ".local-build""#.to_owned(), + ), + ( + "default_profile".to_owned(), + r#"default_profile = "local""#.to_owned(), + ), + ( + "profiles".to_owned(), + r#" +[profiles.local] +staging = true +"# + .to_owned(), + ), + ( + "theme".to_owned(), + r#" +[theme] +repository = "https://github.com/eips-wg/theme.git" +"# + .to_owned(), + ), + ( + format!("theme.{removed_theme_ref_field}"), + format!( + r#" +[theme] +{removed_theme_ref_field} = "3a597d4cd68ec82d36f01c01335492cfa59501ae" +"# + ), + ), + ]; + + for (field, contents) in cases { + let workspace = TestWorkspace::new(); + let config_path = workspace.write_file(LOCAL_CONFIG_FILE, &contents); + let error = LoadedWorkspaceConfig::from_path(&config_path).unwrap_err(); + + assert!( + matches!(error, WorkspaceError::Parse { .. }), + "expected strict parse error for removed field `{field}`, got {error:?}" + ); + } + } + + #[test] + fn discover_path_walks_upward() { + let workspace = TestWorkspace::new(); + let config_path = workspace.write_file(LOCAL_CONFIG_FILE, &default_workspace_config_text()); + let nested = workspace.create_dir("EIPs/content"); + + assert_eq!(discover_path(&nested).unwrap(), config_path); + assert_eq!( + LoadedWorkspaceConfig::discover(&nested) + .unwrap() + .unwrap() + .config_path(), + config_path + ); + } + + #[test] + fn missing_workspace_config_is_not_discovered() { + let workspace = TestWorkspace::new(); + let nested = workspace.create_dir("EIPs/content"); + + assert!(discover_path(&nested).is_none()); + assert!(LoadedWorkspaceConfig::discover(&nested).unwrap().is_none()); + } +} + +#[cfg(test)] +mod active_repo_tests { + use std::path::Path; + + use tempfile::TempDir; + + use super::{ActiveRepo, MANIFEST_FILE}; + + #[test] + fn legacy_repo_manifest_does_not_satisfy_active_repo_loading() { + let tempdir = TempDir::new().unwrap(); + std::fs::write( + tempdir.path().join(".build-eips.repo.toml"), + "repo_id = \"EIPs\"\n", + ) + .unwrap(); + + let error = ActiveRepo::load(tempdir.path()).unwrap_err().to_string(); + + assert!(error.contains("Build.toml"), "{error}"); + } + + #[test] + fn active_repo_loads_manifest_and_normalizes_repository_use() { + let tempdir = TempDir::new().unwrap(); + let manifest_path = tempdir.path().join(MANIFEST_FILE); + std::fs::write( + &manifest_path, + r#" +name = "EIPs" + +[locations.EIPs] +repository = "https://example.test/EIPs.git" +base-url = "https://example.test/EIPs/" + +[locations.ERCs] +repository = "https://example.test/ERCs.git" +base-url = "https://example.test/ERCs/" + +[theme] +repository = "https://example.test/theme.git" +commit = "abc123" +"#, + ) + .unwrap(); + + let active_repo = ActiveRepo::load(Path::new(tempdir.path())).unwrap(); + + assert_eq!(active_repo.title, "EIPs"); + assert_eq!(active_repo.sibling_ids, ["ERCs"]); + assert_eq!(active_repo.repository_use.title, "EIPs"); + assert_eq!( + active_repo.repository_use.location.repository.as_str(), + "https://example.test/EIPs.git" + ); + assert_eq!( + active_repo.repository_use.other_repos["ERCs"].as_str(), + "https://example.test/ERCs.git" + ); + assert_eq!(active_repo.theme.commit, "abc123"); + } +} diff --git a/src/context.rs b/src/context.rs index f371837..7aeee68 100644 --- a/src/context.rs +++ b/src/context.rs @@ -4,17 +4,120 @@ * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ -use std::path::PathBuf; +//! Command context and path resolution helpers. + +use std::path::{Path, PathBuf}; use snafu::{ResultExt, Whatever}; -use crate::{cli::Args, find_root}; +use crate::{cli::Args, config, find_root}; + +#[derive(Debug, Clone)] +pub(crate) struct WorkspaceCommandContext { + pub(crate) search_from: PathBuf, + pub(crate) config_path: Option, +} + +pub(crate) fn resolve_input_path(path: &Path) -> Result { + if path.is_absolute() { + Ok(path.to_path_buf()) + } else { + let cwd = std::env::current_dir().whatever_context("unable to get current directory")?; + Ok(cwd.join(path)) + } +} pub(crate) fn root(args: &Args) -> Result { let dir = match &args.root { - None => find_root::find_root().whatever_context("cannot find root")?, - Some(p) => p.to_path_buf(), + None => find_root::find_root().whatever_context("cannot find repository root")?, + Some(path) => { + if !find_root::is_root(path).whatever_context("invalid root directory")? { + snafu::whatever!("invalid root directory"); + } + path.canonicalize() + .whatever_context("unable to canonicalize root directory")? + } }; find_root::is_root(&dir).whatever_context("invalid root directory")?; Ok(dir) } + +fn workspace_search_start(args: &Args) -> Result { + match &args.root { + Some(path) => { + let path = resolve_input_path(path)?; + path.canonicalize() + .whatever_context("unable to canonicalize workspace search path") + } + None => std::env::current_dir().whatever_context("unable to get current directory"), + } +} + +pub(crate) fn load_workspace_command_context( + args: &Args, +) -> Result { + let search_from = workspace_search_start(args)?; + let config_path = config::discover_path(&search_from); + + Ok(WorkspaceCommandContext { + search_from, + config_path, + }) +} + +#[cfg(test)] +mod tests { + use std::path::Path; + + use clap::Parser; + use tempfile::TempDir; + + use crate::{cli::Args, execution::resolve_execution, find_root}; + + use super::root; + + fn explicit_root_args(path: &Path) -> Args { + Args::try_parse_from(["build-eips", "-C", path.to_str().unwrap(), "changed"]).unwrap() + } + + #[test] + fn explicit_plain_directory_is_rejected_before_execution_manifest_loading() { + let directory = TempDir::new().unwrap(); + let args = explicit_root_args(directory.path()); + + let error = resolve_execution(&args).unwrap_err().to_string(); + + assert_eq!(error, "invalid root directory"); + assert!(!error.contains("unable to load active repository Build.toml")); + assert!(!error.contains("Build.toml")); + } + + #[test] + fn explicit_valid_root_is_accepted() { + let directory = TempDir::new().unwrap(); + std::fs::create_dir_all(directory.path().join("content")).unwrap(); + std::fs::write(directory.path().join("Build.toml"), "").unwrap(); + let args = explicit_root_args(directory.path()); + + assert_eq!( + root(&args).unwrap(), + directory.path().canonicalize().unwrap() + ); + } + + #[test] + fn implicit_root_keeps_auto_discovery_behavior() { + let args = Args::try_parse_from(["build-eips", "changed"]).unwrap(); + + match (root(&args), find_root::find_root()) { + (Ok(root), Ok(discovered)) => assert_eq!(root, discovered), + (Err(error), Err(_)) => { + assert!(error.to_string().contains("cannot find repository root")); + assert!(!error.to_string().contains("invalid root directory")); + } + (root, discovered) => panic!( + "context root result {root:?} did not match automatic discovery {discovered:?}" + ), + } + } +} diff --git a/src/editorial.rs b/src/editorial.rs new file mode 100644 index 0000000..aaa0fe9 --- /dev/null +++ b/src/editorial.rs @@ -0,0 +1,952 @@ +/* + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. + */ + +//! Editorial target selection and runtime helpers. + +use std::{ + collections::BTreeSet, + fs::File, + io::ErrorKind, + path::{Path, PathBuf}, +}; + +use log::info; +use snafu::{OptionExt, ResultExt, Whatever}; + +use crate::{ + cli::EditorialSelectorArgs, + context::resolve_input_path, + execution::ResolvedExecution, + git, + layout::REPO_DIR, + lint, + proposal::{ + classify_editorial_number_selector, is_proposal_path, + resolve_proposal_number_markdown_path, EditorialNumberSelector, + }, +}; + +fn repo_relative_canonical_path( + root_path: &Path, + path: &Path, + canonical_path: &Path, +) -> Result { + if path.is_absolute() { + snafu::whatever!( + "editorial selectors require repo-relative proposal paths, got `{}`", + path.to_string_lossy() + ); + } + + let relative = canonical_path + .strip_prefix(root_path) + .with_whatever_context(|_| { + format!( + "editorial target `{}` escapes the active repository root", + path.to_string_lossy() + ) + })? + .to_path_buf(); + + Ok(relative) +} + +fn validate_editorial_targets( + root_path: &Path, + paths: Vec, + strict: bool, +) -> Result, Whatever> { + let mut unique = BTreeSet::new(); + let mut targets = Vec::new(); + + for path in paths { + if path.is_absolute() { + snafu::whatever!( + "editorial selectors require repo-relative proposal paths, got `{}`", + path.to_string_lossy() + ); + } + + let full_path = root_path.join(&path); + let canonical_path = match full_path.canonicalize() { + Ok(canonical_path) => canonical_path, + Err(error) + if !strict + && matches!(error.kind(), ErrorKind::NotFound | ErrorKind::NotADirectory) => + { + continue; + } + Err(error) => { + return Err(error).with_whatever_context(|_| { + format!( + "unable to resolve editorial target `{}`", + full_path.to_string_lossy() + ) + }); + } + }; + + let relative = repo_relative_canonical_path(root_path, &path, &canonical_path)?; + + if !is_proposal_path(&relative) { + if strict { + snafu::whatever!( + "editorial target `{}` is not a supported proposal path", + relative.to_string_lossy() + ); + } + continue; + } + + if unique.insert(relative.clone()) { + targets.push(relative); + } + } + + if strict && targets.is_empty() { + snafu::whatever!("editorial selector resolved no proposal files"); + } + + Ok(targets) +} + +fn read_editorial_batch(path: &Path) -> Result, Whatever> { + let contents = + std::fs::read_to_string(path).whatever_context("unable to read editorial batch file")?; + let mut paths = Vec::new(); + + for line in contents.lines() { + let line = line.trim(); + if line.is_empty() || line.starts_with('#') { + continue; + } + paths.push(PathBuf::from(line)); + } + + Ok(paths) +} + +fn normalize_editorial_selector(root_path: &Path, path: PathBuf) -> Result { + let Some(selector) = path.as_os_str().to_str() else { + return Ok(path); + }; + + match classify_editorial_number_selector(selector) { + EditorialNumberSelector::Number(proposal_number) => { + resolve_proposal_number_markdown_path(root_path, proposal_number) + } + EditorialNumberSelector::InvalidNumberLike(_failure) => { + snafu::whatever!( + "editorial number selector `{selector}` is invalid; expected a positive proposal number that fits in u32, without signs or commas" + ); + } + EditorialNumberSelector::PathLike => Ok(path), + } +} + +fn normalize_editorial_selectors( + root_path: &Path, + paths: Vec, +) -> Result, Whatever> { + paths + .into_iter() + .map(|path| normalize_editorial_selector(root_path, path)) + .collect::>() +} + +fn prepare_editorial_lint_source( + resolved: &ResolvedExecution, +) -> Result<(PathBuf, git::SourceOnly), Whatever> { + let repo_path = resolved.build_path.join(REPO_DIR); + let source = git::Fresh::new( + &resolved.root_path, + &repo_path, + resolved.repository_use.clone(), + resolved.source_materialization, + ) + .whatever_context("initializing build repo for editorial source preparation")? + .clone_src() + .whatever_context("cloning source repo for editorial source preparation")?; + + Ok((repo_path, source)) +} + +fn prepare_editorial_lint_source_with_upstream( + resolved: &ResolvedExecution, +) -> Result<(PathBuf, git::SourceWithUpstream), Whatever> { + let (repo_path, source) = prepare_editorial_lint_source(resolved)?; + let source = source + .fetch_upstream() + .whatever_context("fetching upstream repo for editorial source preparation")?; + + Ok((repo_path, source)) +} + +fn raw_editorial_targets( + selectors: &EditorialSelectorArgs, + resolved: &ResolvedExecution, + upstream_source: Option<&git::SourceWithUpstream>, +) -> Result, Whatever> { + if selectors.selector_count() != 1 { + snafu::whatever!( + "choose exactly one editorial selector: explicit proposal targets, `--batch`, `--working-tree`, or `--against-upstream`" + ); + } + + let raw_targets = if !selectors.paths.is_empty() { + selectors.paths.clone() + } else if let Some(batch) = selectors.batch.as_deref() { + let batch = resolve_input_path(batch)?; + read_editorial_batch(&batch)? + } else if selectors.working_tree { + git::working_tree_paths(&resolved.root_path) + .whatever_context("unable to resolve working-tree editorial targets")? + } else { + upstream_source + .whatever_context( + "against-upstream editorial target selection requires upstream source", + )? + .changed_files() + .whatever_context("unable to list editorial targets against upstream")? + }; + + Ok(raw_targets) +} + +fn validate_raw_editorial_targets( + selectors: &EditorialSelectorArgs, + resolved: &ResolvedExecution, + raw_targets: Vec, +) -> Result, Whatever> { + let strict = !selectors.paths.is_empty() || selectors.batch.is_some(); + let targets = if strict { + normalize_editorial_selectors(&resolved.root_path, raw_targets)? + } else { + raw_targets + }; + validate_editorial_targets(&resolved.root_path, targets, strict) +} + +pub(crate) fn editorial_targets_from_source( + selectors: &EditorialSelectorArgs, + resolved: &ResolvedExecution, + upstream_source: Option<&git::SourceWithUpstream>, +) -> Result, Whatever> { + let raw_targets = raw_editorial_targets(selectors, resolved, upstream_source)?; + + validate_raw_editorial_targets(selectors, resolved, raw_targets) +} + +pub(crate) fn editorial_targets( + selectors: &EditorialSelectorArgs, + resolved: &ResolvedExecution, +) -> Result, Whatever> { + editorial_targets_from_source(selectors, resolved, None) +} + +fn validate_prepared_editorial_targets( + prepared_repo_path: &Path, + targets: &[PathBuf], +) -> Result<(), Whatever> { + for target in targets { + if target.is_absolute() { + snafu::whatever!( + "editorial selectors require repo-relative proposal paths, got `{}`", + target.to_string_lossy() + ); + } + + let prepared_target = prepared_repo_path.join(target); + let file = match File::open(&prepared_target) { + Ok(file) => file, + Err(error) + if matches!(error.kind(), ErrorKind::NotFound | ErrorKind::NotADirectory) => + { + snafu::whatever!( + "editorial target `{}` exists in the active repo but was not materialized into the prepared source tree; untracked files are not supported", + target.to_string_lossy() + ); + } + Err(error) => { + return Err(error).with_whatever_context(|_| { + format!( + "unable to open prepared editorial target `{}`", + prepared_target.to_string_lossy() + ) + }); + } + }; + let metadata = file.metadata().with_whatever_context(|_| { + format!( + "unable to inspect prepared editorial target `{}`", + prepared_target.to_string_lossy() + ) + })?; + + if !metadata.is_file() { + snafu::whatever!( + "prepared editorial target `{}` is not a file", + target.to_string_lossy() + ); + } + } + + Ok(()) +} + +pub(crate) fn run_editorial_lint( + resolved: &ResolvedExecution, + selectors: &EditorialSelectorArgs, + eipw: lint::CmdArgs, +) -> Result { + if selectors.against_upstream { + let (repo_path, source) = prepare_editorial_lint_source_with_upstream(resolved)?; + let targets = editorial_targets_from_source(selectors, resolved, Some(&source))?; + if targets.is_empty() { + info!("editorial selector resolved no proposal files; skipping editorial lint"); + return Ok(false); + } + source + .merge() + .whatever_context("unable to merge ERC/EIP repositories for editorial lint")?; + validate_prepared_editorial_targets(&repo_path, &targets)?; + + lint::eipw(resolved.theme_path()?, &repo_path, targets, eipw) + .whatever_context("editorial lint failed")?; + + return Ok(true); + } + + let targets = editorial_targets(selectors, resolved)?; + if targets.is_empty() { + info!("editorial selector resolved no proposal files; skipping editorial lint"); + return Ok(false); + } + + let (repo_path, source) = prepare_editorial_lint_source(resolved)?; + source + .merge() + .whatever_context("unable to merge ERC/EIP repositories for editorial lint")?; + validate_prepared_editorial_targets(&repo_path, &targets)?; + + lint::eipw(resolved.theme_path()?, &repo_path, targets, eipw) + .whatever_context("editorial lint failed")?; + + Ok(true) +} + +pub(crate) fn editorial_runtime_execution( + mut resolved: ResolvedExecution, + selectors: &EditorialSelectorArgs, +) -> ResolvedExecution { + if selectors.working_tree { + resolved.source_materialization = git::SourceMaterialization::Dirty; + } + resolved +} + +#[cfg(test)] +mod tests { + use std::path::{Path, PathBuf}; + + use clap::Parser; + use eipw_lint::config::DefaultOptions; + use git2::{IndexAddOption, Repository, Signature}; + use tempfile::TempDir; + use url::Url; + + use crate::{ + cli::{Args, EditorialCommand, EditorialSelectorArgs, RuntimeOperation}, + config::{self, RepositoryUse, ServerBinding}, + execution::{resolve_execution, ResolvedExecution}, + }; + + use super::{ + editorial_runtime_execution, editorial_targets, run_editorial_lint, + validate_editorial_targets, + }; + + struct EditorialWorkspace { + _temp: TempDir, + active_path: PathBuf, + } + + fn write_file(root: &Path, relative: impl AsRef, contents: &str) { + let path = root.join(relative); + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent).unwrap(); + } + std::fs::write(path, contents).unwrap(); + } + + fn commit_all(repo: &Repository, message: &str) { + let mut index = repo.index().unwrap(); + index + .add_all(["*"].iter(), IndexAddOption::DEFAULT, None) + .unwrap(); + index.write().unwrap(); + let tree_oid = index.write_tree().unwrap(); + let tree = repo.find_tree(tree_oid).unwrap(); + let signature = Signature::now("build-eips test", "build-eips@example.test").unwrap(); + let parents = repo + .head() + .ok() + .and_then(|head| head.target()) + .map(|oid| repo.find_commit(oid).unwrap()) + .into_iter() + .collect::>(); + let parent_refs = parents.iter().collect::>(); + + repo.commit( + Some("HEAD"), + &signature, + &signature, + message, + &tree, + &parent_refs, + ) + .unwrap(); + } + + fn init_repo(path: &Path, files: &[(&str, &str)]) -> Repository { + std::fs::create_dir_all(path).unwrap(); + let repo = Repository::init(path).unwrap(); + repo.set_head("refs/heads/master").unwrap(); + for (relative, contents) in files { + write_file(path, relative, contents); + } + commit_all(&repo, "initial"); + repo + } + + fn file_url(path: &Path) -> Url { + Url::from_directory_path(path).unwrap() + } + + fn build_manifest_text(repo_name: &str, repository: &Url, siblings: &[(&str, Url)]) -> String { + let mut manifest = format!( + r#" + name = "{repo_name}" + + [locations.{repo_name}] + repository = "{repository}" + base-url = "https://example.test/{repo_name}/" + + [theme] + repository = "https://example.test/theme.git" + commit = "test-theme-commit" + "# + ); + + for (sibling_id, sibling_repository) in siblings { + manifest.push_str(&format!( + r#" + [locations.{sibling_id}] + repository = "{sibling_repository}" + base-url = "https://example.test/{sibling_id}/" + "# + )); + } + + manifest + } + + fn proposal_markdown(number: u32, category: Option<&str>, body: &str) -> String { + let category = category + .map(|category| format!("category: {category}\n")) + .unwrap_or_default(); + format!( + "---\neip: {number}\ntitle: Proposal {number}\ndescription: Proposal {number}\nauthor: Test Author \ndiscussions-to: https://ethereum-magicians.org/t/test/{number}\nstatus: Draft\ntype: Standards Track\n{category}created: 2025-01-01\n---\n\n{body}\n" + ) + } + + fn write_eipw_config(workspace_root: &Path) { + let schema_version = DefaultOptions::::schema_version(); + write_file( + workspace_root, + "theme/config/eipw.toml", + &format!( + "schema-version = \"{schema_version}\"\n\n[fetch]\nproposal-format = \"{{:05}}\"\n" + ), + ); + } + + fn missing_file_url() -> Url { + let temp = TempDir::new().unwrap(); + file_url(&temp.path().join("missing-upstream")) + } + + fn editorial_workspace_with_upstream( + active_body: &str, + sibling_body: &str, + upstream_url: Option, + ) -> EditorialWorkspace { + let temp = TempDir::new().unwrap(); + let workspace_root = temp.path().join("workspace"); + let active_path = workspace_root.join("EIPs"); + let sibling_path = workspace_root.join("ERCs"); + let active_url = upstream_url.unwrap_or_else(|| file_url(&active_path)); + let sibling_url = file_url(&sibling_path); + let manifest = build_manifest_text("EIPs", &active_url, &[("ERCs", sibling_url)]); + let active_markdown = proposal_markdown(1, None, active_body); + let sibling_markdown = proposal_markdown(2, Some("ERC"), sibling_body); + + write_file(&workspace_root, config::LOCAL_CONFIG_FILE, ""); + write_eipw_config(&workspace_root); + let _active_repo = init_repo( + &active_path, + &[ + (config::MANIFEST_FILE, manifest.as_str()), + ("content/00001.md", active_markdown.as_str()), + ], + ); + let _sibling_repo = init_repo( + &sibling_path, + &[("content/00002.md", sibling_markdown.as_str())], + ); + + EditorialWorkspace { + _temp: temp, + active_path, + } + } + + fn editorial_workspace(active_body: &str, sibling_body: &str) -> EditorialWorkspace { + editorial_workspace_with_upstream(active_body, sibling_body, None) + } + + fn editorial_workspace_with_missing_upstream( + active_body: &str, + sibling_body: &str, + ) -> EditorialWorkspace { + editorial_workspace_with_upstream(active_body, sibling_body, Some(missing_file_url())) + } + + fn parsed_editorial_lint( + active_path: &Path, + lint_args: &[&str], + ) -> ( + ResolvedExecution, + EditorialSelectorArgs, + crate::lint::CmdArgs, + ) { + let active_path = active_path.to_str().unwrap(); + let mut arguments = vec!["build-eips", "-C", active_path, "editorial", "lint"]; + arguments.extend_from_slice(lint_args); + let args = Args::try_parse_from(arguments).unwrap(); + let resolved = resolve_execution(&args).unwrap(); + + match args.operation.runtime_operation().unwrap() { + RuntimeOperation::Editorial { + command: EditorialCommand::Lint { selectors, eipw }, + } => (resolved, selectors, eipw), + _ => panic!("expected editorial lint command"), + } + } + + fn run_lint( + workspace: &EditorialWorkspace, + lint_args: &[&str], + ) -> Result { + let (resolved, selectors, eipw) = parsed_editorial_lint(&workspace.active_path, lint_args); + + run_editorial_lint(&resolved, &selectors, eipw) + } + + fn resolved_execution(root_path: PathBuf) -> ResolvedExecution { + ResolvedExecution { + root_path, + build_path: PathBuf::from("/workspace/build/Core"), + repository_use: RepositoryUse { + title: "Core".to_owned(), + location: config::Location { + repository: "https://example.test/Core.git".parse().unwrap(), + base_url: "https://example.test/Core/".parse().unwrap(), + }, + other_repos: Default::default(), + }, + theme_path: Some(PathBuf::from("/workspace/theme")), + only: None, + source_materialization: crate::git::SourceMaterialization::Clean, + server_binding: ServerBinding::default(), + base_url_override: None, + } + } + + fn explicit_selectors(paths: &[&str]) -> EditorialSelectorArgs { + EditorialSelectorArgs { + paths: paths.iter().map(|path| PathBuf::from(*path)).collect(), + batch: None, + working_tree: false, + against_upstream: false, + } + } + + #[test] + fn editorial_lint_uses_workspace_local_eipw_schema_check() { + let workspace = editorial_workspace("Active proposal.", "Sibling proposal."); + write_file( + workspace.active_path.parent().unwrap(), + "theme/config/eipw.toml", + "schema-version = \"999.0.0\"\n", + ); + + let theme_path = workspace.active_path.parent().unwrap().join("theme"); + let schema_error = crate::lint::eipw_schema_status(&theme_path) + .unwrap_err() + .to_string(); + let error = run_lint(&workspace, &["content/00001.md"]) + .unwrap_err() + .to_string(); + + assert!( + schema_error.contains("eipw configuration"), + "{schema_error}" + ); + assert!( + schema_error.contains("incompatible with this application"), + "{schema_error}" + ); + assert!(error.contains("editorial lint failed"), "{error}"); + } + + #[test] + fn editorial_lint_resolves_sibling_proposals_from_prepared_sources() { + let workspace = editorial_workspace_with_missing_upstream( + "Reference [ERC-2](./00002.md).", + "Sibling proposal.", + ); + + assert!(run_lint( + &workspace, + &[ + "content/00001.md", + "--no-default-lints", + "-D", + "markdown-refs" + ] + ) + .unwrap()); + } + + #[test] + fn editorial_batch_lint_resolves_siblings_without_fetching_active_upstream() { + let workspace = editorial_workspace_with_missing_upstream( + "Reference [ERC-2](./00002.md).", + "Sibling proposal.", + ); + let batch_path = workspace.active_path.join("targets.txt"); + write_file(&workspace.active_path, "targets.txt", "content/00001.md\n"); + let batch_path = batch_path.to_str().unwrap(); + + assert!(run_lint( + &workspace, + &[ + "--batch", + batch_path, + "--no-default-lints", + "-D", + "markdown-refs" + ] + ) + .unwrap()); + } + + #[test] + fn editorial_working_tree_lint_uses_dirty_content_without_fetching_active_upstream() { + let workspace = editorial_workspace_with_missing_upstream( + "Reference [ERC-9999](./09999.md).", + "Sibling proposal.", + ); + write_file( + &workspace.active_path, + "content/00001.md", + &proposal_markdown(1, None, "Reference [ERC-2](./00002.md)."), + ); + + assert!(run_lint( + &workspace, + &[ + "--working-tree", + "--no-default-lints", + "-D", + "markdown-refs" + ] + ) + .unwrap()); + } + + #[test] + fn editorial_against_upstream_lint_still_requires_active_upstream() { + let workspace = + editorial_workspace_with_missing_upstream("Active proposal.", "Sibling proposal."); + + let error = run_lint( + &workspace, + &[ + "--against-upstream", + "--no-default-lints", + "-D", + "markdown-refs", + ], + ) + .unwrap_err() + .to_string(); + + assert!(error.contains("fetching upstream repo for editorial source preparation")); + } + + #[test] + fn editorial_lint_rejects_sibling_only_target_as_non_active_target() { + let workspace = editorial_workspace("Active proposal.", "Sibling proposal."); + + let error = run_lint( + &workspace, + &[ + "content/00002.md", + "--no-default-lints", + "-D", + "markdown-refs", + ], + ) + .unwrap_err() + .to_string(); + + assert!(error.contains("unable to resolve editorial target")); + } + + #[test] + fn editorial_lint_reports_untracked_targets_missing_from_prepared_sources() { + let workspace = editorial_workspace("Active proposal.", "Sibling proposal."); + write_file( + &workspace.active_path, + "content/00003.md", + &proposal_markdown(3, None, "Untracked proposal."), + ); + + let error = run_lint( + &workspace, + &[ + "content/00003.md", + "--no-default-lints", + "-D", + "markdown-refs", + ], + ) + .unwrap_err() + .to_string(); + + assert!(error.contains( + "editorial target `content/00003.md` exists in the active repo but was not materialized into the prepared source tree" + )); + assert!(error.contains("untracked files are not supported")); + } + + #[test] + fn editorial_lint_materializes_tracked_dirty_working_tree_targets() { + let workspace = + editorial_workspace("Reference [ERC-9999](./09999.md).", "Sibling proposal."); + write_file( + &workspace.active_path, + "content/00001.md", + &proposal_markdown(1, None, "Reference [ERC-2](./00002.md)."), + ); + + assert!(run_lint( + &workspace, + &[ + "--working-tree", + "--no-default-lints", + "-D", + "markdown-refs" + ] + ) + .unwrap()); + } + + #[test] + fn editorial_working_tree_check_still_forces_dirty_runtime_materialization() { + let resolved = ResolvedExecution { + root_path: PathBuf::from("/workspace/Core"), + build_path: PathBuf::from("/workspace/build/Core"), + repository_use: RepositoryUse { + title: "Core".to_owned(), + location: config::Location { + repository: "https://example.test/Core.git".parse().unwrap(), + base_url: "https://example.test/Core/".parse().unwrap(), + }, + other_repos: Default::default(), + }, + theme_path: Some(PathBuf::from("/workspace/theme")), + only: None, + source_materialization: crate::git::SourceMaterialization::Clean, + server_binding: ServerBinding::default(), + base_url_override: None, + }; + let selectors = EditorialSelectorArgs { + paths: Vec::new(), + batch: None, + working_tree: true, + against_upstream: false, + }; + + assert_eq!( + editorial_runtime_execution(resolved, &selectors).source_materialization, + crate::git::SourceMaterialization::Dirty + ); + } + + #[test] + fn editorial_explicit_numeric_selectors_resolve_to_markdown_paths() { + let temp = TempDir::new().unwrap(); + write_file(temp.path(), "content/0004.md", ""); + let resolved = resolved_execution(temp.path().to_path_buf()); + + for selector in ["4", "004", "0004"] { + assert_eq!( + editorial_targets(&explicit_selectors(&[selector]), &resolved).unwrap(), + vec![PathBuf::from("content/0004.md")] + ); + } + } + + #[test] + fn editorial_explicit_numeric_selectors_support_multiple_and_dedupe() { + let temp = TempDir::new().unwrap(); + write_file(temp.path(), "content/0004.md", ""); + write_file(temp.path(), "content/0005/index.md", ""); + let resolved = resolved_execution(temp.path().to_path_buf()); + + assert_eq!( + editorial_targets(&explicit_selectors(&["4", "0004", "005"]), &resolved).unwrap(), + vec![ + PathBuf::from("content/0004.md"), + PathBuf::from("content/0005/index.md"), + ] + ); + } + + #[test] + fn editorial_batch_accepts_numbers_paths_comments_and_empty_lines() { + let temp = TempDir::new().unwrap(); + write_file(temp.path(), "content/0004.md", ""); + write_file(temp.path(), "content/0005/index.md", ""); + let batch_path = temp.path().join("targets.txt"); + write_file( + temp.path(), + "targets.txt", + "\n# comment\n \n4\ncontent/0005/index.md\n", + ); + let resolved = resolved_execution(temp.path().to_path_buf()); + let selectors = EditorialSelectorArgs { + paths: Vec::new(), + batch: Some(batch_path), + working_tree: false, + against_upstream: false, + }; + + assert_eq!( + editorial_targets(&selectors, &resolved).unwrap(), + vec![ + PathBuf::from("content/0004.md"), + PathBuf::from("content/0005/index.md"), + ] + ); + } + + #[test] + fn editorial_explicit_repo_relative_path_selectors_still_work() { + let temp = TempDir::new().unwrap(); + write_file(temp.path(), "content/0004.md", ""); + let resolved = resolved_execution(temp.path().to_path_buf()); + + assert_eq!( + editorial_targets(&explicit_selectors(&["content/0004.md"]), &resolved).unwrap(), + vec![PathBuf::from("content/0004.md")] + ); + } + + #[test] + fn editorial_invalid_number_like_selectors_fail_with_editorial_error() { + let temp = TempDir::new().unwrap(); + write_file(temp.path(), "content/0004.md", ""); + let resolved = resolved_execution(temp.path().to_path_buf()); + + for selector in [ + "0", + "+4", + "-4", + "4,5", + "4,,5", + ",4", + "4,", + "+", + "-", + "4294967296", + ] { + let error = editorial_targets(&explicit_selectors(&[selector]), &resolved) + .unwrap_err() + .to_string(); + assert!(error.contains(&format!( + "editorial number selector `{selector}` is invalid" + ))); + assert!(error.contains( + "expected a positive proposal number that fits in u32, without signs or commas" + )); + } + } + + #[test] + fn editorial_path_like_selectors_continue_through_path_validation() { + let temp = TempDir::new().unwrap(); + let resolved = resolved_execution(temp.path().to_path_buf()); + + for selector in ["foo", "draft.md", "4a", "draft-4.md"] { + write_file(temp.path(), selector, ""); + + let error = editorial_targets(&explicit_selectors(&[selector]), &resolved) + .unwrap_err() + .to_string(); + + assert!(error.contains("is not a supported proposal path")); + assert!(!error.contains("editorial number selector")); + } + } + + #[cfg(unix)] + #[test] + fn editorial_non_utf8_selector_continues_through_path_validation() { + use std::{ffi::OsStr, os::unix::ffi::OsStrExt}; + + let temp = TempDir::new().unwrap(); + let resolved = resolved_execution(temp.path().to_path_buf()); + let selectors = EditorialSelectorArgs { + paths: vec![PathBuf::from(OsStr::from_bytes(b"\xff"))], + batch: None, + working_tree: false, + against_upstream: false, + }; + + let error = editorial_targets(&selectors, &resolved) + .unwrap_err() + .to_string(); + + assert!(error.contains("unable to resolve editorial target")); + assert!(!error.contains("editorial number selector")); + } + + #[test] + fn non_strict_editorial_target_validation_does_not_normalize_numeric_paths() { + let temp = TempDir::new().unwrap(); + write_file(temp.path(), "4", ""); + write_file(temp.path(), "content/0004.md", ""); + + assert_eq!( + validate_editorial_targets(temp.path(), vec![PathBuf::from("4")], false).unwrap(), + Vec::::new() + ); + } +} diff --git a/src/execution.rs b/src/execution.rs new file mode 100644 index 0000000..4b804ba --- /dev/null +++ b/src/execution.rs @@ -0,0 +1,742 @@ +/* + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. + */ + +//! Execution source and path resolution. + +use std::{ + collections::BTreeSet, + io::ErrorKind, + path::{Path, PathBuf}, +}; + +use log::{debug, info}; +use snafu::{OptionExt, ResultExt, Whatever}; +use url::Url; + +use crate::{ + cli::{Args, Operation, ServerCliArgs}, + config::{self, ActiveRepo, LoadedWorkspaceConfig, RepositoryUse, ServerBinding}, + context::{resolve_input_path, root}, + git, + layout::BUILD_DIR, + proposal::ProposalNumber, +}; + +#[derive(Debug, Clone)] +pub(crate) struct ResolvedExecution { + pub(crate) root_path: PathBuf, + pub(crate) build_path: PathBuf, + pub(crate) repository_use: RepositoryUse, + pub(crate) theme_path: Option, + pub(crate) only: Option>, + pub(crate) source_materialization: git::SourceMaterialization, + pub(crate) server_binding: ServerBinding, + pub(crate) base_url_override: Option, +} + +impl ResolvedExecution { + pub(crate) fn theme_path(&self) -> Result<&Path, Whatever> { + self.theme_path + .as_deref() + .whatever_context("the selected command requires a resolved workspace-local theme") + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) enum SelectedSource { + WorkspaceLocal, + Remote, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct ExecutionSettings { + pub(crate) build_root: Option, + pub(crate) allow_dirty: bool, + pub(crate) sibling: SelectedSource, +} + +fn has_execution_override_flags(args: &Args) -> bool { + args.remote_siblings || args.build_root.is_some() +} + +pub(crate) fn validate_non_execution_command_flags(args: &Args) -> Result<(), Whatever> { + if args.operation.is_workspace_lifecycle_command() && has_execution_override_flags(args) { + snafu::whatever!("execution override flags cannot be used with `init` or `doctor`"); + } + + if args.operation.is_print_command() && has_execution_override_flags(args) { + snafu::whatever!("execution override flags cannot be used with `print`"); + } + + Ok(()) +} + +fn remote_source_override(force_remote: bool) -> Option { + force_remote.then_some(SelectedSource::Remote) +} + +fn format_sibling_ids(sibling_ids: &[String]) -> String { + sibling_ids.join(", ") +} + +fn cli_only_requested(args: &Args) -> bool { + args.operation + .only_cli_args() + .map(|only| !only.only.is_empty()) + .unwrap_or(false) +} + +fn only_cli_is_applicable(args: &Args) -> bool { + matches!( + args.operation, + Operation::Build { .. } | Operation::Serve { .. } + ) && !args.operation.clean_cli_args().clean + && !args.remote_siblings +} + +pub(crate) fn resolve_execution_settings( + args: &Args, + sibling_ids: &[String], + workspace_config: Option<&LoadedWorkspaceConfig>, +) -> Result { + let build_root = args + .build_root + .as_deref() + .map(resolve_input_path) + .transpose()?; + let sibling_override = remote_source_override(args.remote_siblings); + let clean = args.operation.clean_cli_args().clean; + + if cli_only_requested(args) && !only_cli_is_applicable(args) { + snafu::whatever!("--only is supported only for local dirty build and serve commands"); + } + + let (allow_dirty, default_sibling) = + if args.operation.is_plain_site_command() || args.operation.is_editorial_command() { + (!clean, SelectedSource::WorkspaceLocal) + } else { + (false, SelectedSource::Remote) + }; + + let missing_theme = operation_requires_theme(&args.operation) && workspace_config.is_none(); + let missing_sibling = sibling_override.is_none() + && default_sibling == SelectedSource::WorkspaceLocal + && !sibling_ids.is_empty() + && workspace_config.is_none(); + + match (missing_theme, missing_sibling) { + (true, true) => { + snafu::whatever!( + "the selected command requires a workspace config with local theme and sibling sources, but no `{}` was found.\n\nRun:\n build-eips init \n\nThen retry from that workspace, or pass `--remote-siblings` if you intentionally want remote sibling proposal sources.", + config::LOCAL_CONFIG_FILE + ); + } + (false, true) => { + snafu::whatever!( + "the selected command requires workspace-local sibling sources, but no `{}` was found to provide them.\nResolve this by doing one of the following:\n1. run `build-eips init ` so the workspace config supplies the local sources\n2. pass `--remote-siblings` for remote sibling source overrides", + config::LOCAL_CONFIG_FILE + ); + } + _ => {} + } + + Ok(ExecutionSettings { + build_root, + allow_dirty, + sibling: sibling_override.unwrap_or(default_sibling), + }) +} + +fn dedupe_only_numbers(numbers: &[ProposalNumber]) -> Option> { + let numbers = numbers.iter().copied().collect::>(); + (!numbers.is_empty()).then_some(numbers) +} + +fn resolve_only_selection( + args: &Args, + settings: &ExecutionSettings, + workspace_config: Option<&LoadedWorkspaceConfig>, +) -> Result>, Whatever> { + let applicable = matches!( + args.operation, + Operation::Build { .. } | Operation::Serve { .. } + ) && settings.allow_dirty + && settings.sibling == SelectedSource::WorkspaceLocal; + + if let Some(only) = args.operation.only_cli_args() { + if let Some(numbers) = dedupe_only_numbers(&only.only) { + if !applicable { + snafu::whatever!( + "--only is supported only for local dirty build and serve commands" + ); + } + return Ok(Some(numbers)); + } + } + + if !applicable { + return Ok(None); + } + + Ok(workspace_config + .and_then(|workspace_config| dedupe_only_numbers(&workspace_config.render_settings().only))) +} + +fn local_repo_url(path: &Path) -> Result { + Url::from_directory_path(path) + .ok() + .whatever_context("unable to convert local sibling repository path into a file URL") +} + +fn apply_sibling_sources( + repository_use: &mut RepositoryUse, + sibling_ids: &[String], + workspace_config: Option<&LoadedWorkspaceConfig>, + sibling: &SelectedSource, +) -> Result<(), Whatever> { + match sibling { + SelectedSource::Remote => Ok(()), + SelectedSource::WorkspaceLocal => { + if sibling_ids.is_empty() { + return Ok(()); + } + + let workspace_config = workspace_config.whatever_context( + "workspace-local sibling selection requires a workspace config", + )?; + let mut missing = Vec::new(); + let mut local_repositories = Vec::new(); + + for repo_id in sibling_ids { + let path = workspace_config.local_repo_path(repo_id); + if git::repository_available(&path) { + local_repositories.push((repo_id.clone(), local_repo_url(&path)?)); + } else { + missing.push(repo_id.clone()); + } + } + + if !missing.is_empty() { + snafu::whatever!( + "workspace-local sibling selection requires all declared sibling repos; missing or invalid sibling repo(s): {}", + format_sibling_ids(&missing) + ); + } + + for (repo_id, url) in local_repositories { + repository_use.other_repos.insert(repo_id, url); + } + + Ok(()) + } + } +} + +fn build_path( + root_path: &Path, + repository_use: &RepositoryUse, + workspace_config: Option<&LoadedWorkspaceConfig>, + build_root: Option<&Path>, +) -> PathBuf { + build_root + .map(Path::to_path_buf) + .or_else(|| { + workspace_config.map(|workspace_config| { + workspace_config.workspace_build_root(&repository_use.title) + }) + }) + .unwrap_or_else(|| root_path.join(BUILD_DIR)) +} + +fn operation_requires_theme(operation: &Operation) -> bool { + matches!( + operation, + Operation::Build { .. } + | Operation::Serve { .. } + | Operation::Check { .. } + | Operation::Editorial { .. } + ) +} + +fn resolve_theme_path( + workspace_config: Option<&LoadedWorkspaceConfig>, + operation: &Operation, +) -> Result, Whatever> { + if !operation_requires_theme(operation) { + return Ok(None); + } + + let workspace_config = workspace_config.with_whatever_context(|| { + format!( + "the selected command requires a workspace config with a local theme, but no `{}` was found.\n\nRun:\n build-eips init \n\nThen retry from that workspace.", + config::LOCAL_CONFIG_FILE + ) + })?; + let theme_path = workspace_config.local_theme_path(); + + match std::fs::metadata(&theme_path) { + Ok(_) => Ok(Some(theme_path)), + Err(error) if matches!(error.kind(), ErrorKind::NotFound | ErrorKind::NotADirectory) => { + snafu::whatever!( + "workspace-local theme path `{}` does not exist.\n\nRun `build-eips init ` to bootstrap the workspace, or\nclone/update the theme repository at the configured path.", + theme_path.to_string_lossy() + ); + } + Err(error) => { + snafu::whatever!( + "unable to access workspace-local theme path `{}`: {error}", + theme_path.to_string_lossy() + ); + } + } +} + +fn resolve_server_binding( + workspace_config: Option<&LoadedWorkspaceConfig>, + server_cli: &ServerCliArgs, +) -> ServerBinding { + let mut binding = workspace_config + .map(|workspace_config| ServerBinding::from(workspace_config.server_settings())) + .unwrap_or_default(); + + if let Some(host) = &server_cli.host { + binding.host = host.clone(); + } + + if let Some(port) = server_cli.port { + binding.port = port; + } + + binding +} + +fn resolve_base_url_override( + args: &Args, + workspace_config: Option<&LoadedWorkspaceConfig>, +) -> Result, Whatever> { + if let Some(base_url) = args.operation.base_url_cli_args().base_url { + return Ok(Some(base_url)); + } + + Ok(workspace_config.and_then(|config| config.site_settings().base_url.clone())) +} + +fn clean_active_source_requested(args: &Args) -> bool { + args.operation.clean_cli_args().clean || matches!(args.operation, Operation::Changed { .. }) +} + +pub(crate) fn resolve_execution(args: &Args) -> Result { + let root_path = root(args)?; + + if clean_active_source_requested(args) { + git::check_dirty(&root_path) + .whatever_context("clean active-source mode requires a clean active checkout")?; + } + + let active_repo = ActiveRepo::load(&root_path) + .whatever_context("unable to load active repository Build.toml")?; + let sibling_ids = active_repo.sibling_ids.clone(); + let workspace_config = LoadedWorkspaceConfig::discover(&root_path) + .whatever_context("unable to load workspace config")?; + + if let Some(workspace_config) = workspace_config.as_ref() { + debug!( + "using workspace config `{}`", + workspace_config.config_path().to_string_lossy() + ); + } + + let settings = resolve_execution_settings(args, &sibling_ids, workspace_config.as_ref())?; + let only = resolve_only_selection(args, &settings, workspace_config.as_ref())?; + let theme_path = resolve_theme_path(workspace_config.as_ref(), &args.operation)?; + + let mut repository_use = active_repo.repository_use; + apply_sibling_sources( + &mut repository_use, + &sibling_ids, + workspace_config.as_ref(), + &settings.sibling, + )?; + + let build_path = build_path( + &root_path, + &repository_use, + workspace_config.as_ref(), + settings.build_root.as_deref(), + ); + let source_materialization = if settings.allow_dirty { + info!( + "dirty mode is enabled; tracked working-tree changes from the active content repo will be materialized into the build input" + ); + git::SourceMaterialization::Dirty + } else { + git::SourceMaterialization::Clean + }; + let base_url_override = resolve_base_url_override(args, workspace_config.as_ref())?; + + Ok(ResolvedExecution { + root_path, + build_path, + repository_use, + theme_path, + only, + source_materialization, + server_binding: resolve_server_binding( + workspace_config.as_ref(), + &args.operation.server_cli_args(), + ), + base_url_override, + }) +} + +#[cfg(test)] +mod tests { + use clap::Parser; + use tempfile::TempDir; + + use crate::{ + cli::{Args, ServerCliArgs}, + config::{self, LoadedWorkspaceConfig, ServerBinding}, + }; + + use super::{ + resolve_execution_settings, resolve_only_selection, resolve_server_binding, + resolve_theme_path, SelectedSource, + }; + + fn parse_args(arguments: &[&str]) -> Args { + Args::try_parse_from(arguments).unwrap() + } + + fn load_workspace_config(contents: &str) -> LoadedWorkspaceConfig { + let workspace = TempDir::new().unwrap(); + let config_path = workspace.path().join(config::LOCAL_CONFIG_FILE); + std::fs::write(&config_path, contents).unwrap(); + LoadedWorkspaceConfig::from_path(&config_path).unwrap() + } + + fn assert_theme_only_missing_workspace_error(arguments: &[&str]) { + let args = parse_args(arguments); + let error = resolve_theme_path(None, &args.operation).unwrap_err(); + let message = error.to_string(); + + assert!(message.contains( + "the selected command requires a workspace config with a local theme, but no `.build-eips.toml` was found" + )); + assert!(message.contains("build-eips init ")); + assert!(!message.contains("theme and sibling")); + assert!(!message.contains(concat!("--remote", "-theme"))); + } + + fn assert_combined_missing_workspace_error(arguments: &[&str]) { + let args = parse_args(arguments); + let sibling_ids = vec!["ERCs".to_owned()]; + let error = resolve_execution_settings(&args, &sibling_ids, None).unwrap_err(); + let message = error.to_string(); + + assert!(message.contains( + "the selected command requires a workspace config with local theme and sibling sources" + )); + assert!(message.contains("no `.build-eips.toml` was found")); + assert!(message.contains("build-eips init ")); + assert!(message.contains( + "pass `--remote-siblings` if you intentionally want remote sibling proposal sources" + )); + assert!(!message.contains(concat!("--remote", "-theme"))); + assert!(!message.contains("--profile")); + assert!(!message.contains("--allow-dirty")); + assert!(!message.contains("--theme ")); + assert!(!message.contains("--sibling-repo ")); + } + + fn only_selection_for( + arguments: &[&str], + workspace_config: Option<&LoadedWorkspaceConfig>, + ) -> Option> { + let args = parse_args(arguments); + let settings = resolve_execution_settings(&args, &[], workspace_config).unwrap(); + resolve_only_selection(&args, &settings, workspace_config) + .unwrap() + .map(|numbers| numbers.into_iter().map(|number| number.get()).collect()) + } + + #[test] + fn server_binding_resolution_uses_cli_config_then_defaults() { + assert_eq!( + resolve_server_binding(None, &ServerCliArgs::default()), + ServerBinding { + host: "127.0.0.1".to_owned(), + port: 1111, + } + ); + + let workspace_config = load_workspace_config( + r#" +[server] +host = "0.0.0.0" +port = 8080 +"#, + ); + + assert_eq!( + resolve_server_binding(Some(&workspace_config), &ServerCliArgs::default()), + ServerBinding { + host: "0.0.0.0".to_owned(), + port: 8080, + } + ); + assert_eq!( + resolve_server_binding( + Some(&workspace_config), + &ServerCliArgs { + host: Some("127.0.0.1".to_owned()), + port: Some(4000), + }, + ), + ServerBinding { + host: "127.0.0.1".to_owned(), + port: 4000, + } + ); + assert_eq!( + resolve_server_binding( + Some(&workspace_config), + &ServerCliArgs { + host: None, + port: Some(4000), + }, + ), + ServerBinding { + host: "0.0.0.0".to_owned(), + port: 4000, + } + ); + } + + #[test] + fn non_theme_commands_do_not_require_workspace_local_theme() { + for arguments in [ + &["build-eips", "changed"][..], + &["build-eips", "clean"][..], + &["build-eips", "preview"][..], + &["build-eips", "doctor"][..], + &["build-eips", "print", "schema-version"][..], + ] { + let args = parse_args(arguments); + + assert!(resolve_theme_path(None, &args.operation).unwrap().is_none()); + } + } + + #[test] + fn only_selection_dedupes_and_cli_replaces_config() { + let workspace_config = load_workspace_config( + r#" +[render] +only = [678, 555, 678] +"#, + ); + + assert_eq!( + only_selection_for(&["build-eips", "build"], Some(&workspace_config)).unwrap(), + vec![555, 678] + ); + assert_eq!( + only_selection_for( + &["build-eips", "build", "--only", "00555", "555", "897"], + Some(&workspace_config) + ) + .unwrap(), + vec![555, 897] + ); + assert_eq!( + only_selection_for(&["build-eips", "serve"], Some(&workspace_config)).unwrap(), + vec![555, 678] + ); + assert_eq!( + only_selection_for( + &["build-eips", "serve", "--only", "00555", "555", "897"], + Some(&workspace_config) + ) + .unwrap(), + vec![555, 897] + ); + } + + #[test] + fn missing_render_config_and_empty_only_disable_filtering() { + let missing_render = load_workspace_config(""); + let missing_only = load_workspace_config("[render]\n"); + let empty_only = load_workspace_config( + r#" +[render] +only = [] +"#, + ); + + assert!(only_selection_for(&["build-eips", "build"], Some(&missing_render)).is_none()); + assert!(only_selection_for(&["build-eips", "build"], Some(&missing_only)).is_none()); + assert!(only_selection_for(&["build-eips", "build"], Some(&empty_only)).is_none()); + assert!(only_selection_for(&["build-eips", "serve"], Some(&missing_render)).is_none()); + assert!(only_selection_for(&["build-eips", "serve"], Some(&missing_only)).is_none()); + assert!(only_selection_for(&["build-eips", "serve"], Some(&empty_only)).is_none()); + } + + #[test] + fn local_first_theme_commands_without_workspace_config_report_combined_setup_error() { + for arguments in [ + &["build-eips", "build"][..], + &["build-eips", "serve"][..], + &["build-eips", "check"][..], + &["build-eips", "editorial", "lint", "content/0001.md"][..], + &["build-eips", "editorial", "check", "content/0001.md"][..], + ] { + assert_combined_missing_workspace_error(arguments); + } + } + + #[test] + fn zero_sibling_local_first_without_workspace_config_only_requires_theme_resolution() { + let args = parse_args(&["build-eips", "build"]); + let settings = resolve_execution_settings(&args, &[], None).unwrap(); + + assert_eq!(settings.sibling, SelectedSource::WorkspaceLocal); + assert_theme_only_missing_workspace_error(&["build-eips", "build"]); + } + + #[test] + fn remote_sibling_override_without_workspace_config_only_requires_theme_resolution() { + let args = parse_args(&["build-eips", "--remote-siblings", "build"]); + let sibling_ids = vec!["ERCs".to_owned()]; + let settings = resolve_execution_settings(&args, &sibling_ids, None).unwrap(); + + assert_eq!(settings.sibling, SelectedSource::Remote); + assert_theme_only_missing_workspace_error(&["build-eips", "--remote-siblings", "build"]); + } + + #[test] + fn missing_workspace_theme_path_reports_clear_error() { + let workspace = TempDir::new().unwrap(); + let config_path = workspace.path().join(config::LOCAL_CONFIG_FILE); + std::fs::write(&config_path, "").unwrap(); + let workspace_config = LoadedWorkspaceConfig::from_path(&config_path).unwrap(); + let args = parse_args(&["build-eips", "build"]); + + let error = resolve_theme_path(Some(&workspace_config), &args.operation).unwrap_err(); + let message = error.to_string(); + + assert!(message.contains(&format!( + "workspace-local theme path `{}` does not exist", + workspace + .path() + .join(config::DEFAULT_THEME_DIR) + .to_string_lossy() + ))); + assert!(message.contains("build-eips init ")); + } +} + +#[cfg(test)] +mod active_manifest_clean_tests { + use std::path::Path; + + use clap::Parser; + use git2::{IndexAddOption, Repository, Signature}; + use tempfile::TempDir; + + use crate::cli::Args; + + use super::resolve_execution; + + fn write_file(root: &Path, relative: &str, contents: &str) { + let path = root.join(relative); + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent).unwrap(); + } + std::fs::write(path, contents).unwrap(); + } + + fn commit_all(repo: &Repository, message: &str) { + let mut index = repo.index().unwrap(); + index + .add_all(["*"].iter(), IndexAddOption::DEFAULT, None) + .unwrap(); + index.write().unwrap(); + let tree_oid = index.write_tree().unwrap(); + let tree = repo.find_tree(tree_oid).unwrap(); + let signature = Signature::now("build-eips test", "build-eips@example.test").unwrap(); + repo.commit(Some("HEAD"), &signature, &signature, message, &tree, &[]) + .unwrap(); + } + + fn dirty_active_repo() -> TempDir { + let tempdir = TempDir::new().unwrap(); + let repo = Repository::init(tempdir.path()).unwrap(); + repo.set_head("refs/heads/master").unwrap(); + write_file( + tempdir.path(), + "Build.toml", + r#" +name = "EIPs" + +[locations.EIPs] +repository = "https://example.test/EIPs.git" +base-url = "https://example.test/EIPs/" + +[theme] +repository = "https://example.test/theme.git" +commit = "test-theme-commit" +"#, + ); + write_file(tempdir.path(), "content/00001.md", "# Proposal\n"); + commit_all(&repo, "initial manifest"); + write_file( + tempdir.path(), + "Build.toml", + r#" +name = "EIPs" + +[locations.EIPs] +repository = "https://example.test/EIPs.git" +base-url = "https://dirty.example.test/EIPs/" + +[theme] +repository = "https://example.test/theme.git" +commit = "test-theme-commit" +"#, + ); + tempdir + } + + fn assert_dirty_manifest_is_rejected(command: &[&str]) { + let tempdir = dirty_active_repo(); + let root = tempdir.path().to_string_lossy().to_string(); + let mut arguments = vec!["build-eips", "-C", root.as_str()]; + arguments.extend_from_slice(command); + let args = Args::try_parse_from(arguments).unwrap(); + let error = resolve_execution(&args).unwrap_err(); + + assert!(error + .to_string() + .contains("clean active-source mode requires a clean active checkout")); + assert!(format!("{error:?}").contains("Build.toml")); + } + + #[test] + fn dirty_build_toml_is_rejected_for_clean_mode() { + assert_dirty_manifest_is_rejected(&["build", "--clean"]); + } + + #[test] + fn dirty_build_toml_is_rejected_for_changed() { + assert_dirty_manifest_is_rejected(&["changed"]); + } + + #[test] + fn dirty_build_toml_is_rejected_for_clean_remote_siblings() { + assert_dirty_manifest_is_rejected(&["--remote-siblings", "build", "--clean"]); + } +} diff --git a/src/find_root.rs b/src/find_root.rs index 812ff70..e18f6b3 100644 --- a/src/find_root.rs +++ b/src/find_root.rs @@ -4,8 +4,8 @@ * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ -use super::CONTENT_DIR; use crate::config::MANIFEST_FILE; +use crate::layout::CONTENT_DIR; use snafu::{ResultExt, Snafu}; use std::{ diff --git a/src/git.rs b/src/git.rs index 275c828..7b68aba 100644 --- a/src/git.rs +++ b/src/git.rs @@ -5,24 +5,29 @@ */ use std::{ + collections::BTreeSet, ffi::OsStr, + io::ErrorKind, path::{absolute, Path, PathBuf}, }; +use crate::config::RepositoryUse; use crate::{ - cache::Cache, - config::{NoIdentityError, RepositoryUse}, + layout::{BUILD_DIR, CONTENT_DIR}, progress::{Git, ProgressIteratorExt}, }; use git2::{ build::{CheckoutBuilder, TreeUpdateBuilder}, - BranchType, Commit, FetchOptions, FileMode, ObjectType, Oid, RepositoryOpenFlags, Signature, + Commit, FetchOptions, FileMode, ObjectType, Oid, RepositoryOpenFlags, Signature, Status, StatusOptions, Tree, TreeEntry, TreeWalkResult, }; use log::{debug, info}; use snafu::{ensure, Backtrace, IntoError, OptionExt, ResultExt, Snafu}; use url::Url; +const DIRTY_PATH_DISPLAY_LIMIT: usize = 10; +const CONTENT_INDEX_PATH: &str = "content/_index.md"; + #[derive(Debug, Snafu)] pub enum Error { #[snafu(display("cannot convert path into URL (`{}`)", path.to_string_lossy()))] @@ -39,43 +44,757 @@ pub enum Error { source: git2::Error, backtrace: Backtrace, }, - #[snafu(context(false))] - NoIdentity { - #[snafu(backtrace)] - source: NoIdentityError, + #[snafu(display("{message}"))] + Dirty { + message: String, + backtrace: Backtrace, }, - #[snafu(display("working tree or index has uncommitted modifications"))] - Dirty { backtrace: Backtrace }, + #[snafu(display( + "dirty mode cannot materialize conflicted path `{}`; resolve the conflict and try again", + path.to_string_lossy() + ))] + DirtyConflict { path: PathBuf, backtrace: Backtrace }, + #[snafu(display( + "dirty mode cannot materialize `{}` because it is not a tracked file or symlink in the working tree", + path.to_string_lossy() + ))] + DirtyUnsupportedPath { path: PathBuf, backtrace: Backtrace }, #[snafu(display("unable to update tree ({msg})"))] UpdateTree { msg: String, backtrace: Backtrace }, - #[snafu(context(false))] - Cache { - #[snafu(backtrace)] - source: crate::cache::Error, + #[snafu(display( + "workspace path `{}` already exists but is not a usable git repository", + path.to_string_lossy() + ))] + ExistingWorkspacePath { path: PathBuf, backtrace: Backtrace }, + #[snafu(display( + "workspace repository `{}` is unusable: {reason}", + path.to_string_lossy() + ))] + UnusableWorkspaceRepository { + path: PathBuf, + reason: &'static str, + backtrace: Backtrace, }, + #[snafu(display( + "fresh workspace repository `{}` was missing before checkout", + path.to_string_lossy() + ))] + MissingFreshWorkspaceRepository { path: PathBuf, backtrace: Backtrace }, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum SourceMaterialization { + Clean, + Dirty, +} + +pub fn repository_available(path: &Path) -> bool { + git2::Repository::open(path).is_ok() +} + +/// Open a non-bare repository with a resolvable HEAD without modifying it. +pub fn open_usable_repository(path: &Path) -> Result { + let repository = + git2::Repository::open_ext(path, RepositoryOpenFlags::NO_SEARCH, &[] as &[&OsStr]) + .context(GitSnafu { + what: "open workspace repository", + })?; + ensure_usable_workspace_repository(&repository, path)?; + Ok(repository) +} + +/// Return the commit currently checked out by a repository. +pub fn head_commit_id(repository: &git2::Repository) -> Result { + let head = repository.head().context(GitSnafu { + what: "read repository HEAD", + })?; + let commit = head.peel_to_commit().context(GitSnafu { + what: "resolve repository HEAD commit", + })?; + Ok(commit.id()) +} + +/// Resolve a local commit-ish to its commit object ID without fetching. +pub fn resolve_commit_id(repository: &git2::Repository, commit: &str) -> Result { + let object = repository.revparse_single(commit).context(GitSnafu { + what: "resolve local commit-ish", + })?; + let commit = object.peel_to_commit().context(GitSnafu { + what: "resolve local commit object", + })?; + Ok(commit.id()) +} + +/// Return configured remote URLs without fetching or otherwise modifying a repository. +pub fn remote_urls(repository: &git2::Repository) -> Result, Error> { + let names = repository.remotes().context(GitSnafu { + what: "list repository remotes", + })?; + + names + .iter() + .flatten() + .map(|name| { + let remote = repository.find_remote(name).context(GitSnafu { + what: "read repository remote", + })?; + Ok(remote.url().map(ToOwned::to_owned)) + }) + .collect::, Error>>() + .map(|urls| urls.into_iter().flatten().collect()) +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum CloneOutcome { + Fresh, + Existing, +} + +fn ensure_usable_workspace_repository( + repository: &git2::Repository, + path: &Path, +) -> Result<(), Error> { + if repository.workdir().is_none() { + return UnusableWorkspaceRepositorySnafu { + path: path.to_path_buf(), + reason: "it is bare", + } + .fail(); + } + + if repository.head().is_err() { + return UnusableWorkspaceRepositorySnafu { + path: path.to_path_buf(), + reason: "it has no HEAD commit", + } + .fail(); + } + + Ok(()) +} + +fn open_existing_workspace_repository( + destination: &Path, +) -> Result, Error> { + match git2::Repository::open_ext( + destination, + RepositoryOpenFlags::NO_SEARCH, + &[] as &[&OsStr], + ) { + Ok(repository) => { + ensure_usable_workspace_repository(&repository, destination)?; + Ok(Some(repository)) + } + Err(error) if error.code() == git2::ErrorCode::NotFound => { + match std::fs::symlink_metadata(destination) { + Ok(_) => ExistingWorkspacePathSnafu { + path: destination.to_path_buf(), + } + .fail(), + Err(error) if error.kind() == ErrorKind::NotFound => Ok(None), + Err(error) => Err(IoSnafu { + path: destination.to_path_buf(), + } + .into_error(error)), + } + } + Err(error) => match std::fs::symlink_metadata(destination) { + Ok(_) => ExistingWorkspacePathSnafu { + path: destination.to_path_buf(), + } + .fail(), + Err(metadata_error) if metadata_error.kind() == ErrorKind::NotFound => Err(GitSnafu { + what: "open existing workspace repository", + } + .into_error(error)), + Err(metadata_error) => Err(IoSnafu { + path: destination.to_path_buf(), + } + .into_error(metadata_error)), + }, + } +} + +pub fn clone_missing_repo(url: &str, destination: &Path) -> Result { + if open_existing_workspace_repository(destination)?.is_some() { + info!( + "using existing workspace repo `{}`", + destination.to_string_lossy() + ); + return Ok(CloneOutcome::Existing); + } + + info!("cloning `{url}` into `{}`", destination.to_string_lossy()); + let repository = git2::Repository::clone(url, destination).context(GitSnafu { + what: "clone workspace repository", + })?; + ensure_usable_workspace_repository(&repository, destination)?; + Ok(CloneOutcome::Fresh) +} + +pub fn checkout_fresh_clone_at_commit( + destination: &Path, + repository_url: &str, + commit: &str, +) -> Result<(), Error> { + let Some(repository) = open_existing_workspace_repository(destination)? else { + return MissingFreshWorkspaceRepositorySnafu { + path: destination.to_path_buf(), + } + .fail(); + }; + let commit = match repository + .revparse_single(commit) + .and_then(|object| object.peel_to_commit()) + { + Ok(commit) => commit, + Err(error) + if matches!( + error.code(), + git2::ErrorCode::NotFound | git2::ErrorCode::InvalidSpec + ) => + { + let refspec = format!("+{commit}:refs/build-eips/theme-pin"); + fetch(&repository, repository_url, &refspec)? + } + Err(error) => { + return Err(GitSnafu { + what: "resolve manifest theme commit", + } + .into_error(error)); + } + }; + + repository + .set_head_detached(commit.id()) + .context(GitSnafu { + what: "detach manifest theme commit", + })?; + repository + .checkout_head(Some(CheckoutBuilder::default().force())) + .context(GitSnafu { + what: "checkout manifest theme commit", + })?; + + Ok(()) +} + +fn is_generated_path(path: &Path) -> bool { + path.components() + .next() + .map(|component| component.as_os_str() == OsStr::new(BUILD_DIR)) + .unwrap_or(false) +} + +fn dirty_statuses(repo: &git2::Repository) -> Result, Error> { + let mut options = StatusOptions::default(); + options + .include_untracked(true) + .recurse_untracked_dirs(true) + .renames_head_to_index(true) + .renames_index_to_workdir(true); + repo.statuses(Some(&mut options)).context(GitSnafu { + what: "get root repository status", + }) +} + +fn format_dirty_rejection(tracked_paths: &BTreeSet, untracked_count: usize) -> String { + let mut lines = vec![String::from( + "working tree or index has uncommitted modifications; the selected clean source path requires a clean working tree:", + )]; + + for path in tracked_paths.iter().take(DIRTY_PATH_DISPLAY_LIMIT) { + lines.push(format!("- {}", path.to_string_lossy())); + } + + if tracked_paths.len() > DIRTY_PATH_DISPLAY_LIMIT { + lines.push(format!( + "- ... and {} more tracked path(s)", + tracked_paths.len() - DIRTY_PATH_DISPLAY_LIMIT + )); + } + + if untracked_count > 0 { + lines.push(format!( + "- ... plus {} untracked file(s) not listed", + untracked_count + )); + } + + lines.push(String::new()); + + if untracked_count > 0 { + lines.push(String::from( + "For local build/serve/check commands, run without `--clean` to include tracked local changes. For `--clean` runs, commit or stash tracked changes first. Commit/stash/remove untracked files before retrying.", + )); + } else { + lines.push(String::from( + "For local build/serve/check commands, run without `--clean` to include tracked local changes. For `--clean` runs, commit or stash tracked changes first.", + )); + } + + lines.join("\n") } pub fn check_dirty(root_path: &Path) -> Result<(), Error> { + let (tracked_paths, untracked_count) = + collect_dirty_paths(root_path, |path| !is_generated_path(path))?; + + if tracked_paths.is_empty() && untracked_count == 0 { + Ok(()) + } else { + DirtySnafu { + message: format_dirty_rejection(&tracked_paths, untracked_count), + } + .fail() + } +} + +fn entry_path(entry: &git2::StatusEntry<'_>) -> Option { + entry + .head_to_index() + .and_then(|delta| delta.new_file().path().or_else(|| delta.old_file().path())) + .or_else(|| { + entry + .index_to_workdir() + .and_then(|delta| delta.new_file().path().or_else(|| delta.old_file().path())) + }) + .or_else(|| entry.path().map(Path::new)) + .map(Path::to_path_buf) +} + +fn collect_dirty_paths( + root_path: &Path, + include_path: impl Fn(&Path) -> bool, +) -> Result<(BTreeSet, usize), Error> { let repo = git2::Repository::open(root_path).context(GitSnafu { what: "open root repository", })?; - let mut options = StatusOptions::default(); - options.include_untracked(true); - let statuses = repo.statuses(Some(&mut options)).context(GitSnafu { - what: "get root repository status", + let statuses = dirty_statuses(&repo)?; + let mut paths = BTreeSet::new(); + let mut untracked_count = 0; + + for entry in statuses.iter() { + let status = entry.status(); + let path = entry_path(&entry).unwrap_or_else(|| PathBuf::from("")); + + if status.contains(Status::CONFLICTED) { + return DirtyConflictSnafu { path }.fail(); + } + + if status == Status::CURRENT || status == Status::IGNORED { + continue; + } + + if status == Status::WT_NEW { + if include_path(&path) { + untracked_count += 1; + } + continue; + } + + if let Some(delta) = entry.head_to_index() { + if let Some(old_path) = delta.old_file().path().filter(|path| include_path(path)) { + paths.insert(old_path.to_path_buf()); + } + if let Some(new_path) = delta.new_file().path().filter(|path| include_path(path)) { + paths.insert(new_path.to_path_buf()); + } + } + + if let Some(delta) = entry.index_to_workdir() { + if let Some(old_path) = delta.old_file().path().filter(|path| include_path(path)) { + paths.insert(old_path.to_path_buf()); + } + if let Some(new_path) = delta.new_file().path().filter(|path| include_path(path)) { + paths.insert(new_path.to_path_buf()); + } + } + + if include_path(&path) { + paths.insert(path); + } + } + + Ok((paths, untracked_count)) +} + +pub fn working_tree_paths(root_path: &Path) -> Result, Error> { + let (paths, _) = collect_dirty_paths(root_path, |path| !is_generated_path(path))?; + Ok(paths.into_iter().collect()) +} + +pub fn tracked_working_tree_paths(root_path: &Path) -> Result, Error> { + let (paths, _) = collect_dirty_paths(root_path, |_| true)?; + Ok(paths.into_iter().collect()) +} + +pub fn materialize_working_tree(source_root: &Path, destination_root: &Path) -> Result<(), Error> { + remove_existing_path(destination_root).with_context(|_| IoSnafu { + path: destination_root.to_path_buf(), + })?; + std::fs::create_dir_all(destination_root).with_context(|_| IoSnafu { + path: destination_root.to_path_buf(), + })?; + + let mut paths = tracked_paths(source_root, |_| true)?; + paths.extend(tracked_working_tree_paths(source_root)?); + sync_working_tree_paths(source_root, destination_root, &paths) +} + +pub fn sync_working_tree_paths( + source_root: &Path, + destination_root: &Path, + relative_paths: &BTreeSet, +) -> Result<(), Error> { + for path in relative_paths { + sync_working_tree_path(source_root, destination_root, path)?; + } + + Ok(()) +} + +pub fn index_path(root_path: &Path) -> Result { + let repo = git2::Repository::open(root_path).context(GitSnafu { + what: "open root repository", + })?; + let index = repo.index().context(GitSnafu { + what: "open root repository index", + })?; + index + .path() + .map(Path::to_path_buf) + .with_context(|| UpdateTreeSnafu:: { + msg: "repository index is in-memory".into(), + }) +} + +pub fn sync_materialized_paths( + source_root: &Path, + build_repo_path: &Path, + relative_paths: &BTreeSet, +) -> Result<(), Error> { + if relative_paths.is_empty() { + return Ok(()); + } + + let working_repo = git2::Repository::open(build_repo_path).context(GitSnafu { + what: "open build repository", + })?; + let working_root = working_repo + .workdir() + .with_context(|| UpdateTreeSnafu:: { + msg: "build repository workdir is unavailable".into(), + })?; + let mut index = working_repo.index().context(GitSnafu { + what: "open build repository index", + })?; + + for path in relative_paths { + sync_dirty_path(source_root, working_root, &mut index, path)?; + } + + index.write().context(GitSnafu { + what: "write build repository index", + })?; + + Ok(()) +} + +fn remove_existing_path(path: &Path) -> Result<(), std::io::Error> { + match std::fs::symlink_metadata(path) { + Ok(metadata) if metadata.file_type().is_dir() && !metadata.file_type().is_symlink() => { + std::fs::remove_dir_all(path) + } + Ok(_) => std::fs::remove_file(path), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(error) => Err(error), + } +} + +fn tracked_paths( + root_path: &Path, + include_path: impl Fn(&Path) -> bool, +) -> Result, Error> { + let repo = git2::Repository::open(root_path).context(GitSnafu { + what: "open root repository", + })?; + let head = repo.head().context(GitSnafu { what: "head" })?; + let commit = head.peel_to_commit().context(GitSnafu { + what: "peel head to commit", })?; - let mut statuses = statuses.iter().filter(|x| { - x.path() - .map(|x| !x.trim_end_matches('/').ends_with(super::BUILD_DIR)) - .unwrap_or(false) + let tree = commit.tree().context(GitSnafu { what: "head tree" })?; + let mut paths = BTreeSet::new(); + let mut walk_error = None; + + let walk_result = tree.walk(git2::TreeWalkMode::PreOrder, |prefix, entry| { + let Some(name) = entry.name() else { + walk_error = Some( + UpdateTreeSnafu { + msg: format!("tree entry without name in `{prefix}`"), + } + .build(), + ); + return TreeWalkResult::Abort; + }; + + match entry.kind() { + Some(ObjectType::Blob) => (), + Some(ObjectType::Tree) => return TreeWalkResult::Ok, + kind => { + walk_error = Some( + UpdateTreeSnafu { + msg: format!("unknown blob type `{kind:?}` for `{}{name}`", prefix), + } + .build(), + ); + return TreeWalkResult::Abort; + } + } + + let path = PathBuf::from(format!("{prefix}{name}")); + if include_path(&path) { + paths.insert(path); + } + + TreeWalkResult::Ok }); - if statuses.next().is_some() { - DirtySnafu.fail() + + if let Some(error) = walk_error { + return Err(error); + } + + walk_result.context(GitSnafu { + what: "traverse tree", + })?; + + Ok(paths) +} + +fn remove_index_path(index: &mut git2::Index, path: &Path) -> Result<(), Error> { + match index.remove_path(path) { + Ok(()) => Ok(()), + Err(error) if error.code() == git2::ErrorCode::NotFound => match index.remove_dir(path, -1) + { + Ok(()) => Ok(()), + Err(error) if error.code() == git2::ErrorCode::NotFound => Ok(()), + Err(error) => Err(GitSnafu { + what: "remove dirty path from index", + } + .into_error(error)), + }, + Err(error) => Err(GitSnafu { + what: "remove dirty path from index", + } + .into_error(error)), + } +} + +#[cfg(target_family = "unix")] +fn copy_symlink(source: &Path, destination: &Path) -> Result<(), std::io::Error> { + let target = std::fs::read_link(source)?; + std::os::unix::fs::symlink(target, destination) +} + +#[cfg(target_family = "windows")] +fn copy_symlink(source: &Path, destination: &Path) -> Result<(), std::io::Error> { + let target = std::fs::read_link(source)?; + let resolved_target = source + .parent() + .map(|parent| parent.join(&target)) + .unwrap_or_else(|| target.clone()); + + if std::fs::metadata(&resolved_target) + .map(|metadata| metadata.is_dir()) + .unwrap_or(false) + { + std::os::windows::fs::symlink_dir(target, destination) } else { - Ok(()) + std::os::windows::fs::symlink_file(target, destination) } } +#[cfg(not(any(target_family = "unix", target_family = "windows")))] +fn copy_symlink(_source: &Path, _destination: &Path) -> Result<(), std::io::Error> { + Err(std::io::Error::new( + std::io::ErrorKind::Unsupported, + "no symlink implementation available", + )) +} + +fn sync_dirty_path( + source_root: &Path, + working_root: &Path, + index: &mut git2::Index, + relative_path: &Path, +) -> Result<(), Error> { + let source_path = source_root.join(relative_path); + let working_path = working_root.join(relative_path); + + match std::fs::symlink_metadata(&source_path) { + Ok(metadata) if metadata.file_type().is_dir() && !metadata.file_type().is_symlink() => { + remove_existing_path(&working_path).with_context(|_| IoSnafu { + path: working_path.clone(), + })?; + remove_index_path(index, relative_path)?; + Ok(()) + } + Ok(metadata) if metadata.file_type().is_file() || metadata.file_type().is_symlink() => { + if let Some(parent) = working_path.parent() { + std::fs::create_dir_all(parent).with_context(|_| IoSnafu { + path: parent.to_path_buf(), + })?; + } + + remove_existing_path(&working_path).with_context(|_| IoSnafu { + path: working_path.clone(), + })?; + + if metadata.file_type().is_symlink() { + copy_symlink(&source_path, &working_path).with_context(|_| IoSnafu { + path: working_path.clone(), + })?; + } else { + std::fs::copy(&source_path, &working_path).with_context(|_| IoSnafu { + path: source_path.clone(), + })?; + } + + index.add_path(relative_path).context(GitSnafu { + what: "add dirty path to index", + })?; + Ok(()) + } + Ok(_) => DirtyUnsupportedPathSnafu { + path: relative_path.to_path_buf(), + } + .fail(), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + remove_existing_path(&working_path).with_context(|_| IoSnafu { + path: working_path.clone(), + })?; + remove_index_path(index, relative_path)?; + Ok(()) + } + Err(error) => Err(IoSnafu { path: source_path }.into_error(error)), + } +} + +fn sync_working_tree_path( + source_root: &Path, + destination_root: &Path, + relative_path: &Path, +) -> Result<(), Error> { + let source_path = source_root.join(relative_path); + let destination_path = destination_root.join(relative_path); + + match std::fs::symlink_metadata(&source_path) { + Ok(metadata) if metadata.file_type().is_dir() && !metadata.file_type().is_symlink() => { + remove_existing_path(&destination_path).with_context(|_| IoSnafu { + path: destination_path.clone(), + }) + } + Ok(metadata) if metadata.file_type().is_file() || metadata.file_type().is_symlink() => { + if let Some(parent) = destination_path.parent() { + std::fs::create_dir_all(parent).with_context(|_| IoSnafu { + path: parent.to_path_buf(), + })?; + } + + remove_existing_path(&destination_path).with_context(|_| IoSnafu { + path: destination_path.clone(), + })?; + + if metadata.file_type().is_symlink() { + copy_symlink(&source_path, &destination_path).with_context(|_| IoSnafu { + path: destination_path.clone(), + })?; + } else { + std::fs::copy(&source_path, &destination_path).with_context(|_| IoSnafu { + path: source_path.clone(), + })?; + } + + Ok(()) + } + Ok(_) => DirtyUnsupportedPathSnafu { + path: relative_path.to_path_buf(), + } + .fail(), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + remove_existing_path(&destination_path).with_context(|_| IoSnafu { + path: destination_path, + }) + } + Err(error) => Err(IoSnafu { path: source_path }.into_error(error)), + } +} + +fn materialize_dirty_tree( + source_root: &Path, + working_repo: &git2::Repository, + local_head: Oid, +) -> Result { + let (dirty_paths, untracked_count) = + collect_dirty_paths(source_root, |path| !is_generated_path(path))?; + if untracked_count > 0 { + info!("dirty mode ignores untracked files in the active content repo"); + } + + if dirty_paths.is_empty() { + return Ok(local_head); + } + + let working_root = working_repo + .workdir() + .with_context(|| UpdateTreeSnafu:: { + msg: "build repository workdir is unavailable".into(), + })?; + let mut index = working_repo.index().context(GitSnafu { + what: "open build repository index", + })?; + + for path in dirty_paths { + sync_dirty_path(source_root, working_root, &mut index, &path)?; + } + + index.write().context(GitSnafu { + what: "write build repository index", + })?; + let tree_id = index.write_tree().context(GitSnafu { + what: "write dirty materialization tree", + })?; + let tree = working_repo.find_tree(tree_id).context(GitSnafu { + what: "find dirty materialization tree", + })?; + + let sig = Signature::now("eips-build", "eips-build@eips-build.invalid").context(GitSnafu { + what: "dirty commit signature", + })?; + let parent = working_repo.find_commit(local_head).context(GitSnafu { + what: "find clean local head commit", + })?; + let dirty_head = working_repo + .commit( + Some("HEAD"), + &sig, + &sig, + "Dirty working tree materialization", + &tree, + &[&parent], + ) + .context(GitSnafu { + what: "commit dirty working tree materialization", + })?; + + info!( + "materialized tracked dirty changes from the active content repo into `{}`", + working_root.to_string_lossy() + ); + + Ok(dirty_head) +} + fn check_conflict(master_tree: &Tree, path: &Path, entry: &TreeEntry) -> Result<(), Error> { let original = match master_tree.get_path(path) { Err(_) => return Ok(()), @@ -108,7 +827,9 @@ fn check_conflict(master_tree: &Tree, path: &Path, entry: &TreeEntry) -> Result< pub struct Fresh { src_repo_use: RepositoryUse, + src_repo_path: PathBuf, src_repo_url: Url, + source_materialization: SourceMaterialization, working_repo: git2::Repository, } @@ -116,29 +837,41 @@ pub struct Fresh { impl Fresh { pub fn new( root_path: &Path, - build_path: &Path, + repo_path: &Path, src_repo_use: RepositoryUse, + source_materialization: SourceMaterialization, ) -> Result { - let root_path = absolute(root_path).context(IoSnafu { path: root_path })?; - check_dirty(&root_path)?; - let src_repo_url = Url::from_directory_path(&root_path) - .ok() - .context(PathUrlSnafu { path: root_path })?; + let root_path = absolute(root_path).with_context(|_| IoSnafu { path: root_path })?; + if source_materialization == SourceMaterialization::Clean { + check_dirty(&root_path)?; + } + let src_repo_url = + Url::from_directory_path(&root_path) + .ok() + .with_context(|| PathUrlSnafu { + path: root_path.clone(), + })?; debug!("source repository at `{src_repo_url}`"); - let working_repo = open_or_init(build_path)?; + let working_repo = open_or_init(repo_path)?; Ok(Self { working_repo, + src_repo_path: root_path, src_repo_url, src_repo_use, + source_materialization, }) } pub fn clone_src(self) -> Result { info!("cloning local repository"); - let master = fetch(&self.working_repo, self.src_repo_url.as_str(), "HEAD")?; + let master = fetch( + &self.working_repo, + self.src_repo_url.as_str(), + "+HEAD:refs/build-eips/source-head", + )?; self.working_repo .set_head_detached(master.id()) .context(GitSnafu { what: "detach" })?; @@ -167,9 +900,15 @@ impl Fresh { panic!("submodules not supported yet"); } - let local_head = master.id(); + let mut local_head = master.id(); drop(master); drop(branch); + + if self.source_materialization == SourceMaterialization::Dirty { + local_head = + materialize_dirty_tree(&self.src_repo_path, &self.working_repo, local_head)?; + } + Ok(SourceOnly { local_head, src_repo_use: self.src_repo_use, @@ -186,12 +925,16 @@ pub struct SourceOnly { } impl SourceOnly { + pub fn merge(&self) -> Result<(), Error> { + merge_sibling_repositories(&self.working_repo, &self.src_repo_use, self.local_head) + } + pub fn fetch_upstream(self) -> Result { info!("fetching latest {} repository", self.src_repo_use.title); let latest_master = fetch( &self.working_repo, self.src_repo_use.location.repository.as_str(), - "master", + "+master:refs/build-eips/upstream-head", )?; let upstream_head = latest_master.id(); drop(latest_master); @@ -265,41 +1008,138 @@ impl SourceWithUpstream { Ok(changed_files) } - fn check_ignored(&self, tree: &Tree) -> Result<(), Error> { - let mut walk_error = None; - let walk_result = tree.walk(git2::TreeWalkMode::PreOrder, |a, b| { - if b.kind() != Some(ObjectType::Blob) { - return TreeWalkResult::Ok; + pub fn merge(&self) -> Result<(), Error> { + merge_sibling_repositories(&self.working_repo, &self.src_repo_use, self.local_head) + } +} + +fn check_ignored(working_repo: &git2::Repository, tree: &Tree) -> Result<(), Error> { + let mut walk_error = None; + let walk_result = tree.walk(git2::TreeWalkMode::PreOrder, |a, b| { + if b.kind() != Some(ObjectType::Blob) { + return TreeWalkResult::Ok; + } + + let path = match b.name() { + None => a.to_owned(), + Some(p) => format!("{a}{p}"), + }; + + debug!("checking if `{path}` is ignored"); + + match working_repo.is_path_ignored(&path) { + Ok(false) => TreeWalkResult::Ok, + Ok(true) => { + walk_error = Some( + UpdateTreeSnafu { + msg: format!("contains ignored path `{path}`"), + } + .build(), + ); + TreeWalkResult::Abort } + Err(e) => { + walk_error = Some( + GitSnafu { + what: "check ignored", + } + .into_error(e), + ); + TreeWalkResult::Abort + } + } + }); - let path = match b.name() { - None => a.to_owned(), - Some(p) => format!("{a}{p}"), - }; + if let Some(error) = walk_error { + return Err(error); + } - debug!("checking if `{path}` is ignored"); + walk_result.context(GitSnafu { + what: "traverse tree", + })?; + + Ok(()) +} - match self.working_repo.is_path_ignored(&path) { - Ok(false) => TreeWalkResult::Ok, - Ok(true) => { +fn merge_sibling_repositories( + working_repo: &git2::Repository, + repo_use: &RepositoryUse, + mut local_head: Oid, +) -> Result<(), Error> { + for (index, (other_kind, other_repo)) in repo_use + .other_repos + .iter() + .progress_ext("Merge Repos") + .enumerate() + { + let local_commit = working_repo.find_commit(local_head).context(GitSnafu { + what: "find local head commit", + })?; + let local_tree = local_commit.tree().context(GitSnafu { + what: "getting local head tree", + })?; + info!("fetching {other_kind} repository"); + // Local sibling overrides should follow the checked-out repo HEAD instead of assuming `master`. + let other_ref = format!("refs/build-eips/other-head-{index}"); + let other_refspec = if other_repo.scheme() == "file" { + format!("+HEAD:{other_ref}") + } else { + format!("+master:{other_ref}") + }; + let master_other = fetch(working_repo, other_repo.as_str(), &other_refspec)?; + let other_tree = master_other.tree().context(GitSnafu { + what: "getting other tree", + })?; + + let mut tree_builder = TreeUpdateBuilder::new(); + let prefix = format!("{}/", CONTENT_DIR); + let mut walk_error: Option = None; + let walk_result = other_tree.walk(git2::TreeWalkMode::PreOrder, |a, b| { + if !a.starts_with(&prefix) && (!a.is_empty() || b.name() != Some(CONTENT_DIR)) { + return TreeWalkResult::Skip; + } + + let name = match b.name() { + Some(n) => n, + None => { walk_error = Some( UpdateTreeSnafu { - msg: format!("contains ignored path `{path}`"), + msg: format!("tree entry without name in `{a}`"), } .build(), ); - TreeWalkResult::Abort + return TreeWalkResult::Abort; } - Err(e) => { + }; + + let path = format!("{}{}", a, name); + match b.kind() { + Some(ObjectType::Blob) => (), + Some(ObjectType::Tree) => return TreeWalkResult::Ok, + kind => { walk_error = Some( - GitSnafu { - what: "check ignored", + UpdateTreeSnafu { + msg: format!("unknown blob type `{kind:?}` for `{path}`"), } - .into_error(e), + .build(), ); - TreeWalkResult::Abort + return TreeWalkResult::Abort; } } + + if path == CONTENT_INDEX_PATH { + debug!("skip sibling homepage `{path}`"); + return TreeWalkResult::Ok; + } + + if let Err(e) = check_conflict(&local_tree, Path::new(&path), b) { + walk_error = Some(e); + return TreeWalkResult::Abort; + } + + debug!("upsert `{path}`"); + tree_builder.upsert(path, b.id(), FileMode::Blob); + TreeWalkResult::Ok }); if let Some(error) = walk_error { @@ -310,130 +1150,53 @@ impl SourceWithUpstream { what: "traverse tree", })?; - Ok(()) - } + let merged_tree_oid = tree_builder + .create_updated(working_repo, &local_tree) + .context(GitSnafu { what: "build tree" })?; + let merged_tree = working_repo.find_tree(merged_tree_oid).unwrap(); - pub fn merge(&self) -> Result<(), Error> { - let repo_use = &self.src_repo_use; - let master_tree = self.local_head_tree()?; - let mut local_head = self.local_head; - for (other_kind, other_repo) in repo_use.other_repos.iter().progress_ext("Merge Repos") { - info!("fetching {other_kind} repository"); - let master_other = fetch( - &self.working_repo, - other_repo.as_str(), - "master:master-other", - )?; - let other_tree = master_other.tree().context(GitSnafu { - what: "getting other tree", - })?; + check_ignored(working_repo, &merged_tree)?; - let mut tree_builder = TreeUpdateBuilder::new(); - let prefix = format!("{}/", super::CONTENT_DIR); - let mut walk_error: Option = None; - let walk_result = other_tree.walk(git2::TreeWalkMode::PreOrder, |a, b| { - if !a.starts_with(&prefix) - && (!a.is_empty() || b.name() != Some(super::CONTENT_DIR)) - { - return TreeWalkResult::Skip; - } + let sig = + Signature::now("eips-build", "eips-build@eips-build.invalid").context(GitSnafu { + what: "commit signature", + })?; + let msg = format!("Merge {other_repo}"); + local_head = working_repo + .commit( + Some("HEAD"), + &sig, + &sig, + &msg, + &merged_tree, + &[&local_commit, &master_other], + ) + .context(GitSnafu { what: "committing" })?; - let name = match b.name() { - Some(n) => n, - None => { - walk_error = Some( - UpdateTreeSnafu { - msg: format!("tree entry without name in `{a}`"), - } - .build(), - ); - return TreeWalkResult::Abort; - } - }; - - let path = format!("{}{}", a, name); - match b.kind() { - Some(ObjectType::Blob) => (), - Some(ObjectType::Tree) => return TreeWalkResult::Ok, - kind => { - walk_error = Some( - UpdateTreeSnafu { - msg: format!("unknown blob type `{kind:?}` for `{path}`"), - } - .build(), - ); - return TreeWalkResult::Abort; - } - } + working_repo + .checkout_head(Some(CheckoutBuilder::default().force())) + .context(GitSnafu { + what: "checkout merged", + })?; - if let Err(e) = check_conflict(&master_tree, Path::new(&path), b) { - walk_error = Some(e); - return TreeWalkResult::Abort; + drop(merged_tree); + drop(other_tree); + drop(master_other); + drop(local_tree); + drop(local_commit); + match working_repo.find_reference(&other_ref) { + Ok(mut reference) => { + if let Err(error) = reference.delete() { + debug!("unable to delete temporary sibling ref `{other_ref}`: {error}"); } - - debug!("upsert `{path}`"); - tree_builder.upsert(path, b.id(), FileMode::Blob); - TreeWalkResult::Ok - }); - - if let Some(error) = walk_error { - return Err(error); } - - walk_result.context(GitSnafu { - what: "traverse tree", - })?; - - let merged_tree_oid = tree_builder - .create_updated(&self.working_repo, &master_tree) - .context(GitSnafu { what: "build tree" })?; - let merged_tree = self.working_repo.find_tree(merged_tree_oid).unwrap(); - - self.check_ignored(&merged_tree)?; - - let sig = Signature::now("eips-build", "eips-build@eips-build.invalid").context( - GitSnafu { - what: "commit signature", - }, - )?; - let msg = format!("Merge {other_repo}"); - let master = self - .working_repo - .find_commit(local_head) - .context(GitSnafu { - what: "find local head commit", - })?; - local_head = self - .working_repo - .commit( - Some("HEAD"), - &sig, - &sig, - &msg, - &merged_tree, - &[&master, &master_other], - ) - .context(GitSnafu { what: "committing" })?; - - self.working_repo - .checkout_head(Some(CheckoutBuilder::default().force())) - .context(GitSnafu { - what: "checkout merged", - })?; - - self.working_repo - .find_branch("master-other", BranchType::Local) - .context(GitSnafu { - what: "find master-other", - })? - .delete() - .context(GitSnafu { - what: "delete master-other", - })?; + Err(error) => { + debug!("temporary sibling ref `{other_ref}` was not deleted: {error}"); + } } - - Ok(()) } + + Ok(()) } fn fetch<'a>( @@ -442,7 +1205,19 @@ fn fetch<'a>( refspec: &'_ str, ) -> Result, Error> { debug!("fetching repository at `{url}`"); - let mut remote = repo.remote_anonymous(url).context(GitSnafu { + let remote_name = "__build_eips_fetch"; + match repo.remote_delete(remote_name) { + Ok(()) => (), + Err(error) if error.code() == git2::ErrorCode::NotFound => (), + Err(error) => { + return Err(GitSnafu { + what: "deleting temporary remote", + } + .into_error(error)) + } + } + + let mut remote = repo.remote(remote_name, url).context(GitSnafu { what: "creating remote", })?; { @@ -455,14 +1230,24 @@ fn fetch<'a>( what: "fetching repo", })?; } + drop(remote); + repo.remote_delete(remote_name).context(GitSnafu { + what: "deleting temporary remote", + })?; + + let fetched_ref = refspec + .split_once(':') + .map(|(_, destination)| destination) + .filter(|destination| !destination.is_empty()) + .unwrap_or("FETCH_HEAD"); let commit = repo - .revparse_single("FETCH_HEAD") + .revparse_single(fetched_ref) .context(GitSnafu { - what: "revparse FETCH_HEAD", + what: "revparse fetched ref", })? .peel_to_commit() .context(GitSnafu { - what: "peel FETCH_HEAD", + what: "peel fetched ref", })?; Ok(commit) } @@ -479,36 +1264,327 @@ fn open_or_init(dir: &Path) -> Result { Ok(repo) } -impl Cache { - pub fn repo(&self, url: &str, commit: &str) -> Result { - let key = format!("git\0{url}"); - let dir = self.dir(&key)?; +#[cfg(test)] +mod tests { + use std::path::{Path, PathBuf}; - let repo = open_or_init(&dir)?; - let object = match repo.revparse_single(commit) { - Ok(c) => c, - Err(e) if e.code() == git2::ErrorCode::NotFound => { - fetch(&repo, url, "master")?; - repo.revparse_single(commit).context(GitSnafu { - what: "revparse cached commit", - })? - } - Err(e) => { - return Err(GitSnafu { - what: "revparse cached commit", - } - .into_error(e)) - } + use git2::{IndexAddOption, Repository, Signature}; + use tempfile::TempDir; + + use super::{ + checkout_fresh_clone_at_commit, materialize_working_tree, sync_working_tree_paths, + tracked_working_tree_paths, Fresh, SourceMaterialization, + }; + use crate::config::{Location, RepositoryUse}; + + fn file_url(path: &Path) -> url::Url { + url::Url::from_directory_path(path).unwrap() + } + + fn write_file(root: &Path, relative: impl AsRef, contents: &str) { + let path = root.join(relative); + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent).unwrap(); + } + std::fs::write(path, contents).unwrap(); + } + + fn commit_all(repo: &Repository, message: &str) { + let mut index = repo.index().unwrap(); + index + .add_all(["*"].iter(), IndexAddOption::DEFAULT, None) + .unwrap(); + index.write().unwrap(); + let tree_oid = index.write_tree().unwrap(); + let tree = repo.find_tree(tree_oid).unwrap(); + let signature = Signature::now("build-eips test", "build-eips@example.test").unwrap(); + let parents = repo + .head() + .ok() + .and_then(|head| head.target()) + .map(|oid| repo.find_commit(oid).unwrap()) + .into_iter() + .collect::>(); + let parent_refs = parents.iter().collect::>(); + + repo.commit( + Some("HEAD"), + &signature, + &signature, + message, + &tree, + &parent_refs, + ) + .unwrap(); + } + + fn init_repo(path: &Path, files: &[(&str, &str)]) -> Repository { + std::fs::create_dir_all(path).unwrap(); + let repo = Repository::init(path).unwrap(); + repo.set_head("refs/heads/master").unwrap(); + for (relative, contents) in files { + write_file(path, relative, contents); + } + commit_all(&repo, "initial"); + repo + } + + fn stage_path(repo: &Repository, relative: &str) { + let mut index = repo.index().unwrap(); + index.add_path(Path::new(relative)).unwrap(); + index.write().unwrap(); + } + + #[test] + fn checkout_fresh_clone_missing_destination_returns_error() { + let temp = TempDir::new().unwrap(); + let destination = temp.path().join("missing"); + let error = + checkout_fresh_clone_at_commit(&destination, "https://example.test/theme.git", "main") + .unwrap_err(); + + assert!(matches!( + error, + super::Error::MissingFreshWorkspaceRepository { .. } + )); + } + + #[test] + fn checkout_fresh_clone_fetches_missing_manifest_commit() { + let temp = TempDir::new().unwrap(); + let source_path = temp.path().join("source"); + let source = init_repo( + &source_path, + &[( + "README.md", + "source +", + )], + ); + write_file( + &source_path, + "PINNED.md", + "pinned +", + ); + commit_all(&source, "pinned commit"); + let pinned_commit = source.head().unwrap().target().unwrap().to_string(); + let destination = temp.path().join("destination"); + init_repo( + &destination, + &[( + "README.md", + "destination +", + )], + ); + + checkout_fresh_clone_at_commit( + &destination, + file_url(&source_path).as_str(), + &pinned_commit, + ) + .unwrap(); + + let destination_repo = Repository::open(&destination).unwrap(); + assert_eq!( + destination_repo + .head() + .unwrap() + .target() + .unwrap() + .to_string(), + pinned_commit + ); + assert!(destination_repo + .find_reference("refs/build-eips/theme-pin") + .is_ok()); + assert_eq!( + std::fs::read_to_string(destination.join("PINNED.md")).unwrap(), + "pinned +" + ); + } + + #[test] + fn merge_skips_sibling_homepage_and_keeps_sibling_proposals() { + let temp = TempDir::new().unwrap(); + let active = temp.path().join("active"); + let sibling = temp.path().join("sibling"); + let prepared = temp.path().join("prepared"); + + init_repo( + &active, + &[ + ("content/_index.md", "active homepage\n"), + ("content/00555.md", "# Active proposal\n"), + ], + ); + init_repo( + &sibling, + &[ + ("content/_index.md", "sibling homepage\n"), + ("content/00678.md", "# Sibling proposal\n"), + ], + ); + + let mut other_repos = std::collections::HashMap::new(); + other_repos.insert("ERCs".to_owned(), file_url(&sibling)); + let active_url = file_url(&active); + let repository_use = RepositoryUse { + title: "EIPs".to_owned(), + location: Location { + repository: active_url, + base_url: "https://eips.example.test/".parse().unwrap(), + }, + other_repos, }; - repo.checkout_tree(&object, Some(CheckoutBuilder::new().force())) - .context(GitSnafu { - what: "checkout cached commit", - })?; - repo.set_head_detached(object.id()).context(GitSnafu { - what: "set detached head", - })?; + Fresh::new( + &active, + &prepared, + repository_use, + SourceMaterialization::Clean, + ) + .unwrap() + .clone_src() + .unwrap() + .fetch_upstream() + .unwrap() + .merge() + .unwrap(); + + assert_eq!( + std::fs::read_to_string(prepared.join("content/_index.md")).unwrap(), + "active homepage\n" + ); + assert!(prepared.join("content/00678.md").is_file()); + } + + #[test] + fn materialize_working_tree_uses_tracked_theme_scope() { + let temp = TempDir::new().unwrap(); + let theme = temp.path().join("theme"); + let mounted = temp.path().join("repo/themes/eips-theme"); + let repo = init_repo( + &theme, + &[ + ("config/zola.toml", "title = 'theme'\n"), + ("build/generated.txt", "tracked build path\n"), + ("delete.txt", "delete me\n"), + ("staged.txt", "old staged\n"), + ("tracked.txt", "old tracked\n"), + ], + ); + + write_file(&theme, "tracked.txt", "unstaged tracked edit\n"); + write_file(&theme, "staged.txt", "staged tracked edit\n"); + stage_path(&repo, "staged.txt"); + write_file(&theme, "new-staged.txt", "new staged file\n"); + stage_path(&repo, "new-staged.txt"); + std::fs::remove_file(theme.join("delete.txt")).unwrap(); + let mut index = repo.index().unwrap(); + index.remove_path(Path::new("delete.txt")).unwrap(); + index.write().unwrap(); + write_file( + &theme, + "untracked.txt", + "ignored by theme materialization\n", + ); + + materialize_working_tree(&theme, &mounted).unwrap(); + + assert_eq!( + std::fs::read_to_string(mounted.join("config/zola.toml")).unwrap(), + "title = 'theme'\n" + ); + assert_eq!( + std::fs::read_to_string(mounted.join("build/generated.txt")).unwrap(), + "tracked build path\n" + ); + assert_eq!( + std::fs::read_to_string(mounted.join("tracked.txt")).unwrap(), + "unstaged tracked edit\n" + ); + assert_eq!( + std::fs::read_to_string(mounted.join("staged.txt")).unwrap(), + "staged tracked edit\n" + ); + assert_eq!( + std::fs::read_to_string(mounted.join("new-staged.txt")).unwrap(), + "new staged file\n" + ); + assert!(!mounted.join("delete.txt").exists()); + assert!(!mounted.join("untracked.txt").exists()); + } + + #[test] + fn newly_staged_theme_file_syncs_after_git_index_rescan() { + let temp = TempDir::new().unwrap(); + let theme = temp.path().join("theme"); + let mounted = temp.path().join("repo/themes/eips-theme"); + let repo = init_repo(&theme, &[("config/zola.toml", "title = 'theme'\n")]); + materialize_working_tree(&theme, &mounted).unwrap(); + let mut previous_dirty_paths = tracked_working_tree_paths(&theme) + .unwrap() + .into_iter() + .collect::>(); + + write_file(&theme, "templates/new.html", "new staged template\n"); + stage_path(&repo, "templates/new.html"); + let current_dirty_paths = tracked_working_tree_paths(&theme) + .unwrap() + .into_iter() + .collect::>(); + let affected_paths = previous_dirty_paths + .union(¤t_dirty_paths) + .cloned() + .collect(); + + sync_working_tree_paths(&theme, &mounted, &affected_paths).unwrap(); + previous_dirty_paths = current_dirty_paths; + + assert_eq!( + std::fs::read_to_string(mounted.join("templates/new.html")).unwrap(), + "new staged template\n" + ); + assert!(previous_dirty_paths.contains(Path::new("templates/new.html"))); + } + + #[cfg(target_family = "unix")] + #[test] + fn tracked_theme_symlinks_are_materialized_as_symlinks() { + let temp = TempDir::new().unwrap(); + let theme = temp.path().join("theme"); + let mounted = temp.path().join("repo/themes/eips-theme"); + std::fs::create_dir_all(&theme).unwrap(); + let repo = Repository::init(&theme).unwrap(); + repo.set_head("refs/heads/master").unwrap(); + write_file(&theme, "target.txt", "target\n"); + std::os::unix::fs::symlink("target.txt", theme.join("linked.txt")).unwrap(); + commit_all(&repo, "initial"); + + materialize_working_tree(&theme, &mounted).unwrap(); + + assert!(std::fs::symlink_metadata(mounted.join("linked.txt")) + .unwrap() + .file_type() + .is_symlink()); + assert_eq!( + std::fs::read_link(mounted.join("linked.txt")).unwrap(), + PathBuf::from("target.txt") + ); + } + + #[test] + fn materialize_working_tree_requires_git_repository() { + let temp = TempDir::new().unwrap(); + let theme = temp.path().join("theme"); + let mounted = temp.path().join("repo/themes/eips-theme"); + std::fs::create_dir_all(&theme).unwrap(); + + let error = materialize_working_tree(&theme, &mounted).unwrap_err(); - Ok(dir) + assert!(error.to_string().contains("unable to open root repository")); } } diff --git a/src/layout.rs b/src/layout.rs index cc2faa6..dec1039 100644 --- a/src/layout.rs +++ b/src/layout.rs @@ -4,7 +4,25 @@ * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ +//! Shared build layout names and path helpers. + +use std::path::{Path, PathBuf}; + pub(crate) const CONTENT_DIR: &str = "content"; pub(crate) const BUILD_DIR: &str = "build"; pub(crate) const REPO_DIR: &str = "repo"; -pub(crate) const OUTPUT_DIR: &str = "output"; +const OUTPUT_DIR: &str = "output"; + +pub(crate) fn output_path(build_path: &Path) -> PathBuf { + build_path.join(OUTPUT_DIR) +} + +pub(crate) fn mounted_theme_path(project_path: &Path) -> PathBuf { + project_path.join("themes").join("eips-theme") +} + +pub(crate) fn theme_config_path(theme_path: &Path) -> PathBuf { + [theme_path, Path::new("config"), Path::new("zola.toml")] + .iter() + .collect() +} diff --git a/src/lint.rs b/src/lint.rs index a433b37..1d48054 100644 --- a/src/lint.rs +++ b/src/lint.rs @@ -11,7 +11,6 @@ use clap::ValueEnum; use log::debug; use semver::{Comparator, Op, VersionReq}; -use crate::cache::Cache; use crate::progress::ProgressIteratorExt; use eipw_lint::reporters::{AdditionalHelp, Count, Json, Reporter, Text}; @@ -51,11 +50,6 @@ pub enum Error { source: std::io::Error, }, #[snafu(transparent)] - Git { - #[snafu(backtrace)] - source: crate::git::Error, - }, - #[snafu(transparent)] SchemaVersion { #[snafu(backtrace)] source: SchemaVersionError, @@ -92,17 +86,9 @@ struct Config { eipw: eipw_lint::config::DefaultOptions, } -#[derive(Debug, clap::Args, Serialize, Deserialize)] +#[derive(Debug, Clone, clap::Args, Serialize, Deserialize)] #[serde(rename_all = "kebab-case")] pub struct CmdArgs { - /// Disable linting entirely - #[arg(long, exclusive(true))] - no_lint: bool, - - /// Restrict linting to specific files and/or directories (relative to project root) - #[clap(required(false))] - sources: Vec, - /// Lint output format #[clap(long, value_enum, default_value_t)] format: Format, @@ -255,38 +241,38 @@ fn version_cmp( Ok(()) } -#[tokio::main(flavor = "current_thread")] -pub async fn eipw( - theme_repo: &str, - theme_rev: &str, - cache: &Cache, - root_dir: &Path, - repo_dir: &Path, - changed_paths: Vec, - opts: CmdArgs, -) -> Result<(), Error> { - if opts.no_lint { - return Ok(()); - } - - let mut stdout = std::io::stdout(); - - let mut config_path = cache.repo(theme_repo, theme_rev)?; - - config_path.push("config"); - config_path.push("eipw.toml"); +fn eipw_config_path(theme_path: &Path) -> PathBuf { + theme_path.join("config").join("eipw.toml") +} +/// Check whether a theme's eipw configuration uses a compatible schema. +pub fn eipw_schema_status(theme_path: &Path) -> Result<(), Error> { + let config_path = eipw_config_path(theme_path); let toml_file = Toml::file_exact(&config_path); - let file_version = Figment::new() - .merge(&toml_file) + .merge(toml_file) .extract::() .context(ConfigSnafu)? .schema_version; - let application_version = DefaultOptions::::schema_version(); version_cmp(file_version, application_version)?; + Ok(()) +} + +#[tokio::main(flavor = "current_thread")] +pub async fn eipw( + theme_path: &Path, + repo_dir: &Path, + sources: Vec, + opts: CmdArgs, +) -> Result<(), Error> { + let mut stdout = std::io::stdout(); + + eipw_schema_status(theme_path)?; + + let config_path = eipw_config_path(theme_path); + let toml_file = Toml::file_exact(&config_path); let config: Config = Figment::new() .merge(DefaultOptions::::figment()) @@ -301,36 +287,8 @@ pub async fn eipw( .await .context(FsSnafu { path: repo_dir })?; - let paths = if opts.sources.is_empty() { - changed_paths - } else { - let root_dir = tokio::fs::canonicalize(root_dir) - .await - .context(FsSnafu { path: root_dir })?; - let mut repo_relative_sources = Vec::with_capacity(opts.sources.len()); - for source in &opts.sources { - let root_relative_source = root_dir.join(source); - let full_source = tokio::fs::canonicalize(&root_relative_source) - .await - .context(FsSnafu { - path: root_relative_source, - })?; - - let relative_source = match full_source.strip_prefix(&root_dir) { - Ok(r) => r, - Err(e) => { - let err = std::io::Error::new(std::io::ErrorKind::NotFound, e); - return Err(FsSnafu { path: full_source }.into_error(err)); - } - }; - - repo_relative_sources.push(repo_dir.join(relative_source)); - } - - repo_relative_sources - }; - - let sources = collect_sources(paths).await?; + let sources: Vec<_> = sources.iter().map(|source| repo_dir.join(source)).collect(); + let sources = collect_sources(sources).await?; let reporter = match opts.format { Format::Json => EitherReporter::Json(Json::default()), @@ -395,3 +353,4 @@ pub async fn eipw( Ok(()) } + diff --git a/src/main.rs b/src/main.rs index 0b86029..91a67a2 100644 --- a/src/main.rs +++ b/src/main.rs @@ -4,19 +4,24 @@ * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ -mod cache; mod changed; mod cli; mod config; mod context; +mod editorial; +mod execution; mod find_root; mod git; mod github; mod layout; mod lint; mod markdown; +mod pipeline; +mod preview; +mod proposal; mod print; mod progress; +mod workspace; mod zola; use std::path::{Path, PathBuf}; @@ -27,9 +32,11 @@ use log::{debug, info}; use snafu::{Report, ResultExt, Whatever}; use crate::{ + editorial::{editorial_runtime_execution, run_editorial_lint}, cli::{Args, Operation}, config::{Manifest, RepositoryUse}, layout::{BUILD_DIR, CONTENT_DIR, OUTPUT_DIR, REPO_DIR}, + workspace::{doctor_workspace, init_workspace}, }; fn lock(build_path: &Path) -> Result { @@ -170,6 +177,28 @@ fn run() -> Result<(), Whatever> { return Ok(()); } + if let Operation::Init { path, template } = &args.operation { + init_workspace(&args, path.clone(), *template)?; + return Ok(()); + } + + if let Operation::Doctor = &args.operation { + doctor_workspace(&args)?; + return Ok(()); + } + + if let Operation::Editorial { command } = &args.operation { + let resolved = execution::resolve_execution(&args)?; + match command { + crate::cli::EditorialCommand::Lint { selectors, eipw } => run_editorial_lint(&resolved, selectors, eipw.clone())?, + crate::cli::EditorialCommand::Check { selectors, eipw } => { + run_editorial_lint(&resolved, selectors, eipw.clone())?; + pipeline::Prepared::prepare(editorial_runtime_execution(resolved, selectors))?.check()?; + } + } + return Ok(()); + } + let root_path = context::root(&args)?; let manifest_path = root_path.join(config::MANIFEST_FILE); diff --git a/src/markdown.rs b/src/markdown.rs index 06198e6..6d02a6f 100644 --- a/src/markdown.rs +++ b/src/markdown.rs @@ -22,11 +22,11 @@ use regex::Regex; use serde::{Deserialize, Serialize}; -use std::collections::HashMap; +use std::collections::{BTreeSet, HashMap}; use std::ffi::OsStr; use std::fs::read_to_string; -use std::io::Write; -use std::path::{Path, PathBuf}; +use std::io::{ErrorKind, Write}; +use std::path::{Component, Path, PathBuf}; use snafu::{whatever, OptionExt, ResultExt, Whatever}; @@ -38,7 +38,33 @@ use walkdir::WalkDir; use iref::IriRefBuf; -use crate::progress::ProgressIteratorExt; +use crate::{ + progress::ProgressIteratorExt, + proposal::{ + path_component_proposal_number, proposal_number_from_content_markdown_path, OnlyRenderPlan, + ProposalAssetKind, ProposalNumber, ProposalReference, + }, +}; + +#[derive(Clone, Copy)] +enum MissingPathMode { + Error, + Ignore, +} + +impl MissingPathMode { + fn should_ignore_io_error(self, error: &std::io::Error) -> bool { + matches!(self, Self::Ignore) + && matches!(error.kind(), ErrorKind::NotFound | ErrorKind::NotADirectory) + } + + fn should_ignore_walkdir_error(self, error: &walkdir::Error) -> bool { + error + .io_error() + .map(|error| self.should_ignore_io_error(error)) + .unwrap_or(false) + } +} #[derive(Debug, Serialize, Deserialize)] struct Author { @@ -137,6 +163,19 @@ impl Default for FrontMatter { } } +fn filesystem_modified(p: &Path) -> Result { + let metadata = std::fs::metadata(p) + .with_whatever_context(|e| format!("unable to read metadata for `{}`: {e}", p.display()))?; + let modified = metadata.modified().with_whatever_context(|e| { + format!( + "unable to read filesystem modified time for `{}`: {e}", + p.display() + ) + })?; + let date_time: DateTime = modified.into(); + Ok(date_time.to_rfc3339().parse().unwrap()) +} + fn last_modified(p: &Path) -> Result { // TODO: Replace this with `git2` let mut command = std::process::Command::new("git"); @@ -158,7 +197,16 @@ fn last_modified(p: &Path) -> Result { } let date_str = std::str::from_utf8(&output.stdout) - .with_whatever_context(|e| format!("command {:?} output not UTF-8: {e}", command))?; + .with_whatever_context(|e| format!("command {:?} output not UTF-8: {e}", command))? + .trim(); + + if date_str.is_empty() { + debug!( + "falling back to filesystem modified time for `{}` because git has no timestamp for the current path", + p.to_string_lossy() + ); + return filesystem_modified(p); + } let unix: i64 = date_str.parse().with_whatever_context(|e| { let err_str = std::str::from_utf8(&output.stderr).unwrap_or(""); @@ -232,7 +280,453 @@ fn extract_authors(value: &str) -> Result, Whatever> { Ok(authors) } -pub fn preprocess(root_path: &Path) -> Result<(), Whatever> { +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) enum ProposalAssetPathResolution { + NotAProposalAsset, + ProposalAsset(ProposalAssetPath), +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct ProposalAssetPath { + pub(crate) target_proposal_number: ProposalNumber, + pub(crate) content_relative_asset_path: PathBuf, + pub(crate) asset_relative_path: PathBuf, + pub(crate) kind: ProposalAssetKind, + pub(crate) rendered_target_path: String, +} + +#[derive(Debug)] +struct DecodedPathSegment { + value: String, +} + +pub(crate) fn resolve_proposal_asset_path( + content_root: &Path, + source_md_path: &Path, + iri_path: &str, +) -> Result { + let source_parent = source_md_path.parent().with_whatever_context(|| { + format!( + "source markdown path `{}` has no parent", + source_md_path.to_string_lossy() + ) + })?; + let normalized_root = normalize_path_lexically(content_root); + if !raw_iri_resolves_to_proposal_asset_candidate( + &normalized_root, + content_root, + source_parent, + iri_path, + ) { + return Ok(ProposalAssetPathResolution::NotAProposalAsset); + } + + let decoded_segments = decode_iri_path_segments(iri_path)?; + reject_unsafe_asset_segments(&decoded_segments)?; + + let target_path = + resolve_url_path_lexically(content_root, source_parent, iri_path, &decoded_segments); + let normalized_target = normalize_path_lexically(&target_path); + let Ok(content_relative_path) = normalized_target.strip_prefix(&normalized_root) else { + return Ok(ProposalAssetPathResolution::NotAProposalAsset); + }; + + let Some((target_proposal_number, asset_relative_path)) = + proposal_asset_parts(content_relative_path) + else { + return Ok(ProposalAssetPathResolution::NotAProposalAsset); + }; + + let kind = if iri_path.ends_with(".md") { + ProposalAssetKind::Markdown + } else { + ProposalAssetKind::Static + }; + let rendered_target_path = + rendered_asset_path(target_proposal_number, &asset_relative_path, kind)?; + + Ok(ProposalAssetPathResolution::ProposalAsset( + ProposalAssetPath { + target_proposal_number, + content_relative_asset_path: content_relative_path.to_path_buf(), + asset_relative_path, + kind, + rendered_target_path, + }, + )) +} + +pub(crate) fn absolute_rendered_path_for_content_path( + content_relative_path: &Path, +) -> Result { + if content_relative_path == Path::new("_index.md") { + return Ok("/".to_owned()); + } + + if let Some(proposal_number) = proposal_number_from_content_markdown_path(content_relative_path) + { + return Ok(format!("/{proposal_number}/")); + } + + if let Some((proposal_number, asset_relative_path)) = + proposal_asset_parts(content_relative_path) + { + return rendered_asset_path( + proposal_number, + &asset_relative_path, + ProposalAssetKind::from_path(&asset_relative_path), + ); + } + + snafu::whatever!( + "content path `{}` is not a proposal page or proposal asset", + content_relative_path.to_string_lossy() + ); +} + +pub(crate) fn relative_url_from_rendered_paths( + source_rendered_path: &str, + target_rendered_path: &str, +) -> Result { + let source_segments = rendered_directory_segments(source_rendered_path)?; + let (target_segments, target_is_directory) = rendered_path_segments(target_rendered_path)?; + let common_len = source_segments + .iter() + .zip(target_segments.iter()) + .take_while(|(source, target)| source == target) + .count(); + + let mut relative_segments = Vec::new(); + relative_segments.extend(std::iter::repeat_n( + "..", + source_segments.len() - common_len, + )); + relative_segments.extend(target_segments[common_len..].iter().copied()); + + let mut relative_url = if relative_segments.is_empty() { + ".".to_owned() + } else { + relative_segments.join("/") + }; + if target_is_directory && !relative_url.ends_with('/') { + relative_url.push('/'); + } + + Ok(relative_url) +} + +pub(crate) fn proposal_asset_exists_in_content_tree( + content_root: &Path, + content_relative_asset_path: &Path, +) -> bool { + if !content_relative_asset_path + .components() + .all(|component| matches!(component, Component::Normal(_))) + { + return false; + } + + let Ok(canonical_content_root) = std::fs::canonicalize(content_root) else { + return false; + }; + let Ok(canonical_target) = + std::fs::canonicalize(content_root.join(content_relative_asset_path)) + else { + return false; + }; + + if !canonical_target.starts_with(canonical_content_root) { + return false; + } + + std::fs::metadata(canonical_target) + .map(|metadata| metadata.is_file()) + .unwrap_or(false) +} + +fn decode_iri_path_segments(iri_path: &str) -> Result, Whatever> { + iri_path + .split('/') + .map(|segment| { + Ok(DecodedPathSegment { + value: percent_decode_url_segment(segment)?, + }) + }) + .collect::, _>>() +} + +fn percent_decode_url_segment(segment: &str) -> Result { + let bytes = segment.as_bytes(); + let mut decoded = Vec::with_capacity(bytes.len()); + let mut index = 0; + + while index < bytes.len() { + if bytes[index] != b'%' { + decoded.push(bytes[index]); + index += 1; + continue; + } + + if index + 2 >= bytes.len() { + snafu::whatever!("invalid percent encoding in URL path segment `{segment}`"); + } + + let high = hex_value(bytes[index + 1]).with_whatever_context(|| { + format!("invalid percent encoding in URL path segment `{segment}`") + })?; + let low = hex_value(bytes[index + 2]).with_whatever_context(|| { + format!("invalid percent encoding in URL path segment `{segment}`") + })?; + let value = (high << 4) | low; + if matches!(value, b'/' | b'\\' | b'\0') { + snafu::whatever!("unsafe percent encoding in URL path segment `{segment}`"); + } + decoded.push(value); + index += 3; + } + + String::from_utf8(decoded) + .with_whatever_context(|_| format!("URL path segment `{segment}` is not UTF-8")) +} + +fn hex_value(byte: u8) -> Option { + match byte { + b'0'..=b'9' => Some(byte - b'0'), + b'a'..=b'f' => Some(byte - b'a' + 10), + b'A'..=b'F' => Some(byte - b'A' + 10), + _ => None, + } +} + +fn reject_unsafe_asset_segments(segments: &[DecodedPathSegment]) -> Result<(), Whatever> { + let Some(assets_index) = segments.windows(2).position(|window| { + path_component_proposal_number(Some(OsStr::new(window[0].value.as_str()))).is_some() + && window[1].value == "assets" + }) else { + return Ok(()); + }; + + for segment in &segments[assets_index + 2..] { + if segment.value.is_empty() { + continue; + } + if segment.value == "." || segment.value == ".." { + snafu::whatever!("unsafe proposal asset path segment `{}`", segment.value); + } + if segment.value.contains(['/', '\\', '\0']) { + snafu::whatever!("unsafe proposal asset path segment `{}`", segment.value); + } + } + + Ok(()) +} + +fn resolve_url_path_lexically( + content_root: &Path, + source_parent: &Path, + iri_path: &str, + decoded_segments: &[DecodedPathSegment], +) -> PathBuf { + let mut path = if iri_path.starts_with('/') { + content_root.to_path_buf() + } else { + source_parent.to_path_buf() + }; + + for segment in decoded_segments { + match segment.value.as_str() { + "" | "." => {} + ".." => { + path.pop(); + } + _ => path.push(&segment.value), + } + } + + path +} + +fn raw_iri_resolves_to_proposal_asset_candidate( + normalized_root: &Path, + content_root: &Path, + source_parent: &Path, + iri_path: &str, +) -> bool { + let mut path = if iri_path.starts_with('/') { + content_root.to_path_buf() + } else { + source_parent.to_path_buf() + }; + let raw_segments = iri_path.split('/').collect::>(); + + for (index, segment) in raw_segments.iter().enumerate() { + match *segment { + "" | "." => {} + ".." => { + path.pop(); + } + _ => path.push(segment), + } + + let normalized_path = normalize_path_lexically(&path); + let Ok(content_relative_path) = normalized_path.strip_prefix(normalized_root) else { + continue; + }; + + if proposal_asset_parts(content_relative_path).is_some() { + return true; + } + + if proposal_asset_dir_prefix(content_relative_path).is_some() + && raw_segments[index + 1..] + .iter() + .any(|remaining_segment| !remaining_segment.is_empty()) + { + return true; + } + } + + false +} + +fn normalize_path_lexically(path: &Path) -> PathBuf { + let mut normalized = PathBuf::new(); + for component in path.components() { + match component { + Component::Prefix(prefix) => normalized.push(prefix.as_os_str()), + Component::RootDir => normalized.push(component.as_os_str()), + Component::CurDir => {} + Component::ParentDir => { + normalized.pop(); + } + Component::Normal(part) => normalized.push(part), + } + } + normalized +} + +fn proposal_asset_dir_prefix(content_relative_path: &Path) -> Option { + let mut components = content_relative_path.components(); + let proposal_component = components.next()?; + let assets_component = components.next()?; + if components.next().is_some() || assets_component.as_os_str() != OsStr::new("assets") { + return None; + } + + path_component_proposal_number(Some(proposal_component.as_os_str())) +} + +fn proposal_asset_parts(content_relative_path: &Path) -> Option<(ProposalNumber, PathBuf)> { + let mut components = content_relative_path.components(); + let proposal_component = components.next()?; + let assets_component = components.next()?; + if assets_component.as_os_str() != OsStr::new("assets") { + return None; + } + + let proposal_number = path_component_proposal_number(Some(proposal_component.as_os_str()))?; + let asset_relative_path = components.as_path(); + if asset_relative_path.as_os_str().is_empty() { + return None; + } + if !asset_relative_path + .components() + .all(|component| matches!(component, Component::Normal(_))) + { + return None; + } + + Some((proposal_number, asset_relative_path.to_path_buf())) +} + +fn rendered_asset_path( + proposal_number: ProposalNumber, + asset_relative_path: &Path, + kind: ProposalAssetKind, +) -> Result { + let mut segments = vec![proposal_number.to_string(), "assets".to_owned()]; + let asset_segments = asset_relative_path + .components() + .map(|component| match component { + Component::Normal(part) => { + part.to_str().map(str::to_owned).with_whatever_context(|| { + format!( + "non-UTF-8 proposal asset path `{}`", + asset_relative_path.to_string_lossy() + ) + }) + } + _ => snafu::whatever!( + "unsupported proposal asset path component in `{}`", + asset_relative_path.to_string_lossy() + ), + }) + .collect::, _>>()?; + + let last_index = asset_segments.len().saturating_sub(1); + for (index, mut segment) in asset_segments.into_iter().enumerate() { + if kind == ProposalAssetKind::Markdown && index == last_index { + segment = segment + .strip_suffix(".md") + .with_whatever_context(|| { + format!( + "proposal asset markdown path `{}` does not end in `.md`", + asset_relative_path.to_string_lossy() + ) + })? + .to_owned(); + } + segments.push(percent_encode_url_segment(&segment)); + } + + let mut rendered_path = format!("/{}", segments.join("/")); + if kind == ProposalAssetKind::Markdown { + rendered_path.push('/'); + } + + Ok(rendered_path) +} + +fn percent_encode_url_segment(segment: &str) -> String { + let mut encoded = String::with_capacity(segment.len()); + for byte in segment.bytes() { + if byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'.' | b'_' | b'~') { + encoded.push(char::from(byte)); + } else { + encoded.push_str(&format!("%{byte:02X}")); + } + } + encoded +} + +fn rendered_directory_segments(rendered_path: &str) -> Result, Whatever> { + let (mut segments, is_directory) = rendered_path_segments(rendered_path)?; + if !is_directory { + segments.pop(); + } + Ok(segments) +} + +fn rendered_path_segments(rendered_path: &str) -> Result<(Vec<&str>, bool), Whatever> { + if !rendered_path.starts_with('/') { + snafu::whatever!("rendered path `{rendered_path}` is not absolute"); + } + + let is_directory = rendered_path.ends_with('/'); + let segments = rendered_path + .trim_matches('/') + .split('/') + .filter(|segment| !segment.is_empty()) + .collect(); + + Ok((segments, is_directory)) +} + +fn is_generated_zola_markdown(contents: &str) -> bool { + contents.starts_with("+++\n") +} + +pub fn preprocess(root_path: &Path, only_plan: Option<&OnlyRenderPlan>) -> Result<(), Whatever> { let dir = std::fs::read_dir(root_path).with_whatever_context(|_| { format!("could not read directory `{}`", root_path.to_string_lossy()) })?; @@ -268,16 +762,109 @@ pub fn preprocess(root_path: &Path) -> Result<(), Whatever> { } if file_type.is_dir() { - process_eip(root_path, &entry_path.join("index.md"))?; - process_assets(root_path, &entry_path)?; + let relative_path = entry_path + .strip_prefix(root_path) + .with_whatever_context(|_| { + format!( + "content directory entry `{}` is outside `{}`", + entry_path.to_string_lossy(), + root_path.to_string_lossy() + ) + })?; + if only_plan + .map(|plan| plan.should_process_proposal_dir(relative_path)) + .unwrap_or(true) + { + let index_path = entry_path.join("index.md"); + if let Some(plan) = only_plan { + let index_relative_path = relative_path.join("index.md"); + if plan.should_preprocess_markdown(&index_relative_path) { + process_eip(root_path, &index_path, only_plan, MissingPathMode::Error)?; + } + } else { + process_eip(root_path, &index_path, only_plan, MissingPathMode::Error)?; + } + process_assets(root_path, &entry_path, only_plan, MissingPathMode::Error)?; + } } else if entry_path.extension().and_then(OsStr::to_str) == Some("md") { - process_eip(root_path, &entry_path)?; + let relative_path = entry_path + .strip_prefix(root_path) + .with_whatever_context(|_| { + format!( + "content file `{}` is outside `{}`", + entry_path.to_string_lossy(), + root_path.to_string_lossy() + ) + })?; + if only_plan + .map(|plan| plan.should_preprocess_markdown(relative_path)) + .unwrap_or(true) + { + process_eip(root_path, &entry_path, only_plan, MissingPathMode::Error)?; + } } } Ok(()) } +pub fn preprocess_paths( + root_path: &Path, + relative_paths: &BTreeSet, + only_plan: Option<&OnlyRenderPlan>, +) -> Result<(), Whatever> { + let mut eips = BTreeSet::new(); + let mut asset_dirs = BTreeSet::new(); + + for relative_path in relative_paths { + let Ok(content_relative_path) = relative_path.strip_prefix("content") else { + continue; + }; + + if content_relative_path.as_os_str().is_empty() { + continue; + } + + if content_relative_path.extension().and_then(OsStr::to_str) != Some("md") { + continue; + } + + if only_plan + .map(|plan| !plan.should_sync_dirty_path(relative_path)) + .unwrap_or(false) + { + continue; + } + + let mut components = content_relative_path.components(); + let Some(first_component) = components.next() else { + continue; + }; + + if matches!( + components.next(), + Some(component) if component.as_os_str() == OsStr::new("assets") + ) { + let proposal_dir = root_path.join(first_component.as_os_str()); + asset_dirs.insert(proposal_dir); + continue; + } + + let path = root_path.join(content_relative_path); + eips.insert(path); + } + + for path in eips { + process_eip(root_path, &path, only_plan, MissingPathMode::Ignore)?; + } + + for path in asset_dirs { + process_assets(root_path, &path, only_plan, MissingPathMode::Ignore)?; + } + + Ok(()) +} + fn path_to_at(root: &Path, parent: &Path, input: &str) -> Result { let croot = std::fs::canonicalize(root).with_whatever_context(|_| { format!("could not canonicalize `{}`", root.to_string_lossy()) @@ -333,9 +920,108 @@ fn canonicalize_md(path: &Path) -> Result { }) } +enum AssetLinkRewrite { + Rewrite(String), + FallThrough, +} + +fn resolve_asset_link_rewrite( + root: &Path, + source_md_path: &Path, + only_plan: Option<&OnlyRenderPlan>, + iri_path: &str, +) -> Result, Whatever> { + if iri_path.is_empty() { + return Ok(None); + } + + let ProposalAssetPathResolution::ProposalAsset(asset_path) = + resolve_proposal_asset_path(root, source_md_path, iri_path)? + else { + return Ok(None); + }; + + if let Some(plan) = only_plan { + if let Some(public_url) = + plan.public_url_for_omitted_proposal_asset(&asset_path.content_relative_asset_path) + { + return Ok(Some(AssetLinkRewrite::Rewrite(public_url))); + } + + if plan.has_proposal_asset(&asset_path.content_relative_asset_path) { + validate_local_proposal_asset(root, source_md_path, iri_path, &asset_path)?; + return local_asset_link_rewrite(root, source_md_path, asset_path).map(Some); + } + + snafu::whatever!( + "proposal asset link `{iri_path}` in `{}` resolved to `{}` but was not found in targeted render inventory", + source_md_path.to_string_lossy(), + asset_path.content_relative_asset_path.to_string_lossy() + ); + } + + validate_local_proposal_asset(root, source_md_path, iri_path, &asset_path)?; + local_asset_link_rewrite(root, source_md_path, asset_path).map(Some) +} + +fn validate_local_proposal_asset( + root: &Path, + source_md_path: &Path, + iri_path: &str, + asset_path: &ProposalAssetPath, +) -> Result<(), Whatever> { + if proposal_asset_exists_in_content_tree(root, &asset_path.content_relative_asset_path) { + return Ok(()); + } + + snafu::whatever!( + "proposal asset link `{iri_path}` in `{}` resolved to missing asset `{}`", + source_md_path.to_string_lossy(), + asset_path.content_relative_asset_path.to_string_lossy() + ); +} + +fn local_asset_link_rewrite( + root: &Path, + source_md_path: &Path, + asset_path: ProposalAssetPath, +) -> Result { + if asset_path.kind == ProposalAssetKind::Markdown { + return Ok(AssetLinkRewrite::FallThrough); + } + + let source_relative_path = source_md_path + .strip_prefix(root) + .with_whatever_context(|_| { + format!( + "source markdown `{}` is outside content root `{}`", + source_md_path.to_string_lossy(), + root.to_string_lossy() + ) + })?; + let source_rendered_path = absolute_rendered_path_for_content_path(source_relative_path)?; + let relative_url = + relative_url_from_rendered_paths(&source_rendered_path, &asset_path.rendered_target_path)?; + + Ok(AssetLinkRewrite::Rewrite(relative_url)) +} + +fn append_query_and_fragment(mut url: String, iri_ref: &IriRefBuf) -> String { + if let Some(query) = iri_ref.query() { + url.push('?'); + url.push_str(query.as_str()); + } + if let Some(fragment) = iri_ref.fragment() { + url.push('#'); + url.push_str(fragment.as_str()); + } + url +} + fn fix_links<'a, 'b>( root: &'a Path, - parent: &'a Path, + source_md_path: &'a Path, + only_plan: Option<&'a OnlyRenderPlan>, mut e: Event<'b>, ) -> Result, Whatever> { match &mut e { @@ -344,16 +1030,67 @@ fn fix_links<'a, 'b>( .map_err(|e| e.to_string()) .whatever_context("invalid URL in image/link")?; - if iri_ref.authority().is_some() { + if iri_ref.scheme().is_some() || iri_ref.authority().is_some() { // Is a protocol-relative or absolute URL. return Ok(e); } + let iri_path: &str = iri_ref.path().as_ref(); + match resolve_asset_link_rewrite(root, source_md_path, only_plan, iri_path)? { + Some(AssetLinkRewrite::Rewrite(url)) => { + *dest_url = CowStr::from(append_query_and_fragment(url, &iri_ref)); + return Ok(e); + } + Some(AssetLinkRewrite::FallThrough) | None => {} + } + if !iri_ref.path().ends_with(".md") { // Only markdown files need the `@` syntax. return Ok(e); } + let parent = source_md_path.parent().with_whatever_context(|| { + format!( + "source markdown path `{}` has no parent", + source_md_path.to_string_lossy() + ) + })?; + let child = if iri_path.starts_with("/") { + let mut path = Path::new(iri_path); + path = path.strip_prefix("/").unwrap(); + root.join(path) + } else { + parent.join(Path::new(iri_path)) + }; + let normalized_root = normalize_path_lexically(root); + let normalized_child = normalize_path_lexically(&child); + if let Some(public_url) = only_plan.and_then(|plan| { + normalized_child + .strip_prefix(&normalized_root) + .ok() + .and_then(|relative_path| plan.external_url_for_content_target(relative_path)) + }) { + let mut external_url = public_url.to_owned(); + if let Some(query) = iri_ref.query() { + external_url.push('?'); + external_url.push_str(query.as_str()); + } + if let Some(fragment) = iri_ref.fragment() { + external_url.push('#'); + external_url.push_str(fragment.as_str()); + } + *dest_url = CowStr::from(external_url); + return Ok(e); + } + let canonicalized = canonicalize_md(&child)?; + if let Some(public_url) = + only_plan.and_then(|plan| plan.external_url_for_canonical_target(&canonicalized)) + { + *dest_url = + CowStr::from(append_query_and_fragment(public_url.to_owned(), &iri_ref)); + return Ok(e); + } + let canonicalized = path_to_at(root, parent, iri_ref.path())?; let path = iref::iri::Path::new(&canonicalized).expect("path is valid IRI"); iri_ref.set_path(path); @@ -429,7 +1166,12 @@ impl RenderCsl { } } -fn transform_markdown(root: &Path, path: &Path, body: &str) -> Result { +fn transform_markdown( + root: &Path, + path: &Path, + body: &str, + only_plan: Option<&OnlyRenderPlan>, +) -> Result { let mut opts = Options::empty(); opts.insert(Options::ENABLE_TABLES); opts.insert(Options::ENABLE_FOOTNOTES); @@ -437,11 +1179,10 @@ fn transform_markdown(root: &Path, path: &Path, body: &str) -> Result csl.render_csl(e).transpose(), err => Some(err), @@ -456,8 +1197,67 @@ fn transform_markdown(root: &Path, path: &Path, body: &str) -> Result Result<(), Whatever> { +fn process_assets( + root: &Path, + path: &Path, + only_plan: Option<&OnlyRenderPlan>, + missing_path_mode: MissingPathMode, +) -> Result<(), Whatever> { let canon_root = std::fs::canonicalize(root).whatever_context("could not canonicalize root")?; + let assets_dir = path.join("assets"); + + let mut entries = Vec::new(); + let mut ignored_missing_path = false; + + for entry in WalkDir::new(&assets_dir).follow_links(true).into_iter() { + let entry = match entry { + Ok(entry) => entry, + Err(error) if missing_path_mode.should_ignore_walkdir_error(&error) => { + ignored_missing_path = true; + continue; + } + Err(error) => { + return Err(error).with_whatever_context(|_| { + format!("couldn't read entry in `{}`", assets_dir.to_string_lossy()) + }); + } + }; + + if !entry.file_type().is_file() { + continue; + } + + if entry.path().extension().and_then(OsStr::to_str) != Some("md") { + continue; + } + + let candidate = match std::fs::canonicalize(entry.path()) { + Ok(c) => c, + Err(e) => { + warn!( + "unable to canonicalize `{}`: {e}", + entry.path().to_string_lossy() + ); + continue; + } + }; + + let in_root = candidate.starts_with(&canon_root); + if !in_root { + warn!( + "asset `{}` not in root, skipping", + entry.path().to_string_lossy() + ); + continue; + } + + entries.push(entry); + } + + if entries.is_empty() && ignored_missing_path { + return Ok(()); + } + let number_txt = path .file_name() .with_whatever_context(|| format!("no file name for `{}`", path.to_string_lossy()))? @@ -468,60 +1268,28 @@ fn process_assets(root: &Path, path: &Path) -> Result<(), Whatever> { format!("can't parse number for `{}`", path.to_string_lossy()) })?; - let assets_dir = path.join("assets"); - - let dir = WalkDir::new(&assets_dir) - .follow_links(true) - .into_iter() - .filter(|e| match e { - Ok(f) if !f.file_type().is_file() => false, - Ok(f) => f.path().extension().and_then(OsStr::to_str) == Some("md"), - Err(_) => true, - }) - .filter(|e| { - let f = match e { - Ok(f) => f, - _ => return true, - }; - - let candidate = match std::fs::canonicalize(f.path()) { - Ok(c) => c, - Err(e) => { - warn!( - "unable to canonicalize `{}`: {e}", - f.path().to_string_lossy() - ); - return false; - } - }; - - let in_root = candidate.starts_with(&canon_root); - if !in_root { - warn!( - "asset `{}` not in root, skipping", - f.path().to_string_lossy() - ); + for entry in entries.into_iter().progress_ext("Assets") { + let path = entry.path(); + let contents = match read_to_string(path) { + Ok(contents) => contents, + Err(error) if missing_path_mode.should_ignore_io_error(&error) => continue, + Err(error) => { + return Err(error).with_whatever_context(|_| { + format!("could not read file `{}`", path.to_string_lossy()) + }); } - in_root - }); - let dirs: Vec<_> = dir.collect(); - - for entry in dirs.into_iter().progress_ext("Assets") { - let entry = entry.with_whatever_context(|_| { - format!("couldn't read entry in `{}`", assets_dir.to_string_lossy()) - })?; + }; + if is_generated_zola_markdown(&contents) { + continue; + } - let path = entry.path(); - let contents = read_to_string(path).with_whatever_context(|_| { - format!("could not read file `{}`", path.to_string_lossy()) - })?; - - let contents = transform_markdown(root, path, &contents).with_whatever_context(|_| { - format!( - "unable to transform markdown for `{}`", - path.to_string_lossy() - ) - })?; + let contents = + transform_markdown(root, path, &contents, only_plan).with_whatever_context(|_| { + format!( + "unable to transform markdown for `{}`", + path.to_string_lossy() + ) + })?; let relative_path = path.strip_prefix(&assets_dir).unwrap(); let relative_path = relative_path.with_file_name(relative_path.file_stem().unwrap()); @@ -550,21 +1318,39 @@ fn process_assets(root: &Path, path: &Path) -> Result<(), Whatever> { ..Default::default() }; - write_file(path, front_matter, &contents).whatever_context("couldn't write file")?; + match write_file(path, front_matter, &contents) { + Ok(()) => {} + Err(error) if missing_path_mode.should_ignore_io_error(&error) => continue, + Err(error) => return Err(error).whatever_context("couldn't write file"), + } } Ok(()) } -fn process_eip(root: &Path, path: &Path) -> Result<(), Whatever> { +fn process_eip( + root: &Path, + path: &Path, + only_plan: Option<&OnlyRenderPlan>, + missing_path_mode: MissingPathMode, +) -> Result<(), Whatever> { let path_lossy = path.to_string_lossy(); - let contents = read_to_string(path) - .with_whatever_context(|_| format!("could not read file `{}`", path_lossy))?; + let contents = match read_to_string(path) { + Ok(contents) => contents, + Err(error) if missing_path_mode.should_ignore_io_error(&error) => return Ok(()), + Err(error) => { + return Err(error) + .with_whatever_context(|_| format!("could not read file `{}`", path_lossy)); + } + }; + if is_generated_zola_markdown(&contents) { + return Ok(()); + } let (preamble, body) = Preamble::split(&contents) .with_whatever_context(|_| format!("couldn't split preamble for `{}`", path_lossy))?; - let body = transform_markdown(root, path, body) + let body = transform_markdown(root, path, body, only_plan) .with_whatever_context(|_| format!("unable to transform markdown for `{path_lossy}`"))?; let preamble = Preamble::parse(Some(&path_lossy), preamble) @@ -654,8 +1440,24 @@ fn process_eip(root: &Path, path: &Path) -> Result<(), Whatever> { .whatever_context("could not parse requires")? .into_iter() .map(|eip| { - let path = format!("/{eip:0>5}.md"); - path_to_at(root, root, &path) + let proposal_number = match ProposalNumber::from_u32(eip) { + Ok(proposal_number) => proposal_number, + Err(()) => snafu::whatever!("could not parse requires"), + }; + match only_plan { + Some(plan) => { + match plan.reference_for_required_number(proposal_number)? { + ProposalReference::Internal(path) => Ok(path), + ProposalReference::External(public_url) => { + Ok(public_url.to_owned()) + } + } + } + None => { + let path = format!("/{eip:0>5}.md"); + path_to_at(root, root, &path) + } + } }) .collect::>()?; front_matter @@ -669,7 +1471,1034 @@ fn process_eip(root: &Path, path: &Path) -> Result<(), Whatever> { } } - write_file(Path::new(&path), front_matter, &body).whatever_context("couldn't write file")?; + match write_file(path, front_matter, &body) { + Ok(()) => {} + Err(error) if missing_path_mode.should_ignore_io_error(&error) => return Ok(()), + Err(error) => return Err(error).whatever_context("couldn't write file"), + } Ok(()) } + +#[cfg(test)] +mod tests { + use std::collections::BTreeSet; + use std::path::{Path, PathBuf}; + + use git2::{IndexAddOption, Repository, Signature}; + use snafu::Report; + use tempfile::TempDir; + use toml::Value as TomlValue; + + use super::{ + absolute_rendered_path_for_content_path, preprocess, preprocess_paths, + proposal_asset_exists_in_content_tree, relative_url_from_rendered_paths, + resolve_proposal_asset_path, ProposalAssetPath, ProposalAssetPathResolution, + }; + use crate::proposal::ProposalAssetKind; + use crate::proposal::{OnlyRenderPlan, ProposalNumber}; + + fn number(value: u32) -> ProposalNumber { + ProposalNumber::from_u32(value).unwrap() + } + + fn write_file(root: &Path, relative: &str, contents: impl AsRef) { + let path = root.join(relative); + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent).unwrap(); + } + std::fs::write(path, contents.as_ref()).unwrap(); + } + + fn commit_all(repo: &Repository) { + let mut index = repo.index().unwrap(); + index + .add_all(["content"].iter(), IndexAddOption::DEFAULT, None) + .unwrap(); + index.write().unwrap(); + let tree_oid = index.write_tree().unwrap(); + let tree = repo.find_tree(tree_oid).unwrap(); + let signature = Signature::now("build-eips test", "build-eips@example.test").unwrap(); + + repo.commit(Some("HEAD"), &signature, &signature, "initial", &tree, &[]) + .unwrap(); + } + + fn content_repo(files: &[(&str, String)]) -> (TempDir, PathBuf) { + let temp = TempDir::new().unwrap(); + let repo_root = temp.path().join("repo"); + let content_root = repo_root.join("content"); + std::fs::create_dir_all(&content_root).unwrap(); + let repo = Repository::init(&repo_root).unwrap(); + repo.set_head("refs/heads/master").unwrap(); + + for (relative, contents) in files { + write_file(&content_root, relative, contents); + } + + commit_all(&repo); + (temp, content_root) + } + + fn proposal_markdown( + proposal_number: u32, + category: Option<&str>, + extra_preamble: &str, + body: &str, + ) -> String { + let category = category + .map(|category| format!("category: {category}\n")) + .unwrap_or_default(); + format!( + "---\neip: {proposal_number}\ntitle: Proposal {proposal_number}\n{category}{extra_preamble}---\n{body}\n" + ) + } + + fn only_plan(content_root: &Path, selected: &[u32]) -> OnlyRenderPlan { + let selected = selected + .iter() + .copied() + .map(number) + .collect::>(); + OnlyRenderPlan::build(content_root, selected).unwrap() + } + + fn repo_paths(paths: &[&str]) -> BTreeSet { + paths.iter().map(PathBuf::from).collect() + } + + fn rendered_body(path: &Path) -> String { + let contents = std::fs::read_to_string(path).unwrap(); + contents.split_once("\n+++\n").unwrap().1.to_owned() + } + + fn rendered_front_matter(path: &Path) -> TomlValue { + let contents = std::fs::read_to_string(path).unwrap(); + let front_matter = contents + .strip_prefix("+++\n") + .unwrap() + .split_once("\n+++\n") + .unwrap() + .0; + toml::from_str(front_matter).unwrap() + } + + fn resolved_asset( + content_root: &Path, + source_md_path: &Path, + iri_path: &str, + ) -> ProposalAssetPath { + match resolve_proposal_asset_path(content_root, source_md_path, iri_path).unwrap() { + ProposalAssetPathResolution::ProposalAsset(asset_path) => asset_path, + ProposalAssetPathResolution::NotAProposalAsset => { + panic!("expected `{iri_path}` to resolve as proposal asset") + } + } + } + + #[test] + fn resolver_detects_flat_source_cross_proposal_static_asset() { + let temp = TempDir::new().unwrap(); + let content = temp.path().join("content"); + let source = content.join("00555.md"); + + let asset_path = resolved_asset(&content, &source, "./00678/assets/foo.pdf"); + + assert_eq!(asset_path.target_proposal_number, number(678)); + assert_eq!( + asset_path.content_relative_asset_path, + Path::new("00678/assets/foo.pdf") + ); + assert_eq!(asset_path.asset_relative_path, Path::new("foo.pdf")); + assert_eq!(asset_path.kind, ProposalAssetKind::Static); + assert_eq!(asset_path.rendered_target_path, "/678/assets/foo.pdf"); + } + + #[test] + fn resolver_detects_directory_source_cross_proposal_static_asset() { + let temp = TempDir::new().unwrap(); + let content = temp.path().join("content"); + let source = content.join("00555/index.md"); + + let asset_path = resolved_asset(&content, &source, "../00678/assets/foo.pdf"); + + assert_eq!( + asset_path.content_relative_asset_path, + Path::new("00678/assets/foo.pdf") + ); + assert_eq!(asset_path.rendered_target_path, "/678/assets/foo.pdf"); + } + + #[test] + fn resolver_detects_asset_markdown_lexically_without_filesystem() { + let temp = TempDir::new().unwrap(); + let content = temp.path().join("content"); + let source = content.join("00555/assets/guide.md"); + + let asset_path = resolved_asset(&content, &source, "../../00678/assets/guide.md"); + + assert_eq!(asset_path.kind, ProposalAssetKind::Markdown); + assert_eq!( + asset_path.content_relative_asset_path, + Path::new("00678/assets/guide.md") + ); + assert_eq!(asset_path.rendered_target_path, "/678/assets/guide/"); + } + + #[test] + fn resolver_decodes_safe_percent_paths_and_renders_encoded_urls() { + let temp = TempDir::new().unwrap(); + let content = temp.path().join("content"); + let source = content.join("00555.md"); + + let asset_path = resolved_asset( + &content, + &source, + "./00678/assets/Contract%20Interactions%20diagram.svg", + ); + + assert_eq!( + asset_path.content_relative_asset_path, + Path::new("00678/assets/Contract Interactions diagram.svg") + ); + assert_eq!( + asset_path.rendered_target_path, + "/678/assets/Contract%20Interactions%20diagram.svg" + ); + } + + #[test] + fn resolver_rejects_unsafe_percent_and_asset_segments() { + let temp = TempDir::new().unwrap(); + let content = temp.path().join("content"); + let source = content.join("00555.md"); + + for iri_path in [ + "./00678/assets/foo%2Fbar.pdf", + "./00678/assets/foo%5Cbar.pdf", + "./00678/assets/foo%00bar.pdf", + "./00678/assets/.", + "./00678/assets/..", + "./00678/assets/%2E", + "./00678/assets/%2E%2E", + ] { + let error = resolve_proposal_asset_path(&content, &source, iri_path) + .unwrap_err() + .to_string(); + assert!( + error.contains("unsafe"), + "expected unsafe path error for `{iri_path}`, got `{error}`" + ); + } + } + + #[test] + fn resolver_allows_unsafe_percent_encodings_for_non_proposal_paths() { + let temp = TempDir::new().unwrap(); + let content = temp.path().join("content"); + let source = content.join("00555.md"); + + let resolution = + resolve_proposal_asset_path(&content, &source, "./images/foo%2Fbar.pdf").unwrap(); + + assert_eq!(resolution, ProposalAssetPathResolution::NotAProposalAsset); + } + + #[test] + fn resolver_still_rejects_unsafe_percent_encodings_for_proposal_assets() { + let temp = TempDir::new().unwrap(); + let content = temp.path().join("content"); + let source = content.join("00555.md"); + + let error = resolve_proposal_asset_path(&content, &source, "./00678/assets/foo%2Fbar.pdf") + .unwrap_err() + .to_string(); + + assert!(error.contains("unsafe")); + } + + #[test] + fn resolver_returns_passthrough_for_outside_root_without_error() { + let temp = TempDir::new().unwrap(); + let content = temp.path().join("content"); + let source = content.join("00555.md"); + + let resolution = + resolve_proposal_asset_path(&content, &source, "../elsewhere/foo.pdf").unwrap(); + + assert_eq!(resolution, ProposalAssetPathResolution::NotAProposalAsset); + } + + #[test] + fn resolver_returns_passthrough_for_non_proposal_asset_paths() { + let temp = TempDir::new().unwrap(); + let content = temp.path().join("content"); + let source = content.join("00555.md"); + + let resolution = + resolve_proposal_asset_path(&content, &source, "./images/foo.pdf").unwrap(); + + assert_eq!(resolution, ProposalAssetPathResolution::NotAProposalAsset); + } + + #[test] + fn rendered_path_helper_maps_proposal_and_asset_content_paths() { + for (content_relative_path, expected_rendered_path) in [ + ("_index.md", "/"), + ("00555.md", "/555/"), + ("00555/index.md", "/555/"), + ("00555/assets/guide.md", "/555/assets/guide/"), + ("00555/assets/README.md", "/555/assets/README/"), + ("00555/assets/index.md", "/555/assets/index/"), + ("00678/assets/foo.pdf", "/678/assets/foo.pdf"), + ( + "00678/assets/Contract Interactions diagram.svg", + "/678/assets/Contract%20Interactions%20diagram.svg", + ), + ] { + assert_eq!( + absolute_rendered_path_for_content_path(Path::new(content_relative_path)).unwrap(), + expected_rendered_path + ); + } + } + + #[test] + fn relative_url_helper_uses_rendered_paths() { + assert_eq!( + relative_url_from_rendered_paths("/555/", "/678/assets/foo.pdf").unwrap(), + "../678/assets/foo.pdf" + ); + assert_eq!( + relative_url_from_rendered_paths("/555/assets/guide/", "/678/assets/foo.pdf").unwrap(), + "../../../678/assets/foo.pdf" + ); + assert_eq!( + relative_url_from_rendered_paths("/555/", "/678/assets/guide/").unwrap(), + "../678/assets/guide/" + ); + } + + #[test] + fn filesystem_validator_checks_content_relative_assets() { + let temp = TempDir::new().unwrap(); + let content = temp.path().join("content"); + write_file(&content, "00678/assets/foo.pdf", ""); + + assert!(proposal_asset_exists_in_content_tree( + &content, + Path::new("00678/assets/foo.pdf") + )); + assert!(!proposal_asset_exists_in_content_tree( + &content, + Path::new("00678/assets/missing.pdf") + )); + assert!(!proposal_asset_exists_in_content_tree( + &content, + Path::new("../00678/assets/foo.pdf") + )); + } + + #[cfg(unix)] + #[test] + fn filesystem_validator_rejects_symlink_targets_outside_content_root() { + let temp = TempDir::new().unwrap(); + let content = temp.path().join("content"); + write_file(&content, "00678/assets/placeholder", ""); + let outside = temp.path().join("outside.pdf"); + std::fs::write(&outside, "").unwrap(); + std::os::unix::fs::symlink(&outside, content.join("00678/assets/outside.pdf")).unwrap(); + + assert!(!proposal_asset_exists_in_content_tree( + &content, + Path::new("00678/assets/outside.pdf") + )); + } + + #[test] + fn preprocess_rewrites_flat_source_cross_proposal_static_asset() { + let (_temp, content) = content_repo(&[ + ( + "00555.md", + proposal_markdown(555, None, "", "See [asset](./00678/assets/foo.pdf)."), + ), + ( + "00678/index.md", + proposal_markdown(678, None, "", "Target."), + ), + ("00678/assets/foo.pdf", "".to_owned()), + ]); + + preprocess(&content, None).unwrap(); + + let body = rendered_body(&content.join("00555.md")); + assert!(body.contains("(../678/assets/foo.pdf)")); + } + + #[test] + fn preprocess_rewrites_directory_source_cross_proposal_static_asset() { + let (_temp, content) = content_repo(&[ + ( + "00555/index.md", + proposal_markdown(555, None, "", "See [asset](../00678/assets/foo.pdf)."), + ), + ("00555/assets/.keep", "".to_owned()), + ( + "00678/index.md", + proposal_markdown(678, None, "", "Target."), + ), + ("00678/assets/foo.pdf", "".to_owned()), + ]); + + preprocess(&content, None).unwrap(); + + let body = rendered_body(&content.join("00555/index.md")); + assert!(body.contains("(../678/assets/foo.pdf)")); + } + + #[test] + fn preprocess_rewrites_root_index_source_cross_proposal_static_asset() { + let (_temp, content) = content_repo(&[ + ( + "_index.md", + "---\ntitle: Home\n---\nSee [asset](/00678/assets/foo.pdf).\n".to_owned(), + ), + ( + "00678/index.md", + proposal_markdown(678, None, "", "Target."), + ), + ("00678/assets/foo.pdf", "".to_owned()), + ]); + + preprocess(&content, None).unwrap(); + + let body = rendered_body(&content.join("_index.md")); + assert!(body.contains("(678/assets/foo.pdf)")); + } + + #[test] + fn preprocess_rewrites_asset_markdown_source_using_rendered_source_path() { + let (_temp, content) = content_repo(&[ + ( + "00555/index.md", + proposal_markdown(555, None, "", "Source."), + ), + ( + "00555/assets/guide.md", + "See [asset](../../00678/assets/foo.pdf).".to_owned(), + ), + ( + "00678/index.md", + proposal_markdown(678, None, "", "Target."), + ), + ("00678/assets/foo.pdf", "".to_owned()), + ]); + + preprocess(&content, None).unwrap(); + + let body = rendered_body(&content.join("00555/assets/guide.md")); + assert!(body.contains("(../../../678/assets/foo.pdf)")); + } + + #[test] + fn preprocess_rewrites_source_root_absolute_asset_path() { + let (_temp, content) = content_repo(&[ + ( + "00555.md", + proposal_markdown(555, None, "", "See [asset](/00678/assets/foo.pdf)."), + ), + ( + "00678/index.md", + proposal_markdown(678, None, "", "Target."), + ), + ("00678/assets/foo.pdf", "".to_owned()), + ]); + + preprocess(&content, None).unwrap(); + + let body = rendered_body(&content.join("00555.md")); + assert!(body.contains("(../678/assets/foo.pdf)")); + } + + #[test] + fn preprocess_rewrites_cross_proposal_image_links() { + let (_temp, content) = content_repo(&[ + ( + "00555.md", + proposal_markdown(555, None, "", "![diagram](./00678/assets/diagram.png)"), + ), + ( + "00678/index.md", + proposal_markdown(678, None, "", "Target."), + ), + ("00678/assets/diagram.png", "".to_owned()), + ]); + + preprocess(&content, None).unwrap(); + + let body = rendered_body(&content.join("00555.md")); + assert!(body.contains("![diagram](../678/assets/diagram.png)")); + } + + #[test] + fn preprocess_preserves_query_and_fragment_on_asset_links() { + let (_temp, content) = content_repo(&[ + ( + "00555.md", + proposal_markdown( + 555, + None, + "", + "See [asset](./00678/assets/foo.pdf?download=1#page=2).", + ), + ), + ( + "00678/index.md", + proposal_markdown(678, None, "", "Target."), + ), + ("00678/assets/foo.pdf", "".to_owned()), + ]); + + preprocess(&content, None).unwrap(); + + let body = rendered_body(&content.join("00555.md")); + assert!(body.contains("(../678/assets/foo.pdf?download=1#page=2)")); + } + + #[test] + fn preprocess_decodes_asset_paths_and_keeps_generated_urls_encoded() { + let (_temp, content) = content_repo(&[ + ( + "00555.md", + proposal_markdown( + 555, + None, + "", + "See [asset](./00678/assets/Contract%20Interactions%20diagram.svg).", + ), + ), + ( + "00678/index.md", + proposal_markdown(678, None, "", "Target."), + ), + ( + "00678/assets/Contract Interactions diagram.svg", + "".to_owned(), + ), + ]); + + preprocess(&content, None).unwrap(); + + let body = rendered_body(&content.join("00555.md")); + assert!(body.contains("../678/assets/Contract%20Interactions%20diagram.svg")); + } + + #[test] + fn preprocess_keeps_selected_or_full_asset_markdown_links_on_existing_md_path() { + let (_temp, content) = content_repo(&[ + ( + "00555.md", + proposal_markdown(555, None, "", "See [guide](./00678/assets/guide.md)."), + ), + ( + "00678/index.md", + proposal_markdown(678, None, "", "Target."), + ), + ("00678/assets/guide.md", "Guide.".to_owned()), + ]); + + preprocess(&content, None).unwrap(); + + let body = rendered_body(&content.join("00555.md")); + assert!(body.contains("(@/00678/assets/guide.md)")); + } + + #[test] + fn preprocess_keeps_asset_markdown_fragment_links_unchanged() { + let (_temp, content) = content_repo(&[ + ( + "00555/index.md", + proposal_markdown(555, None, "", "Source."), + ), + ( + "00555/assets/guide.md", + "See [heading](#heading).\n\n## Heading\n".to_owned(), + ), + ]); + + preprocess(&content, None).unwrap(); + + let body = rendered_body(&content.join("00555/assets/guide.md")); + assert!(body.contains("[heading](#heading)")); + } + + #[test] + fn preprocess_keeps_ordinary_proposal_markdown_links_on_existing_path() { + let (_temp, content) = content_repo(&[ + ( + "00555.md", + proposal_markdown(555, None, "", "See [proposal](./00678.md)."), + ), + ("00678.md", proposal_markdown(678, None, "", "Target.")), + ]); + + preprocess(&content, None).unwrap(); + + let body = rendered_body(&content.join("00555.md")); + assert!(body.contains("(@/00678.md)")); + } + + #[test] + fn targeted_preprocess_rewrites_omitted_static_asset_to_public_url() { + let (_temp, content) = content_repo(&[ + ( + "00555.md", + proposal_markdown(555, None, "", "See [asset](./00678/assets/foo.pdf)."), + ), + ( + "00678/index.md", + proposal_markdown(678, None, "", "Target."), + ), + ("00678/assets/foo.pdf", "".to_owned()), + ]); + let plan = only_plan(&content, &[555]); + + preprocess(&content, Some(&plan)).unwrap(); + + let body = rendered_body(&content.join("00555.md")); + assert!(body.contains("(https://eips.ethereum.org/678/assets/foo.pdf)")); + } + + #[test] + fn targeted_preprocess_rewrites_omitted_asset_markdown_to_public_url() { + let (_temp, content) = content_repo(&[ + ( + "00555.md", + proposal_markdown(555, None, "", "See [guide](./00678/assets/guide.md)."), + ), + ( + "00678/index.md", + proposal_markdown(678, None, "", "Target."), + ), + ("00678/assets/guide.md", "Guide.".to_owned()), + ]); + let plan = only_plan(&content, &[555]); + + preprocess(&content, Some(&plan)).unwrap(); + + let body = rendered_body(&content.join("00555.md")); + assert!(body.contains("(https://eips.ethereum.org/678/assets/guide/)")); + } + + #[test] + fn targeted_preprocess_rewrites_omitted_readme_and_index_asset_markdown_public_urls() { + let (_temp, content) = content_repo(&[ + ( + "00555.md", + proposal_markdown( + 555, + None, + "", + "See [readme](./00678/assets/README.md) and [index](./00678/assets/index.md).", + ), + ), + ( + "00678/index.md", + proposal_markdown(678, None, "", "Target."), + ), + ("00678/assets/README.md", "Readme.".to_owned()), + ("00678/assets/index.md", "Index.".to_owned()), + ]); + let plan = only_plan(&content, &[555]); + + preprocess(&content, Some(&plan)).unwrap(); + + let body = rendered_body(&content.join("00555.md")); + assert!(body.contains("(https://eips.ethereum.org/678/assets/README/)")); + assert!(body.contains("(https://eips.ethereum.org/678/assets/index/)")); + } + + #[test] + fn targeted_preprocess_keeps_selected_static_asset_local() { + let (_temp, content) = content_repo(&[ + ( + "00555.md", + proposal_markdown(555, None, "", "See [asset](./00678/assets/foo.pdf)."), + ), + ( + "00678/index.md", + proposal_markdown(678, None, "", "Target."), + ), + ("00678/assets/foo.pdf", "".to_owned()), + ]); + let plan = only_plan(&content, &[555, 678]); + + preprocess(&content, Some(&plan)).unwrap(); + + let body = rendered_body(&content.join("00555.md")); + assert!(body.contains("(../678/assets/foo.pdf)")); + assert!(!body.contains("https://eips.ethereum.org/678/assets/foo.pdf")); + } + + #[test] + fn targeted_dirty_preprocess_uses_inventory_after_omitted_target_is_pruned() { + let (_temp, content) = content_repo(&[ + ( + "00555.md", + proposal_markdown(555, None, "", "See [asset](./00678/assets/foo.pdf)."), + ), + ( + "00678/index.md", + proposal_markdown(678, None, "", "Target."), + ), + ("00678/assets/foo.pdf", "".to_owned()), + ]); + let plan = only_plan(&content, &[555]); + plan.prune_content(&content).unwrap(); + + preprocess(&content, Some(&plan)).unwrap(); + + let body = rendered_body(&content.join("00555.md")); + assert!(body.contains("(https://eips.ethereum.org/678/assets/foo.pdf)")); + } + + #[test] + fn preprocess_errors_clearly_for_missing_selected_or_full_asset_target() { + let (_temp, content) = content_repo(&[ + ( + "00555.md", + proposal_markdown(555, None, "", "See [asset](./00678/assets/missing.pdf)."), + ), + ( + "00678/index.md", + proposal_markdown(678, None, "", "Target."), + ), + ("00678/assets/.keep", "".to_owned()), + ]); + + let error = Report::from_error(preprocess(&content, None).unwrap_err()).to_string(); + + assert!(error.contains("proposal asset link")); + assert!(error.contains("00555.md")); + assert!(error.contains("./00678/assets/missing.pdf")); + assert!(error.contains("00678/assets/missing.pdf")); + } + + #[test] + fn preprocess_skips_generated_zola_markdown_files() { + let original = + "+++\ntitle = \"Generated\"\n+++\nSee [asset](./00678/assets/missing.pdf).\n"; + let (_temp, content) = content_repo(&[("00555.md", original.to_owned())]); + + preprocess(&content, None).unwrap(); + + assert_eq!( + std::fs::read_to_string(content.join("00555.md")).unwrap(), + original + ); + } + + #[test] + fn process_assets_skips_only_generated_asset_markdown_file() { + let generated = + "+++\ntitle = \"Generated\"\n+++\nSee [missing](../../00678/assets/missing.pdf).\n"; + let (_temp, content) = content_repo(&[ + ( + "00555/index.md", + proposal_markdown(555, None, "", "Source."), + ), + ("00555/assets/generated.md", generated.to_owned()), + ("00555/assets/fresh.md", "Fresh asset markdown.".to_owned()), + ]); + + preprocess(&content, None).unwrap(); + + assert_eq!( + std::fs::read_to_string(content.join("00555/assets/generated.md")).unwrap(), + generated + ); + assert!( + std::fs::read_to_string(content.join("00555/assets/fresh.md")) + .unwrap() + .starts_with("+++\n") + ); + } + + #[test] + fn preprocess_leaves_non_proposal_relative_asset_links_unchanged() { + let (_temp, content) = content_repo(&[( + "00555.md", + proposal_markdown(555, None, "", "See [local](./images/foo.pdf)."), + )]); + + preprocess(&content, None).unwrap(); + + let body = rendered_body(&content.join("00555.md")); + assert!(body.contains("(./images/foo.pdf)")); + } + + #[test] + fn preprocess_leaves_raw_html_asset_references_unchanged() { + let (_temp, content) = content_repo(&[ + ( + "00555.md", + proposal_markdown( + 555, + None, + "", + r#"asset"#, + ), + ), + ( + "00678/index.md", + proposal_markdown(678, None, "", "Target."), + ), + ("00678/assets/foo.pdf", "".to_owned()), + ]); + + preprocess(&content, None).unwrap(); + + let body = rendered_body(&content.join("00555.md")); + assert!(body.contains(r#"asset"#)); + } + + #[test] + fn targeted_preprocess_rewrites_selected_body_links_to_unselected_public_urls() { + let (_temp, content) = content_repo(&[ + ( + "00555.md", + proposal_markdown(555, None, "", "See [ERC-678](/00678.md)."), + ), + ( + "00678.md", + proposal_markdown(678, Some("ERC"), "", "Target."), + ), + ]); + let plan = only_plan(&content, &[555]); + + preprocess(&content, Some(&plan)).unwrap(); + + let body = rendered_body(&content.join("00555.md")); + assert!(body.contains("https://ercs.ethereum.org/ERCS/erc-678")); + assert!(!body.contains("@/00678.md")); + } + + #[test] + fn targeted_preprocess_paths_rewrites_selected_dirty_markdown_with_plan() { + let (_temp, content) = content_repo(&[ + ( + "00555.md", + proposal_markdown( + 555, + None, + "requires: 155\n", + "See [EIP-155](./00155.md#list-of-chain-id-s).", + ), + ), + ("00155.md", proposal_markdown(155, None, "", "Unselected.")), + ]); + let plan = only_plan(&content, &[555]); + + preprocess_paths(&content, &repo_paths(&["content/00555.md"]), Some(&plan)).unwrap(); + + let body = rendered_body(&content.join("00555.md")); + let front_matter = rendered_front_matter(&content.join("00555.md")); + let requires = front_matter["extra"]["requires"].as_array().unwrap(); + assert!(body.contains("https://eips.ethereum.org/EIPS/eip-155#list-of-chain-id-s")); + assert_eq!( + requires[0].as_str().unwrap(), + "https://eips.ethereum.org/EIPS/eip-155" + ); + } + + #[test] + fn targeted_preprocess_paths_ignores_deleted_dirty_markdown() { + let (_temp, content) = + content_repo(&[("00555.md", proposal_markdown(555, None, "", "Selected."))]); + let plan = only_plan(&content, &[555]); + std::fs::remove_file(content.join("00555.md")).unwrap(); + + preprocess_paths(&content, &repo_paths(&["content/00555.md"]), Some(&plan)).unwrap(); + + assert!(!content.join("00555.md").exists()); + } + + #[test] + fn targeted_preprocess_rewrites_retained_non_proposal_links_to_public_urls() { + let (_temp, content) = content_repo(&[ + ( + "_index.md", + "---\ntitle: Home\n---\nSee [EIP-678](/00678.md).\n".to_owned(), + ), + ("00555.md", proposal_markdown(555, None, "", "Selected.")), + ("00678.md", proposal_markdown(678, None, "", "Unselected.")), + ]); + let plan = only_plan(&content, &[555]); + + preprocess(&content, Some(&plan)).unwrap(); + + let body = rendered_body(&content.join("_index.md")); + assert!(body.contains("https://eips.ethereum.org/EIPS/eip-678")); + assert!(!body.contains("@/00678.md")); + } + + #[test] + fn targeted_preprocess_paths_rewrites_retained_non_proposal_markdown_with_plan() { + let (_temp, content) = content_repo(&[ + ( + "_index.md", + "---\ntitle: Home\n---\nSee [EIP-678](/00678.md).\n".to_owned(), + ), + ("00555.md", proposal_markdown(555, None, "", "Selected.")), + ("00678.md", proposal_markdown(678, None, "", "Unselected.")), + ]); + let plan = only_plan(&content, &[555]); + + preprocess_paths(&content, &repo_paths(&["content/_index.md"]), Some(&plan)).unwrap(); + + let body = rendered_body(&content.join("_index.md")); + assert!(body.contains("https://eips.ethereum.org/EIPS/eip-678")); + assert!(!body.contains("@/00678.md")); + } + + #[test] + fn targeted_preprocess_paths_rewrites_selected_asset_markdown_with_plan() { + let (_temp, content) = content_repo(&[ + ("00555.md", proposal_markdown(555, None, "", "Selected.")), + ( + "00555/assets/guide.md", + "See [EIP-678](/00678.md).\n".to_owned(), + ), + ("00555/assets/diagram.png", "image\n".to_owned()), + ( + "00678.md", + proposal_markdown(678, Some("ERC"), "", "Unselected."), + ), + ]); + let plan = only_plan(&content, &[555]); + + preprocess_paths( + &content, + &repo_paths(&["content/00555/assets/guide.md"]), + Some(&plan), + ) + .unwrap(); + + let body = rendered_body(&content.join("00555/assets/guide.md")); + assert!(body.contains("https://ercs.ethereum.org/ERCS/erc-678")); + assert_eq!( + std::fs::read_to_string(content.join("00555/assets/diagram.png")).unwrap(), + "image\n" + ); + } + + #[test] + fn targeted_preprocess_paths_ignores_deleted_dirty_asset_dir() { + let (_temp, content) = content_repo(&[ + ("00555.md", proposal_markdown(555, None, "", "Selected.")), + ( + "00555/assets/guide.md", + "See [EIP-678](/00678.md).\n".to_owned(), + ), + ("00678.md", proposal_markdown(678, None, "", "Unselected.")), + ]); + let plan = only_plan(&content, &[555]); + std::fs::remove_dir_all(content.join("00555/assets")).unwrap(); + + preprocess_paths( + &content, + &repo_paths(&["content/00555/assets/guide.md"]), + Some(&plan), + ) + .unwrap(); + + assert!(!content.join("00555/assets").exists()); + } + + #[test] + fn targeted_preprocess_preserves_query_and_fragment_on_external_links() { + let (_temp, content) = content_repo(&[ + ( + "00555.md", + proposal_markdown( + 555, + None, + "", + "See [Fragment](./00155.md#list-of-chain-id-s).\nSee [Query](./00155.md?foo=bar#list-of-chain-id-s).", + ), + ), + ( + "00155.md", + proposal_markdown(155, None, "", "Unselected."), + ), + ]); + let plan = only_plan(&content, &[555]); + + preprocess(&content, Some(&plan)).unwrap(); + + let body = rendered_body(&content.join("00555.md")); + assert!(body.contains("https://eips.ethereum.org/EIPS/eip-155#list-of-chain-id-s")); + assert!(body.contains("https://eips.ethereum.org/EIPS/eip-155?foo=bar#list-of-chain-id-s")); + } + + #[test] + fn targeted_preprocess_rewrites_requires_to_unselected_public_urls() { + let (_temp, content) = content_repo(&[ + ( + "00555.md", + proposal_markdown(555, None, "requires: 678\n", "Selected."), + ), + ( + "00678.md", + proposal_markdown(678, Some("ERC"), "", "Target."), + ), + ]); + let plan = only_plan(&content, &[555]); + + preprocess(&content, Some(&plan)).unwrap(); + + let front_matter = rendered_front_matter(&content.join("00555.md")); + let requires = front_matter["extra"]["requires"].as_array().unwrap(); + assert_eq!( + requires[0].as_str().unwrap(), + "https://ercs.ethereum.org/ERCS/erc-678" + ); + } + + #[test] + fn targeted_preprocess_keeps_internal_references_between_selected_proposals() { + let (_temp, content) = content_repo(&[ + ( + "00555.md", + proposal_markdown(555, None, "requires: 678\n", "See [EIP-678](/00678.md)."), + ), + ( + "00678.md", + proposal_markdown(678, Some("ERC"), "", "Target."), + ), + ]); + let plan = only_plan(&content, &[555, 678]); + + preprocess(&content, Some(&plan)).unwrap(); + + let body = rendered_body(&content.join("00555.md")); + let front_matter = rendered_front_matter(&content.join("00555.md")); + let requires = front_matter["extra"]["requires"].as_array().unwrap(); + assert!(body.contains("@/00678.md")); + assert_eq!(requires[0].as_str().unwrap(), "@/00678.md"); + } + + #[test] + fn targeted_preprocess_does_not_mask_missing_body_link_targets() { + let (_temp, content) = content_repo(&[( + "00555.md", + proposal_markdown(555, None, "", "See [Missing](/00678.md)."), + )]); + let plan = only_plan(&content, &[555]); + + let error = Report::from_error(preprocess(&content, Some(&plan)).unwrap_err()).to_string(); + + assert!(error.contains("could not canonicalize")); + assert!(error.contains("00678.md")); + } +} diff --git a/src/pipeline.rs b/src/pipeline.rs new file mode 100644 index 0000000..d5f16d1 --- /dev/null +++ b/src/pipeline.rs @@ -0,0 +1,510 @@ +/* + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. + */ + +//! Prepared Zola runtime pipeline. + +use std::path::{Path, PathBuf}; + +use snafu::{OptionExt, ResultExt, Whatever}; +use url::Url; + +use crate::{ + config::{RepositoryUse, ServerBinding}, + execution::ResolvedExecution, + git, + layout::{mounted_theme_path, output_path, CONTENT_DIR, REPO_DIR}, + markdown, + proposal::OnlyRenderPlan, + serve::{serve_sync_config, DirtyServeWatcher, LocalThemeServeSync}, + zola, +}; + +fn prepare_theme_for_zola( + theme_path: PathBuf, + repo_path: &Path, +) -> Result<(PathBuf, LocalThemeServeSync), Whatever> { + let mounted_theme_dir = mounted_theme_path(repo_path); + git::materialize_working_tree(&theme_path, &mounted_theme_dir) + .whatever_context("unable to materialize workspace-local theme")?; + let theme_index_path = git::index_path(&theme_path) + .whatever_context("unable to resolve workspace-local theme Git index path")?; + + Ok(( + mounted_theme_dir.clone(), + LocalThemeServeSync { + theme_source_root: theme_path, + mounted_theme_dir, + theme_index_path, + }, + )) +} + +fn prepare_runtime_source( + root_path: &Path, + repo_path: &Path, + repository_use: &RepositoryUse, + source_materialization: git::SourceMaterialization, +) -> Result<(), Whatever> { + let source = git::Fresh::new( + root_path, + repo_path, + repository_use.clone(), + source_materialization, + ) + .whatever_context("initializing build repo")? + .clone_src() + .whatever_context("cloning source repo")?; + + source + .merge() + .whatever_context("unable to merge ERC/EIP repositories")?; + + Ok(()) +} + +#[derive(Debug)] +pub(crate) struct Prepared { + repo_path: PathBuf, + output_path: PathBuf, + repository_use: RepositoryUse, + theme_path: PathBuf, + local_theme_sync: Option, + only_plan: Option, + source_root: PathBuf, + source_materialization: git::SourceMaterialization, + server_binding: ServerBinding, + base_url_override: Option, +} + +impl Prepared { + pub(crate) fn prepare(resolved: ResolvedExecution) -> Result { + zola::find_zola().whatever_context("unable to find suitable zola binary")?; + + let ResolvedExecution { + root_path, + build_path, + repository_use, + theme_path, + only, + source_materialization, + server_binding, + base_url_override, + } = resolved; + let theme_path = + theme_path.whatever_context("Zola runtime requires a workspace-local theme path")?; + + let repo_path = build_path.join(REPO_DIR); + let content_path = repo_path.join(CONTENT_DIR); + let output_path = output_path(&build_path); + + prepare_runtime_source( + &root_path, + &repo_path, + &repository_use, + source_materialization, + )?; + + let only_plan = only + .map(|selected_numbers| OnlyRenderPlan::build(&content_path, selected_numbers)) + .transpose() + .whatever_context("unable to build targeted render plan")?; + markdown::preprocess(&content_path, only_plan.as_ref()) + .whatever_context("unable to preprocess markdown")?; + if let Some(only_plan) = &only_plan { + only_plan + .prune_content(&content_path) + .whatever_context("unable to prune unselected proposals")?; + } + let (theme_path, local_theme_sync) = prepare_theme_for_zola(theme_path, &repo_path)?; + + Ok(Prepared { + repository_use, + theme_path, + local_theme_sync: Some(local_theme_sync), + repo_path, + output_path, + only_plan, + source_root: root_path, + source_materialization, + server_binding, + base_url_override, + }) + } + + pub(crate) fn build(self) -> Result<(), Whatever> { + let base_url = self + .base_url_override + .as_ref() + .unwrap_or(&self.repository_use.location.base_url); + zola::build( + &self.theme_path, + &self.repo_path, + &self.output_path, + base_url.as_str(), + ) + .whatever_context("zola build failed")?; + Ok(()) + } + + pub(crate) fn serve(self) -> Result<(), Whatever> { + let sync_config = serve_sync_config( + self.source_materialization, + &self.source_root, + &self.repo_path, + self.only_plan.clone(), + self.local_theme_sync.clone(), + ); + let dirty_watcher = if sync_config.has_targets() { + Some( + DirtyServeWatcher::start(sync_config) + .whatever_context("unable to start dirty serve watcher")?, + ) + } else { + None + }; + + let result = zola::serve( + &self.theme_path, + &self.repo_path, + &self.output_path, + &self.server_binding, + self.base_url_override.as_ref(), + ) + .whatever_context("zola serve failed"); + + if let Some(dirty_watcher) = dirty_watcher { + dirty_watcher.stop(); + } + + result + } + + pub(crate) fn check(self) -> Result<(), Whatever> { + zola::check(&self.theme_path, &self.repo_path).whatever_context("zola check failed")?; + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use std::path::{Path, PathBuf}; + + use clap::Parser; + use git2::{IndexAddOption, Repository, Signature}; + use tempfile::TempDir; + use url::Url; + + use crate::{ + changed, + cli::{Args, ChangedFormat, EditorialCommand, RuntimeOperation}, + config, + editorial::editorial_runtime_execution, + execution::{resolve_execution, ResolvedExecution}, + git::SourceMaterialization, + layout::{mounted_theme_path, theme_config_path, REPO_DIR}, + }; + + use super::{prepare_runtime_source, prepare_theme_for_zola, Prepared}; + + struct RuntimeWorkspace { + _temp: TempDir, + active_path: PathBuf, + } + + fn write_file(root: &Path, relative: impl AsRef, contents: &str) { + let path = root.join(relative); + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent).unwrap(); + } + std::fs::write(path, contents).unwrap(); + } + + fn commit_all(repo: &Repository, message: &str) { + let mut index = repo.index().unwrap(); + index + .add_all(["*"].iter(), IndexAddOption::DEFAULT, None) + .unwrap(); + index.write().unwrap(); + let tree_oid = index.write_tree().unwrap(); + let tree = repo.find_tree(tree_oid).unwrap(); + let signature = Signature::now("build-eips test", "build-eips@example.test").unwrap(); + let parents = repo + .head() + .ok() + .and_then(|head| head.target()) + .map(|oid| repo.find_commit(oid).unwrap()) + .into_iter() + .collect::>(); + let parent_refs = parents.iter().collect::>(); + + repo.commit( + Some("HEAD"), + &signature, + &signature, + message, + &tree, + &parent_refs, + ) + .unwrap(); + } + + fn init_repo(path: &Path, files: &[(&str, &str)]) -> Repository { + std::fs::create_dir_all(path).unwrap(); + let repo = Repository::init(path).unwrap(); + repo.set_head("refs/heads/master").unwrap(); + for (relative, contents) in files { + write_file(path, relative, contents); + } + commit_all(&repo, "initial"); + repo + } + + fn file_url(path: &Path) -> Url { + Url::from_directory_path(path).unwrap() + } + + const ACTIVE_PROPOSAL: &str = "---\neip: 1\ntitle: Active proposal\ndescription: Active proposal\nauthor: Test Author \ndiscussions-to: https://ethereum-magicians.org/t/test/1\nstatus: Draft\ntype: Standards Track\ncreated: 2025-01-01\n---\n\nActive proposal.\n"; + + fn build_manifest_text(repo_name: &str, repository: &Url, siblings: &[(&str, Url)]) -> String { + let mut manifest = format!( + r#" + name = "{repo_name}" + + [locations.{repo_name}] + repository = "{repository}" + base-url = "https://example.test/{repo_name}/" + + [theme] + repository = "https://example.test/theme.git" + commit = "test-theme-commit" + "# + ); + + for (sibling_id, sibling_repository) in siblings { + manifest.push_str(&format!( + r#" + [locations.{sibling_id}] + repository = "{sibling_repository}" + base-url = "https://example.test/{sibling_id}/" + "# + )); + } + + manifest + } + + fn runtime_workspace(with_sibling: bool) -> RuntimeWorkspace { + let temp = TempDir::new().unwrap(); + let workspace_root = temp.path().join("workspace"); + let active_path = workspace_root.join("EIPs"); + let sibling_path = workspace_root.join("ERCs"); + let missing_upstream = file_url(&temp.path().join("missing-upstream")); + let siblings = with_sibling.then(|| ("ERCs", file_url(&sibling_path))); + let siblings = siblings.into_iter().collect::>(); + let manifest = build_manifest_text("EIPs", &missing_upstream, &siblings); + + write_file(&workspace_root, config::LOCAL_CONFIG_FILE, ""); + let _theme_repo = init_repo( + &workspace_root.join(config::DEFAULT_THEME_DIR), + &[ + ("config/zola.toml", "title = 'local theme'\n"), + ("templates/index.html", "workspace local theme\n"), + ], + ); + let _active_repo = init_repo( + &active_path, + &[ + (config::MANIFEST_FILE, manifest.as_str()), + ("content/00001.md", ACTIVE_PROPOSAL), + ], + ); + + if with_sibling { + let _sibling_repo = init_repo(&sibling_path, &[("content/00002.md", "sibling\n")]); + } + + RuntimeWorkspace { + _temp: temp, + active_path, + } + } + + fn resolved_runtime(workspace: &RuntimeWorkspace, command: &[&str]) -> ResolvedExecution { + let active_path = workspace.active_path.to_str().unwrap(); + let mut arguments = vec!["build-eips", "-C", active_path]; + arguments.extend_from_slice(command); + let args = Args::try_parse_from(arguments).unwrap(); + + resolve_execution(&args).unwrap() + } + + fn prepare_resolved_source(resolved: &ResolvedExecution) -> Result<(), snafu::Whatever> { + std::fs::create_dir_all(&resolved.build_path).unwrap(); + let repo_path = resolved.build_path.join(REPO_DIR); + prepare_runtime_source( + &resolved.root_path, + &repo_path, + &resolved.repository_use, + resolved.source_materialization, + ) + } + + fn prepared_path(resolved: &ResolvedExecution, relative: impl AsRef) -> PathBuf { + resolved.build_path.join(REPO_DIR).join(relative) + } + + #[test] + fn workspace_local_theme_dirty_edits_are_materialized_as_mounted_theme_for_zola() { + let temp = TempDir::new().unwrap(); + let theme_root = temp.path().join("workspace/theme"); + init_repo( + &theme_root, + &[ + ("config/zola.toml", "title = 'theme'\n"), + ("templates/index.html", "committed local theme\n"), + ], + ); + write_file(&theme_root, "templates/index.html", "dirty local theme\n"); + let repo_path = temp.path().join("workspace/.local-build/Core/repo"); + + let (theme_path, sync) = prepare_theme_for_zola(theme_root.clone(), &repo_path).unwrap(); + + let mounted_theme_dir = mounted_theme_path(&repo_path); + assert_eq!(theme_path, mounted_theme_dir); + assert_eq!( + theme_config_path(&mounted_theme_dir), + repo_path.join("themes/eips-theme/config/zola.toml") + ); + assert_eq!( + std::fs::read_to_string(mounted_theme_dir.join("templates/index.html")).unwrap(), + "dirty local theme\n" + ); + assert_eq!(sync.theme_source_root, theme_root); + assert_eq!(sync.mounted_theme_dir, mounted_theme_dir); + assert!(sync.theme_index_path.ends_with(".git/index")); + } + + #[test] + fn runtime_preparation_uses_workspace_local_theme_without_manifest_theme_network_access() { + for command in [&["build"][..], &["check"][..], &["serve"][..]] { + let workspace = runtime_workspace(false); + let resolved = resolved_runtime(&workspace, command); + let prepared = Prepared::prepare(resolved).unwrap(); + + assert_eq!( + std::fs::read_to_string(prepared.theme_path.join("templates/index.html")).unwrap(), + "workspace local theme\n" + ); + } + } + + #[test] + fn prepared_runtime_source_succeeds_with_unreachable_active_upstream() { + for command in [&["build"][..], &["check"][..], &["serve"][..]] { + let workspace = runtime_workspace(false); + let resolved = resolved_runtime(&workspace, command); + + prepare_resolved_source(&resolved).unwrap(); + + assert_eq!( + std::fs::read_to_string(prepared_path(&resolved, "content/00001.md")).unwrap(), + ACTIVE_PROPOSAL + ); + } + } + + #[test] + fn prepared_runtime_source_uses_remote_siblings_without_active_upstream_fetch() { + let workspace = runtime_workspace(true); + let resolved = resolved_runtime(&workspace, &["--remote-siblings", "build"]); + + assert_eq!( + resolved.repository_use.other_repos["ERCs"], + file_url(&workspace.active_path.parent().unwrap().join("ERCs")) + ); + prepare_resolved_source(&resolved).unwrap(); + + assert_eq!( + std::fs::read_to_string(prepared_path(&resolved, "content/00001.md")).unwrap(), + ACTIVE_PROPOSAL + ); + assert_eq!( + std::fs::read_to_string(prepared_path(&resolved, "content/00002.md")).unwrap(), + "sibling\n" + ); + } + + #[test] + fn clean_runtime_source_prep_keeps_local_active_checkout() { + for command in [ + &["build", "--clean"][..], + &["--remote-siblings", "build", "--clean"][..], + &["check", "--clean"][..], + ] { + let workspace = runtime_workspace(false); + let resolved = resolved_runtime(&workspace, command); + + assert_eq!( + resolved.source_materialization, + SourceMaterialization::Clean + ); + prepare_resolved_source(&resolved).unwrap(); + + assert_eq!( + std::fs::read_to_string(prepared_path(&resolved, "content/00001.md")).unwrap(), + ACTIVE_PROPOSAL + ); + } + } + + #[test] + fn changed_still_requires_active_upstream() { + let workspace = runtime_workspace(false); + let resolved = resolved_runtime(&workspace, &["changed"]); + std::fs::create_dir_all(&resolved.build_path).unwrap(); + + let error = changed::run( + &resolved, + &resolved.build_path, + false, + &ChangedFormat::Newline, + ) + .unwrap_err() + .to_string(); + + assert!(error.contains("fetching upstream repo")); + } + + #[test] + fn editorial_check_site_phase_source_prep_does_not_fetch_active_upstream() { + let workspace = runtime_workspace(false); + let active_path = workspace.active_path.to_str().unwrap(); + let args = Args::try_parse_from([ + "build-eips", + "-C", + active_path, + "editorial", + "check", + "--against-upstream", + ]) + .unwrap(); + let resolved = resolve_execution(&args).unwrap(); + let RuntimeOperation::Editorial { + command: EditorialCommand::Check { selectors, .. }, + } = args.operation.runtime_operation().unwrap() + else { + panic!("expected editorial check runtime operation"); + }; + let resolved = editorial_runtime_execution(resolved, &selectors); + + prepare_resolved_source(&resolved).unwrap(); + + assert_eq!( + std::fs::read_to_string(prepared_path(&resolved, "content/00001.md")).unwrap(), + ACTIVE_PROPOSAL + ); + } +} diff --git a/src/preview.rs b/src/preview.rs new file mode 100644 index 0000000..4f91335 --- /dev/null +++ b/src/preview.rs @@ -0,0 +1,263 @@ +/* + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. + */ + +use std::{ + fs::File, + io::ErrorKind, + path::{Component, Path, PathBuf}, +}; + +use log::info; +use snafu::{ResultExt, Whatever}; +use tiny_http::{Header, Method, Request, Response, Server, StatusCode}; + +use crate::config::ServerBinding; + +const INDEX_HTML: &str = "index.html"; + +pub fn serve(output_path: &Path, server_binding: &ServerBinding) -> Result<(), Whatever> { + if !output_path.is_dir() { + snafu::whatever!( + "preview output directory `{}` is missing; run `build-eips build` for this profile first", + output_path.to_string_lossy() + ); + } + + let server = match Server::http((server_binding.host.as_str(), server_binding.port)) { + Ok(server) => server, + Err(error) => { + snafu::whatever!("unable to bind preview server on {server_binding}: {error}") + } + }; + + info!( + "serving static preview from `{}` at http://{server_binding}/", + output_path.to_string_lossy() + ); + + for request in server.incoming_requests() { + handle_request(output_path, request)?; + } + + Ok(()) +} + +fn handle_request(output_path: &Path, request: Request) -> Result<(), Whatever> { + match *request.method() { + Method::Get | Method::Head => {} + _ => { + request + .respond(Response::empty(StatusCode(405))) + .whatever_context("unable to send preview method error response")?; + return Ok(()); + } + } + + let Some(paths) = resolve_request_paths(output_path, request.url()) else { + request + .respond(Response::empty(StatusCode(400))) + .whatever_context("unable to send preview bad request response")?; + return Ok(()); + }; + + let Some((path, file)) = open_preview_asset(paths)? else { + request + .respond(Response::empty(StatusCode(404))) + .whatever_context("unable to send preview not found response")?; + return Ok(()); + }; + + let response = if let Some(value) = content_type(&path) { + Response::from_file(file).with_header(content_type_header(value)) + } else { + Response::from_file(file) + }; + + request + .respond(response) + .with_whatever_context(|e| format!("unable to send preview response: {e}"))?; + + Ok(()) +} + +fn open_preview_asset(paths: Vec) -> Result, Whatever> { + for path in paths { + let file = match File::open(&path) { + Ok(file) => file, + Err(error) + if matches!(error.kind(), ErrorKind::NotFound | ErrorKind::NotADirectory) => + { + continue; + } + Err(error) => { + snafu::whatever!( + "unable to open preview asset `{}`: {error}", + path.to_string_lossy() + ); + } + }; + + if !file + .metadata() + .with_whatever_context(|e| { + format!( + "unable to inspect preview asset `{}`: {e}", + path.to_string_lossy() + ) + })? + .is_file() + { + continue; + } + + return Ok(Some((path, file))); + } + + Ok(None) +} + +fn resolve_request_paths(output_path: &Path, url: &str) -> Option> { + let raw_path = url.split('?').next().unwrap_or("/"); + let mut resolved = output_path.to_path_buf(); + let mut saw_normal_component = false; + + for component in Path::new(raw_path.trim_start_matches('/')).components() { + match component { + Component::CurDir | Component::RootDir => {} + Component::Normal(component) => { + saw_normal_component = true; + resolved.push(component); + } + Component::ParentDir | Component::Prefix(_) => return None, + } + } + + if raw_path.ends_with('/') || !saw_normal_component { + return Some(vec![resolved.join(INDEX_HTML)]); + } + + if resolved.extension().is_none() { + return Some(vec![resolved.join(INDEX_HTML), resolved]); + } + + Some(vec![resolved]) +} + +fn content_type(path: &Path) -> Option<&'static str> { + match path.extension().and_then(|extension| extension.to_str()) { + Some("css") => Some("text/css; charset=utf-8"), + Some("gif") => Some("image/gif"), + Some("htm" | "html") => Some("text/html; charset=utf-8"), + Some("ico") => Some("image/x-icon"), + Some("jpeg" | "jpg") => Some("image/jpeg"), + Some("js") => Some("application/javascript; charset=utf-8"), + Some("json") => Some("application/json; charset=utf-8"), + Some("mjs") => Some("application/javascript; charset=utf-8"), + Some("png") => Some("image/png"), + Some("svg") => Some("image/svg+xml"), + Some("txt") => Some("text/plain; charset=utf-8"), + Some("webp") => Some("image/webp"), + Some("xml") => Some("application/xml; charset=utf-8"), + _ => None, + } +} + +fn content_type_header(value: &str) -> Header { + Header::from_bytes(b"Content-Type", value.as_bytes()) + .expect("hard-coded content-type headers must be valid") +} + +#[cfg(test)] +mod tests { + use std::path::{Path, PathBuf}; + + use tempfile::TempDir; + + use super::{open_preview_asset, resolve_request_paths, INDEX_HTML}; + + fn write_file(root: &Path, relative: &str, contents: &str) { + let path = root.join(relative); + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent).unwrap(); + } + std::fs::write(path, contents).unwrap(); + } + + fn candidates(root: &Path, url: &str) -> Vec { + resolve_request_paths(root, url).unwrap() + } + + fn selected_path(root: &Path, url: &str) -> Option { + open_preview_asset(candidates(root, url)) + .unwrap() + .map(|(path, _file)| path) + } + + #[test] + fn slash_path_resolves_to_index_candidate() { + let temp = TempDir::new().unwrap(); + let root = temp.path(); + + assert_eq!( + candidates(root, "/foo/"), + vec![root.join("foo").join(INDEX_HTML)] + ); + } + + #[test] + fn extensionless_path_prefers_index_then_file_candidate() { + let temp = TempDir::new().unwrap(); + let root = temp.path(); + + assert_eq!( + candidates(root, "/foo"), + vec![root.join("foo").join(INDEX_HTML), root.join("foo")] + ); + } + + #[test] + fn extension_path_resolves_to_file_candidate() { + let temp = TempDir::new().unwrap(); + let root = temp.path(); + + assert_eq!(candidates(root, "/foo.css"), vec![root.join("foo.css")]); + } + + #[test] + fn parent_traversal_is_rejected() { + let temp = TempDir::new().unwrap(); + + assert!(resolve_request_paths(temp.path(), "/../secret.txt").is_none()); + } + + #[test] + fn extensionless_path_serves_index_when_present() { + let temp = TempDir::new().unwrap(); + let root = temp.path(); + write_file(root, "foo/index.html", "index"); + + assert_eq!( + selected_path(root, "/foo"), + Some(root.join("foo").join(INDEX_HTML)) + ); + } + + #[test] + fn extensionless_path_serves_file_without_index() { + let temp = TempDir::new().unwrap(); + let root = temp.path(); + write_file(root, "foo", "file"); + + assert_eq!(selected_path(root, "/foo"), Some(root.join("foo"))); + } + + #[test] + fn missing_path_returns_no_asset() { + let temp = TempDir::new().unwrap(); + + assert!(selected_path(temp.path(), "/missing").is_none()); + } +} diff --git a/src/proposal.rs b/src/proposal.rs new file mode 100644 index 0000000..787f3ba --- /dev/null +++ b/src/proposal.rs @@ -0,0 +1,1384 @@ +/* + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. + */ + +//! Proposal path classification and targeted render policy. + +use std::{ + collections::{BTreeMap, BTreeSet}, + ffi::OsStr, + fmt, + num::NonZeroU32, + path::{Path, PathBuf}, +}; + +use eipw_preamble::Preamble; +use log::warn; +use serde::{ + de::{self, Unexpected, Visitor}, + Deserialize, Deserializer, Serialize, Serializer, +}; +use snafu::{OptionExt, ResultExt, Whatever}; +use walkdir::WalkDir; + +use crate::layout::CONTENT_DIR; + +/// Positive proposal number used by CLI selectors and `[render].only` config. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct ProposalNumber(NonZeroU32); + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum ProposalNumberParseFailure { + Empty, + NonDigit, + Zero, + Overflow, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum EditorialNumberSelector { + Number(ProposalNumber), + InvalidNumberLike(ProposalNumberParseFailure), + PathLike, +} + +impl ProposalNumber { + fn parse_selector(selector: &str) -> Result { + if selector.is_empty() { + return Err(ProposalNumberParseFailure::Empty); + } + + if !selector.bytes().all(|byte| byte.is_ascii_digit()) { + return Err(ProposalNumberParseFailure::NonDigit); + } + + let number = selector + .parse::() + .map_err(|_| ProposalNumberParseFailure::Overflow)?; + + Self::from_u32(number).map_err(|_| ProposalNumberParseFailure::Zero) + } + + pub(crate) fn parse_cli_selector(selector: &str) -> Result { + Self::parse_selector(selector).map_err(|_| { + format!( + "`{selector}` is not a valid --only selector; expected a positive proposal number" + ) + }) + } + + pub(crate) fn from_u32(number: u32) -> Result { + NonZeroU32::new(number).map(Self).ok_or(()) + } + + pub(crate) fn get(self) -> u32 { + self.0.get() + } +} + +pub(crate) fn classify_editorial_number_selector(selector: &str) -> EditorialNumberSelector { + match ProposalNumber::parse_selector(selector) { + Ok(number) => EditorialNumberSelector::Number(number), + Err(failure) if is_number_like_selector(selector) => { + EditorialNumberSelector::InvalidNumberLike(failure) + } + Err(_) => EditorialNumberSelector::PathLike, + } +} + +fn is_number_like_selector(selector: &str) -> bool { + !selector.is_empty() + && selector + .bytes() + .all(|byte| byte.is_ascii_digit() || matches!(byte, b'+' | b'-' | b',')) +} + +impl fmt::Display for ProposalNumber { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(formatter, "{}", self.get()) + } +} + +impl Serialize for ProposalNumber { + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + serializer.serialize_u32(self.get()) + } +} + +impl<'de> Deserialize<'de> for ProposalNumber { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + struct ProposalNumberVisitor; + + impl Visitor<'_> for ProposalNumberVisitor { + type Value = ProposalNumber; + + fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("a positive proposal number") + } + + fn visit_i64(self, value: i64) -> Result + where + E: de::Error, + { + let value = u32::try_from(value).map_err(|_| { + E::invalid_value(Unexpected::Signed(value), &"a positive u32 proposal number") + })?; + ProposalNumber::from_u32(value).map_err(|_| { + E::invalid_value( + Unexpected::Unsigned(u64::from(value)), + &"a non-zero proposal number", + ) + }) + } + + fn visit_u64(self, value: u64) -> Result + where + E: de::Error, + { + let value = u32::try_from(value).map_err(|_| { + E::invalid_value( + Unexpected::Unsigned(value), + &"a positive u32 proposal number", + ) + })?; + ProposalNumber::from_u32(value).map_err(|_| { + E::invalid_value( + Unexpected::Unsigned(u64::from(value)), + &"a non-zero proposal number", + ) + }) + } + + fn visit_str(self, value: &str) -> Result + where + E: de::Error, + { + Err(E::invalid_type(Unexpected::Str(value), &self)) + } + } + + deserializer.deserialize_any(ProposalNumberVisitor) + } +} + +#[derive(Debug)] +pub(crate) enum ProposalReference<'a> { + Internal(String), + External(&'a str), +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum ProposalAssetKind { + Static, + Markdown, +} + +impl ProposalAssetKind { + pub(crate) fn from_path(path: &Path) -> Self { + if path.extension().and_then(OsStr::to_str) == Some("md") { + Self::Markdown + } else { + Self::Static + } + } +} + +#[derive(Debug, Clone)] +#[allow(dead_code)] +struct ProposalAssetInventoryEntry { + proposal_number: ProposalNumber, + site: ProposalPublicSite, + asset_relative_path: PathBuf, + kind: ProposalAssetKind, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum ProposalPublicSite { + Eips, + Ercs, +} + +impl ProposalPublicSite { + fn proposal_url(self, proposal_number: ProposalNumber) -> String { + match self { + Self::Eips => format!( + "https://eips.ethereum.org/EIPS/eip-{}", + proposal_number.get() + ), + Self::Ercs => format!( + "https://ercs.ethereum.org/ERCS/erc-{}", + proposal_number.get() + ), + } + } + + fn asset_base_url(self) -> &'static str { + match self { + Self::Eips => "https://eips.ethereum.org", + Self::Ercs => "https://ercs.ethereum.org", + } + } +} + +#[derive(Debug, Clone)] +pub(crate) struct OnlyRenderPlan { + selected_numbers: BTreeSet, + asset_inventory: BTreeMap, + canonical_proposal_numbers: BTreeMap, + markdown_paths_by_number: BTreeMap>, + public_sites_by_number: BTreeMap, + public_urls_by_number: BTreeMap, +} + +impl OnlyRenderPlan { + pub(crate) fn build( + content_root: &Path, + selected_numbers: BTreeSet, + ) -> Result { + let mut plan = Self { + selected_numbers, + asset_inventory: BTreeMap::new(), + canonical_proposal_numbers: BTreeMap::new(), + markdown_paths_by_number: BTreeMap::new(), + public_sites_by_number: BTreeMap::new(), + public_urls_by_number: BTreeMap::new(), + }; + + let entries = std::fs::read_dir(content_root).with_whatever_context(|_| { + format!( + "unable to read materialized content directory `{}`", + content_root.to_string_lossy() + ) + })?; + + for entry in entries { + let entry = entry.with_whatever_context(|_| { + format!( + "unable to read materialized content directory entry in `{}`", + content_root.to_string_lossy() + ) + })?; + let entry_path = entry.path(); + let file_type = entry.file_type().with_whatever_context(|_| { + format!( + "unable to inspect materialized content path `{}`", + entry_path.to_string_lossy() + ) + })?; + + if file_type.is_file() { + let Some(number) = flat_proposal_number(&entry_path) else { + continue; + }; + plan.record_markdown_path(content_root, number, &entry_path)?; + } else if file_type.is_dir() { + let Some(number) = path_component_proposal_number(entry_path.file_name()) else { + continue; + }; + let index_path = entry_path.join("index.md"); + match std::fs::read_to_string(&index_path) { + Ok(contents) => { + plan.record_markdown_contents(content_root, number, &index_path, &contents)? + } + Err(error) + if matches!( + error.kind(), + std::io::ErrorKind::NotFound | std::io::ErrorKind::NotADirectory + ) => {} + Err(error) => { + snafu::whatever!( + "unable to read proposal markdown `{}`: {error}", + index_path.to_string_lossy() + ); + } + } + } + } + + plan.inventory_assets(content_root)?; + + for selected_number in &plan.selected_numbers { + if !plan.markdown_paths_by_number.contains_key(selected_number) { + snafu::whatever!("selected proposal `{selected_number}` was not found"); + } + } + + Ok(plan) + } + + fn record_markdown_path( + &mut self, + content_root: &Path, + proposal_number: ProposalNumber, + markdown_path: &Path, + ) -> Result<(), Whatever> { + let contents = std::fs::read_to_string(markdown_path).with_whatever_context(|_| { + format!( + "unable to read proposal markdown `{}`", + markdown_path.to_string_lossy() + ) + })?; + self.record_markdown_contents(content_root, proposal_number, markdown_path, &contents) + } + + fn record_markdown_contents( + &mut self, + content_root: &Path, + proposal_number: ProposalNumber, + markdown_path: &Path, + contents: &str, + ) -> Result<(), Whatever> { + let relative_path = markdown_path + .strip_prefix(content_root) + .with_whatever_context(|_| { + format!( + "proposal markdown `{}` is outside content root `{}`", + markdown_path.to_string_lossy(), + content_root.to_string_lossy() + ) + })? + .to_path_buf(); + let canonical_path = std::fs::canonicalize(markdown_path).with_whatever_context(|_| { + format!( + "unable to canonicalize proposal markdown `{}`", + markdown_path.to_string_lossy() + ) + })?; + let site = public_site_for_markdown(markdown_path, contents)?; + let public_url = site.proposal_url(proposal_number); + + match self.public_urls_by_number.get(&proposal_number) { + Some(existing_url) if existing_url != &public_url => { + snafu::whatever!( + "proposal `{proposal_number}` has conflicting public URLs `{existing_url}` and `{public_url}`" + ); + } + Some(_) => {} + None => { + self.public_sites_by_number.insert(proposal_number, site); + self.public_urls_by_number + .insert(proposal_number, public_url); + } + } + + self.canonical_proposal_numbers + .insert(canonical_path, proposal_number); + self.markdown_paths_by_number + .entry(proposal_number) + .or_default() + .insert(relative_path); + + Ok(()) + } + + fn inventory_assets(&mut self, content_root: &Path) -> Result<(), Whatever> { + let canon_root = std::fs::canonicalize(content_root).with_whatever_context(|_| { + format!( + "unable to canonicalize content root `{}` for proposal asset inventory", + content_root.to_string_lossy() + ) + })?; + + let proposal_assets = self + .markdown_paths_by_number + .iter() + .flat_map(|(proposal_number, markdown_paths)| { + markdown_paths.iter().map(move |markdown_path| { + ( + *proposal_number, + markdown_path.clone(), + asset_dir_for_markdown_path(markdown_path), + ) + }) + }) + .collect::>(); + + for (proposal_number, markdown_path, asset_dir) in proposal_assets { + let Some(site) = self.public_sites_by_number.get(&proposal_number).copied() else { + continue; + }; + + let absolute_asset_dir = content_root.join(&asset_dir); + for entry in WalkDir::new(&absolute_asset_dir) + .follow_links(true) + .into_iter() + { + let entry = match entry { + Ok(entry) => entry, + Err(error) if missing_asset_dir(&error) => continue, + Err(error) => { + return Err(error).with_whatever_context(|_| { + format!( + "couldn't read proposal asset inventory entry in `{}`", + absolute_asset_dir.to_string_lossy() + ) + }); + } + }; + + if !entry.file_type().is_file() { + continue; + } + + let candidate = match std::fs::canonicalize(entry.path()) { + Ok(candidate) => candidate, + Err(error) => { + warn!( + "unable to canonicalize `{}`: {error}", + entry.path().to_string_lossy() + ); + continue; + } + }; + + if !candidate.starts_with(&canon_root) { + warn!( + "asset `{}` not in root, skipping", + entry.path().to_string_lossy() + ); + continue; + } + + let content_relative_path = entry + .path() + .strip_prefix(content_root) + .with_whatever_context(|_| { + format!( + "proposal asset `{}` for `{}` is outside content root `{}`", + entry.path().to_string_lossy(), + markdown_path.to_string_lossy(), + content_root.to_string_lossy() + ) + })?; + let asset_relative_path = entry + .path() + .strip_prefix(&absolute_asset_dir) + .with_whatever_context(|_| { + format!( + "proposal asset `{}` is outside asset directory `{}`", + entry.path().to_string_lossy(), + absolute_asset_dir.to_string_lossy() + ) + })?; + let entry = ProposalAssetInventoryEntry { + proposal_number, + site, + asset_relative_path: asset_relative_path.to_path_buf(), + kind: ProposalAssetKind::from_path(asset_relative_path), + }; + + self.asset_inventory + .insert(content_relative_path.to_path_buf(), entry); + } + } + + Ok(()) + } + + pub(crate) fn has_proposal_asset(&self, content_relative_asset_path: &Path) -> bool { + self.asset_inventory + .contains_key(content_relative_asset_path) + } + + pub(crate) fn public_url_for_omitted_proposal_asset( + &self, + content_relative_asset_path: &Path, + ) -> Option { + let entry = self.asset_inventory.get(content_relative_asset_path)?; + if self.selected_numbers.contains(&entry.proposal_number) { + return None; + } + + Some(public_asset_url(entry)) + } + + pub(crate) fn external_url_for_canonical_target( + &self, + canonical_target: &Path, + ) -> Option<&str> { + let proposal_number = self.canonical_proposal_numbers.get(canonical_target)?; + if self.selected_numbers.contains(proposal_number) { + return None; + } + + self.public_urls_by_number + .get(proposal_number) + .map(String::as_str) + } + + pub(crate) fn external_url_for_content_target( + &self, + content_relative_path: &Path, + ) -> Option<&str> { + let proposal_number = proposal_number_from_content_markdown_path(content_relative_path)?; + if self.selected_numbers.contains(&proposal_number) { + return None; + } + + self.public_urls_by_number + .get(&proposal_number) + .map(String::as_str) + } + + pub(crate) fn reference_for_required_number( + &self, + proposal_number: ProposalNumber, + ) -> Result, Whatever> { + if self.selected_numbers.contains(&proposal_number) { + let markdown_path = self + .markdown_paths_by_number + .get(&proposal_number) + .and_then(|paths| paths.iter().next()) + .with_whatever_context(|| { + format!("required selected proposal `{proposal_number}` was not found") + })?; + return Ok(ProposalReference::Internal(format!( + "@/{}", + markdown_path.to_string_lossy() + ))); + } + + let public_url = self + .public_urls_by_number + .get(&proposal_number) + .with_whatever_context(|| { + format!("required proposal `{proposal_number}` was not found") + })?; + Ok(ProposalReference::External(public_url)) + } + + pub(crate) fn should_preprocess_markdown(&self, content_relative_path: &Path) -> bool { + match proposal_number_from_content_markdown_path(content_relative_path) { + Some(proposal_number) => { + self.selected_numbers.contains(&proposal_number) + && self + .markdown_paths_by_number + .get(&proposal_number) + .map(|paths| paths.contains(content_relative_path)) + .unwrap_or(false) + } + None => true, + } + } + + pub(crate) fn should_process_proposal_dir(&self, content_relative_path: &Path) -> bool { + path_component_proposal_number(content_relative_path.file_name()) + .map(|proposal_number| self.selected_numbers.contains(&proposal_number)) + .unwrap_or(true) + } + + pub(crate) fn should_sync_dirty_path(&self, repo_relative_path: &Path) -> bool { + let Ok(content_relative_path) = repo_relative_path.strip_prefix(CONTENT_DIR) else { + return true; + }; + + self.should_sync_content_dirty_path(content_relative_path) + } + + pub(crate) fn is_selected_proposal_markdown_path(&self, repo_relative_path: &Path) -> bool { + let Ok(content_relative_path) = repo_relative_path.strip_prefix(CONTENT_DIR) else { + return false; + }; + + self.is_selected_content_proposal_markdown_path(content_relative_path) + } + + fn is_selected_content_proposal_markdown_path(&self, content_relative_path: &Path) -> bool { + let Some(proposal_number) = + proposal_number_from_content_markdown_path(content_relative_path) + else { + return false; + }; + + self.selected_numbers.contains(&proposal_number) + && self + .markdown_paths_by_number + .get(&proposal_number) + .map(|paths| paths.contains(content_relative_path)) + .unwrap_or(false) + } + + fn should_sync_content_dirty_path(&self, content_relative_path: &Path) -> bool { + if proposal_number_from_content_markdown_path(content_relative_path).is_some() { + return self.is_selected_content_proposal_markdown_path(content_relative_path); + } + + let mut components = content_relative_path.components(); + let Some(first) = components.next() else { + return true; + }; + + path_component_proposal_number(Some(first.as_os_str())) + .map(|proposal_number| self.selected_numbers.contains(&proposal_number)) + .unwrap_or(true) + } + + pub(crate) fn prune_content(&self, content_root: &Path) -> Result<(), Whatever> { + let entries = std::fs::read_dir(content_root).with_whatever_context(|_| { + format!( + "unable to read materialized content directory `{}` for pruning", + content_root.to_string_lossy() + ) + })?; + + for entry in entries { + let entry = entry.with_whatever_context(|_| { + format!( + "unable to read materialized content directory entry in `{}` for pruning", + content_root.to_string_lossy() + ) + })?; + let entry_path = entry.path(); + let file_type = entry.file_type().with_whatever_context(|_| { + format!( + "unable to inspect materialized content path `{}` for pruning", + entry_path.to_string_lossy() + ) + })?; + + if file_type.is_file() { + let Some(number) = flat_proposal_number(&entry_path) else { + continue; + }; + if !self.selected_numbers.contains(&number) { + remove_file_if_present(&entry_path)?; + } + } else if file_type.is_dir() { + let Some(number) = path_component_proposal_number(entry_path.file_name()) else { + continue; + }; + if !self.selected_numbers.contains(&number) { + remove_dir_if_present(&entry_path)?; + } + } + } + + Ok(()) + } +} + +fn missing_asset_dir(error: &walkdir::Error) -> bool { + error.depth() == 0 + && error.io_error().is_some_and(|io_error| { + matches!( + io_error.kind(), + std::io::ErrorKind::NotFound | std::io::ErrorKind::NotADirectory + ) + }) +} + +fn asset_dir_for_markdown_path(markdown_path: &Path) -> PathBuf { + if markdown_path.file_name() == Some(OsStr::new("index.md")) { + markdown_path + .parent() + .map(|proposal_dir| proposal_dir.join("assets")) + .expect("index path has proposal parent") + } else { + markdown_path.with_extension("").join("assets") + } +} + +fn remove_file_if_present(path: &Path) -> Result<(), Whatever> { + match std::fs::remove_file(path) { + Ok(()) => Ok(()), + Err(error) + if matches!( + error.kind(), + std::io::ErrorKind::NotFound | std::io::ErrorKind::NotADirectory + ) => + { + Ok(()) + } + Err(error) => { + snafu::whatever!( + "unable to prune unselected proposal file `{}`: {error}", + path.to_string_lossy() + ); + } + } +} + +fn remove_dir_if_present(path: &Path) -> Result<(), Whatever> { + match std::fs::remove_dir_all(path) { + Ok(()) => Ok(()), + Err(error) + if matches!( + error.kind(), + std::io::ErrorKind::NotFound | std::io::ErrorKind::NotADirectory + ) => + { + Ok(()) + } + Err(error) => { + snafu::whatever!( + "unable to prune unselected proposal directory `{}`: {error}", + path.to_string_lossy() + ); + } + } +} + +fn public_site_for_markdown( + markdown_path: &Path, + contents: &str, +) -> Result { + let path_lossy = markdown_path.to_string_lossy(); + let (preamble, _) = Preamble::split(contents) + .with_whatever_context(|_| format!("couldn't split preamble for `{path_lossy}`"))?; + let preamble = Preamble::parse(Some(&path_lossy), preamble) + .ok() + .with_whatever_context(|| format!("couldn't parse preamble in `{path_lossy}`"))?; + let is_erc = preamble + .fields() + .any(|field| field.name() == "category" && field.value().trim() == "ERC"); + + if is_erc { + Ok(ProposalPublicSite::Ercs) + } else { + Ok(ProposalPublicSite::Eips) + } +} + +#[allow(dead_code)] +fn public_asset_url(entry: &ProposalAssetInventoryEntry) -> String { + let mut url = format!( + "{}/{}/assets", + entry.site.asset_base_url(), + entry.proposal_number.get() + ); + for component in entry.asset_relative_path.components() { + let std::path::Component::Normal(component) = component else { + continue; + }; + url.push('/'); + url.push_str(&percent_encode_url_segment(&component.to_string_lossy())); + } + + if entry.kind == ProposalAssetKind::Markdown { + url.truncate(url.len() - ".md".len()); + url.push('/'); + } + + url +} + +#[allow(dead_code)] +fn percent_encode_url_segment(segment: &str) -> String { + let mut encoded = String::with_capacity(segment.len()); + for byte in segment.bytes() { + if byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'.' | b'_' | b'~') { + encoded.push(char::from(byte)); + } else { + encoded.push_str(&format!("%{byte:02X}")); + } + } + encoded +} + +pub(crate) fn flat_proposal_number(path: &Path) -> Option { + if path.extension().and_then(OsStr::to_str) != Some("md") { + return None; + } + + path_component_proposal_number(path.file_stem()) +} + +pub(crate) fn path_component_proposal_number(component: Option<&OsStr>) -> Option { + let name = component?.to_str()?; + if name.is_empty() || !name.bytes().all(|byte| byte.is_ascii_digit()) { + return None; + } + + name.parse::() + .ok() + .and_then(|number| NonZeroU32::new(number).map(ProposalNumber)) +} + +pub(crate) fn proposal_number_from_content_markdown_path( + content_relative_path: &Path, +) -> Option { + let mut components = content_relative_path.components(); + let first = components.next()?; + let first_path = Path::new(first.as_os_str()); + + match components.next() { + None => flat_proposal_number(first_path), + Some(component) + if component.as_os_str() == OsStr::new("index.md") && components.next().is_none() => + { + path_component_proposal_number(Some(first.as_os_str())) + } + Some(_) => None, + } +} + +pub(crate) fn is_proposal_path(path: &Path) -> bool { + let Ok(content_relative_path) = path.strip_prefix(CONTENT_DIR) else { + return false; + }; + + proposal_number_from_content_markdown_path(content_relative_path).is_some() +} + +pub(crate) fn resolve_proposal_number_markdown_path( + active_repo_root: &Path, + proposal_number: ProposalNumber, +) -> Result { + let content_root = active_repo_root.join(CONTENT_DIR); + let mut matches = BTreeSet::new(); + let entries = std::fs::read_dir(&content_root).with_whatever_context(|_| { + format!( + "unable to read active repository content directory `{}`", + content_root.to_string_lossy() + ) + })?; + + for entry in entries { + let entry = entry.with_whatever_context(|_| { + format!( + "unable to read active repository content directory entry in `{}`", + content_root.to_string_lossy() + ) + })?; + let entry_path = entry.path(); + let file_type = entry.file_type().with_whatever_context(|_| { + format!( + "unable to inspect active repository content path `{}`", + entry_path.to_string_lossy() + ) + })?; + + if file_type.is_file() { + if flat_proposal_number(&entry_path) == Some(proposal_number) { + matches.insert(PathBuf::from(CONTENT_DIR).join(entry.file_name())); + } + } else if file_type.is_dir() + && path_component_proposal_number(Some(entry.file_name().as_os_str())) + == Some(proposal_number) + { + let index_path = entry_path.join("index.md"); + match std::fs::metadata(&index_path) { + Ok(metadata) if metadata.is_file() => { + matches.insert( + PathBuf::from(CONTENT_DIR) + .join(entry.file_name()) + .join("index.md"), + ); + } + Ok(_) => {} + Err(error) + if matches!( + error.kind(), + std::io::ErrorKind::NotFound | std::io::ErrorKind::NotADirectory + ) => {} + Err(error) => { + snafu::whatever!( + "unable to inspect proposal markdown `{}`: {error}", + index_path.to_string_lossy() + ); + } + } + } + } + + match matches.len() { + 0 => snafu::whatever!( + "proposal `{proposal_number}` was not found in active repository content" + ), + 1 => Ok(matches.into_iter().next().expect("one proposal path")), + _ => { + let paths = matches + .iter() + .map(|path| format!("`{}`", path.to_string_lossy())) + .collect::>() + .join(", "); + snafu::whatever!( + "proposal `{proposal_number}` has more than one markdown path in active repository content: {paths}" + ); + } + } +} + +#[cfg(test)] +mod tests { + use std::path::Path; + + use tempfile::TempDir; + + use super::{ + classify_editorial_number_selector, is_proposal_path, + proposal_number_from_content_markdown_path, resolve_proposal_number_markdown_path, + EditorialNumberSelector, OnlyRenderPlan, ProposalNumber, ProposalNumberParseFailure, + }; + + fn number(value: u32) -> ProposalNumber { + ProposalNumber::from_u32(value).unwrap() + } + + fn write_file(root: &Path, relative: &str, contents: &str) { + let path = root.join(relative); + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent).unwrap(); + } + std::fs::write(path, contents).unwrap(); + } + + fn proposal_markdown(number: u32, category: Option<&str>) -> String { + let category = category + .map(|category| format!("category: {category}\n")) + .unwrap_or_default(); + format!("---\neip: {number}\ntitle: Test\n{category}---\nBody\n") + } + + #[test] + fn proposal_numbers_parse_cli_selectors_strictly() { + assert_eq!( + ProposalNumber::parse_cli_selector("555").unwrap(), + number(555) + ); + assert_eq!( + ProposalNumber::parse_cli_selector("00555").unwrap(), + number(555) + ); + + for selector in ["+555", "0", "-555", "abc", "555,678", "content/00555.md"] { + let error = ProposalNumber::parse_cli_selector(selector).unwrap_err(); + assert_eq!( + error, + format!( + "`{selector}` is not a valid --only selector; expected a positive proposal number" + ) + ); + } + + let error = ProposalNumber::parse_cli_selector("4294967296").unwrap_err(); + assert_eq!( + error, + "`4294967296` is not a valid --only selector; expected a positive proposal number" + ); + } + + #[test] + fn editorial_number_selector_classifier_splits_numbers_invalid_numbers_and_paths() { + assert_eq!( + classify_editorial_number_selector("000555"), + EditorialNumberSelector::Number(number(555)) + ); + + for (selector, expected_failure) in [ + ("0", ProposalNumberParseFailure::Zero), + ("+555", ProposalNumberParseFailure::NonDigit), + ("-555", ProposalNumberParseFailure::NonDigit), + ("555,678", ProposalNumberParseFailure::NonDigit), + ("4294967296", ProposalNumberParseFailure::Overflow), + ] { + assert_eq!( + classify_editorial_number_selector(selector), + EditorialNumberSelector::InvalidNumberLike(expected_failure) + ); + } + + for selector in ["foo", "draft.md", "4a", "draft-4.md", "content/00555.md"] { + assert_eq!( + classify_editorial_number_selector(selector), + EditorialNumberSelector::PathLike + ); + } + } + + #[test] + fn proposal_path_matching_normalizes_numeric_paths() { + assert_eq!( + proposal_number_from_content_markdown_path(Path::new("555.md")), + Some(number(555)) + ); + assert_eq!( + proposal_number_from_content_markdown_path(Path::new("00555.md")), + Some(number(555)) + ); + assert_eq!( + proposal_number_from_content_markdown_path(Path::new("000555/index.md")), + Some(number(555)) + ); + assert!(is_proposal_path(Path::new("content/000555/index.md"))); + assert!(!is_proposal_path(Path::new( + "content/000555/assets/readme.md" + ))); + } + + #[test] + fn proposal_number_resolver_returns_exact_flat_markdown_path() { + for (selector, existing_path) in [ + ("4", "content/4.md"), + ("004", "content/0004.md"), + ("0004", "content/004.md"), + ] { + let temp = TempDir::new().unwrap(); + write_file(temp.path(), existing_path, ""); + + assert_eq!( + resolve_proposal_number_markdown_path( + temp.path(), + ProposalNumber::parse_cli_selector(selector).unwrap(), + ) + .unwrap(), + Path::new(existing_path) + ); + } + } + + #[test] + fn proposal_number_resolver_returns_exact_directory_index_path() { + for (selector, existing_path) in [ + ("4", "content/4/index.md"), + ("004", "content/0004/index.md"), + ("0004", "content/004/index.md"), + ] { + let temp = TempDir::new().unwrap(); + write_file(temp.path(), existing_path, ""); + + assert_eq!( + resolve_proposal_number_markdown_path( + temp.path(), + ProposalNumber::parse_cli_selector(selector).unwrap(), + ) + .unwrap(), + Path::new(existing_path) + ); + } + } + + #[test] + fn proposal_number_resolver_reports_missing_and_ignores_assets_only_dirs() { + let missing = TempDir::new().unwrap(); + write_file(missing.path(), "content/0005.md", ""); + let error = resolve_proposal_number_markdown_path(missing.path(), number(4)) + .unwrap_err() + .to_string(); + assert!(error.contains("proposal `4` was not found in active repository content")); + + let assets_only = TempDir::new().unwrap(); + write_file(assets_only.path(), "content/0004/assets/foo.png", ""); + let error = resolve_proposal_number_markdown_path(assets_only.path(), number(4)) + .unwrap_err() + .to_string(); + assert!(error.contains("proposal `4` was not found in active repository content")); + } + + #[test] + fn proposal_number_resolver_reports_ambiguous_markdown_paths() { + for paths in [ + &["content/4.md", "content/0004/index.md"][..], + &["content/4.md", "content/0004.md"][..], + &["content/4/index.md", "content/0004/index.md"][..], + ] { + let temp = TempDir::new().unwrap(); + for path in paths { + write_file(temp.path(), path, ""); + } + + let error = resolve_proposal_number_markdown_path(temp.path(), number(4)) + .unwrap_err() + .to_string(); + + assert!(error.contains( + "proposal `4` has more than one markdown path in active repository content" + )); + } + } + + #[test] + fn proposal_number_resolver_searches_only_active_repo_content() { + let temp = TempDir::new().unwrap(); + let active_repo = temp.path().join("active"); + let sibling_repo = temp.path().join("sibling"); + write_file(&active_repo, "content/0005.md", ""); + write_file(&sibling_repo, "content/0004.md", ""); + + let error = resolve_proposal_number_markdown_path(&active_repo, number(4)) + .unwrap_err() + .to_string(); + + assert!(error.contains("proposal `4` was not found in active repository content")); + } + + #[test] + fn only_render_plan_requires_selected_markdown_not_assets_only() { + let temp = TempDir::new().unwrap(); + let content = temp.path(); + write_file(content, "00555/assets/foo.png", ""); + + let error = OnlyRenderPlan::build(content, [number(555)].into_iter().collect()) + .unwrap_err() + .to_string(); + + assert!(error.contains("selected proposal `555` was not found")); + } + + #[test] + fn only_render_plan_reports_missing_selected_proposal() { + let temp = TempDir::new().unwrap(); + let content = temp.path(); + write_file(content, "00555.md", &proposal_markdown(555, None)); + + let error = OnlyRenderPlan::build(content, [number(678)].into_iter().collect()) + .unwrap_err() + .to_string(); + + assert!(error.contains("selected proposal `678` was not found")); + } + + #[test] + fn only_render_plan_records_exact_public_urls_by_category() { + let temp = TempDir::new().unwrap(); + let content = temp.path(); + write_file(content, "00555.md", &proposal_markdown(555, None)); + write_file(content, "00678.md", &proposal_markdown(678, Some("ERC"))); + write_file( + content, + "00777.md", + &proposal_markdown(777, Some("Standards Track")), + ); + + let plan = OnlyRenderPlan::build(content, [number(555)].into_iter().collect()).unwrap(); + + assert_eq!( + plan.public_urls_by_number.get(&number(555)).unwrap(), + "https://eips.ethereum.org/EIPS/eip-555" + ); + assert_eq!( + plan.public_urls_by_number.get(&number(678)).unwrap(), + "https://ercs.ethereum.org/ERCS/erc-678" + ); + assert_eq!( + plan.public_urls_by_number.get(&number(777)).unwrap(), + "https://eips.ethereum.org/EIPS/eip-777" + ); + } + + #[test] + fn only_render_plan_inventories_directory_proposal_assets() { + let temp = TempDir::new().unwrap(); + let content = temp.path(); + write_file(content, "00555.md", &proposal_markdown(555, None)); + write_file( + content, + "00678/index.md", + &proposal_markdown(678, Some("ERC")), + ); + write_file(content, "00678/assets/foo.pdf", ""); + write_file(content, "00678/assets/guide.md", ""); + + let plan = OnlyRenderPlan::build(content, [number(555)].into_iter().collect()).unwrap(); + + assert!(plan.has_proposal_asset(Path::new("00678/assets/foo.pdf"))); + assert!(plan.has_proposal_asset(Path::new("00678/assets/guide.md"))); + assert_eq!( + plan.public_url_for_omitted_proposal_asset(Path::new("00678/assets/foo.pdf")) + .unwrap(), + "https://ercs.ethereum.org/678/assets/foo.pdf" + ); + assert_eq!( + plan.public_url_for_omitted_proposal_asset(Path::new("00678/assets/guide.md")) + .unwrap(), + "https://ercs.ethereum.org/678/assets/guide/" + ); + } + + #[test] + fn only_render_plan_inventories_flat_proposal_assets_when_directory_exists() { + let temp = TempDir::new().unwrap(); + let content = temp.path(); + write_file(content, "00555.md", &proposal_markdown(555, None)); + write_file(content, "00678.md", &proposal_markdown(678, None)); + write_file(content, "00678/assets/README.md", ""); + write_file(content, "00678/assets/index.md", ""); + write_file( + content, + "00678/assets/Contract Interactions diagram.svg", + "", + ); + + let plan = OnlyRenderPlan::build(content, [number(555)].into_iter().collect()).unwrap(); + + assert_eq!( + plan.public_url_for_omitted_proposal_asset(Path::new("00678/assets/README.md")) + .unwrap(), + "https://eips.ethereum.org/678/assets/README/" + ); + assert_eq!( + plan.public_url_for_omitted_proposal_asset(Path::new("00678/assets/index.md")) + .unwrap(), + "https://eips.ethereum.org/678/assets/index/" + ); + assert_eq!( + plan.public_url_for_omitted_proposal_asset(Path::new( + "00678/assets/Contract Interactions diagram.svg" + )) + .unwrap(), + "https://eips.ethereum.org/678/assets/Contract%20Interactions%20diagram.svg" + ); + } + + #[test] + fn only_render_plan_records_flat_proposals_without_assets_as_empty() { + let temp = TempDir::new().unwrap(); + let content = temp.path(); + write_file(content, "00555.md", &proposal_markdown(555, None)); + write_file(content, "00678.md", &proposal_markdown(678, None)); + + let plan = OnlyRenderPlan::build(content, [number(555)].into_iter().collect()).unwrap(); + + assert!(!plan.has_proposal_asset(Path::new("00678/assets/foo.pdf"))); + assert!(plan + .public_url_for_omitted_proposal_asset(Path::new("00678/assets/foo.pdf")) + .is_none()); + } + + #[test] + fn only_render_plan_does_not_inventory_assets_only_numeric_dirs() { + let temp = TempDir::new().unwrap(); + let content = temp.path(); + write_file(content, "00555.md", &proposal_markdown(555, None)); + write_file(content, "00678/assets/foo.pdf", ""); + + let plan = OnlyRenderPlan::build(content, [number(555)].into_iter().collect()).unwrap(); + + assert!(!plan.has_proposal_asset(Path::new("00678/assets/foo.pdf"))); + assert!(plan + .public_url_for_omitted_proposal_asset(Path::new("00678/assets/foo.pdf")) + .is_none()); + } + + #[test] + fn only_render_plan_does_not_construct_public_asset_urls_for_selected_targets() { + let temp = TempDir::new().unwrap(); + let content = temp.path(); + write_file(content, "00555.md", &proposal_markdown(555, None)); + write_file(content, "00678.md", &proposal_markdown(678, None)); + write_file(content, "00678/assets/foo.pdf", ""); + + let plan = OnlyRenderPlan::build(content, [number(555), number(678)].into_iter().collect()) + .unwrap(); + + assert!(plan.has_proposal_asset(Path::new("00678/assets/foo.pdf"))); + assert!(plan + .public_url_for_omitted_proposal_asset(Path::new("00678/assets/foo.pdf")) + .is_none()); + } + + #[test] + fn only_render_plan_errors_on_malformed_proposal_before_asset_inventory_policy() { + let temp = TempDir::new().unwrap(); + let content = temp.path(); + write_file(content, "00555.md", &proposal_markdown(555, None)); + write_file(content, "00678.md", "not front matter"); + write_file(content, "00678/assets/foo.pdf", ""); + + let error = OnlyRenderPlan::build(content, [number(555)].into_iter().collect()) + .unwrap_err() + .to_string(); + + assert!(error.contains("couldn't split preamble")); + } + + #[test] + fn only_render_plan_does_not_mask_missing_required_targets() { + let temp = TempDir::new().unwrap(); + let content = temp.path(); + write_file(content, "00555.md", &proposal_markdown(555, None)); + let plan = OnlyRenderPlan::build(content, [number(555)].into_iter().collect()).unwrap(); + + let error = plan + .reference_for_required_number(number(678)) + .unwrap_err() + .to_string(); + + assert!(error.contains("required proposal `678` was not found")); + } + + #[test] + fn only_render_plan_does_not_mask_malformed_target_front_matter() { + let temp = TempDir::new().unwrap(); + let content = temp.path(); + write_file(content, "00555.md", &proposal_markdown(555, None)); + write_file(content, "00678.md", "not front matter"); + + let error = OnlyRenderPlan::build(content, [number(555)].into_iter().collect()) + .unwrap_err() + .to_string(); + + assert!(error.contains("couldn't split preamble")); + } + + #[test] + fn only_render_plan_detects_conflicting_public_urls() { + let temp = TempDir::new().unwrap(); + let content = temp.path(); + write_file(content, "00555.md", &proposal_markdown(555, None)); + write_file( + content, + "00555/index.md", + &proposal_markdown(555, Some("ERC")), + ); + + let error = OnlyRenderPlan::build(content, [number(555)].into_iter().collect()) + .unwrap_err() + .to_string(); + + assert!(error.contains("conflicting public URLs")); + } + + #[test] + fn only_render_plan_prunes_unselected_proposal_content_only() { + let temp = TempDir::new().unwrap(); + let content = temp.path(); + write_file(content, "00555.md", &proposal_markdown(555, None)); + write_file(content, "00555/assets/foo.png", ""); + write_file(content, "00678.md", &proposal_markdown(678, None)); + write_file(content, "00777/index.md", &proposal_markdown(777, None)); + write_file(content, "00777/assets/foo.png", ""); + write_file(content, "_index.md", "+++\ntitle = \"Home\"\n+++\n"); + + let plan = OnlyRenderPlan::build(content, [number(555)].into_iter().collect()).unwrap(); + plan.prune_content(content).unwrap(); + + assert!(content.join("00555.md").is_file()); + assert!(content.join("00555/assets/foo.png").is_file()); + assert!(!content.join("00678.md").exists()); + assert!(!content.join("00777").exists()); + assert!(content.join("_index.md").is_file()); + } + + #[test] + fn only_render_plan_filters_dirty_paths_without_filesystem_state() { + let temp = TempDir::new().unwrap(); + let content = temp.path(); + write_file(content, "00555.md", &proposal_markdown(555, None)); + write_file(content, "00678.md", &proposal_markdown(678, None)); + let plan = OnlyRenderPlan::build(content, [number(555)].into_iter().collect()).unwrap(); + + assert!(plan.should_sync_dirty_path(Path::new("content/00555.md"))); + assert!(plan.should_sync_dirty_path(Path::new("content/00555/assets/diagram.png"))); + assert!(plan.should_sync_dirty_path(Path::new("content/_index.md"))); + assert!(plan.should_sync_dirty_path(Path::new(".build-eips.repo.toml"))); + assert!(!plan.should_sync_dirty_path(Path::new("content/00678.md"))); + assert!(!plan.should_sync_dirty_path(Path::new("content/00678/assets/diagram.png"))); + assert!(!plan.should_sync_dirty_path(Path::new("content/00999.md"))); + + assert!(plan.is_selected_proposal_markdown_path(Path::new("content/00555.md"))); + assert!( + !plan.is_selected_proposal_markdown_path(Path::new("content/00555/assets/diagram.png")) + ); + assert!(!plan.is_selected_proposal_markdown_path(Path::new("content/_index.md"))); + assert!(!plan.is_selected_proposal_markdown_path(Path::new("content/00678.md"))); + } +} diff --git a/src/serve.rs b/src/serve.rs new file mode 100644 index 0000000..9bab36b --- /dev/null +++ b/src/serve.rs @@ -0,0 +1,841 @@ +/* + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. + */ + +//! Dirty active-repo and local-theme serve synchronization. + +use std::{ + collections::BTreeSet, + ffi::OsStr, + path::{Path, PathBuf}, + sync::{ + atomic::{AtomicBool, Ordering}, + mpsc::{self, RecvTimeoutError}, + Arc, + }, + thread::{self, JoinHandle}, + time::Duration, +}; + +use log::{debug, info, warn}; +use notify::{Event, RecursiveMode, Watcher}; +use snafu::{Report, ResultExt, Whatever}; + +use crate::{git, layout::CONTENT_DIR, markdown, proposal::OnlyRenderPlan}; + +#[derive(Debug)] +pub(crate) struct DirtyServeWatcher { + stop: Arc, + thread: JoinHandle<()>, +} + +#[derive(Debug, Clone)] +struct ActiveRepoServeSync { + source_root: PathBuf, + build_repo_path: PathBuf, + only_plan: Option, +} + +#[derive(Debug, Clone)] +pub(crate) struct LocalThemeServeSync { + pub(crate) theme_source_root: PathBuf, + pub(crate) mounted_theme_dir: PathBuf, + pub(crate) theme_index_path: PathBuf, +} + +#[derive(Debug, Clone, Default)] +pub(crate) struct ServeSyncConfig { + active_repo: Option, + local_theme: Option, +} + +impl ServeSyncConfig { + pub(crate) fn has_targets(&self) -> bool { + self.active_repo.is_some() || self.local_theme.is_some() + } +} + +impl DirtyServeWatcher { + pub(crate) fn start(sync_config: ServeSyncConfig) -> Result { + let stop = Arc::new(AtomicBool::new(false)); + let stop_thread = stop.clone(); + let (ready_tx, ready_rx) = mpsc::channel(); + let thread = + thread::spawn(move || dirty_serve_sync_loop(sync_config, stop_thread, ready_tx)); + + match ready_rx + .recv() + .whatever_context("dirty serve watcher exited before initialization")? + { + Ok(()) => Ok(Self { stop, thread }), + Err(message) => { + stop.store(true, Ordering::Relaxed); + let _ = thread.join(); + snafu::whatever!("{message}"); + } + } + } + + pub(crate) fn stop(self) { + self.stop.store(true, Ordering::Relaxed); + let _ = self.thread.join(); + } +} + +fn path_is_watched_source_path(root_path: &Path, path: &Path) -> bool { + let Ok(relative_path) = path.strip_prefix(root_path) else { + return false; + }; + + relative_path + .components() + .next() + .map(|component| component.as_os_str() != OsStr::new(".git")) + .unwrap_or(false) +} + +fn event_has_watched_source_path(root_path: &Path, event: &Event) -> bool { + event + .paths + .iter() + .any(|path| path_is_watched_source_path(root_path, path)) +} + +fn index_lock_path(index_path: &Path) -> Option { + let file_name = index_path.file_name()?.to_string_lossy(); + Some(index_path.with_file_name(format!("{file_name}.lock"))) +} + +fn event_has_theme_index_path(index_path: &Path, event: &Event) -> bool { + let lock_path = index_lock_path(index_path); + event.paths.iter().any(|path| { + path == index_path + || lock_path + .as_ref() + .map(|lock_path| path == lock_path) + .unwrap_or(false) + }) +} + +fn sync_dirty_serve_state( + source_root: &Path, + build_repo_path: &Path, + only_plan: Option<&OnlyRenderPlan>, + previous_dirty_paths: &mut BTreeSet, +) -> Result<(), Whatever> { + let current_dirty_paths = filter_dirty_paths( + git::working_tree_paths(source_root) + .whatever_context("unable to list tracked dirty paths for dirty serve")?, + only_plan, + ); + + let affected_paths = affected_dirty_paths(previous_dirty_paths, ¤t_dirty_paths); + + if affected_paths.is_empty() { + *previous_dirty_paths = current_dirty_paths; + return Ok(()); + } + + for path in selected_deleted_proposal_markdown_paths(source_root, &affected_paths, only_plan) { + warn!( + "selected proposal path `{}` was removed from the source repo; removing it from the targeted serve build input", + path.to_string_lossy() + ); + } + + git::sync_materialized_paths(source_root, build_repo_path, &affected_paths) + .whatever_context("unable to synchronize tracked paths into the materialized repo")?; + markdown::preprocess_paths( + &build_repo_path.join(CONTENT_DIR), + &affected_paths, + only_plan, + ) + .whatever_context("unable to preprocess synchronized markdown during dirty serve")?; + + info!( + "synchronized {} tracked path(s) into the materialized repo for dirty serve", + affected_paths.len() + ); + + *previous_dirty_paths = current_dirty_paths; + Ok(()) +} + +fn filter_dirty_paths( + dirty_paths: impl IntoIterator, + only_plan: Option<&OnlyRenderPlan>, +) -> BTreeSet { + dirty_paths + .into_iter() + .filter(|path| { + only_plan + .map(|plan| plan.should_sync_dirty_path(path)) + .unwrap_or(true) + }) + .collect() +} + +fn affected_dirty_paths( + previous_dirty_paths: &BTreeSet, + current_dirty_paths: &BTreeSet, +) -> BTreeSet { + previous_dirty_paths + .union(current_dirty_paths) + .cloned() + .collect() +} + +fn selected_deleted_proposal_markdown_paths( + source_root: &Path, + affected_paths: &BTreeSet, + only_plan: Option<&OnlyRenderPlan>, +) -> Vec { + let Some(only_plan) = only_plan else { + return Vec::new(); + }; + + affected_paths + .iter() + .filter(|path| only_plan.is_selected_proposal_markdown_path(path)) + .filter( + |path| match std::fs::symlink_metadata(source_root.join(path)) { + Ok(_) => false, + Err(error) + if matches!( + error.kind(), + std::io::ErrorKind::NotFound | std::io::ErrorKind::NotADirectory + ) => + { + true + } + Err(_) => false, + }, + ) + .cloned() + .collect() +} + +fn capture_active_dirty_paths( + source_root: &Path, + only_plan: Option<&OnlyRenderPlan>, +) -> Result, Whatever> { + Ok(filter_dirty_paths( + git::working_tree_paths(source_root) + .whatever_context("unable to list tracked dirty paths for dirty serve")?, + only_plan, + )) +} + +fn sync_theme_serve_state( + theme_source_root: &Path, + mounted_theme_dir: &Path, + previous_dirty_paths: &mut BTreeSet, +) -> Result<(), Whatever> { + let current_dirty_paths: BTreeSet<_> = git::tracked_working_tree_paths(theme_source_root) + .whatever_context("unable to list tracked dirty paths for local theme serve")? + .into_iter() + .collect(); + + let affected_paths: BTreeSet<_> = previous_dirty_paths + .union(¤t_dirty_paths) + .cloned() + .collect(); + + if affected_paths.is_empty() { + *previous_dirty_paths = current_dirty_paths; + return Ok(()); + } + + git::sync_working_tree_paths(theme_source_root, mounted_theme_dir, &affected_paths) + .whatever_context("unable to synchronize tracked local theme paths")?; + + info!( + "synchronized {} tracked path(s) into the mounted local theme for serve", + affected_paths.len() + ); + + *previous_dirty_paths = current_dirty_paths; + Ok(()) +} + +fn watch_theme_index( + watcher: &mut notify::RecommendedWatcher, + theme_index_path: &Path, +) -> Result<(), String> { + let file_result = watcher.watch(theme_index_path, RecursiveMode::NonRecursive); + let Some(parent) = theme_index_path.parent() else { + return file_result.map_err(|file_error| { + format!( + "unable to watch local theme Git index `{}`: {file_error}", + theme_index_path.to_string_lossy() + ) + }); + }; + let parent_result = watcher.watch(parent, RecursiveMode::NonRecursive); + + match (file_result, parent_result) { + (Ok(()), Ok(())) => Ok(()), + (Ok(()), Err(parent_error)) => { + debug!( + "unable to watch local theme Git index parent `{}`: {parent_error}", + parent.to_string_lossy() + ); + Ok(()) + } + (Err(file_error), Ok(())) => { + debug!( + "using local theme Git index parent watch for `{}` after file watch failed: {file_error}", + theme_index_path.to_string_lossy() + ); + Ok(()) + } + (Err(file_error), Err(parent_error)) => Err(format!( + "unable to watch local theme Git index `{}`: {file_error}; fallback watch on `{}` also failed: {parent_error}", + theme_index_path.to_string_lossy(), + parent.to_string_lossy() + )), + } +} + +fn dirty_serve_sync_loop( + sync_config: ServeSyncConfig, + stop: Arc, + ready_tx: mpsc::Sender>, +) { + let (event_tx, event_rx) = mpsc::channel(); + let mut watcher = match notify::recommended_watcher(move |result| { + let _ = event_tx.send(result); + }) { + Ok(watcher) => watcher, + Err(error) => { + let _ = ready_tx.send(Err(format!("unable to start dirty serve watcher: {error}"))); + return; + } + }; + + if let Some(active_repo) = &sync_config.active_repo { + if let Err(error) = watcher.watch(&active_repo.source_root, RecursiveMode::Recursive) { + let _ = ready_tx.send(Err(format!( + "unable to watch `{}` for dirty serve changes: {error}", + active_repo.source_root.to_string_lossy() + ))); + return; + } + } + + if let Some(local_theme) = &sync_config.local_theme { + if let Err(error) = watcher.watch(&local_theme.theme_source_root, RecursiveMode::Recursive) + { + let _ = ready_tx.send(Err(format!( + "unable to watch local theme `{}` for serve changes: {error}", + local_theme.theme_source_root.to_string_lossy() + ))); + return; + } + + if let Err(message) = watch_theme_index(&mut watcher, &local_theme.theme_index_path) { + let _ = ready_tx.send(Err(message)); + return; + } + } + + let mut previous_active_dirty_paths: BTreeSet<_> = match &sync_config.active_repo { + Some(active_repo) => match capture_active_dirty_paths( + &active_repo.source_root, + active_repo.only_plan.as_ref(), + ) { + Ok(paths) => paths, + Err(error) => { + let _ = ready_tx.send(Err(format!( + "unable to capture initial dirty serve state: {}", + Report::from_error(error) + ))); + return; + } + }, + None => BTreeSet::new(), + }; + + let mut previous_theme_dirty_paths: BTreeSet<_> = match &sync_config.local_theme { + Some(local_theme) => { + if let Err(error) = git::materialize_working_tree( + &local_theme.theme_source_root, + &local_theme.mounted_theme_dir, + ) { + let _ = ready_tx.send(Err(format!( + "unable to synchronize initial local theme state after watcher setup: {}", + Report::from_error(error) + ))); + return; + } + + match git::tracked_working_tree_paths(&local_theme.theme_source_root) { + Ok(paths) => paths.into_iter().collect(), + Err(error) => { + let _ = ready_tx.send(Err(format!( + "unable to capture initial local theme dirty state: {}", + Report::from_error(error) + ))); + return; + } + } + } + None => BTreeSet::new(), + }; + + if let Some(active_repo) = &sync_config.active_repo { + info!( + "watching `{}` for dirty serve changes", + active_repo.source_root.to_string_lossy() + ); + } + if let Some(local_theme) = &sync_config.local_theme { + info!( + "watching `{}` for local theme serve changes", + local_theme.theme_source_root.to_string_lossy() + ); + } + let _ = ready_tx.send(Ok(())); + + while !stop.load(Ordering::Relaxed) { + let first_event = match event_rx.recv_timeout(Duration::from_millis(250)) { + Ok(event) => Some(event), + Err(RecvTimeoutError::Timeout) => None, + Err(RecvTimeoutError::Disconnected) => break, + }; + + let Some(first_event) = first_event else { + continue; + }; + + let mut saw_active_event = false; + let mut saw_theme_event = false; + + match first_event { + Ok(event) => { + if let Some(active_repo) = &sync_config.active_repo { + saw_active_event |= + event_has_watched_source_path(&active_repo.source_root, &event); + } + if let Some(local_theme) = &sync_config.local_theme { + saw_theme_event |= + event_has_watched_source_path(&local_theme.theme_source_root, &event) + || event_has_theme_index_path(&local_theme.theme_index_path, &event); + } + } + Err(error) => { + warn!("filesystem watcher error: {error}"); + } + } + + loop { + match event_rx.recv_timeout(Duration::from_millis(75)) { + Ok(Ok(event)) => { + if let Some(active_repo) = &sync_config.active_repo { + saw_active_event |= + event_has_watched_source_path(&active_repo.source_root, &event); + } + if let Some(local_theme) = &sync_config.local_theme { + saw_theme_event |= + event_has_watched_source_path(&local_theme.theme_source_root, &event) + || event_has_theme_index_path( + &local_theme.theme_index_path, + &event, + ); + } + } + Ok(Err(error)) => warn!("filesystem watcher error: {error}"), + Err(RecvTimeoutError::Timeout) => break, + Err(RecvTimeoutError::Disconnected) => return, + } + } + + if saw_active_event { + if let Some(active_repo) = &sync_config.active_repo { + if let Err(error) = sync_dirty_serve_state( + &active_repo.source_root, + &active_repo.build_repo_path, + active_repo.only_plan.as_ref(), + &mut previous_active_dirty_paths, + ) { + warn!( + "unable to synchronize dirty serve changes: {}", + Report::from_error(error) + ); + } + } + } + + if saw_theme_event { + if let Some(local_theme) = &sync_config.local_theme { + if let Err(error) = sync_theme_serve_state( + &local_theme.theme_source_root, + &local_theme.mounted_theme_dir, + &mut previous_theme_dirty_paths, + ) { + warn!( + "unable to synchronize local theme serve changes: {}", + Report::from_error(error) + ); + } + } + } + } +} + +pub(crate) fn serve_sync_config( + source_materialization: git::SourceMaterialization, + source_root: &Path, + repo_path: &Path, + only_plan: Option, + local_theme_sync: Option, +) -> ServeSyncConfig { + ServeSyncConfig { + active_repo: (source_materialization == git::SourceMaterialization::Dirty).then(|| { + ActiveRepoServeSync { + source_root: source_root.to_path_buf(), + build_repo_path: repo_path.to_path_buf(), + only_plan, + } + }), + local_theme: local_theme_sync, + } +} + +#[cfg(test)] +mod tests { + use std::{ + collections::BTreeSet, + path::{Path, PathBuf}, + }; + + use git2::{IndexAddOption, Repository, Signature}; + use notify::{Event, EventKind}; + use tempfile::TempDir; + + use crate::{ + git::{self, SourceMaterialization}, + proposal::{OnlyRenderPlan, ProposalNumber}, + }; + + use super::{ + affected_dirty_paths, event_has_theme_index_path, filter_dirty_paths, + selected_deleted_proposal_markdown_paths, serve_sync_config, sync_dirty_serve_state, + sync_theme_serve_state, LocalThemeServeSync, + }; + + fn fake_theme_sync(root: &Path) -> LocalThemeServeSync { + LocalThemeServeSync { + theme_source_root: root.join("theme"), + mounted_theme_dir: root.join("repo/themes/eips-theme"), + theme_index_path: root.join("theme/.git/index"), + } + } + + fn number(value: u32) -> ProposalNumber { + ProposalNumber::from_u32(value).unwrap() + } + + fn write_file(root: &Path, relative: &str, contents: &str) { + let path = root.join(relative); + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent).unwrap(); + } + std::fs::write(path, contents).unwrap(); + } + + fn commit_all(repo: &Repository, message: &str) { + let mut index = repo.index().unwrap(); + index + .add_all(["*"].iter(), IndexAddOption::DEFAULT, None) + .unwrap(); + index.write().unwrap(); + let tree_oid = index.write_tree().unwrap(); + let tree = repo.find_tree(tree_oid).unwrap(); + let signature = Signature::now("build-eips test", "build-eips@example.test").unwrap(); + let parents = repo + .head() + .ok() + .and_then(|head| head.target()) + .map(|oid| repo.find_commit(oid).unwrap()) + .into_iter() + .collect::>(); + let parent_refs = parents.iter().collect::>(); + + repo.commit( + Some("HEAD"), + &signature, + &signature, + message, + &tree, + &parent_refs, + ) + .unwrap(); + } + + fn init_repo(root: &Path, files: &[(&str, &str)]) -> Repository { + std::fs::create_dir_all(root).unwrap(); + let repo = Repository::init(root).unwrap(); + repo.set_head("refs/heads/master").unwrap(); + for (relative, contents) in files { + write_file(root, relative, contents); + } + commit_all(&repo, "initial"); + repo + } + + fn proposal_markdown(value: u32, extra_preamble: &str, body: &str) -> String { + format!("---\neip: {value}\ntitle: Proposal {value}\n{extra_preamble}---\n{body}\n") + } + + fn only_plan(root: &Path) -> OnlyRenderPlan { + let content = root.join("content"); + write_file(&content, "00555.md", &proposal_markdown(555, "", "Body")); + write_file(&content, "00678.md", &proposal_markdown(678, "", "Body")); + OnlyRenderPlan::build(&content, [number(555)].into_iter().collect()).unwrap() + } + + fn paths(paths: &[&str]) -> BTreeSet { + paths.iter().map(PathBuf::from).collect() + } + + fn rendered_body(path: &Path) -> String { + let contents = std::fs::read_to_string(path).unwrap(); + contents.split_once("\n+++\n").unwrap().1.to_owned() + } + + fn dirty_sync_fixture() -> (TempDir, PathBuf, PathBuf, OnlyRenderPlan) { + let temp = TempDir::new().unwrap(); + let source = temp.path().join("source"); + let build = temp.path().join("build/repo"); + let selected = proposal_markdown(555, "requires: 678\n", "See [EIP-678](/00678.md)."); + let unselected = proposal_markdown(678, "", "Unselected."); + init_repo( + &source, + &[ + ("content/00555.md", selected.as_str()), + ("content/00555/assets/diagram.png", "selected image\n"), + ("content/00678.md", unselected.as_str()), + ("content/00678/assets/diagram.png", "unselected image\n"), + ( + "content/_index.md", + "---\ntitle: Home\n---\nSee [EIP-678](/00678.md).\n", + ), + ], + ); + let plan = + OnlyRenderPlan::build(&source.join("content"), [number(555)].into_iter().collect()) + .unwrap(); + init_repo( + &build, + &[ + ("content/00555.md", selected.as_str()), + ("content/00555/assets/diagram.png", "selected image\n"), + ( + "content/_index.md", + "---\ntitle: Home\n---\nSee [EIP-678](/00678.md).\n", + ), + ], + ); + (temp, source, build, plan) + } + + #[test] + fn local_theme_serve_syncs_tracked_edits_into_mounted_theme() { + let temp = TempDir::new().unwrap(); + let theme_root = temp.path().join("workspace/theme"); + let mounted_theme_dir = temp.path().join("build/repo/themes/eips-theme"); + init_repo( + &theme_root, + &[ + ("config/zola.toml", "title = 'theme'\n"), + ("templates/index.html", "committed local theme\n"), + ], + ); + git::materialize_working_tree(&theme_root, &mounted_theme_dir).unwrap(); + write_file(&theme_root, "templates/index.html", "dirty local theme\n"); + let mut previous_dirty_paths = BTreeSet::new(); + + sync_theme_serve_state(&theme_root, &mounted_theme_dir, &mut previous_dirty_paths).unwrap(); + + assert_eq!( + std::fs::read_to_string(mounted_theme_dir.join("templates/index.html")).unwrap(), + "dirty local theme\n" + ); + } + + #[test] + fn local_theme_index_events_trigger_rescan() { + let index_path = PathBuf::from("/workspace/theme/.git/index"); + let index_event = Event::new(EventKind::Any).add_path(index_path.clone()); + let lock_event = + Event::new(EventKind::Any).add_path(PathBuf::from("/workspace/theme/.git/index.lock")); + let unrelated_event = + Event::new(EventKind::Any).add_path(PathBuf::from("/workspace/theme/.git/config")); + + assert!(event_has_theme_index_path(&index_path, &index_event)); + assert!(event_has_theme_index_path(&index_path, &lock_event)); + assert!(!event_has_theme_index_path(&index_path, &unrelated_event)); + } + + #[test] + fn local_serve_syncs_theme_and_dirty_active_repo() { + let temp = TempDir::new().unwrap(); + let plan = only_plan(temp.path()); + + let sync_config = serve_sync_config( + SourceMaterialization::Dirty, + &temp.path().join("Core"), + &temp.path().join(".local-build/Core/repo"), + Some(plan), + Some(fake_theme_sync(temp.path())), + ); + + assert!(sync_config.active_repo.is_some()); + assert!(sync_config + .active_repo + .as_ref() + .unwrap() + .only_plan + .is_some()); + assert!(sync_config.local_theme.is_some()); + } + + #[test] + fn clean_local_serve_keeps_theme_sync_but_disables_active_repo_dirty_sync() { + let temp = TempDir::new().unwrap(); + + let sync_config = serve_sync_config( + SourceMaterialization::Clean, + &temp.path().join("Core"), + &temp.path().join(".local-build/Core/repo"), + None, + Some(fake_theme_sync(temp.path())), + ); + + assert!(sync_config.active_repo.is_none()); + assert!(sync_config.local_theme.is_some()); + } + + #[test] + fn only_dirty_path_filter_runs_before_union_and_keeps_selected_deletions() { + let temp = TempDir::new().unwrap(); + let plan = only_plan(temp.path()); + let previous_raw = paths(&[ + "content/00555.md", + "content/00678.md", + "content/00678/assets/diagram.png", + ]); + let current_raw = paths(&["content/00555/assets/diagram.png", "content/00999.md"]); + + let previous_filtered = filter_dirty_paths(previous_raw, Some(&plan)); + let current_filtered = filter_dirty_paths(current_raw, Some(&plan)); + let affected = affected_dirty_paths(&previous_filtered, ¤t_filtered); + + assert_eq!( + affected, + paths(&["content/00555.md", "content/00555/assets/diagram.png"]) + ); + } + + #[test] + fn selected_deleted_proposal_markdown_paths_reports_only_selected_markdown_deletions() { + let (_temp, source, _build, plan) = dirty_sync_fixture(); + std::fs::remove_file(source.join("content/00555.md")).unwrap(); + std::fs::remove_file(source.join("content/00555/assets/diagram.png")).unwrap(); + std::fs::remove_file(source.join("content/_index.md")).unwrap(); + + let affected_paths = paths(&[ + "content/00555.md", + "content/00555/assets/diagram.png", + "content/_index.md", + "content/00678.md", + ]); + + assert_eq!( + selected_deleted_proposal_markdown_paths(&source, &affected_paths, Some(&plan)), + vec![PathBuf::from("content/00555.md")] + ); + assert!( + selected_deleted_proposal_markdown_paths(&source, &affected_paths, None).is_empty() + ); + } + + #[test] + fn only_dirty_sync_does_not_reintroduce_unselected_markdown_or_assets() { + let (_temp, source, build, plan) = dirty_sync_fixture(); + write_file( + &source, + "content/00678.md", + &proposal_markdown(678, "", "Dirty unselected."), + ); + write_file( + &source, + "content/00678/assets/diagram.png", + "dirty unselected image\n", + ); + let mut previous_dirty_paths = BTreeSet::new(); + + sync_dirty_serve_state(&source, &build, Some(&plan), &mut previous_dirty_paths).unwrap(); + + assert!(!build.join("content/00678.md").exists()); + assert!(!build.join("content/00678/assets/diagram.png").exists()); + assert!(previous_dirty_paths.is_empty()); + } + + #[test] + fn only_dirty_sync_copies_selected_assets_without_markdown_preprocessing() { + let (_temp, source, build, plan) = dirty_sync_fixture(); + write_file( + &source, + "content/00555/assets/diagram.png", + "dirty selected image\n", + ); + let mut previous_dirty_paths = BTreeSet::new(); + + sync_dirty_serve_state(&source, &build, Some(&plan), &mut previous_dirty_paths).unwrap(); + + assert_eq!( + std::fs::read_to_string(build.join("content/00555/assets/diagram.png")).unwrap(), + "dirty selected image\n" + ); + assert!(previous_dirty_paths.contains(Path::new("content/00555/assets/diagram.png"))); + } + + #[test] + fn only_dirty_sync_preprocesses_selected_and_retained_markdown_with_plan() { + let (_temp, source, build, plan) = dirty_sync_fixture(); + write_file( + &source, + "content/00555.md", + &proposal_markdown(555, "requires: 678\n", "Dirty [EIP-678](/00678.md)."), + ); + write_file( + &source, + "content/_index.md", + "---\ntitle: Home\n---\nDirty [EIP-678](/00678.md).\n", + ); + let mut previous_dirty_paths = BTreeSet::new(); + + sync_dirty_serve_state(&source, &build, Some(&plan), &mut previous_dirty_paths).unwrap(); + + let selected = std::fs::read_to_string(build.join("content/00555.md")).unwrap(); + let index_body = rendered_body(&build.join("content/_index.md")); + assert!(selected.contains("https://eips.ethereum.org/EIPS/eip-678")); + assert!(index_body.contains("https://eips.ethereum.org/EIPS/eip-678")); + } + + #[test] + fn only_dirty_sync_propagates_selected_proposal_deletion() { + let (_temp, source, build, plan) = dirty_sync_fixture(); + std::fs::remove_file(source.join("content/00555.md")).unwrap(); + let mut previous_dirty_paths = BTreeSet::new(); + + sync_dirty_serve_state(&source, &build, Some(&plan), &mut previous_dirty_paths).unwrap(); + + assert!(!build.join("content/00555.md").exists()); + assert!(previous_dirty_paths.contains(Path::new("content/00555.md"))); + } +} diff --git a/src/workspace.rs b/src/workspace.rs new file mode 100644 index 0000000..8475cad --- /dev/null +++ b/src/workspace.rs @@ -0,0 +1,790 @@ +/* + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. + */ + +//! Local workspace setup and diagnostics. + +use std::{ + fmt, + fs::OpenOptions, + io::{ErrorKind, Write}, + path::{Path, PathBuf}, +}; + +use log::info; +use snafu::{Report, ResultExt, Whatever}; +use url::Url; + +use crate::{ + cli::Args, + config::{self, ActiveRepo, LoadedWorkspaceConfig, Manifest}, + context::{load_workspace_command_context, resolve_input_path, root}, + git, +}; + +const PROPOSAL_TEMPLATE_URL: &str = "https://github.com/eips-wg/template.git"; +const WORKSPACE_DOC_FILE: &str = "WORKSPACE.md"; + +#[derive(Debug, Clone, Copy)] +enum DoctorStatus { + Ok, + Warn, + Fail, +} + +#[derive(Debug, Default)] +struct DoctorReport { + warnings: usize, + failures: usize, +} + +struct WorkspaceInitToolingRepositories<'a> { + template: &'a Url, +} + +impl fmt::Display for DoctorStatus { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let label = match self { + Self::Ok => "ok", + Self::Warn => "warn", + Self::Fail => "fail", + }; + + f.write_str(label) + } +} + +impl DoctorReport { + fn record(&mut self, status: DoctorStatus, message: impl AsRef) { + match status { + DoctorStatus::Ok => (), + DoctorStatus::Warn => self.warnings += 1, + DoctorStatus::Fail => self.failures += 1, + } + + println!("[{status}] {}", message.as_ref()); + } +} + +#[cfg(unix)] +fn is_command_candidate(path: &Path) -> bool { + use std::os::unix::fs::PermissionsExt; + + path.metadata() + .map(|metadata| metadata.is_file() && metadata.permissions().mode() & 0o111 != 0) + .unwrap_or(false) +} + +#[cfg(all(not(unix), not(windows)))] +fn is_command_candidate(path: &Path) -> bool { + path.is_file() +} + +#[cfg(windows)] +fn is_command_candidate(path: &Path) -> bool { + path.is_file() +} + +fn command_path(command: &str) -> Option { + let path = std::env::var_os("PATH")?; + + #[cfg(not(windows))] + let candidates = [command.to_owned()]; + + #[cfg(windows)] + { + use std::ffi::OsString; + + let mut candidates = vec![command.to_owned()]; + let command = OsString::from(command); + let path_exts = std::env::var_os("PATHEXT") + .unwrap_or_default() + .to_string_lossy() + .split(';') + .filter(|ext| !ext.is_empty()) + .map(|ext| format!("{}{}", command.to_string_lossy(), ext)) + .collect::>(); + candidates.extend(path_exts); + std::env::split_paths(&path).find_map(|entry| { + candidates + .iter() + .map(|candidate| entry.join(candidate)) + .find(|candidate| is_command_candidate(candidate)) + }) + } + + #[cfg(not(windows))] + { + std::env::split_paths(&path).find_map(|entry| { + candidates + .iter() + .map(|candidate| entry.join(candidate)) + .find(|candidate| is_command_candidate(candidate)) + }) + } +} + +fn check_workspace_repo( + report: &mut DoctorReport, + workspace_root: &Path, + name: &str, +) -> Option { + let path = workspace_root.join(name); + match git::open_usable_repository(&path) { + Ok(repository) => { + report.record( + DoctorStatus::Ok, + format!( + "found usable workspace repo `{}` at `{}`", + name, + path.to_string_lossy() + ), + ); + Some(repository) + } + Err(_) if !path.exists() => { + report.record( + DoctorStatus::Fail, + format!( + "expected workspace repo `{}` at `{}`", + name, + path.to_string_lossy() + ), + ); + None + } + Err(error) => { + report.record( + DoctorStatus::Fail, + format!( + "expected `{}` to be a usable git repository at `{}`: {}", + name, + path.to_string_lossy(), + Report::from_error(error) + ), + ); + None + } + } +} + +fn check_sibling_manifest_id( + report: &mut DoctorReport, + sibling_path: &Path, + expected_repo_id: &str, +) { + let manifest_path = sibling_path.join(config::MANIFEST_FILE); + match Manifest::load(&manifest_path) { + Ok(manifest) if manifest.name == *expected_repo_id => report.record( + DoctorStatus::Ok, + format!("sibling `{expected_repo_id}` Build.toml name matches workspace key"), + ), + Ok(manifest) => report.record( + DoctorStatus::Fail, + format!( + "sibling `{expected_repo_id}` Build.toml declares name `{}`", + manifest.name + ), + ), + Err(config::Error::Io { source, .. }) + if matches!( + source.kind(), + ErrorKind::NotFound | ErrorKind::NotADirectory + ) => report.record( + DoctorStatus::Warn, + format!( + "sibling `{expected_repo_id}` has no Build.toml yet; this is a transitional rollout warning" + ), + ), + Err(error) => report.record( + DoctorStatus::Fail, + format!( + "sibling `{expected_repo_id}` Build.toml could not be loaded: {}", + Report::from_error(error) + ), + ), + } +} + +fn check_theme_zola_config(report: &mut DoctorReport, theme_path: &Path) { + let zola_config = theme_path.join("config").join("zola.toml"); + match std::fs::read_to_string(&zola_config) { + Ok(contents) => match toml::from_str::(&contents) { + Ok(_) => report.record( + DoctorStatus::Ok, + format!( + "workspace theme Zola config parses at `{}`", + zola_config.to_string_lossy() + ), + ), + Err(error) => report.record( + DoctorStatus::Fail, + format!( + "workspace theme Zola config is invalid at `{}`: {error}", + zola_config.to_string_lossy() + ), + ), + }, + Err(error) => report.record( + DoctorStatus::Fail, + format!( + "workspace theme Zola config could not be read at `{}`: {error}", + zola_config.to_string_lossy() + ), + ), + } +} + +fn check_theme_eipw_schema(report: &mut DoctorReport, theme_path: &Path) { + match crate::lint::eipw_schema_status(theme_path) { + Ok(()) => report.record( + DoctorStatus::Ok, + "workspace theme eipw config schema is compatible", + ), + Err(error) => report.record( + DoctorStatus::Fail, + format!("workspace theme eipw config is not usable: {error}"), + ), + } +} + +fn normalized_repository_path(path: &str) -> String { + path.trim_matches('/') + .strip_suffix(".git") + .unwrap_or(path.trim_matches('/')) + .to_owned() +} + +fn repository_identity(value: &str) -> Option<(String, String)> { + if let Ok(url) = Url::parse(value) { + return Some(( + url.host_str()?.to_ascii_lowercase(), + normalized_repository_path(url.path()), + )); + } + + let (user_host, path) = value.split_once(':')?; + let (_, host) = user_host.rsplit_once('@')?; + Some((host.to_ascii_lowercase(), normalized_repository_path(path))) +} + +fn theme_remote_matches_manifest(remote: &str, repository: &Url) -> bool { + remote == repository.as_str() + || matches!( + ( + repository_identity(remote), + repository_identity(repository.as_str()) + ), + (Some(remote), Some(manifest)) if remote == manifest + ) +} + +fn check_theme_remote( + report: &mut DoctorReport, + theme_repo: &git2::Repository, + theme_repository: &Url, +) { + match git::remote_urls(theme_repo) { + Ok(remotes) + if remotes + .iter() + .any(|remote| theme_remote_matches_manifest(remote, theme_repository)) => + { + report.record( + DoctorStatus::Ok, + "workspace theme has a remote matching active Build.toml theme.repository", + ); + } + Ok(remotes) => report.record( + DoctorStatus::Warn, + format!( + "workspace theme has no remote matching active Build.toml theme.repository `{theme_repository}` (checked {} configured remote(s))", + remotes.len() + ), + ), + Err(error) => report.record( + DoctorStatus::Warn, + format!("workspace theme remotes could not be inspected: {error}"), + ), + } +} + +fn check_theme_pin(report: &mut DoctorReport, theme_repo: &git2::Repository, manifest_pin: &str) { + let head = match git::head_commit_id(theme_repo) { + Ok(head) => head, + Err(error) => { + report.record( + DoctorStatus::Fail, + format!("workspace theme HEAD could not be resolved: {error}"), + ); + return; + } + }; + + match git::resolve_commit_id(theme_repo, manifest_pin) { + Ok(pin) if pin == head => report.record( + DoctorStatus::Ok, + format!("workspace theme HEAD matches active Build.toml pin `{manifest_pin}`"), + ), + Ok(pin) => report.record( + DoctorStatus::Warn, + format!( + "workspace theme HEAD `{head}` does not match active Build.toml pin `{manifest_pin}` (resolved locally as `{pin}`)" + ), + ), + Err(error) => report.record( + DoctorStatus::Warn, + format!( + "active Build.toml theme pin `{manifest_pin}` could not be resolved locally; doctor did not fetch it: {error}" + ), + ), + } +} + +fn check_theme_readiness( + report: &mut DoctorReport, + workspace_root: &Path, + theme_repository: &Url, + theme_pin: &str, +) { + let theme_path = workspace_root.join(config::DEFAULT_THEME_DIR); + let Some(theme_repo) = check_workspace_repo(report, workspace_root, config::DEFAULT_THEME_DIR) + else { + return; + }; + + check_theme_zola_config(report, &theme_path); + check_theme_eipw_schema(report, &theme_path); + check_theme_remote(report, &theme_repo, theme_repository); + check_theme_pin(report, &theme_repo, theme_pin); +} + +fn doctor_root(args: &Args) -> Result { + root(args) +} + +fn check_tool(report: &mut DoctorReport, command: &str, why: &str) -> bool { + match command_path(command) { + Some(path) => { + report.record( + DoctorStatus::Ok, + format!( + "found required tool `{}` at `{}`", + command, + path.to_string_lossy() + ), + ); + true + } + None => { + report.record( + DoctorStatus::Fail, + format!("missing required tool `{}`: {}", command, why), + ); + false + } + } +} + +#[cfg(windows)] +fn check_default_windows_build_eips_path(report: &mut DoctorReport) { + let Some(local_app_data) = std::env::var_os("LOCALAPPDATA") else { + return; + }; + + let install_dir = PathBuf::from(local_app_data).join("build-eips").join("bin"); + let build_eips_path = install_dir.join("build-eips.exe"); + + if build_eips_path.is_file() { + report.record( + DoctorStatus::Warn, + format!( + "found build-eips at the default user-local install path `{}`, but `{}` is not on PATH", + build_eips_path.to_string_lossy(), + install_dir.to_string_lossy() + ), + ); + } +} + +#[cfg(not(windows))] +fn check_default_windows_build_eips_path(_report: &mut DoctorReport) {} + +#[cfg(not(windows))] +fn check_optional_download_tool(report: &mut DoctorReport) { + let curl = command_path("curl"); + let wget = command_path("wget"); + + record_optional_download_tool(report, curl.as_deref(), wget.as_deref()); +} + +#[cfg(not(windows))] +fn record_optional_download_tool( + report: &mut DoctorReport, + curl: Option<&Path>, + wget: Option<&Path>, +) { + match (curl, wget) { + (Some(path), _) => report.record( + DoctorStatus::Ok, + format!( + "found front-door download helper `curl` at `{}`", + path.to_string_lossy() + ), + ), + (None, Some(path)) => report.record( + DoctorStatus::Ok, + format!( + "found front-door download helper `wget` at `{}`", + path.to_string_lossy() + ), + ), + (None, None) => report.record( + DoctorStatus::Warn, + "missing both `curl` and `wget`; `scripts/dev-setup` will not be able to download a release binary", + ), + } +} + +#[cfg(not(windows))] +fn check_front_door_archive_tool(report: &mut DoctorReport) { + let tar = command_path("tar"); + record_front_door_archive_tool(report, tar.as_deref()); +} + +#[cfg(not(windows))] +fn record_front_door_archive_tool(report: &mut DoctorReport, tar: Option<&Path>) { + match tar { + Some(path) => report.record( + DoctorStatus::Ok, + format!( + "found front-door archive tool `tar` at `{}`", + path.to_string_lossy() + ), + ), + None => report.record( + DoctorStatus::Warn, + "missing `tar`; `scripts/dev-setup` will not be able to unpack the release binary", + ), + } +} + +#[cfg(not(windows))] +fn check_front_door_setup_tools(report: &mut DoctorReport) { + check_optional_download_tool(report); + check_front_door_archive_tool(report); +} + +#[cfg(windows)] +fn check_front_door_setup_tools(_report: &mut DoctorReport) {} + +fn collect_doctor_report(args: &Args, check_tools: bool) -> Result { + let context = load_workspace_command_context(args)?; + let mut report = DoctorReport::default(); + let (root_path, active_repo) = match doctor_root(args) { + Ok(root_path) => match ActiveRepo::load(&root_path) { + Ok(active_repo) => { + report.record( + DoctorStatus::Ok, + format!("loaded active repo Build.toml for `{}`", active_repo.title), + ); + report.record( + DoctorStatus::Ok, + format!( + "Build.toml parses at `{}`", + root_path.join(config::MANIFEST_FILE).to_string_lossy() + ), + ); + (Some(root_path), Some(active_repo)) + } + Err(error) => { + report.record( + DoctorStatus::Fail, + format!("active repo Build.toml could not be loaded: {error}"), + ); + (Some(root_path), None) + } + }, + Err(error) => { + report.record( + DoctorStatus::Fail, + format!( + "active repo root could not be resolved: {}", + Report::from_error(error) + ), + ); + (None, None) + } + }; + + match context.config_path.as_ref() { + Some(path) => report.record( + DoctorStatus::Ok, + format!( + "found workspace config candidate `{}`", + path.to_string_lossy() + ), + ), + None => report.record( + DoctorStatus::Fail, + format!( + "could not find `{}` while searching upward from `{}`", + config::LOCAL_CONFIG_FILE, + context.search_from.to_string_lossy() + ), + ), + } + + let parsed_config = context + .config_path + .as_deref() + .map(LoadedWorkspaceConfig::from_path) + .transpose(); + + match parsed_config { + Ok(Some(config)) => { + report.record( + DoctorStatus::Ok, + format!( + "workspace config parses at `{}`", + config.config_path().to_string_lossy() + ), + ); + + let workspace_root = config.workspace_root(); + if workspace_root.is_dir() { + report.record( + DoctorStatus::Ok, + format!( + "workspace root exists at `{}`", + workspace_root.to_string_lossy() + ), + ); + } else { + report.record( + DoctorStatus::Fail, + format!( + "workspace root is missing at `{}`", + workspace_root.to_string_lossy() + ), + ); + } + + if let (Some(root_path), Some(active_repo)) = (root_path.as_ref(), active_repo.as_ref()) + { + let expected_root = workspace_root.join(&active_repo.title); + if root_path == &expected_root { + report.record( + DoctorStatus::Ok, + format!( + "active repo `{}` is checked out at `{}`", + &active_repo.title, + expected_root.to_string_lossy() + ), + ); + } else { + report.record( + DoctorStatus::Fail, + format!( + "active repo `{}` should be checked out at `{}`, found `{}`", + &active_repo.title, + expected_root.to_string_lossy(), + root_path.to_string_lossy() + ), + ); + } + + check_workspace_repo(&mut report, workspace_root, &active_repo.title); + for sibling_id in &active_repo.sibling_ids { + let sibling_path = workspace_root.join(sibling_id); + if check_workspace_repo(&mut report, workspace_root, sibling_id).is_some() { + check_sibling_manifest_id(&mut report, &sibling_path, sibling_id); + } + } + + check_theme_readiness( + &mut report, + workspace_root, + &active_repo.theme.repository, + &active_repo.theme.commit, + ); + } else { + report.record( + DoctorStatus::Warn, + "workspace repo layout checks were skipped because active Build.toml was unavailable", + ); + } + } + Err(error) => { + report.record( + DoctorStatus::Fail, + format!( + "workspace config could not be parsed: {}", + Report::from_error(error) + ), + ); + } + Ok(None) => (), + } + + if check_tools { + if !check_tool( + &mut report, + "build-eips", + "workspace bootstrap and build-eips commands expect `build-eips` on PATH", + ) { + check_default_windows_build_eips_path(&mut report); + } + check_tool( + &mut report, + "git", + "workspace bootstrap and build-eips commands expect git to be available", + ); + check_tool( + &mut report, + "zola", + "build, check, and serve commands need a working zola binary", + ); + check_front_door_setup_tools(&mut report); + } + + Ok(report) +} + +pub(crate) fn doctor_workspace(args: &Args) -> Result<(), Whatever> { + let report = collect_doctor_report(args, true)?; + + if report.failures > 0 { + snafu::whatever!("doctor found {} failing check(s)", report.failures); + } + + Ok(()) +} + +pub(crate) fn init_workspace( + args: &Args, + workspace_root: PathBuf, + include_template: bool, +) -> Result<(), Whatever> { + let template_repository = Url::parse(PROPOSAL_TEMPLATE_URL) + .whatever_context("invalid proposal template repository URL")?; + let repositories = WorkspaceInitToolingRepositories { + template: &template_repository, + }; + + init_workspace_with_repositories( + args, + workspace_root, + include_template, + &repositories, + ) +} + +fn init_workspace_with_repositories( + args: &Args, + workspace_root: PathBuf, + include_template: bool, + repositories: &WorkspaceInitToolingRepositories<'_>, +) -> Result<(), Whatever> { + let root_path = root(args) + .whatever_context("workspace init requires an active Build.toml repository root")?; + let active_repo = ActiveRepo::load(&root_path) + .whatever_context("unable to load active repository Build.toml")?; + let workspace_root = resolve_input_path(&workspace_root)?; + std::fs::create_dir_all(&workspace_root) + .whatever_context("unable to create workspace root directory")?; + let workspace_root = workspace_root + .canonicalize() + .whatever_context("unable to canonicalize workspace root directory")?; + + let repository_use = active_repo.repository_use; + let theme = active_repo.theme; + + for (sibling_id, sibling_url) in repository_use.other_repos { + let sibling_path = workspace_root.join(&sibling_id); + git::clone_missing_repo(sibling_url.as_str(), &sibling_path).with_whatever_context(|_| { + format!( + "unable to prepare workspace sibling repo `{sibling_id}` at `{}`; destination must be missing or a usable git repository", + sibling_path.to_string_lossy() + ) + })?; + } + + let theme_path = workspace_root.join(config::DEFAULT_THEME_DIR); + if git::clone_missing_repo(theme.repository.as_str(), &theme_path) + .with_whatever_context(|_| { + format!( + "unable to prepare workspace theme repo at `{}`; destination must be missing or a usable git repository", + theme_path.to_string_lossy() + ) + })? + == git::CloneOutcome::Fresh + { + git::checkout_fresh_clone_at_commit(&theme_path, theme.repository.as_str(), &theme.commit) + .with_whatever_context(|_| { + format!( + "unable to fetch or check out active Build.toml theme commit `{}`", + theme.commit + ) + })?; + } + + if include_template { + let template_path = workspace_root.join("template"); + git::clone_missing_repo(repositories.template.as_str(), &template_path) + .with_whatever_context(|_| { + format!( + "unable to prepare workspace template repo at `{}`; destination must be missing or a usable git repository", + template_path.to_string_lossy() + ) + })?; + } + + std::fs::create_dir_all(workspace_root.join(config::DEFAULT_BUILD_ROOT_BASE)) + .whatever_context("unable to create local build root")?; + + let config_path = workspace_root.join(config::LOCAL_CONFIG_FILE); + match OpenOptions::new() + .write(true) + .create_new(true) + .open(&config_path) + { + Ok(mut config_file) => { + config_file + .write_all(config::default_workspace_config_text().as_bytes()) + .whatever_context("unable to write workspace config")?; + } + Err(error) if error.kind() == ErrorKind::AlreadyExists => { + info!( + "leaving existing workspace config `{}` in place", + config_path.to_string_lossy() + ); + } + Err(error) => { + return Err(error).whatever_context("unable to write workspace config"); + } + } + + write_workspace_doc(&workspace_root)?; + + Ok(()) +} + +fn workspace_doc_text() -> &'static str { + include_str!("workspace_doc.md") +} + +fn write_workspace_doc(workspace_root: &Path) -> Result<(), Whatever> { + let doc_path = workspace_root.join(WORKSPACE_DOC_FILE); + std::fs::write(&doc_path, workspace_doc_text()).with_whatever_context(|_| { + format!( + "unable to write workspace document `{}`", + doc_path.to_string_lossy() + ) + })?; + + Ok(()) +} + diff --git a/src/workspace_doc.md b/src/workspace_doc.md new file mode 100644 index 0000000..c37443f --- /dev/null +++ b/src/workspace_doc.md @@ -0,0 +1,19 @@ +# build-eips Workspace + +This guide is generated as `WORKSPACE.md` by `build-eips init`. + +A workspace keeps proposal repositories, an editable local theme, generated build output, and workspace-local settings together. + +The active proposal repository must contain a valid `Build.toml`. Its locations define sibling proposal repositories, and `[theme]` supplies the repository and pin used when `init` creates a missing theme checkout. + +```sh +build-eips init .. +build-eips init .. --template +``` + +`init` preserves existing usable repositories and writes `.build-eips.toml` only when it is missing. + + +## Render Specific Proposals Only + +Use `--only` with local dirty builds and serves to render selected proposal numbers. A workspace `[render]` section can provide the same defaults. diff --git a/src/zola.rs b/src/zola.rs index fc65638..eb185af 100644 --- a/src/zola.rs +++ b/src/zola.rs @@ -15,7 +15,10 @@ use semver::Version; use snafu::{ensure, Backtrace, IntoError, Report, ResultExt, Snafu}; use url::Url; -use crate::{cache::Cache, git}; +use crate::{ + config::ServerBinding, + layout::{mounted_theme_path, theme_config_path}, +}; const MINIMUM_VERSION: Version = Version::new(0, 22, 1); @@ -38,8 +41,11 @@ fn symlink_dir(original: &Path, link: &Path) -> Result<(), std::io::Error> { } fn force_symlink_dir(original: &Path, link: &Path) -> Result<(), std::io::Error> { - match std::fs::remove_file(link) { - Ok(()) => (), + match std::fs::symlink_metadata(link) { + Ok(metadata) if metadata.file_type().is_dir() && !metadata.file_type().is_symlink() => { + std::fs::remove_dir_all(link)?; + } + Ok(_) => std::fs::remove_file(link)?, Err(e) if e.kind() == ErrorKind::NotFound => (), Err(e) => return Err(e), } @@ -47,6 +53,19 @@ fn force_symlink_dir(original: &Path, link: &Path) -> Result<(), std::io::Error> symlink_dir(original, link) } +fn mount_theme(theme_dir: &Path, project_path: &Path) -> Result { + let mounted_theme_path = mounted_theme_path(project_path); + if theme_dir == mounted_theme_path { + return Ok(mounted_theme_path); + } + + if let Some(parent) = mounted_theme_path.parent() { + std::fs::create_dir_all(parent)?; + } + force_symlink_dir(theme_dir, &mounted_theme_path)?; + Ok(mounted_theme_path) +} + #[derive(Debug, Snafu)] pub enum Error { #[snafu(display("could not find zola binary (requires at least version {MINIMUM_VERSION})"))] @@ -71,11 +90,6 @@ pub enum Error { backtrace: Backtrace, source: std::io::Error, }, - #[snafu(context(false))] - Git { - #[snafu(backtrace)] - source: git::Error, - }, } pub fn find_zola() -> Result<(), Error> { @@ -96,21 +110,14 @@ pub fn find_zola() -> Result<(), Error> { Ok(()) } -pub fn check( - theme_repo: &str, - theme_rev: &str, - cache: &Cache, - project_path: &Path, -) -> Result<(), Error> { +pub fn check(theme_dir: &Path, project_path: &Path) -> Result<(), Error> { let args = ["check", "--drafts", "--skip-external-links"]; - spawn_log(theme_repo, theme_rev, cache, project_path, args)?; + spawn_log(theme_dir, project_path, args)?; Ok(()) } pub fn build( - theme_repo: &str, - theme_rev: &str, - cache: &Cache, + theme_dir: &Path, project_path: &Path, output_path: &Path, base_url: &str, @@ -120,7 +127,7 @@ pub fn build( .map(OsString::from) .into_iter() .chain(std::iter::once(output_path.into())); - spawn_log(theme_repo, theme_rev, cache, project_path, args)?; + spawn_log(theme_dir, project_path, args)?; if let Ok(url) = Url::from_file_path(output_path) { info!("HTML output to: {}", url); } @@ -128,23 +135,50 @@ pub fn build( } pub fn serve( - theme_repo: &str, - theme_rev: &str, - cache: &Cache, + theme_dir: &Path, project_path: &Path, output_path: &Path, + server_binding: &ServerBinding, + base_url_override: Option<&Url>, ) -> Result<(), Error> { // TODO: Properly kill the child process when we receive ctrl-c. - warn!("live reloading is not implemented"); remove_output(output_path); - let args = ["serve", "--drafts", "-o"] - .map(OsString::from) - .into_iter() - .chain(std::iter::once(output_path.into())); - spawn_log(theme_repo, theme_rev, cache, project_path, args)?; + let args = serve_args(server_binding, output_path, base_url_override); + spawn_log(theme_dir, project_path, args)?; Ok(()) } +fn serve_args( + server_binding: &ServerBinding, + output_path: &Path, + base_url_override: Option<&Url>, +) -> Vec { + let mut args = [ + "serve", + "--drafts", + "--fast", + "--force", + "--interface", + server_binding.host.as_str(), + "--port", + ] + .map(OsString::from) + .to_vec(); + + args.push(OsString::from(server_binding.port.to_string())); + + if let Some(base_url) = base_url_override { + args.extend([ + OsString::from("-u"), + OsString::from(base_url.as_str()), + OsString::from("--no-port-append"), + ]); + } + + args.extend([OsString::from("-o"), output_path.as_os_str().to_os_string()]); + args +} + fn remove_output(output_path: &Path) { if let Err(e) = std::fs::remove_dir_all(output_path) { debug!( @@ -154,13 +188,7 @@ fn remove_output(output_path: &Path) { } } -fn spawn_log( - theme_repo: &str, - theme_rev: &str, - cache: &Cache, - project_path: &Path, - args: U, -) -> Result<(), Error> +fn spawn_log(theme_dir: &Path, project_path: &Path, args: U) -> Result<(), Error> where U: IntoIterator, I: Into, @@ -173,18 +201,9 @@ where find_zola()?; - let theme_dir = cache.repo(theme_repo, theme_rev)?; - - let mut themes_dir = project_path.join("themes"); - if let Err(e) = std::fs::create_dir(&themes_dir) { - debug!("got while creating themes dir: {}", Report::from_error(e)); - } - themes_dir.push("eips-theme"); - force_symlink_dir(&theme_dir, &themes_dir).context(FsSnafu { path: &themes_dir })?; - - let config_path: PathBuf = [&theme_dir, Path::new("config"), Path::new("zola.toml")] - .iter() - .collect(); + let mounted_theme_path = + mount_theme(theme_dir, project_path).context(FsSnafu { path: theme_dir })?; + let config_path = theme_config_path(&mounted_theme_path); let prefix = [OsString::from("-c"), config_path.into()].into_iter(); let args = prefix.chain(args.into_iter().map(Into::into)); @@ -221,3 +240,188 @@ where Ok(()) } + +#[cfg(test)] +mod tests { + use std::{ + ffi::OsString, + fs, + path::{Path, PathBuf}, + process::{Command, ExitStatus}, + }; + + use crate::{ + config::ServerBinding, + layout::{mounted_theme_path, theme_config_path}, + }; + use tempfile::TempDir; + + use super::{find_zola, mount_theme, serve_args}; + + fn write_file(root: &Path, relative: &str, contents: &str) { + let path = root.join(relative); + if let Some(parent) = path.parent() { + fs::create_dir_all(parent).unwrap(); + } + fs::write(path, contents).unwrap(); + } + + fn zola_build_status(project_root: &Path) -> ExitStatus { + Command::new("zola") + .arg("build") + .arg("--drafts") + .current_dir(project_root) + .status() + .unwrap() + } + + #[test] + fn zola_rejects_missing_internal_links_but_accepts_external_links() { + if find_zola().is_err() { + eprintln!("skipping zola link behavior fixture because zola is not installed"); + return; + } + + let temp = TempDir::new().unwrap(); + let internal = temp.path().join("internal"); + write_file( + &internal, + "config.toml", + "base_url = \"https://example.test\"\n", + ); + write_file( + &internal, + "content/_index.md", + "+++\ntitle = \"Internal\"\n+++\n[Missing](@/missing.md)\n", + ); + let external = temp.path().join("external"); + write_file( + &external, + "config.toml", + "base_url = \"https://example.test\"\n", + ); + write_file( + &external, + "content/_index.md", + "+++\ntitle = \"External\"\n+++\n[External](https://eips.ethereum.org/EIPS/eip-1)\n", + ); + + assert!(!zola_build_status(&internal).success()); + assert!(zola_build_status(&external).success()); + } + + #[test] + fn serve_args_include_configured_interface_and_port() { + let server_binding = ServerBinding { + host: "0.0.0.0".to_owned(), + port: 8080, + }; + + assert_eq!( + serve_args(&server_binding, Path::new("/tmp/build-output"), None), + vec![ + OsString::from("serve"), + OsString::from("--drafts"), + OsString::from("--fast"), + OsString::from("--force"), + OsString::from("--interface"), + OsString::from("0.0.0.0"), + OsString::from("--port"), + OsString::from("8080"), + OsString::from("-o"), + OsString::from("/tmp/build-output"), + ] + ); + } + + #[test] + fn serve_args_include_base_url_override_when_present() { + let server_binding = ServerBinding { + host: "127.0.0.1".to_owned(), + port: 1111, + }; + let base_url = "http://127.0.0.1:1111".parse().unwrap(); + + assert_eq!( + serve_args( + &server_binding, + Path::new("/tmp/build-output"), + Some(&base_url) + ), + vec![ + OsString::from("serve"), + OsString::from("--drafts"), + OsString::from("--fast"), + OsString::from("--force"), + OsString::from("--interface"), + OsString::from("127.0.0.1"), + OsString::from("--port"), + OsString::from("1111"), + OsString::from("-u"), + OsString::from("http://127.0.0.1:1111/"), + OsString::from("--no-port-append"), + OsString::from("-o"), + OsString::from("/tmp/build-output"), + ] + ); + } + + #[test] + fn mounted_theme_paths_are_under_project_themes_directory() { + let project_path = PathBuf::from("/tmp/project"); + let mounted_theme = mounted_theme_path(&project_path); + + assert_eq!( + mounted_theme, + PathBuf::from("/tmp/project/themes/eips-theme") + ); + assert_eq!( + theme_config_path(&mounted_theme), + PathBuf::from("/tmp/project/themes/eips-theme/config/zola.toml") + ); + } + + #[test] + fn mount_theme_does_not_symlink_mounted_local_theme_onto_itself() { + let temp = TempDir::new().unwrap(); + let project_path = temp.path().join("repo"); + let mounted_theme = mounted_theme_path(&project_path); + fs::create_dir_all(mounted_theme.join("config")).unwrap(); + fs::write(mounted_theme.join("config/zola.toml"), "title = 'local'\n").unwrap(); + + let result = mount_theme(&mounted_theme, &project_path).unwrap(); + + assert_eq!(result, mounted_theme); + assert!(mounted_theme.join("config/zola.toml").is_file()); + assert!(!fs::symlink_metadata(&mounted_theme) + .unwrap() + .file_type() + .is_symlink()); + } + + #[cfg(target_family = "unix")] + #[test] + fn theme_mount_replaces_prior_real_mounted_theme_directory() { + let temp = TempDir::new().unwrap(); + let project_path = temp.path().join("repo"); + let source_theme = temp.path().join("source-theme"); + fs::create_dir_all(source_theme.join("config")).unwrap(); + fs::write(source_theme.join("config/zola.toml"), "title = 'source'\n").unwrap(); + + let mounted_theme = mounted_theme_path(&project_path); + fs::create_dir_all(&mounted_theme).unwrap(); + fs::write(mounted_theme.join("stale.txt"), "stale").unwrap(); + + let result = mount_theme(&source_theme, &project_path).unwrap(); + + assert_eq!(result, mounted_theme); + assert!(fs::symlink_metadata(&mounted_theme) + .unwrap() + .file_type() + .is_symlink()); + assert_eq!( + fs::read_to_string(theme_config_path(&mounted_theme)).unwrap(), + "title = 'source'\n" + ); + } +}