Adds html-relative Passthrough Copy mode for relative asset references in HTML #3552

package.json

@@ -125,7 +125,7 @@
 		"@11ty/eleventy-utils": "2.0.0-alpha.2",
 		"@11ty/lodash-custom": "^4.17.21",
 		"@11ty/posthtml-urls": "^1.0.0",
-		"@11ty/recursive-copy": "^3.0.0",
+		"@11ty/recursive-copy": "^3.0.1",
 		"@sindresorhus/slugify": "^2.2.1",
 		"bcp-47-normalize": "^2.3.0",
 		"chardet": "^2.0.0",

src/Eleventy.js

@@ -518,9 +518,9 @@ class Eleventy {
 		}
 		this.templateData.setFileSystemSearch(this.fileSystemSearch);
 
-		// TODO swap this to getters
 		this.passthroughManager = new TemplatePassthroughManager(this.eleventyConfig);
 		this.passthroughManager.setRunMode(this.runMode);
+		this.passthroughManager.setDryRun(this.isDryRun);
 		this.passthroughManager.extensionMap = this.extensionMap;
 		this.passthroughManager.setFileSystemSearch(this.fileSystemSearch);
 

src/Plugins/HtmlBasePlugin.js

@@ -137,7 +137,7 @@ function eleventyHtmlBasePlugin(eleventyConfig, defaultOptions = {}) {
 			});
 		},
 		{
-			priority: -1, // run last (especially after PathToUrl transform)
+			priority: -2, // priority is descending, so this runs last (especially after AutoCopy and InputPathToUrl transform)
 			enabled: function (context) {
 				// Enabled when pathPrefix is non-default or via renderTransforms
 				return Boolean(context.baseHref) || opts.baseHref !== "/";

src/Plugins/HtmlRelativeCopyPlugin.js

@@ -0,0 +1,52 @@
+import { HtmlRelativeCopy } from "../Util/HtmlRelativeCopy.js";
+
+// one HtmlRelativeCopy instance per entry
+function init(eleventyConfig, options) {
+	let opts = Object.assign(
+		{
+			extensions: "html",
+			match: false, // can be one glob string or an array of globs
+			paths: [], // directories to also look in for files
+			failOnError: true, // fails when a path matches (via `match`) but not found on file system
+			copyOptions: undefined,
+		},
+		options,
+	);
+
+	let htmlrel = new HtmlRelativeCopy();
+	htmlrel.setUserConfig(eleventyConfig);
+	htmlrel.addMatchingGlob(opts.match);
+	htmlrel.setFailOnError(opts.failOnError);
+	htmlrel.setCopyOptions(opts.copyOptions);
+
+	eleventyConfig.htmlTransformer.addUrlTransform(
+		opts.extensions,
+		function (targetFilepathOrUrl) {
+			// @ts-ignore
+			htmlrel.copy(targetFilepathOrUrl, this.page.inputPath, this.page.outputPath);
+
+			// TODO front matter option for manual copy
+			return targetFilepathOrUrl;
+		},
+		{
+			enabled: () => htmlrel.isEnabled(),
+			// - MUST run after other plugins but BEFORE HtmlBase plugin
+			priority: -1,
+		},
+	);
+
+	htmlrel.addPaths(opts.paths);
+}
+
+function HtmlRelativeCopyPlugin(eleventyConfig) {
+	// Important: if this is empty, no URL transforms are added
+	for (let options of eleventyConfig.passthroughCopiesHtmlRelative) {
+		init(eleventyConfig, options);
+	}
+}
+
+Object.defineProperty(HtmlRelativeCopyPlugin, "eleventyPackage", {
+	value: "@11ty/eleventy/html-relative-copy-plugin",
+});
+
+export { HtmlRelativeCopyPlugin };

src/TemplatePassthrough.js

@@ -18,21 +18,19 @@ class TemplatePassthrough {
 	#isInputPathGlob;
 	#benchmarks;
 	#isAlreadyNormalized = false;
+	#projectDirCheck = false;
 
-	// paths already guaranteed (probably from the autocopy plugin)
-	static normalizedFactory(inputPath, outputPath, opts = {}) {
+	// paths already guaranteed from the autocopy plugin
+	static factory(inputPath, outputPath, opts = {}) {
 		let p = new TemplatePassthrough(
 			{
 				inputPath,
 				outputPath,
+				copyOptions: opts.copyOptions,
 			},
 			opts.templateConfig,
 		);
 
-		if (opts.normalized) {
-			p.setIsAlreadyNormalized(true);
-		}
-
 		return p;
 	}
 
@@ -69,6 +67,10 @@ class TemplatePassthrough {
 		return this.templateConfig.getConfig();
 	}
 
+	get directories() {
+		return this.templateConfig.directories;
+	}
+
 	// inputDir is used when stripping from output path in `getOutputPath`
 	get inputDir() {
 		return this.templateConfig.directories.input;
@@ -78,10 +80,15 @@ class TemplatePassthrough {
 		return this.templateConfig.directories.output;
 	}
 
+	// Skips `getFiles()` normalization
 	setIsAlreadyNormalized(isNormalized) {
 		this.#isAlreadyNormalized = Boolean(isNormalized);
 	}
 
+	setCheckSourceDirectory(check) {
+		this.#projectDirCheck = Boolean(check);
+	}
+
 	/* { inputPath, outputPath } though outputPath is *not* the full path: just the output directory */
 	getPath() {
 		return this.rawPath;
@@ -239,11 +246,15 @@ class TemplatePassthrough {
 	 * 3. individual file
 	 */
 	async copy(src, dest, copyOptions) {
-		if (
-			!TemplatePath.stripLeadingDotSlash(dest).startsWith(
-				TemplatePath.stripLeadingDotSlash(this.outputDir),
-			)
-		) {
+		if (this.#projectDirCheck && !this.directories.isFileInProjectFolder(src)) {
+			return Promise.reject(
+				new TemplatePassthroughError(
+					"Source file is not in the project directory. Check your passthrough paths.",
+				),
+			);
+		}
+
+		if (!this.directories.isFileInOutputFolder(dest)) {
 			return Promise.reject(
 				new TemplatePassthroughError(
 					"Destination is not in the site output directory. Check your passthrough paths.",
@@ -255,6 +266,7 @@ class TemplatePassthrough {
 		let fileSizeCount = 0;
 		let map = {};
 		let b = this.benchmarks.aggregate.get("Passthrough Copy File");
+
 		// returns a promise
 		return copy(src, dest, copyOptions)
 			.on(copy.events.COPY_FILE_START, (copyOp) => {
@@ -268,13 +280,28 @@ class TemplatePassthrough {
 				fileSizeCount += copyOp.stats.size;
 				b.after();
 			})
-			.then(() => {
-				return {
-					count: fileCopyCount,
-					size: fileSizeCount,
-					map,
-				};
-			});
+			.then(
+				() => {
+					return {
+						count: fileCopyCount,
+						size: fileSizeCount,
+						map,
+					};
+				},
+				(error) => {
+					if (copyOptions.overwrite === false && error.code === "EEXIST") {
+						// just ignore if the output already exists and overwrite: false
+						debug("Overwrite error ignored: %O", error);
+						return {
+							count: 0,
+							size: 0,
+							map,
+						};
+					}
+
+					return Promise.reject(error);
+				},
+			);
 	}
 
 	async write() {

src/TemplatePassthroughManager.js

@@ -7,22 +7,39 @@ import EleventyBaseError from "./Errors/EleventyBaseError.js";
 import TemplatePassthrough from "./TemplatePassthrough.js";
 import checkPassthroughCopyBehavior from "./Util/PassthroughCopyBehaviorCheck.js";
 import { isGlobMatch } from "./Util/GlobMatcher.js";
+import { withResolvers } from "./Util/PromiseUtil.js";
 
 const debug = debugUtil("Eleventy:TemplatePassthroughManager");
 
-class TemplatePassthroughManagerConfigError extends EleventyBaseError {}
 class TemplatePassthroughManagerCopyError extends EleventyBaseError {}
 
 class TemplatePassthroughManager {
 	#isDryRun = false;
+	#afterBuild;
+	#queue = new Map();
 
 	constructor(templateConfig) {
 		if (!templateConfig || templateConfig.constructor.name !== "TemplateConfig") {
-			throw new TemplatePassthroughManagerConfigError("Missing or invalid `config` argument.");
+			throw new Error("Internal error: Missing or invalid `templateConfig` argument.");
 		}
 
 		this.templateConfig = templateConfig;
 		this.config = templateConfig.getConfig();
+
+		// eleventy# event listeners are removed on each build
+		this.config.events.on("eleventy#copy", ({ source, target, options }) => {
+			this.enqueueCopy(source, target, options);
+		});
+
+		this.config.events.on("eleventy#beforerender", () => {
+			this.#afterBuild = withResolvers();
+		});
+
+		this.config.events.on("eleventy#render", () => {
+			let { resolve } = this.#afterBuild;
+			resolve();
+		});
+
 		this.reset();
 	}
 
@@ -31,7 +48,8 @@ class TemplatePassthroughManager {
 		this.size = 0;
 		this.conflictMap = {};
 		this.incrementalFile;
-		debug("Resetting counts to 0");
+
+		this.#queue = new Map();
 	}
 
 	set extensionMap(extensionMap) {
@@ -288,16 +306,50 @@ class TemplatePassthroughManager {
 		return entries;
 	}
 
-	// Performance note: these can actually take a fair bit of time, but aren’t a
-	// bottleneck to eleventy. The copies are performed asynchronously and don’t affect eleventy
-	// write times in a significant way.
+	async #waitForTemplatesRendered() {
+		if (!this.#afterBuild) {
+			return Promise.resolve(); // immediately resolve
+		}
+
+		let { promise } = this.#afterBuild;
+		return promise;
+	}
+
+	enqueueCopy(source, target, copyOptions) {
+		let key = `${source}=>${target}`;
+
+		// light de-dupe the same source/target combo (might be in the same file, might be viaTransforms)
+		if (this.#queue.has(key)) {
+			return;
+		}
+
+		let passthrough = TemplatePassthrough.factory(source, target, {
+			templateConfig: this.templateConfig,
+			copyOptions,
+		});
+
+		passthrough.setCheckSourceDirectory(true);
+		passthrough.setIsAlreadyNormalized(true);
+		passthrough.setRunMode(this.runMode);
+		passthrough.setDryRun(this.#isDryRun);
+
+		this.#queue.set(key, this.copyPassthrough(passthrough));
+	}
+
 	async copyAll(templateExtensionPaths) {
 		debug("TemplatePassthrough copy started.");
 		let normalizedPaths = this.getAllNormalizedPaths(templateExtensionPaths);
 
 		let passthroughs = normalizedPaths.map((path) => this.getTemplatePassthroughForPath(path));
 
 		let promises = passthroughs.map((pass) => this.copyPassthrough(pass));
+
+		await this.#waitForTemplatesRendered();
+
+		for (let [key, afterBuildCopyPromises] of this.#queue) {
+			promises.push(afterBuildCopyPromises);
+		}
+
 		return Promise.all(promises).then(async (results) => {
 			let aliases = this.getAliasesFromPassthroughResults(results);
 			await this.config.events.emit("eleventy.passthrough", {

src/TemplateWriter.js

@@ -429,18 +429,20 @@ class TemplateWriter {
 
 	async write() {
 		let paths = await this._getAllPaths();
-		let promises = [];
 
-		// The ordering here is important to destructuring in Eleventy->_watch
-		promises.push(this.writePassthroughCopy(paths));
+		// This must happen before writePassthroughCopy
+		this.templateConfig.userConfig.emit("eleventy#beforerender");
 
-		promises.push(...(await this.generateTemplates(paths)));
+		let aggregatePassthroughCopyPromise = this.writePassthroughCopy(paths);
 
-		return Promise.all(promises).then(
-			async ([passthroughCopyResults, ...templateResults]) => {
-				// TODO wait for afterBuildCopy to finish
-				// console.log( "AFTER??", passthroughCopyResults );
+		let templatesPromise = Promise.all(await this.generateTemplates(paths)).then((results) => {
+			this.templateConfig.userConfig.emit("eleventy#render");
+
+			return results;
+		});
 
+		return Promise.all([aggregatePassthroughCopyPromise, templatesPromise]).then(
+			async ([passthroughCopyResults, templateResults]) => {
 				return {
 					passthroughCopy: passthroughCopyResults,
 					// New in 3.0: flatten and filter out falsy templates

src/UserConfig.js

@@ -145,6 +145,7 @@ class UserConfig {
 
 		/** @type {object} */
 		this.passthroughCopies = {};
+		this.passthroughCopiesHtmlRelative = new Set();
 
 		/** @type {object} */
 		this.layoutAliases = {};
@@ -791,7 +792,18 @@ class UserConfig {
 	 * @returns {any} a reference to the `EleventyConfig` object.
 	 */
 	addPassthroughCopy(fileOrDir, copyOptions = {}) {
-		if (typeof fileOrDir === "string") {
+		if (copyOptions.mode === "html-relative") {
+			if (isPlainObject(fileOrDir)) {
+				throw new Error(
+					"mode: 'html-relative' does not yet support passthrough copy objects (input -> output mapping). Use a string glob or an Array of string globs.",
+				);
+			}
+
+			this.passthroughCopiesHtmlRelative?.add({
+				match: fileOrDir,
+				...copyOptions,
+			});
+		} else if (typeof fileOrDir === "string") {
 			this.passthroughCopies[fileOrDir] = { outputPath: true, copyOptions };
 		} else {
 			for (let [inputPath, outputPath] of Object.entries(fileOrDir)) {
@@ -1215,6 +1227,7 @@ class UserConfig {
 			globalData: this.globalData,
 			layoutAliases: this.layoutAliases,
 			layoutResolution: this.layoutResolution,
+			passthroughCopiesHtmlRelative: this.passthroughCopiesHtmlRelative,
 			passthroughCopies: this.passthroughCopies,
 
 			// Liquid