diff --git a/packages/cli/src/lib/fs-utils.js b/packages/cli/src/lib/fs-utils.js new file mode 100644 index 000000000..a81651686 --- /dev/null +++ b/packages/cli/src/lib/fs-utils.js @@ -0,0 +1,33 @@ +import fs from 'node:fs/promises'; + +// Small helper to retry copyFile on transient errors (EBUSY on Windows CI). +// Attempts: number of tries (default 5) +// baseDelay: starting delay in ms for exponential backoff (default 100ms) +async function copyFileWithRetry(source, target, { attempts = 5, baseDelay = 100 } = {}) { + let lastError; + + for (let i = 0; i < attempts; i++) { + try { + return await fs.copyFile(source, target); + } catch (err) { + lastError = err; + + // Only retry for transient EBUSY / EPERM on Windows or generic resource busy errors + if ((err.code === 'EBUSY' || err.code === 'EPERM' || err.code === 'EACCES') && i < attempts - 1) { + const delay = baseDelay * Math.pow(2, i); + // add small jitter + const jitter = Math.floor(Math.random() * baseDelay); + await new Promise((res) => setTimeout(res, delay + jitter)); + continue; + } + + // non-retryable or last attempt -> rethrow + throw err; + } + } + + // if we exhausted loop, throw last seen error + throw lastError; +} + +export { copyFileWithRetry }; diff --git a/packages/cli/src/lifecycles/copy.js b/packages/cli/src/lifecycles/copy.js index d3f852653..02e64c126 100644 --- a/packages/cli/src/lifecycles/copy.js +++ b/packages/cli/src/lifecycles/copy.js @@ -1,6 +1,23 @@ import fs from "node:fs/promises"; import { checkResourceExists } from "../lib/resource-utils.js"; import { asyncForEach } from "../lib/async-utils.js"; +import { copyFileWithRetry } from "../lib/fs-utils.js"; + +// simple concurrency-limited mapper using chunked batches +async function mapWithConcurrency(items, mapper, concurrency = 8) { + const results = []; + + for (let i = 0; i < items.length; i += concurrency) { + const chunk = items.slice(i, i + concurrency); + const chunkPromises = chunk.map((item) => mapper(item)); + // wait for this batch to finish before continuing to the next + // preserve individual rejections + const chunkResults = await Promise.all(chunkPromises); + results.push(...chunkResults); + } + + return results; +} async function rreaddir(dir, allFiles = []) { const files = (await fs.readdir(dir)).map((f) => new URL(`./${f}`, dir)); @@ -20,10 +37,9 @@ async function rreaddir(dir, allFiles = []) { async function copyFile(source, target, projectDirectory) { try { console.info(`copying file... ${source.pathname.replace(projectDirectory.pathname, "")}`); - - await fs.copyFile(source, target); + await copyFileWithRetry(source, target); } catch (error) { - console.error("ERROR", error); + console.error("ERROR copying file", source.href, "->", target.href, error); } } @@ -39,7 +55,7 @@ async function copyDirectory(fromUrl, toUrl, projectDirectory) { }); } - await asyncForEach(files, async (fileUrl) => { + await mapWithConcurrency(files, async (fileUrl) => { const targetUrl = new URL( `file://${fileUrl.pathname.replace(fromUrl.pathname, toUrl.pathname)}`, ); @@ -60,7 +76,7 @@ async function copyDirectory(fromUrl, toUrl, projectDirectory) { } await copyFile(fileUrl, targetUrl, projectDirectory); - }); + }, 8); } } catch (e) { console.error("ERROR", e); @@ -71,10 +87,10 @@ const copyAssets = async (compilation) => { const copyPlugins = compilation.config.plugins.filter((plugin) => plugin.type === "copy"); const { projectDirectory } = compilation.context; - await asyncForEach(copyPlugins, async (plugin) => { + await mapWithConcurrency(copyPlugins, async (plugin) => { const locations = await plugin.provider(compilation); - await asyncForEach(locations, async (location) => { + await mapWithConcurrency(locations, async (location) => { const { from, to } = location; if (from.pathname.endsWith("/")) { @@ -82,8 +98,8 @@ const copyAssets = async (compilation) => { } else { await copyFile(from, to, projectDirectory); } - }); - }); + }, 4); + }, 2); }; export { copyAssets }; diff --git a/packages/cli/src/plugins/resource/plugin-standard-css.js b/packages/cli/src/plugins/resource/plugin-standard-css.js index 18919a714..5f1b35d57 100644 --- a/packages/cli/src/plugins/resource/plugin-standard-css.js +++ b/packages/cli/src/plugins/resource/plugin-standard-css.js @@ -5,6 +5,7 @@ * */ import fs from "node:fs"; +import { copyFileWithRetry } from "../../lib/fs-utils.js"; import path from "node:path"; import { parse, walk } from "css-tree"; import { hashString } from "../../lib/hashing-utils.js"; @@ -134,7 +135,11 @@ function bundleCss(body, sourceUrl, compilation, workingUrl) { recursive: true, }); - fs.promises.copyFile(resolvedUrl, new URL(`.${finalValue}`, outputDir)); + // Use copy helper with retry to avoid intermittent EBUSY on Windows CI + // bundleCss is synchronous, so we don't await here; log any copy errors. + copyFileWithRetry(resolvedUrl, new URL(`.${finalValue}`, outputDir)).catch((err) => + console.error('ERROR copying asset during CSS bundling', resolvedUrl.href, err), + ); } optimizedCss += `url('${basePath}${finalValue}')`; diff --git a/packages/init/test/cases/develop.default/develop.default.spec.js b/packages/init/test/cases/develop.default/develop.default.spec.js index c4c715724..67601d6dd 100644 --- a/packages/init/test/cases/develop.default/develop.default.spec.js +++ b/packages/init/test/cases/develop.default/develop.default.spec.js @@ -15,7 +15,7 @@ import chai from "chai"; import { JSDOM } from "jsdom"; import path from "node:path"; import { Runner } from "gallinago"; -import { runSmokeTest } from "../../../../../test/smoke-test.js"; +import { runSmokeTest, safeTeardown } from "../../../../../test/smoke-test.js"; import { fileURLToPath } from "node:url"; const expect = chai.expect; @@ -93,8 +93,11 @@ describe("Initialize a new Greenwood project: ", function () { }); }); - after(function () { + after(async function () { runner.stopCommand(); - runner.teardown([initOutputPath]); + // give the process a moment to release file handles (helps on Windows) + await new Promise((resolve) => setTimeout(resolve, 500)); + // increase attempts and baseDelay for teardown to be more resilient on CI + await safeTeardown(runner, [initOutputPath], 6, 200); }); }); diff --git a/test/smoke-test.js b/test/smoke-test.js index c5049f9d9..b5f5e7476 100644 --- a/test/smoke-test.js +++ b/test/smoke-test.js @@ -252,3 +252,71 @@ async function runSmokeTest(testCases, label) { } export { runSmokeTest }; + +// Shared helper for tests to teardown runner paths safely with retries on transient Windows file locks +import fsPromises from 'node:fs/promises'; + +async function safeRmPath(pathStr, attempts = 5, baseDelay = 100, verbose = false) { + for (let i = 0; i < attempts; i++) { + try { + if (verbose) console.info(`[safeRmPath] rm attempt ${i + 1}/${attempts} ${pathStr}`); + // Node 14+ supports fs.promises.rm with recursive and force options + await fsPromises.rm(pathStr, { recursive: true, force: false }); + if (verbose) console.info(`[safeRmPath] rm succeeded ${pathStr}`); + return; + } catch (err) { + if (verbose) console.warn(`[safeRmPath] rm attempt ${i + 1} failed for ${pathStr}:`, err && err.code ? err.code : err); + + if ((err.code === 'EBUSY' || err.code === 'EPERM' || err.code === 'EACCES') && i < attempts - 1) { + const delay = baseDelay * Math.pow(2, i); + await new Promise((resolve) => setTimeout(resolve, delay)); + continue; + } + + // rethrow if non-retryable or last attempt + throw err; + } + } +} + +async function safeTeardown(runner, paths = [], attempts = 5, baseDelay = 100) { + const verbose = process.env.GWD_RETRY_VERBOSE === 'true'; + + for (let i = 0; i < attempts; i++) { + try { + if (verbose) console.info(`[safeTeardown] attempt ${i + 1}/${attempts} teardown paths=${JSON.stringify(paths)}`); + runner.teardown(paths); + if (verbose) console.info(`[safeTeardown] teardown succeeded`); + return; + } catch (err) { + if (verbose) console.warn(`[safeTeardown] attempt ${i + 1} failed:`, err && err.code ? err.code : err); + + if ((err.code === 'EBUSY' || err.code === 'EPERM' || err.code === 'EACCES') && i < attempts - 1) { + const delay = baseDelay * Math.pow(2, i); + if (verbose) console.info(`[safeTeardown] retrying in ${delay}ms`); + await new Promise((resolve) => setTimeout(resolve, delay)); + continue; + } + + // If runner.teardown failed and we've exhausted retries, attempt per-path removal as fallback + if (err.code === 'EBUSY' || err.code === 'EPERM' || err.code === 'EACCES') { + if (verbose) console.info('[safeTeardown] falling back to per-path rm'); + for (const p of paths) { + try { + await safeRmPath(p, attempts, baseDelay, verbose); + } catch (rmErr) { + // if fallback removal fails, surface original error for debugging + if (verbose) console.error('[safeTeardown] fallback rm failed for', p, rmErr); + throw rmErr; + } + } + + return; + } + + throw err; + } + } +} + +export { safeTeardown };