From 0e563f485bee9277f45c0c8e3ab39cbcbef4ca83 Mon Sep 17 00:00:00 2001 From: timonwong Date: Wed, 23 Sep 2026 09:24:00 +0000 Subject: [PATCH 1/6] feat!: compile build info into the package as BUILD_INFO readBuildInfo() read dist/build-info.json with node:fs, which ties the build info to Node and to a file that has to travel with the module. `npm run build` now turns dist/build-info.json into a typed BUILD_INFO constant, exported from the package entry, and fails if the JSON does not describe dist/shellcheck.wasm or the pinned ShellCheck version. build-info.json stays a GitHub Release asset but leaves the tarball. BREAKING CHANGE: the ./build-info subpath and readBuildInfo() are removed; import BUILD_INFO from the package entry instead. --- package.json | 8 ++--- scripts/gen-build-info.mjs | 72 ++++++++++++++++++++++++++++++++++++++ src/build-info.ts | 19 ++-------- src/index.ts | 2 ++ src/node.ts | 6 ++-- test/package.test.ts | 51 ++++++++++++--------------- 6 files changed, 105 insertions(+), 53 deletions(-) create mode 100644 scripts/gen-build-info.mjs diff --git a/package.json b/package.json index 0dc9dd9..581dcfb 100644 --- a/package.json +++ b/package.json @@ -9,6 +9,7 @@ }, "files": [ "dist", + "!dist/build-info.json", "README.md", "LICENSE" ], @@ -24,10 +25,6 @@ "default": "./dist/node.js" }, "./shellcheck.wasm": "./dist/shellcheck.wasm", - "./build-info": { - "types": "./dist/build-info.d.ts", - "default": "./dist/build-info.js" - }, "./package.json": "./package.json" }, "publishConfig": { @@ -35,7 +32,8 @@ }, "scripts": { "gen:version": "node scripts/gen-version.mjs", - "build": "npm run gen:version && tsc -p tsconfig.build.json", + "gen:build-info": "node scripts/gen-build-info.mjs", + "build": "npm run gen:version && npm run gen:build-info && tsc -p tsconfig.build.json", "build:wasm": "scripts/build-wasm.sh", "fetch:native": "scripts/fetch-native-shellcheck.sh", "lint": "oxlint", diff --git a/scripts/gen-build-info.mjs b/scripts/gen-build-info.mjs new file mode 100644 index 0000000..51badcf --- /dev/null +++ b/scripts/gen-build-info.mjs @@ -0,0 +1,72 @@ +import { createHash } from "node:crypto"; +import { mkdirSync, readFileSync, writeFileSync } from "node:fs"; +import { dirname, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +const repoRoot = resolve(dirname(fileURLToPath(import.meta.url)), ".."); +const infoFile = resolve(repoRoot, "dist/build-info.json"); +const wasmFile = resolve(repoRoot, "dist/shellcheck.wasm"); +const versionFile = resolve(repoRoot, "buildtools/wasm/version.txt"); +const outFile = resolve(repoRoot, "src/generated/build-info.ts"); + +/** Field name → expected `typeof`, in the order `src/build-info.ts` declares them. */ +const FIELDS = { + shellcheckVersion: "string", + ghcWasmMetaCommit: "string", + ghcVersion: "string", + cabalVersion: "string", + wasmOptVersion: "string", + cflags: "string", + targetFeatures: "object", + sha256: "string", + size: "number", +}; + +function fail(message) { + console.error(`gen-build-info: ${message}`); + process.exit(1); +} + +let info; +let wasm; +try { + info = JSON.parse(readFileSync(infoFile, "utf8")); + wasm = readFileSync(wasmFile); +} catch (error) { + fail( + `${error.message}\nPut the artifact and its build-info.json in dist/: \`npm run build:wasm\` (Docker) or download both from a GitHub Release.`, + ); +} + +const keys = Object.keys(info).toSorted(); +if (JSON.stringify(keys) !== JSON.stringify(Object.keys(FIELDS).toSorted())) { + fail(`${infoFile} has fields ${keys.join(", ")}; update src/build-info.ts and this script`); +} +for (const [field, type] of Object.entries(FIELDS)) { + if (typeof info[field] !== type) fail(`${field} in ${infoFile} is not a ${type}`); +} +if (!Array.isArray(info.targetFeatures) || info.targetFeatures.some((f) => typeof f !== "string")) { + fail(`targetFeatures in ${infoFile} is not a list of strings`); +} + +// A build-info.json from a different build than the artifact would describe the wrong module. +const sha256 = createHash("sha256").update(wasm).digest("hex"); +if (info.sha256 !== sha256 || info.size !== wasm.byteLength) { + fail( + `${infoFile} describes a different shellcheck.wasm (sha256 ${info.sha256}, actual ${sha256})`, + ); +} +const version = readFileSync(versionFile, "utf8").trim(); +if (info.shellcheckVersion !== version) { + fail(`${infoFile} is for ShellCheck ${info.shellcheckVersion}, version.txt pins ${version}`); +} + +const ordered = Object.fromEntries(Object.keys(FIELDS).map((field) => [field, info[field]])); +mkdirSync(dirname(outFile), { recursive: true }); +writeFileSync( + outFile, + `import type { BuildInfo } from "../build-info.js"; + +export const BUILD_INFO: BuildInfo = ${JSON.stringify(ordered, null, 2)}; +`, +); diff --git a/src/build-info.ts b/src/build-info.ts index dc4c73e..bd06fe0 100644 --- a/src/build-info.ts +++ b/src/build-info.ts @@ -1,6 +1,4 @@ -import { readFileSync } from "node:fs"; - -/** How the bundled shellcheck.wasm was built; mirrors `dist/build-info.json`. */ +/** How the bundled shellcheck.wasm was built, as recorded by the wasm build. */ export interface BuildInfo { /** ShellCheck release tag, e.g. `v0.11.0`. */ readonly shellcheckVersion: string; @@ -10,19 +8,8 @@ export interface BuildInfo { readonly wasmOptVersion: string; readonly cflags: string; readonly targetFeatures: readonly string[]; + /** Hex sha256 of shellcheck.wasm. */ readonly sha256: string; + /** Size of shellcheck.wasm in bytes. */ readonly size: number; } - -let cached: BuildInfo | undefined; - -/** - * Reads the build info shipped next to shellcheck.wasm. Kept free of runner imports so - * Hosts can read it without pulling the runner into their bundle. - */ -export function readBuildInfo(): BuildInfo { - cached ??= JSON.parse( - readFileSync(new URL("./build-info.json", import.meta.url), "utf8"), - ) as BuildInfo; - return cached; -} diff --git a/src/index.ts b/src/index.ts index d086d0b..c24f58b 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,6 +1,8 @@ export { run } from "./runner.js"; export type { RunOptions, RunResult } from "./runner.js"; export { SHELLCHECK_VERSION } from "./generated/version.js"; +export { BUILD_INFO } from "./generated/build-info.js"; +export type { BuildInfo } from "./build-info.js"; /** URL of the bundled shellcheck.wasm, resolved relative to this module. */ export const wasmUrl: URL = new URL("./shellcheck.wasm", import.meta.url); diff --git a/src/node.ts b/src/node.ts index df25877..fa3b236 100644 --- a/src/node.ts +++ b/src/node.ts @@ -2,12 +2,10 @@ import { readFile } from "node:fs/promises"; import { fileURLToPath } from "node:url"; import { wasmUrl } from "./index.js"; -export { run, SHELLCHECK_VERSION, wasmUrl } from "./index.js"; -export type { RunOptions, RunResult } from "./index.js"; +export { BUILD_INFO, run, SHELLCHECK_VERSION, wasmUrl } from "./index.js"; +export type { BuildInfo, RunOptions, RunResult } from "./index.js"; export { createReadOnlyPreopen } from "./host-preopen.js"; export type { ReadOnlyPreopen } from "./host-preopen.js"; -export { readBuildInfo } from "./build-info.js"; -export type { BuildInfo } from "./build-info.js"; /** Filesystem path of the bundled shellcheck.wasm. */ export const wasmPath: string = fileURLToPath(wasmUrl); diff --git a/test/package.test.ts b/test/package.test.ts index 878301c..61338cf 100644 --- a/test/package.test.ts +++ b/test/package.test.ts @@ -1,4 +1,5 @@ import { spawnSync } from "node:child_process"; +import { createHash } from "node:crypto"; import { existsSync, readFileSync } from "node:fs"; import path from "node:path"; import { describe, expect, it } from "vitest"; @@ -11,6 +12,7 @@ import { repoRoot, skipHint, wasmHint, + wasmPath, } from "./helpers.js"; const packageName = "@vscode-shellcheck/shellcheck-wasm"; @@ -61,8 +63,7 @@ describe("package", () => { "dist/index.d.ts", "dist/node.js", "dist/node.d.ts", - "dist/build-info.js", - "dist/build-info.d.ts", + "dist/generated/build-info.js", "package.json", "README.md", ]), @@ -80,8 +81,8 @@ describe("package", () => { }, ); - it.skipIf(skipHint(!hasBuildInfo, buildInfoHint))("ships build-info.json", () => { - expect(npmPackDryRun().files.map((file) => file.path)).toContain("dist/build-info.json"); + it("does not ship build-info.json, which is compiled in", () => { + expect(npmPackDryRun().files.map((file) => file.path)).not.toContain("dist/build-info.json"); }); }); @@ -102,27 +103,21 @@ describe("package", () => { expect(typeof node.loadModule).toBe("function"); expect(typeof node.createReadOnlyPreopen).toBe("function"); expect(node.wasmPath).toBe(path.join(repoRoot, "dist", "shellcheck.wasm")); - expect(typeof node.readBuildInfo).toBe("function"); + expect(node.BUILD_INFO.shellcheckVersion).toBe(SHELLCHECK_VERSION); }); - it("keeps dist/build-info.js free of the runner", () => { - const source = readFileSync(path.join(repoRoot, "dist", "build-info.js"), "utf8"); - const imports = [...source.matchAll(/^import .* from "([^"]+)";$/gm)].map((m) => m[1]); - expect(imports).toEqual(["node:fs"]); - }); - - it.skipIf(skipHint(!hasBuildInfo, buildInfoHint))( - "reads build info for the same ShellCheck through the published entry point", + it.skipIf(skipHint(!hasBuildInfo, buildInfoHint) || skipHint(!hasWasm, wasmHint))( + "compiles in the build info of the artifact", () => { const output = nodeEval( - `import { readBuildInfo } from ${JSON.stringify(`${packageName}/build-info`)}; - const info = readBuildInfo(); - console.log(JSON.stringify([info === readBuildInfo(), info]));`, + `import { BUILD_INFO } from ${JSON.stringify(packageName)}; + console.log(JSON.stringify(BUILD_INFO));`, ); - const [cached, info] = JSON.parse(output) as [boolean, unknown]; - expect(cached).toBe(true); + const info = JSON.parse(output) as { sha256: string; size: number }; expect(info).toEqual(JSON.parse(readFileSync(buildInfoPath, "utf8"))); - expect((info as { shellcheckVersion: string }).shellcheckVersion).toBe(SHELLCHECK_VERSION); + const wasm = readFileSync(wasmPath); + expect(info.sha256).toBe(createHash("sha256").update(wasm).digest("hex")); + expect(info.size).toBe(wasm.byteLength); }, ); @@ -131,25 +126,25 @@ describe("package", () => { `console.log(JSON.stringify([ import.meta.resolve(${JSON.stringify(packageName)}), import.meta.resolve(${JSON.stringify(`${packageName}/node`)}), - import.meta.resolve(${JSON.stringify(`${packageName}/build-info`)}), import.meta.resolve(${JSON.stringify(`${packageName}/shellcheck.wasm`)}), import.meta.resolve(${JSON.stringify(`${packageName}/package.json`)}), ]))`, ); - const [index, node, buildInfo, wasm, pkg] = JSON.parse(resolved) as string[]; + const [index, node, wasm, pkg] = JSON.parse(resolved) as string[]; expect(index).toBe(`file://${path.join(repoRoot, "dist", "index.js")}`); expect(node).toBe(`file://${path.join(repoRoot, "dist", "node.js")}`); - expect(buildInfo).toBe(`file://${path.join(repoRoot, "dist", "build-info.js")}`); expect(wasm).toBe(`file://${path.join(repoRoot, "dist", "shellcheck.wasm")}`); expect(pkg).toBe(`file://${path.join(repoRoot, "package.json")}`); }); - it("no longer exports build-info.json", () => { - const code = nodeEval( - `try { import.meta.resolve(${JSON.stringify(`${packageName}/build-info.json`)}); console.log("resolved"); } - catch (error) { console.log(error.code); }`, - ); - expect(code).toBe("ERR_PACKAGE_PATH_NOT_EXPORTED"); + it("no longer exports build-info.json or ./build-info", () => { + for (const subpath of ["/build-info.json", "/build-info"]) { + const code = nodeEval( + `try { import.meta.resolve(${JSON.stringify(packageName + subpath)}); console.log("resolved"); } + catch (error) { console.log(error.code); }`, + ); + expect(code, subpath).toBe("ERR_PACKAGE_PATH_NOT_EXPORTED"); + } }); it.skipIf(skipHint(!hasWasm, wasmHint))( From ae2f13b62baa14f060fe0a56d2b46ccbf15d91f5 Mon Sep 17 00:00:00 2001 From: timonwong Date: Wed, 23 Sep 2026 09:24:34 +0000 Subject: [PATCH 2/6] feat!: lint in a host-supplied worker through an injected file system The extension is moving to lint documents of any URI scheme by reading them through vscode.workspace.fs, and later to run on the web, so the package can no longer reach for node:fs or hand back a synchronous runner that blocks the caller. createShellCheck({ module, createWorker }) now owns one Worker at a time and runs lints through it in FIFO order. Each lint may carry a ShellCheckFileSystem (async stat, readFile, readDirectory) that is mounted read-only at guest `/`. The guest's synchronous WASI calls reach it through a SharedArrayBuffer bridge: the Worker posts a request and waits in Atomics.wait while the caller's thread runs the async call, with payloads over 1 MiB sent in chunks and answers cached for the rest of the lint. `..` is folded lexically and may not leave `/`; symlink containment is now the file system's job. An AbortSignal drops a queued lint or terminates the Worker of the running one; a lost Worker is replaced on the next lint. The Worker side ships as `./worker` (startWorker), and no shipped module imports a Node built-in. BREAKING CHANGE: the `./node` entry (loadModule, createReadOnlyPreopen, wasmPath) and the synchronous run() export are removed. Use createShellCheck() with a Worker whose entry calls startWorker(), and pass the files ShellCheck may read as LintRequest.fs. LintResult carries stdout and stderr as strings. --- .oxlintrc.jsonc | 4 + package.json | 8 +- src/bridge.ts | 107 +++++++ src/client.ts | 350 +++++++++++++++++++++++ src/file-system.ts | 32 +++ src/host-preopen.ts | 495 --------------------------------- src/index.ts | 21 +- src/node.ts | 16 -- src/preopen.ts | 319 +++++++++++++++++++++ src/protocol.ts | 68 +++++ src/runner.ts | 2 +- src/worker.ts | 55 ++++ test/bridge.test.ts | 131 +++++++++ test/helpers.ts | 89 +++++- test/lint.test.ts | 96 +++++++ test/package.test.ts | 215 +++++++++----- test/parity.test.ts | 75 ++--- test/preopen.test.ts | 455 ++++++++++++++---------------- test/runner.test.ts | 86 ------ test/scheduler.test.ts | 204 ++++++++++++++ test/support/bridge-worker.mjs | 30 ++ test/support/crash-worker.mjs | 16 ++ test/support/file-systems.ts | 132 +++++++++ test/support/node-worker.mjs | 7 + 24 files changed, 2044 insertions(+), 969 deletions(-) create mode 100644 src/bridge.ts create mode 100644 src/client.ts create mode 100644 src/file-system.ts delete mode 100644 src/host-preopen.ts delete mode 100644 src/node.ts create mode 100644 src/preopen.ts create mode 100644 src/protocol.ts create mode 100644 src/worker.ts create mode 100644 test/bridge.test.ts create mode 100644 test/lint.test.ts delete mode 100644 test/runner.test.ts create mode 100644 test/scheduler.test.ts create mode 100644 test/support/bridge-worker.mjs create mode 100644 test/support/crash-worker.mjs create mode 100644 test/support/file-systems.ts create mode 100644 test/support/node-worker.mjs diff --git a/.oxlintrc.jsonc b/.oxlintrc.jsonc index 7d3d2c1..93fdb1c 100644 --- a/.oxlintrc.jsonc +++ b/.oxlintrc.jsonc @@ -9,5 +9,9 @@ "builtin": true, "es2022": true, }, + "rules": { + // Aimed at window.postMessage; Worker and MessagePort postMessage take no targetOrigin. + "unicorn/require-post-message-target-origin": "off", + }, "ignorePatterns": ["dist/", "node_modules/", ".cache/", "src/generated/"], } diff --git a/package.json b/package.json index 581dcfb..62b2d0e 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "@vscode-shellcheck/shellcheck-wasm", "version": "0.1.1", - "description": "ShellCheck compiled to WebAssembly (WASI command module) with a minimal runner", + "description": "ShellCheck compiled to WebAssembly (WASI command module) with a Worker-based runner", "license": "GPL-3.0-or-later", "repository": { "type": "git", @@ -20,9 +20,9 @@ "types": "./dist/index.d.ts", "default": "./dist/index.js" }, - "./node": { - "types": "./dist/node.d.ts", - "default": "./dist/node.js" + "./worker": { + "types": "./dist/worker.d.ts", + "default": "./dist/worker.js" }, "./shellcheck.wasm": "./dist/shellcheck.wasm", "./package.json": "./package.json" diff --git a/src/bridge.ts b/src/bridge.ts new file mode 100644 index 0000000..1892c80 --- /dev/null +++ b/src/bridge.ts @@ -0,0 +1,107 @@ +import type { FileStat, FileType } from "./file-system.js"; +import { + CHUNK_BYTES, + ERRNO_SUCCESS, + HEADER_BYTES, + LENGTH, + PENDING, + STATE, + STATUS, + TOTAL, + type FileSystemOp, + type FromWorker, +} from "./protocol.js"; + +export type DirectoryEntry = readonly [name: string, type: FileType]; + +/** The Host's file system as the guest needs it: blocking, with a WASI errno for failures. */ +export interface SyncFileSystem { + stat(path: string): FileStat | number; + readFile(path: string): Uint8Array | number; + readDirectory(path: string): readonly DirectoryEntry[] | number; +} + +const decoder = new TextDecoder(); + +/** + * How long the Worker polls for an answer before sleeping in `Atomics.wait`. Answers + * usually take well under a millisecond, and a thread that sleeps for each of them runs + * the guest measurably slower after waking (about 4% of a 100 ms lint in `npm run bench`). + */ +const SPIN_MS = 1; + +function decodeJson(bytes: Uint8Array): T { + return JSON.parse(decoder.decode(bytes)) as T; +} + +/** + * Worker side of the bridge, for one lint. Results, failures included, are cached for the + * whole lint: ShellCheck probes the same rc paths repeatedly, and every miss costs a round + * trip to the caller's thread. A fresh lint gets a fresh bridge, so edits between lints show. + */ +export class Bridge implements SyncFileSystem { + private readonly control: Int32Array; + private readonly data: Uint8Array; + private readonly cache = new Map(); + + constructor( + shared: SharedArrayBuffer, + private readonly post: (message: FromWorker) => void, + ) { + this.control = new Int32Array(shared, 0, HEADER_BYTES / Int32Array.BYTES_PER_ELEMENT); + this.data = new Uint8Array(shared, HEADER_BYTES, CHUNK_BYTES); + } + + stat(path: string): FileStat | number { + return this.cached("stat", path, decodeJson); + } + + readFile(path: string): Uint8Array | number { + return this.cached("readFile", path, (bytes) => bytes); + } + + readDirectory(path: string): readonly DirectoryEntry[] | number { + return this.cached("readDirectory", path, decodeJson); + } + + private cached(op: FileSystemOp, path: string, decode: (bytes: Uint8Array) => T): T | number { + const key = `${op}\0${path}`; + if (this.cache.has(key)) return this.cache.get(key) as T | number; + const bytes = this.request({ type: "fs", op, path }); + const value = typeof bytes === "number" ? bytes : decode(bytes); + this.cache.set(key, value); + return value; + } + + private request(message: FromWorker): Uint8Array | number { + this.exchange(message); + const status = this.control[STATUS]!; + if (status !== ERRNO_SUCCESS) return status; + const out = new Uint8Array(this.control[TOTAL]!); + let received = this.take(out, 0); + while (received < out.byteLength) { + this.exchange({ type: "fs-next" }); + received += this.take(out, received); + } + return out; + } + + private exchange(message: FromWorker): void { + // Reset before posting: the caller may answer before this thread reaches wait(). + Atomics.store(this.control, STATE, PENDING); + this.post(message); + const spinUntil = performance.now() + SPIN_MS; + while (Atomics.load(this.control, STATE) === PENDING && performance.now() < spinUntil) { + // Busy-wait; see SPIN_MS. + } + while (Atomics.load(this.control, STATE) === PENDING) { + Atomics.wait(this.control, STATE, PENDING); + } + } + + private take(out: Uint8Array, offset: number): number { + const length = this.control[LENGTH]!; + out.set(this.data.subarray(0, length), offset); + return length; + } +} diff --git a/src/client.ts b/src/client.ts new file mode 100644 index 0000000..bf27796 --- /dev/null +++ b/src/client.ts @@ -0,0 +1,350 @@ +import type { FileType, ShellCheckFileSystem } from "./file-system.js"; +import { + CHUNK_BYTES, + ERRNO_IO, + ERRNO_SUCCESS, + HEADER_BYTES, + LENGTH, + READY, + STATE, + STATUS, + TOTAL, + errnoOf, + type FileSystemOp, + type FromWorker, + type LintMessage, + type ToWorker, +} from "./protocol.js"; + +/** + * A Worker running `startWorker` from `@vscode-shellcheck/shellcheck-wasm/worker`, seen + * from the thread that created it. `onMessage` receives the posted value itself (unwrap + * `MessageEvent.data` for a web Worker). + */ +export interface WorkerPort { + postMessage(message: unknown): void; + onMessage(listener: (message: unknown) => void): void; + onError(listener: (error: unknown) => void): void; + /** Omit where the platform has no exit event, such as a web Worker. */ + onExit?(listener: (exitCode: number) => void): void; + terminate(): void | Promise; +} + +export interface ShellCheckOptions { + /** The compiled `shellcheck.wasm`; sent to each Worker once, when it starts. */ + module: WebAssembly.Module | PromiseLike; + /** Starts a Worker. Called lazily for the first lint and again after a Worker is lost. */ + createWorker(): WorkerPort; +} + +export interface LintRequest { + /** ShellCheck arguments, without argv[0]. Example: `["-f", "json1", "-s", "bash", "-"]`. */ + args: readonly string[]; + /** Script or other data for stdin. Default: empty. */ + stdin?: string | Uint8Array; + /** + * Environment for the guest. `PWD` is the guest working directory: the GHC RTS chdir()s + * there at startup, so it must name a directory in `fs` or be left unset. Pointing it + * elsewhere makes the RTS print `hs_init_ghc: chdir(...) failed` and the lint ends with + * no stdout. + */ + env?: Readonly>; + /** Mounted read-only at guest `/`. Without it ShellCheck sees no files at all. */ + fs?: ShellCheckFileSystem; +} + +export interface LintResult { + stdout: string; + stderr: string; + /** ShellCheck's own: `0` no findings, `1` findings, `2` or higher for invalid input. */ + exitCode: number; +} + +export interface LintOptions { + /** + * Aborting a queued lint drops it; aborting the running one terminates its Worker, and + * the next lint starts a new one. Either way the promise rejects with `signal.reason`. + */ + signal?: AbortSignal; +} + +export interface ShellCheck { + /** Queues a lint; lints run one at a time in call order. */ + lint(request: LintRequest, options?: LintOptions): Promise; + /** Rejects pending lints and terminates the Worker. Further lints reject. */ + dispose(): Promise; +} + +interface Job { + readonly request: LintRequest; + readonly resolve: (result: LintResult) => void; + readonly reject: (error: unknown) => void; + session?: Session; +} + +const encoder = new TextEncoder(); +const FILE_TYPES: ReadonlySet = new Set(["file", "directory", "other"]); + +function abortReason(signal: AbortSignal): unknown { + return signal.reason ?? new DOMException("The operation was aborted.", "AbortError"); +} + +function integer(value: unknown): number { + if (typeof value !== "number" || !Number.isFinite(value)) { + throw new TypeError(`expected a finite number, got ${String(value)}`); + } + return Math.max(0, Math.trunc(value)); +} + +function fileType(value: unknown): FileType { + if (!FILE_TYPES.has(value)) throw new TypeError(`unknown file type ${String(value)}`); + return value as FileType; +} + +/** Runs one file-system call and encodes the answer as the bridge payload. */ +async function call(fs: ShellCheckFileSystem, op: FileSystemOp, path: string): Promise { + switch (op) { + case "stat": { + const { type, size, mtime } = await fs.stat(path); + const stat = { type: fileType(type), size: integer(size), mtime: integer(mtime) }; + return encoder.encode(JSON.stringify(stat)); + } + case "readFile": { + const data = await fs.readFile(path); + if (!(data instanceof Uint8Array)) throw new TypeError("readFile must return a Uint8Array"); + return data; + } + case "readDirectory": { + const entries = Array.from(await fs.readDirectory(path), ([name, type]) => { + if (typeof name !== "string") throw new TypeError("entry names must be strings"); + return [name, fileType(type)]; + }); + return encoder.encode(JSON.stringify(entries)); + } + } +} + +function describe(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} + +/** One Worker and its bridge memory, which lives exactly as long as the Worker. */ +class Session { + closed = false; + private readonly control: Int32Array; + private readonly data: Uint8Array; + private active: Job | undefined; + private payload: Uint8Array | undefined; + private sent = 0; + + constructor( + private readonly port: WorkerPort, + module: WebAssembly.Module, + ) { + const shared = new SharedArrayBuffer(HEADER_BYTES + CHUNK_BYTES); + this.control = new Int32Array(shared, 0, HEADER_BYTES / Int32Array.BYTES_PER_ELEMENT); + this.data = new Uint8Array(shared, HEADER_BYTES, CHUNK_BYTES); + port.onMessage((message) => this.receive(message as FromWorker)); + port.onError((error) => { + this.fail(new Error(`ShellCheck worker failed: ${describe(error)}`, { cause: error })); + }); + port.onExit?.((exitCode) => { + this.fail(new Error(`ShellCheck worker exited unexpectedly with code ${exitCode}`)); + }); + this.post({ type: "init", module, shared }); + } + + lint(job: Job): void { + if (this.closed) throw new Error("ShellCheck worker is gone"); + const { args, stdin = "", env = {}, fs } = job.request; + const message: LintMessage = { + type: "lint", + args: [...args], + stdin, + env: { ...env }, + mounted: fs !== undefined, + }; + this.post(message); + this.active = job; + job.session = this; + } + + terminate(): Promise { + if (this.closed) return Promise.resolve(); + this.closed = true; + this.active = undefined; + return Promise.resolve(this.port.terminate()).then( + () => undefined, + () => undefined, + ); + } + + private post(message: ToWorker): void { + this.port.postMessage(message); + } + + private fail(error: Error): void { + if (this.closed) return; + const job = this.active; + void this.terminate(); + job?.reject(error); + } + + private receive(message: FromWorker): void { + if (this.closed) return; + switch (message.type) { + case "fs": + void this.serve(message.op, message.path); + return; + case "fs-next": + this.answer(); + return; + case "done": + case "failed": { + const job = this.active; + this.active = undefined; + if (message.type === "done") { + const { stdout, stderr, exitCode } = message; + job?.resolve({ stdout, stderr, exitCode }); + } else { + job?.reject(new Error(`ShellCheck failed: ${message.name}: ${message.message}`)); + } + return; + } + } + } + + private async serve(op: FileSystemOp, path: string): Promise { + const job = this.active; + let status = ERRNO_SUCCESS; + let payload: Uint8Array = new Uint8Array(); + try { + const fs = job?.request.fs; + if (fs === undefined) throw new Error("no file system for this lint"); + payload = await call(fs, op, path); + } catch (error) { + status = errnoOf(error); + } + // The lint may have been aborted, and its Worker terminated, while fs was busy. + if (this.closed || this.active !== job) return; + this.control[STATUS] = status; + this.control[TOTAL] = payload.byteLength; + this.payload = payload; + this.sent = 0; + this.answer(); + } + + /** Writes the next chunk of the current payload and wakes the Worker. */ + private answer(): void { + const payload = this.payload; + if (payload === undefined) { + this.control[STATUS] = ERRNO_IO; + this.control[LENGTH] = 0; + } else { + const chunk = payload.subarray(this.sent, this.sent + CHUNK_BYTES); + this.data.set(chunk); + this.control[LENGTH] = chunk.byteLength; + this.sent += chunk.byteLength; + if (this.sent >= payload.byteLength) this.payload = undefined; + } + Atomics.store(this.control, STATE, READY); + Atomics.notify(this.control, STATE); + } +} + +class Scheduler implements ShellCheck { + private readonly queue: Job[] = []; + private running: Job | undefined; + private session: Session | undefined; + private spawning: Promise | undefined; + private disposed = false; + + constructor(private readonly options: ShellCheckOptions) {} + + lint(request: LintRequest, { signal }: LintOptions = {}): Promise { + if (this.disposed) return Promise.reject(new Error("ShellCheck instance is disposed")); + if (signal?.aborted) return Promise.reject(abortReason(signal)); + return new Promise((resolve, reject) => { + let settled = false; + const settle = (): boolean => { + if (settled) return false; + settled = true; + signal?.removeEventListener("abort", onAbort); + if (this.running === job) { + this.running = undefined; + void this.pump(); + } + return true; + }; + const job: Job = { + request, + resolve: (result) => settle() && resolve(result), + reject: (error) => settle() && reject(error), + }; + const onAbort = (): void => { + const queued = this.queue.indexOf(job); + if (queued !== -1) this.queue.splice(queued, 1); + if (this.running === job) void job.session?.terminate(); + job.reject(abortReason(signal!)); + }; + signal?.addEventListener("abort", onAbort, { once: true }); + this.queue.push(job); + void this.pump(); + }); + } + + async dispose(): Promise { + if (this.disposed) return; + this.disposed = true; + const error = new Error("ShellCheck instance is disposed"); + for (const job of [...this.queue.splice(0), this.running]) job?.reject(error); + const session = this.session ?? (await this.spawning?.catch(() => undefined)); + this.session = undefined; + await session?.terminate(); + } + + private async pump(): Promise { + if (this.running !== undefined || this.disposed) return; + const job = this.queue.shift(); + if (job === undefined) return; + this.running = job; + try { + const session = await this.ensureSession(); + // Aborted or disposed while the Worker was starting. + if (this.running === job) session.lint(job); + } catch (error) { + job.reject(error); + } + } + + private ensureSession(): Promise { + if (this.session !== undefined && !this.session.closed) return Promise.resolve(this.session); + this.spawning ??= this.spawn().finally(() => { + this.spawning = undefined; + }); + return this.spawning; + } + + private async spawn(): Promise { + const module = await this.options.module; + if (this.disposed) throw new Error("ShellCheck instance is disposed"); + const port = this.options.createWorker(); + try { + this.session = new Session(port, module); + } catch (error) { + await port.terminate(); + throw error; + } + return this.session; + } +} + +/** + * Runs ShellCheck in a Worker the Host supplies, one lint at a time. The Worker starts on + * the first lint and is replaced after it is lost or a running lint is aborted. + * Scheduling policy beyond first-in-first-out, such as timeouts or dropping stale lints, + * belongs to the Host, which can pass `AbortSignal.timeout(ms)` or its own signal. + */ +export function createShellCheck(options: ShellCheckOptions): ShellCheck { + return new Scheduler(options); +} diff --git a/src/file-system.ts b/src/file-system.ts new file mode 100644 index 0000000..90a4aeb --- /dev/null +++ b/src/file-system.ts @@ -0,0 +1,32 @@ +export type FileType = "file" | "directory" | "other"; + +export interface FileStat { + readonly type: FileType; + /** Size in bytes. */ + readonly size: number; + /** Modification time in milliseconds since the Unix epoch. */ + readonly mtime: number; +} + +/** + * Codes a {@link ShellCheckFileSystem} puts on the `code` property of what it throws; they + * mirror `vscode.FileSystemError`. Anything else reaches ShellCheck as `EIO`. + */ +export type FileSystemErrorCode = + | "FileNotFound" + | "FileNotADirectory" + | "FileIsADirectory" + | "NoPermissions" + | "Unavailable"; + +/** + * The files ShellCheck may read during one lint, served on the thread that called `lint()`. + * Every `path` is a normalized absolute POSIX path in the guest's view, such as `/` or + * `/dir/.shellcheckrc`; the implementation decides what backs it and must keep symlinks + * from leading anywhere the guest may not see. + */ +export interface ShellCheckFileSystem { + stat(path: string): Promise; + readFile(path: string): Promise; + readDirectory(path: string): Promise>; +} diff --git a/src/host-preopen.ts b/src/host-preopen.ts deleted file mode 100644 index 8fbe599..0000000 --- a/src/host-preopen.ts +++ /dev/null @@ -1,495 +0,0 @@ -import fs from "node:fs"; -import path from "node:path"; -import { Fd, wasi } from "@bjorn3/browser_wasi_shim"; - -/** A read-only WASI preopen backed by a host directory. */ -export interface ReadOnlyPreopen extends Fd { - /** Closes every host file descriptor the guest left open under this preopen. */ - dispose(): void; -} - -const MAX_OPEN_FILES = 256; - -/** preview1 `lookupflags::symlink_follow`; the shim does not export it. */ -const LOOKUPFLAGS_SYMLINK_FOLLOW = 1; - -type FdstatResult = { ret: number; fdstat: wasi.Fdstat | null }; -type FilestatResult = { ret: number; filestat: wasi.Filestat | null }; -type ReadResult = { ret: number; data: Uint8Array }; -type WriteResult = { ret: number; nwritten: number }; -type SeekResult = { ret: number; offset: bigint }; -type OpenResult = { ret: number; fd_obj: Fd | null }; -type ReaddirResult = { ret: number; dirent: wasi.Dirent | null }; -type ReadlinkResult = { ret: number; data: string | null }; -type PrestatResult = { ret: number; prestat: wasi.Prestat | null }; -type Resolved = { ret: number; hostPath: string | null }; - -const EMPTY = new Uint8Array(); - -/** - * Node exceptions must become errnos: an exception escaping an fd method unwinds - * through the guest's wasm stack instead of reaching its error handling. - */ -function errnoOf(error: unknown): number { - switch ((error as NodeJS.ErrnoException | undefined)?.code) { - case "ENOENT": - return wasi.ERRNO_NOENT; - case "EACCES": - return wasi.ERRNO_ACCES; - case "EPERM": - return wasi.ERRNO_PERM; - case "ENOTDIR": - return wasi.ERRNO_NOTDIR; - case "EISDIR": - return wasi.ERRNO_ISDIR; - case "EINVAL": - return wasi.ERRNO_INVAL; - case "ELOOP": - return wasi.ERRNO_LOOP; - case "ENAMETOOLONG": - return wasi.ERRNO_NAMETOOLONG; - case "EMFILE": - return wasi.ERRNO_MFILE; - case "ENFILE": - return wasi.ERRNO_NFILE; - case "EBUSY": - return wasi.ERRNO_BUSY; - case "EBADF": - return wasi.ERRNO_BADF; - default: - return wasi.ERRNO_IO; - } -} - -function filetypeOf(stats: fs.BigIntStats): number { - if (stats.isDirectory()) return wasi.FILETYPE_DIRECTORY; - if (stats.isSymbolicLink()) return wasi.FILETYPE_SYMBOLIC_LINK; - if (stats.isFile()) return wasi.FILETYPE_REGULAR_FILE; - if (stats.isCharacterDevice()) return wasi.FILETYPE_CHARACTER_DEVICE; - if (stats.isBlockDevice()) return wasi.FILETYPE_BLOCK_DEVICE; - return wasi.FILETYPE_UNKNOWN; -} - -function filestatOf(stats: fs.BigIntStats): wasi.Filestat { - const filestat = new wasi.Filestat(stats.ino, filetypeOf(stats), stats.size); - filestat.dev = stats.dev; - filestat.nlink = stats.nlink; - filestat.atim = stats.atimeNs; - filestat.mtim = stats.mtimeNs; - filestat.ctim = stats.ctimeNs; - return filestat; -} - -function isInside(root: string, candidate: string): boolean { - return candidate === root || candidate.startsWith(root + path.sep); -} - -/** State shared by every fd derived from one preopen: the containment root and the open host fds. */ -class PreopenRoot { - readonly root: string; - private readonly openFiles = new Set(); - - constructor(hostDir: string) { - // Containment compares real paths, so the root has to be one too (e.g. /tmp on macOS). - this.root = fs.realpathSync.native(hostDir); - if (!fs.statSync(this.root).isDirectory()) { - throw new Error(`createReadOnlyPreopen: ${hostDir} is not a directory`); - } - } - - /** - * Maps a guest path (relative to `baseDir`, itself inside the root) to a host path. - * `..` is folded textually first, then the real path must stay under the root, so a - * symlink pointing outside is indistinguishable from a path the guest may not name. - */ - resolve(baseDir: string, guestPath: string): Resolved { - if (guestPath.includes("\0")) return { ret: wasi.ERRNO_INVAL, hostPath: null }; - // wasi-libc turns absolute paths into (preopen fd, relative path) itself; one that - // reaches us bypassed that and has no preopen to be relative to. - if (guestPath.startsWith("/")) return { ret: wasi.ERRNO_NOTCAPABLE, hostPath: null }; - - const hostPath = path.resolve(baseDir, guestPath); - if (!isInside(this.root, hostPath)) return { ret: wasi.ERRNO_NOTCAPABLE, hostPath: null }; - - let realPath: string; - try { - realPath = fs.realpathSync.native(hostPath); - } catch (error) { - return { ret: errnoOf(error), hostPath: null }; - } - if (!isInside(this.root, realPath)) return { ret: wasi.ERRNO_NOTCAPABLE, hostPath: null }; - return { ret: wasi.ERRNO_SUCCESS, hostPath }; - } - - openFile(hostPath: string): OpenResult { - if (this.openFiles.size >= MAX_OPEN_FILES) return { ret: wasi.ERRNO_NFILE, fd_obj: null }; - let fd: number; - try { - fd = fs.openSync(hostPath, fs.constants.O_RDONLY); - } catch (error) { - return { ret: errnoOf(error), fd_obj: null }; - } - let stats: fs.BigIntStats; - try { - stats = fs.fstatSync(fd, { bigint: true }); - } catch (error) { - // Not in openFiles yet, so dispose() could never reclaim it. - try { - fs.closeSync(fd); - } catch { - // The fstat errno is the one the guest needs. - } - return { ret: errnoOf(error), fd_obj: null }; - } - const file = new HostFile(this, fd, filetypeOf(stats)); - this.openFiles.add(file); - return { ret: wasi.ERRNO_SUCCESS, fd_obj: file }; - } - - release(file: HostFile): void { - this.openFiles.delete(file); - } - - dispose(): void { - for (const file of Array.from(this.openFiles)) file.fd_close(); - } -} - -class HostFile extends Fd { - private fd: number | null; - private position = 0n; - - constructor( - private readonly owner: PreopenRoot, - fd: number, - private readonly filetype: number, - ) { - super(); - this.fd = fd; - } - - override fd_fdstat_get(): FdstatResult { - return { ret: wasi.ERRNO_SUCCESS, fdstat: new wasi.Fdstat(this.filetype, 0) }; - } - - override fd_fdstat_set_flags(): number { - return wasi.ERRNO_SUCCESS; - } - - override fd_filestat_get(): FilestatResult { - if (this.fd === null) return { ret: wasi.ERRNO_BADF, filestat: null }; - try { - return { - ret: wasi.ERRNO_SUCCESS, - filestat: filestatOf(fs.fstatSync(this.fd, { bigint: true })), - }; - } catch (error) { - return { ret: errnoOf(error), filestat: null }; - } - } - - override fd_read(size: number): ReadResult { - const result = this.readAt(size, this.position); - this.position += BigInt(result.data.byteLength); - return result; - } - - override fd_pread(size: number, offset: bigint): ReadResult { - return this.readAt(size, offset); - } - - override fd_seek(offset: bigint, whence: number): SeekResult { - if (this.fd === null) return { ret: wasi.ERRNO_BADF, offset: 0n }; - let target: bigint; - switch (whence) { - case wasi.WHENCE_SET: - target = offset; - break; - case wasi.WHENCE_CUR: - target = this.position + offset; - break; - case wasi.WHENCE_END: - try { - target = fs.fstatSync(this.fd, { bigint: true }).size + offset; - } catch (error) { - return { ret: errnoOf(error), offset: 0n }; - } - break; - default: - return { ret: wasi.ERRNO_INVAL, offset: 0n }; - } - if (target < 0n) return { ret: wasi.ERRNO_INVAL, offset: 0n }; - this.position = target; - return { ret: wasi.ERRNO_SUCCESS, offset: target }; - } - - override fd_tell(): SeekResult { - if (this.fd === null) return { ret: wasi.ERRNO_BADF, offset: 0n }; - return { ret: wasi.ERRNO_SUCCESS, offset: this.position }; - } - - override fd_write(): WriteResult { - return { ret: wasi.ERRNO_ROFS, nwritten: 0 }; - } - - override fd_pwrite(): WriteResult { - return { ret: wasi.ERRNO_ROFS, nwritten: 0 }; - } - - override fd_allocate(): number { - return wasi.ERRNO_ROFS; - } - - override fd_filestat_set_size(): number { - return wasi.ERRNO_ROFS; - } - - override fd_filestat_set_times(): number { - return wasi.ERRNO_ROFS; - } - - override fd_close(): number { - const fd = this.fd; - this.fd = null; - this.owner.release(this); - if (fd === null) return wasi.ERRNO_BADF; - try { - fs.closeSync(fd); - } catch (error) { - return errnoOf(error); - } - return wasi.ERRNO_SUCCESS; - } - - private readAt(size: number, offset: bigint): ReadResult { - if (this.fd === null) return { ret: wasi.ERRNO_BADF, data: EMPTY }; - if (size === 0) return { ret: wasi.ERRNO_SUCCESS, data: EMPTY }; - const buffer = new Uint8Array(size); - try { - const read = fs.readSync(this.fd, buffer, 0, size, Number(offset)); - return { ret: wasi.ERRNO_SUCCESS, data: buffer.subarray(0, read) }; - } catch (error) { - return { ret: errnoOf(error), data: EMPTY }; - } - } -} - -class HostDirectory extends Fd { - private entries: fs.Dirent[] | null = null; - - constructor( - protected readonly owner: PreopenRoot, - protected readonly hostPath: string, - ) { - super(); - } - - override fd_fdstat_get(): FdstatResult { - return { ret: wasi.ERRNO_SUCCESS, fdstat: new wasi.Fdstat(wasi.FILETYPE_DIRECTORY, 0) }; - } - - override fd_fdstat_set_flags(): number { - return wasi.ERRNO_SUCCESS; - } - - override fd_filestat_get(): FilestatResult { - try { - return { - ret: wasi.ERRNO_SUCCESS, - filestat: filestatOf(fs.statSync(this.hostPath, { bigint: true })), - }; - } catch (error) { - return { ret: errnoOf(error), filestat: null }; - } - } - - override fd_read(): ReadResult { - return { ret: wasi.ERRNO_ISDIR, data: EMPTY }; - } - - override fd_pread(): ReadResult { - return { ret: wasi.ERRNO_ISDIR, data: EMPTY }; - } - - override fd_seek(): SeekResult { - return { ret: wasi.ERRNO_BADF, offset: 0n }; - } - - override fd_tell(): SeekResult { - return { ret: wasi.ERRNO_BADF, offset: 0n }; - } - - override fd_close(): number { - this.entries = null; - return wasi.ERRNO_SUCCESS; - } - - override fd_readdir_single(cookie: bigint): ReaddirResult { - if (cookie === 0n) { - // A rewind re-lists; later cookies keep the snapshot so a directory changing - // underneath a multi-call readdir cannot skip or repeat entries. - this.entries = null; - return this.dirent(1n, this.hostPath, "."); - } - if (cookie === 1n) return this.dirent(2n, path.dirname(this.hostPath), ".."); - if (this.entries === null) { - try { - this.entries = fs.readdirSync(this.hostPath, { withFileTypes: true }); - } catch (error) { - return { ret: errnoOf(error), dirent: null }; - } - } - const entry = this.entries[Number(cookie - 2n)]; - if (entry === undefined) return { ret: wasi.ERRNO_SUCCESS, dirent: null }; - return this.dirent(cookie + 1n, path.join(this.hostPath, entry.name), entry.name); - } - - override path_filestat_get(flags: number, guestPath: string): FilestatResult { - const { ret, hostPath } = this.owner.resolve(this.hostPath, guestPath); - if (hostPath === null) return { ret, filestat: null }; - try { - const stats = - (flags & LOOKUPFLAGS_SYMLINK_FOLLOW) !== 0 - ? fs.statSync(hostPath, { bigint: true }) - : fs.lstatSync(hostPath, { bigint: true }); - return { ret: wasi.ERRNO_SUCCESS, filestat: filestatOf(stats) }; - } catch (error) { - return { ret: errnoOf(error), filestat: null }; - } - } - - override path_readlink(guestPath: string): ReadlinkResult { - const { ret, hostPath } = this.owner.resolve(this.hostPath, guestPath); - if (hostPath === null) return { ret, data: null }; - try { - return { ret: wasi.ERRNO_SUCCESS, data: fs.readlinkSync(hostPath, "utf8") }; - } catch (error) { - return { ret: errnoOf(error), data: null }; - } - } - - override path_open( - _dirflags: number, - guestPath: string, - oflags: number, - fsRightsBase: bigint, - _fsRightsInheriting: bigint, - _fdFlags: number, - ): OpenResult { - const writeRights = BigInt(wasi.RIGHTS_FD_WRITE); - if ( - (oflags & (wasi.OFLAGS_CREAT | wasi.OFLAGS_TRUNC)) !== 0 || - (fsRightsBase & writeRights) !== 0n - ) { - return { ret: wasi.ERRNO_ROFS, fd_obj: null }; - } - const { ret, hostPath } = this.owner.resolve(this.hostPath, guestPath); - if (hostPath === null) return { ret, fd_obj: null }; - let stats: fs.BigIntStats; - try { - stats = fs.statSync(hostPath, { bigint: true }); - } catch (error) { - return { ret: errnoOf(error), fd_obj: null }; - } - if (stats.isDirectory()) { - return { ret: wasi.ERRNO_SUCCESS, fd_obj: new HostDirectory(this.owner, hostPath) }; - } - if ((oflags & wasi.OFLAGS_DIRECTORY) !== 0) return { ret: wasi.ERRNO_NOTDIR, fd_obj: null }; - return this.owner.openFile(hostPath); - } - - override fd_write(): WriteResult { - return { ret: wasi.ERRNO_ROFS, nwritten: 0 }; - } - - override fd_pwrite(): WriteResult { - return { ret: wasi.ERRNO_ROFS, nwritten: 0 }; - } - - override fd_allocate(): number { - return wasi.ERRNO_ROFS; - } - - override fd_filestat_set_size(): number { - return wasi.ERRNO_ROFS; - } - - override fd_filestat_set_times(): number { - return wasi.ERRNO_ROFS; - } - - override path_create_directory(): number { - return wasi.ERRNO_ROFS; - } - - override path_filestat_set_times(): number { - return wasi.ERRNO_ROFS; - } - - override path_link(): number { - return wasi.ERRNO_ROFS; - } - - override path_unlink(): { ret: number; inode_obj: null } { - return { ret: wasi.ERRNO_ROFS, inode_obj: null }; - } - - override path_lookup(): { ret: number; inode_obj: null } { - return { ret: wasi.ERRNO_ROFS, inode_obj: null }; - } - - override path_remove_directory(): number { - return wasi.ERRNO_ROFS; - } - - override path_rename(): number { - return wasi.ERRNO_ROFS; - } - - override path_unlink_file(): number { - return wasi.ERRNO_ROFS; - } - - private dirent(next: bigint, hostPath: string, name: string): ReaddirResult { - let ino = 0n; - let filetype = wasi.FILETYPE_UNKNOWN; - try { - const stats = fs.lstatSync(hostPath, { bigint: true }); - ino = stats.ino; - filetype = filetypeOf(stats); - } catch { - // The entry vanished between readdir and lstat; report it with unknown type - // rather than fail the whole listing. - } - return { ret: wasi.ERRNO_SUCCESS, dirent: new wasi.Dirent(next, ino, name, filetype) }; - } -} - -class PreopenDirectory extends HostDirectory implements ReadOnlyPreopen { - constructor( - owner: PreopenRoot, - private readonly guestPath: string, - ) { - super(owner, owner.root); - } - - override fd_prestat_get(): PrestatResult { - return { ret: wasi.ERRNO_SUCCESS, prestat: wasi.Prestat.dir(this.guestPath) }; - } - - dispose(): void { - this.owner.dispose(); - } -} - -/** - * A read-only WASI preopen serving `hostDir` to the guest at `guestPath` (default `/`). - * Guest paths are contained by real path: anything that resolves outside `hostDir`, - * including through a symlink, is refused with `ENOTCAPABLE`. The check is not atomic with - * the open that follows it, so a process rewriting symlinks under `hostDir` during a run can - * race it; Node has no `openat` to close that window. Every mutating call fails with `EROFS`. - * Call `dispose()` after the run, even if it threw, to close host fds the guest left open. - */ -export function createReadOnlyPreopen(hostDir: string, guestPath = "/"): ReadOnlyPreopen { - return new PreopenDirectory(new PreopenRoot(hostDir), guestPath); -} diff --git a/src/index.ts b/src/index.ts index c24f58b..90a1bea 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,8 +1,21 @@ -export { run } from "./runner.js"; -export type { RunOptions, RunResult } from "./runner.js"; -export { SHELLCHECK_VERSION } from "./generated/version.js"; -export { BUILD_INFO } from "./generated/build-info.js"; +export { createShellCheck } from "./client.js"; +export type { + LintOptions, + LintRequest, + LintResult, + ShellCheck, + ShellCheckOptions, + WorkerPort, +} from "./client.js"; +export type { + FileStat, + FileSystemErrorCode, + FileType, + ShellCheckFileSystem, +} from "./file-system.js"; export type { BuildInfo } from "./build-info.js"; +export { BUILD_INFO } from "./generated/build-info.js"; +export { SHELLCHECK_VERSION } from "./generated/version.js"; /** URL of the bundled shellcheck.wasm, resolved relative to this module. */ export const wasmUrl: URL = new URL("./shellcheck.wasm", import.meta.url); diff --git a/src/node.ts b/src/node.ts deleted file mode 100644 index fa3b236..0000000 --- a/src/node.ts +++ /dev/null @@ -1,16 +0,0 @@ -import { readFile } from "node:fs/promises"; -import { fileURLToPath } from "node:url"; -import { wasmUrl } from "./index.js"; - -export { BUILD_INFO, run, SHELLCHECK_VERSION, wasmUrl } from "./index.js"; -export type { BuildInfo, RunOptions, RunResult } from "./index.js"; -export { createReadOnlyPreopen } from "./host-preopen.js"; -export type { ReadOnlyPreopen } from "./host-preopen.js"; - -/** Filesystem path of the bundled shellcheck.wasm. */ -export const wasmPath: string = fileURLToPath(wasmUrl); - -/** Reads and compiles the bundled artifact. Hosts should call this once and reuse the Module. */ -export async function loadModule(): Promise { - return WebAssembly.compile(await readFile(wasmPath)); -} diff --git a/src/preopen.ts b/src/preopen.ts new file mode 100644 index 0000000..569b7fd --- /dev/null +++ b/src/preopen.ts @@ -0,0 +1,319 @@ +import { Fd, File, OpenFile, wasi } from "@bjorn3/browser_wasi_shim"; +import type { DirectoryEntry, SyncFileSystem } from "./bridge.js"; +import type { FileStat, FileType } from "./file-system.js"; + +type FdstatResult = { ret: number; fdstat: wasi.Fdstat | null }; +type FilestatResult = { ret: number; filestat: wasi.Filestat | null }; +type ReadResult = { ret: number; data: Uint8Array }; +type WriteResult = { ret: number; nwritten: number }; +type SeekResult = { ret: number; offset: bigint }; +type OpenResult = { ret: number; fd_obj: Fd | null }; +type ReaddirResult = { ret: number; dirent: wasi.Dirent | null }; +type ReadlinkResult = { ret: number; data: string | null }; +type PrestatResult = { ret: number; prestat: wasi.Prestat | null }; +type Resolved = { ret: number; path: string | null; directoryOnly: boolean }; + +const EMPTY = new Uint8Array(); + +function filetypeOf(type: FileType): number { + switch (type) { + case "file": + return wasi.FILETYPE_REGULAR_FILE; + case "directory": + return wasi.FILETYPE_DIRECTORY; + default: + return wasi.FILETYPE_UNKNOWN; + } +} + +function parentOf(path: string): string { + const slash = path.lastIndexOf("/"); + return slash <= 0 ? "/" : path.slice(0, slash); +} + +function childOf(path: string, name: string): string { + return path === "/" ? `/${name}` : `${path}/${name}`; +} + +/** + * Folds `guestPath` onto the guest-absolute directory `base` without asking the file + * system, as `path.resolve` would. A `..` above the mount root has nothing to name, so it + * fails with `ENOTCAPABLE` like any other attempt to leave the mount. + */ +function resolve(base: string, guestPath: string): Resolved { + if (guestPath.includes("\0")) return { ret: wasi.ERRNO_INVAL, path: null, directoryOnly: false }; + // wasi-libc turns absolute paths into (preopen fd, relative path) itself; one that + // reaches us bypassed that and has no preopen to be relative to. + if (guestPath.startsWith("/")) { + return { ret: wasi.ERRNO_NOTCAPABLE, path: null, directoryOnly: false }; + } + const parts = base === "/" ? [] : base.slice(1).split("/"); + for (const part of guestPath.split("/")) { + if (part === "" || part === ".") continue; + if (part !== "..") { + parts.push(part); + } else if (parts.pop() === undefined) { + return { ret: wasi.ERRNO_NOTCAPABLE, path: null, directoryOnly: false }; + } + } + return { + ret: wasi.ERRNO_SUCCESS, + path: `/${parts.join("/")}`, + directoryOnly: guestPath.endsWith("/"), + }; +} + +/** Per-lint state shared by every fd under one preopen. */ +class Mount { + private readonly inodes = new Map(); + + constructor(readonly fs: SyncFileSystem) {} + + /** Stable per path within the lint, so path and fd stats of one file agree. */ + ino(path: string): bigint { + let ino = this.inodes.get(path); + if (ino === undefined) { + ino = BigInt(this.inodes.size + 1); + this.inodes.set(path, ino); + } + return ino; + } + + filestat(path: string, stat: FileStat): wasi.Filestat { + const filestat = new wasi.Filestat(this.ino(path), filetypeOf(stat.type), BigInt(stat.size)); + const time = BigInt(stat.mtime) * 1_000_000n; + filestat.atim = time; + filestat.mtim = time; + filestat.ctim = time; + return filestat; + } + + /** Stats `guestPath` under `base`; `ENOTDIR` for a trailing slash on a non-directory. */ + lookup(base: string, guestPath: string): { ret: number; path: string; stat: FileStat | null } { + const resolved = resolve(base, guestPath); + if (resolved.path === null) return { ret: resolved.ret, path: "", stat: null }; + const stat = this.fs.stat(resolved.path); + if (typeof stat === "number") return { ret: stat, path: resolved.path, stat: null }; + if (resolved.directoryOnly && stat.type !== "directory") { + return { ret: wasi.ERRNO_NOTDIR, path: resolved.path, stat: null }; + } + return { ret: wasi.ERRNO_SUCCESS, path: resolved.path, stat }; + } +} + +class ReadOnlyFile extends OpenFile { + constructor( + private readonly mount: Mount, + private readonly path: string, + private readonly stat: FileStat, + data: Uint8Array, + ) { + super(new File(data, { readonly: true })); + } + + override fd_fdstat_set_flags(): number { + return wasi.ERRNO_SUCCESS; + } + + override fd_filestat_get(): { ret: number; filestat: wasi.Filestat } { + const stat = { ...this.stat, size: this.file.data.byteLength }; + return { ret: wasi.ERRNO_SUCCESS, filestat: this.mount.filestat(this.path, stat) }; + } + + override fd_write(): WriteResult { + return { ret: wasi.ERRNO_ROFS, nwritten: 0 }; + } + + override fd_pwrite(): WriteResult { + return { ret: wasi.ERRNO_ROFS, nwritten: 0 }; + } + + override fd_allocate(): number { + return wasi.ERRNO_ROFS; + } + + override fd_filestat_set_size(): number { + return wasi.ERRNO_ROFS; + } + + override fd_filestat_set_times(): number { + return wasi.ERRNO_ROFS; + } +} + +class ReadOnlyDirectory extends Fd { + private entries: readonly DirectoryEntry[] | null = null; + + constructor( + protected readonly mount: Mount, + protected readonly path: string, + ) { + super(); + } + + override fd_fdstat_get(): FdstatResult { + return { ret: wasi.ERRNO_SUCCESS, fdstat: new wasi.Fdstat(wasi.FILETYPE_DIRECTORY, 0) }; + } + + override fd_fdstat_set_flags(): number { + return wasi.ERRNO_SUCCESS; + } + + override fd_filestat_get(): FilestatResult { + const stat = this.mount.fs.stat(this.path); + if (typeof stat === "number") return { ret: stat, filestat: null }; + return { ret: wasi.ERRNO_SUCCESS, filestat: this.mount.filestat(this.path, stat) }; + } + + override fd_read(): ReadResult { + return { ret: wasi.ERRNO_ISDIR, data: EMPTY }; + } + + override fd_pread(): ReadResult { + return { ret: wasi.ERRNO_ISDIR, data: EMPTY }; + } + + override fd_seek(): SeekResult { + return { ret: wasi.ERRNO_BADF, offset: 0n }; + } + + override fd_tell(): SeekResult { + return { ret: wasi.ERRNO_BADF, offset: 0n }; + } + + override fd_close(): number { + this.entries = null; + return wasi.ERRNO_SUCCESS; + } + + override fd_readdir_single(cookie: bigint): ReaddirResult { + if (cookie === 0n) { + // A rewind re-lists; later cookies keep the snapshot so a multi-call readdir + // cannot skip or repeat entries. + this.entries = null; + return this.dirent(1n, this.path, ".", wasi.FILETYPE_DIRECTORY); + } + if (cookie === 1n) return this.dirent(2n, parentOf(this.path), "..", wasi.FILETYPE_DIRECTORY); + if (this.entries === null) { + const entries = this.mount.fs.readDirectory(this.path); + if (typeof entries === "number") return { ret: entries, dirent: null }; + this.entries = entries; + } + const entry = this.entries[Number(cookie - 2n)]; + if (entry === undefined) return { ret: wasi.ERRNO_SUCCESS, dirent: null }; + const [name, type] = entry; + return this.dirent(cookie + 1n, childOf(this.path, name), name, filetypeOf(type)); + } + + override path_filestat_get(_flags: number, guestPath: string): FilestatResult { + const { ret, path, stat } = this.mount.lookup(this.path, guestPath); + if (stat === null) return { ret, filestat: null }; + return { ret, filestat: this.mount.filestat(path, stat) }; + } + + override path_readlink(guestPath: string): ReadlinkResult { + // The file system follows symlinks itself, so from here nothing is one. + const { ret, stat } = this.mount.lookup(this.path, guestPath); + return { ret: stat === null ? ret : wasi.ERRNO_INVAL, data: null }; + } + + override path_open( + _dirflags: number, + guestPath: string, + oflags: number, + fsRightsBase: bigint, + _fsRightsInheriting: bigint, + _fdFlags: number, + ): OpenResult { + const writeRights = BigInt(wasi.RIGHTS_FD_WRITE); + if ( + (oflags & (wasi.OFLAGS_CREAT | wasi.OFLAGS_TRUNC)) !== 0 || + (fsRightsBase & writeRights) !== 0n + ) { + return { ret: wasi.ERRNO_ROFS, fd_obj: null }; + } + const { ret, path, stat } = this.mount.lookup(this.path, guestPath); + if (stat === null) return { ret, fd_obj: null }; + if (stat.type === "directory") { + return { ret: wasi.ERRNO_SUCCESS, fd_obj: new ReadOnlyDirectory(this.mount, path) }; + } + if ((oflags & wasi.OFLAGS_DIRECTORY) !== 0) return { ret: wasi.ERRNO_NOTDIR, fd_obj: null }; + const data = this.mount.fs.readFile(path); + if (typeof data === "number") return { ret: data, fd_obj: null }; + return { ret: wasi.ERRNO_SUCCESS, fd_obj: new ReadOnlyFile(this.mount, path, stat, data) }; + } + + override fd_write(): WriteResult { + return { ret: wasi.ERRNO_ROFS, nwritten: 0 }; + } + + override fd_pwrite(): WriteResult { + return { ret: wasi.ERRNO_ROFS, nwritten: 0 }; + } + + override fd_allocate(): number { + return wasi.ERRNO_ROFS; + } + + override fd_filestat_set_size(): number { + return wasi.ERRNO_ROFS; + } + + override fd_filestat_set_times(): number { + return wasi.ERRNO_ROFS; + } + + override path_create_directory(): number { + return wasi.ERRNO_ROFS; + } + + override path_filestat_set_times(): number { + return wasi.ERRNO_ROFS; + } + + override path_link(): number { + return wasi.ERRNO_ROFS; + } + + override path_unlink(): { ret: number; inode_obj: null } { + return { ret: wasi.ERRNO_ROFS, inode_obj: null }; + } + + override path_lookup(): { ret: number; inode_obj: null } { + return { ret: wasi.ERRNO_ROFS, inode_obj: null }; + } + + override path_remove_directory(): number { + return wasi.ERRNO_ROFS; + } + + override path_rename(): number { + return wasi.ERRNO_ROFS; + } + + override path_unlink_file(): number { + return wasi.ERRNO_ROFS; + } + + private dirent(next: bigint, path: string, name: string, filetype: number): ReaddirResult { + return { + ret: wasi.ERRNO_SUCCESS, + dirent: new wasi.Dirent(next, this.mount.ino(path), name, filetype), + }; + } +} + +class PreopenRoot extends ReadOnlyDirectory { + override fd_prestat_get(): PrestatResult { + return { ret: wasi.ERRNO_SUCCESS, prestat: wasi.Prestat.dir("/") }; + } +} + +/** + * The read-only WASI preopen at guest `/` for one lint. Every lookup goes through `fs`, so + * the tree is only ever as deep as what ShellCheck asks for; every mutation fails with + * `EROFS`. + */ +export function createReadOnlyPreopen(fs: SyncFileSystem): Fd { + return new PreopenRoot(new Mount(fs), "/"); +} diff --git a/src/protocol.ts b/src/protocol.ts new file mode 100644 index 0000000..a6b5310 --- /dev/null +++ b/src/protocol.ts @@ -0,0 +1,68 @@ +/** + * Messages and shared-memory layout between `createShellCheck` and `startWorker`. Both sides + * ship in the same package version, so the protocol carries no version of its own. + * + * The bridge: while the guest is inside a WASI call, the worker posts an `fs` request and + * blocks in `Atomics.wait` on `STATE`. The caller runs the async file system, writes status, + * total size and the first chunk, then flips `STATE` to `READY`. Payloads larger than the + * data region take one `fs-next` round trip per further chunk. + */ + +/** Int32 slots of the control header. */ +export const STATE = 0; +export const STATUS = 1; +export const TOTAL = 2; +export const LENGTH = 3; + +export const HEADER_BYTES = 16; +export const CHUNK_BYTES = 1 << 20; + +export const PENDING = 0; +export const READY = 1; + +/** WASI preview1 errnos. Spelled out so the caller-side entry does not load the WASI shim. */ +export const ERRNO_SUCCESS = 0; +export const ERRNO_ACCES = 2; +export const ERRNO_IO = 29; +export const ERRNO_ISDIR = 31; +export const ERRNO_NOENT = 44; +export const ERRNO_NOTDIR = 54; + +export type FileSystemOp = "stat" | "readFile" | "readDirectory"; + +export interface InitMessage { + type: "init"; + module: WebAssembly.Module; + shared: SharedArrayBuffer; +} + +export interface LintMessage { + type: "lint"; + args: string[]; + stdin: string | Uint8Array; + env: Record; + mounted: boolean; +} + +export type ToWorker = InitMessage | LintMessage; + +export type FromWorker = + | { type: "fs"; op: FileSystemOp; path: string } + | { type: "fs-next" } + | { type: "done"; stdout: string; stderr: string; exitCode: number } + | { type: "failed"; name: string; message: string }; + +export function errnoOf(error: unknown): number { + switch ((error as { code?: unknown } | null | undefined)?.code) { + case "FileNotFound": + return ERRNO_NOENT; + case "FileNotADirectory": + return ERRNO_NOTDIR; + case "FileIsADirectory": + return ERRNO_ISDIR; + case "NoPermissions": + return ERRNO_ACCES; + default: + return ERRNO_IO; + } +} diff --git a/src/runner.ts b/src/runner.ts index 2d75537..7e93981 100644 --- a/src/runner.ts +++ b/src/runner.ts @@ -13,7 +13,7 @@ export interface RunOptions { * `hs_init_ghc: chdir(...) failed` and the run ends with no stdout. */ env?: Readonly>; - /** Preopened directories, assigned fd 3, 4, … in order. See `createReadOnlyPreopen` in `./node`. */ + /** Preopened directories, assigned fd 3, 4, … in order. */ preopens?: readonly Fd[]; } diff --git a/src/worker.ts b/src/worker.ts new file mode 100644 index 0000000..73c5194 --- /dev/null +++ b/src/worker.ts @@ -0,0 +1,55 @@ +import { Bridge } from "./bridge.js"; +import { createReadOnlyPreopen } from "./preopen.js"; +import type { FromWorker, ToWorker } from "./protocol.js"; +import { run } from "./runner.js"; + +/** + * The worker's end of its channel to the thread that created it: Node's `parentPort`, or + * a dedicated Worker's global scope with `onMessage` unwrapping `MessageEvent.data`. + */ +export interface ParentPort { + postMessage(message: unknown): void; + onMessage(listener: (message: unknown) => void): void; +} + +// ignoreBOM keeps a leading U+FEFF in the output instead of silently dropping it. +const decoder = new TextDecoder("utf-8", { ignoreBOM: true }); + +/** + * Serves `createShellCheck` from inside a Worker; call it once from the Worker's entry + * module. Each lint runs synchronously on a fresh `WebAssembly.Instance`, blocking this + * thread while the caller's thread answers file-system requests. + */ +export function startWorker(port: ParentPort): void { + let module: WebAssembly.Module | undefined; + let shared: SharedArrayBuffer | undefined; + const post = (message: FromWorker): void => port.postMessage(message); + + port.onMessage((data) => { + const request = data as ToWorker; + if (request.type === "init") { + ({ module, shared } = request); + return; + } + try { + if (module === undefined || shared === undefined) { + throw new Error("received a lint before init"); + } + const result = run(module, { + args: request.args, + stdin: request.stdin, + env: request.env, + preopens: request.mounted ? [createReadOnlyPreopen(new Bridge(shared, post))] : [], + }); + post({ + type: "done", + stdout: decoder.decode(result.stdout), + stderr: decoder.decode(result.stderr), + exitCode: result.exitCode, + }); + } catch (error) { + const { name = "Error", message = String(error) } = (error ?? {}) as Partial; + post({ type: "failed", name, message }); + } + }); +} diff --git a/test/bridge.test.ts b/test/bridge.test.ts new file mode 100644 index 0000000..7d3d0ba --- /dev/null +++ b/test/bridge.test.ts @@ -0,0 +1,131 @@ +import { createHash } from "node:crypto"; +import { wasi } from "@bjorn3/browser_wasi_shim"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import type { ShellCheckFileSystem } from "../src/index.js"; +import { CHUNK_BYTES, errnoOf } from "../src/protocol.js"; +import { + createTestShellCheck, + nodeWorkerPort, + skipWorkerTests, + supportWorkerUrl, + type TestShellCheck, +} from "./helpers.js"; +import { fileSystemError, memoryFileSystem } from "./support/file-systems.js"; + +type BridgeResult = { errno: number } | { size: number; sha256: string } | { value: unknown }; + +function bytes(size: number): Uint8Array { + const data = new Uint8Array(size); + for (let i = 0; i < size; i += 1) data[i] = (i * 31 + (i >> 11)) & 0xff; + return data; +} + +const digest = (data: Uint8Array): { size: number; sha256: string } => ({ + size: data.byteLength, + sha256: createHash("sha256").update(data).digest("hex"), +}); + +describe("errnoOf", () => { + it("maps file-system error codes to WASI errnos", () => { + expect(errnoOf(fileSystemError("FileNotFound"))).toBe(wasi.ERRNO_NOENT); + expect(errnoOf(fileSystemError("FileNotADirectory"))).toBe(wasi.ERRNO_NOTDIR); + expect(errnoOf(fileSystemError("FileIsADirectory"))).toBe(wasi.ERRNO_ISDIR); + expect(errnoOf(fileSystemError("NoPermissions"))).toBe(wasi.ERRNO_ACCES); + expect(errnoOf(fileSystemError("Unavailable"))).toBe(wasi.ERRNO_IO); + expect(errnoOf(fileSystemError("ENOENT"))).toBe(wasi.ERRNO_IO); + expect(errnoOf(new Error("plain"))).toBe(wasi.ERRNO_IO); + expect(errnoOf(undefined)).toBe(wasi.ERRNO_IO); + expect(errnoOf("FileNotFound")).toBe(wasi.ERRNO_IO); + }); +}); + +describe.skipIf(skipWorkerTests())("bridge", () => { + let shellcheck: TestShellCheck; + + beforeAll(() => { + shellcheck = createTestShellCheck({ + createWorker: () => nodeWorkerPort(supportWorkerUrl("bridge-worker.mjs")), + }); + }); + + afterAll(async () => { + await shellcheck.dispose(); + }); + + /** Each request is " ", answered by the bridge inside one lint. */ + async function ask(fs: ShellCheckFileSystem, ...requests: string[]): Promise { + const result = await shellcheck.lint({ args: requests, fs }); + return JSON.parse(result.stdout) as BridgeResult[]; + } + + it("transfers files of any size, including across chunk boundaries", async () => { + const sizes = [0, 1, CHUNK_BYTES - 1, CHUNK_BYTES, CHUNK_BYTES + 1, 2 * CHUNK_BYTES + 17]; + const files = Object.fromEntries(sizes.map((size) => [`/f${size}`, bytes(size)])); + const fs = memoryFileSystem(files); + const results = await ask(fs, ...sizes.map((size) => `readFile /f${size}`)); + expect(results).toEqual(sizes.map((size) => digest(files[`/f${size}`]!))); + }); + + it("round-trips stat and directory listings", async () => { + const fs = memoryFileSystem({ "/dir/a.sh": "a", "/dir/sub/b.sh": "bb" }); + expect(await ask(fs, "stat /dir/sub/b.sh", "stat /dir", "readDirectory /dir")).toEqual([ + { value: { type: "file", size: 2, mtime: 0 } }, + { value: { type: "directory", size: 0, mtime: 0 } }, + { + value: [ + ["a.sh", "file"], + ["sub", "directory"], + ], + }, + ]); + }); + + it("normalizes stat numbers and rejects malformed answers with EIO", async () => { + const fs: ShellCheckFileSystem = { + stat: async (path) => { + if (path === "/odd") return { type: "file", size: 12.9, mtime: 1_700_000_000_123.4 }; + return { type: "symlink" as never, size: 0, mtime: 0 }; + }, + readFile: async () => "text" as unknown as Uint8Array, + readDirectory: async () => [[42 as unknown as string, "file"]], + }; + expect(await ask(fs, "stat /odd", "stat /bad", "readFile /x", "readDirectory /")).toEqual([ + { value: { type: "file", size: 12, mtime: 1_700_000_000_123 } }, + { errno: wasi.ERRNO_IO }, + { errno: wasi.ERRNO_IO }, + { errno: wasi.ERRNO_IO }, + ]); + }); + + it("passes file-system errors through as errnos", async () => { + const codes = ["FileNotFound", "FileNotADirectory", "FileIsADirectory", "NoPermissions"]; + const fs = memoryFileSystem( + {}, + { + readFile: async (path) => { + const code = path.slice(1); + if (code === "plain") throw new Error("boom"); + throw fileSystemError(code); + }, + }, + ); + const results = await ask( + fs, + ...[...codes, "Unavailable", "plain"].map((c) => `readFile /${c}`), + ); + expect(results).toEqual( + [wasi.ERRNO_NOENT, wasi.ERRNO_NOTDIR, wasi.ERRNO_ISDIR, wasi.ERRNO_ACCES] + .concat(wasi.ERRNO_IO, wasi.ERRNO_IO) + .map((errno) => ({ errno })), + ); + }); + + it("asks the file system once per path and operation within a lint, not across lints", async () => { + const fs = memoryFileSystem({ "/a.sh": "a" }); + const requests = ["stat /a.sh", "stat /missing", "readFile /a.sh", "readDirectory /"]; + await ask(fs, ...requests, ...requests); + expect(fs.calls).toEqual(requests); + await ask(fs, "stat /a.sh"); + expect(fs.calls).toEqual([...requests, "stat /a.sh"]); + }); +}); diff --git a/test/helpers.ts b/test/helpers.ts index 588ec22..1b773f4 100644 --- a/test/helpers.ts +++ b/test/helpers.ts @@ -1,21 +1,28 @@ -import { existsSync, readFileSync } from "node:fs"; +import { existsSync, readdirSync, readFileSync, statSync } from "node:fs"; import path from "node:path"; -import { fileURLToPath } from "node:url"; +import { fileURLToPath, pathToFileURL } from "node:url"; +import { Worker } from "node:worker_threads"; +import { + createShellCheck, + type ShellCheck, + type ShellCheckOptions, + type WorkerPort, +} from "../src/index.js"; export const repoRoot: string = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); export const fixtureRoot: string = path.join(repoRoot, "test", "fixtures"); export const wasmPath: string = path.join(repoRoot, "dist", "shellcheck.wasm"); -export const buildInfoPath: string = path.join(repoRoot, "dist", "build-info.json"); export const nativePath: string = process.env["SHELLCHECK_NATIVE"] ?? path.join(repoRoot, ".cache", "native", "shellcheck"); +const distWorkerPath = path.join(repoRoot, "dist", "worker.js"); export const hasWasm: boolean = existsSync(wasmPath); export const hasNative: boolean = existsSync(nativePath); -export const hasBuildInfo: boolean = existsSync(buildInfoPath); +export const hasDist: boolean = existsSync(distWorkerPath); -export const wasmHint = `dist/shellcheck.wasm is missing; run \`npm run build:wasm\` (Docker) or use the dev stand-in from AGENTS.md`; +export const wasmHint = `dist/shellcheck.wasm is missing; run \`npm run build:wasm\` (Docker) or download it from a GitHub Release (see AGENTS.md)`; export const nativeHint = `${nativePath} is missing; run \`npm run fetch:native\` or set SHELLCHECK_NATIVE`; -export const buildInfoHint = `dist/build-info.json is missing; it is produced by \`npm run build:wasm\` (Docker) or downloaded from a GitHub Release`; +export const distHint = "dist/worker.js is missing; run `npm run build` first"; const printedHints = new Set(); @@ -34,18 +41,74 @@ export function skipHint(condition: boolean, hint: string): boolean { return condition; } +/** True when tests that start a Worker must skip: they need the artifact and dist/worker.js. */ +export function skipWorkerTests(): boolean { + return skipHint(!hasWasm, wasmHint) || skipHint(!hasDist, distHint); +} + +/** + * Workers load dist/ while the tests load src/; a dist older than src would test the last + * build's worker against the current caller. + */ +function assertDistIsFresh(): void { + const built = statSync(distWorkerPath).mtimeMs; + const sourceDir = path.join(repoRoot, "src"); + for (const name of readdirSync(sourceDir)) { + if (name.endsWith(".ts") && statSync(path.join(sourceDir, name)).mtimeMs > built) { + throw new Error(`src/${name} is newer than dist/; run \`npm run build\``); + } + } +} + let modulePromise: Promise | undefined; -/** Compiles dist/shellcheck.wasm once per worker; runs always get a fresh Instance. */ +/** Compiles dist/shellcheck.wasm once per test file; lints always get a fresh Instance. */ export function loadTestModule(): Promise { modulePromise ??= WebAssembly.compile(readFileSync(wasmPath)); return modulePromise; } -const decoder = new TextDecoder(); +/** A test-only Worker entry under test/support; `node-worker.mjs` is the plain one. */ +export function supportWorkerUrl(name: string): URL { + return pathToFileURL(path.join(repoRoot, "test", "support", name)); +} + +/** The adapter a Node Host writes around `node:worker_threads`. */ +export function nodeWorkerPort(url: URL = supportWorkerUrl("node-worker.mjs")): WorkerPort { + const worker = new Worker(url); + return { + postMessage: (message) => worker.postMessage(message), + onMessage: (listener) => worker.on("message", listener), + onError: (listener) => worker.on("error", listener), + onExit: (listener) => worker.on("exit", listener), + terminate: () => worker.terminate(), + }; +} + +export interface TestShellCheck extends ShellCheck { + /** How many Workers have been started so far. */ + readonly spawned: number; +} -export function text(bytes: Uint8Array): string { - return decoder.decode(bytes); +/** A ShellCheck on the plain test worker unless overridden; dispose it after the test. */ +export function createTestShellCheck(options: Partial = {}): TestShellCheck { + assertDistIsFresh(); + const createWorker = options.createWorker ?? (() => nodeWorkerPort()); + let spawned = 0; + const shellcheck = createShellCheck({ + module: options.module ?? loadTestModule(), + createWorker: () => { + spawned += 1; + return createWorker(); + }, + }); + return { + lint: (request, lintOptions) => shellcheck.lint(request, lintOptions), + dispose: () => shellcheck.dispose(), + get spawned() { + return spawned; + }, + }; } export interface Json1Comment { @@ -55,11 +118,11 @@ export interface Json1Comment { message: string; } -export function json1Comments(stdout: Uint8Array): Json1Comment[] { - const parsed = JSON.parse(text(stdout)) as { comments: Json1Comment[] }; +export function json1Comments(stdout: string): Json1Comment[] { + const parsed = JSON.parse(stdout) as { comments: Json1Comment[] }; return parsed.comments; } -export function codesOf(stdout: Uint8Array): number[] { +export function codesOf(stdout: string): number[] { return json1Comments(stdout).map((comment) => comment.code); } diff --git a/test/lint.test.ts b/test/lint.test.ts new file mode 100644 index 0000000..63b78a3 --- /dev/null +++ b/test/lint.test.ts @@ -0,0 +1,96 @@ +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { SHELLCHECK_VERSION } from "../src/index.js"; +import { codesOf, createTestShellCheck, skipWorkerTests, type TestShellCheck } from "./helpers.js"; + +const JSON1_STDIN = ["-f", "json1", "-s", "bash", "-"]; + +describe.skipIf(skipWorkerTests())("lint without a file system", () => { + let shellcheck: TestShellCheck; + + beforeAll(() => { + shellcheck = createTestShellCheck(); + }); + + afterAll(async () => { + await shellcheck.dispose(); + }); + + const lint: TestShellCheck["lint"] = (request, options) => shellcheck.lint(request, options); + + it("reports the version recorded in version.txt", async () => { + const result = await lint({ args: ["--version"] }); + expect(result.exitCode).toBe(0); + expect(result.stdout).toContain(`version: ${SHELLCHECK_VERSION.slice(1)}`); + expect(result.stderr).toBe(""); + }); + + it("reports SC2086 as json1 with exit code 1", async () => { + const result = await lint({ args: JSON1_STDIN, stdin: "echo $x\n" }); + expect(result.exitCode).toBe(1); + expect(codesOf(result.stdout)).toContain(2086); + expect(result.stderr).toBe(""); + }); + + it("exits 0 with an empty comment list for a clean script", async () => { + const result = await lint({ args: JSON1_STDIN, stdin: 'echo "$1"\n' }); + expect(result.exitCode).toBe(0); + expect(result.stdout).toBe('{"comments":[]}\n'); + }); + + it("rejects an unknown flag on stderr with exit code >= 2", async () => { + const result = await lint({ args: ["--no-such-flag"] }); + expect(result.exitCode).toBeGreaterThanOrEqual(2); + expect(result.stderr).toContain("no-such-flag"); + }); + + it("treats string and Uint8Array stdin alike", async () => { + const script = "echo $x\n"; + const asString = await lint({ args: JSON1_STDIN, stdin: script }); + const asBytes = await lint({ args: JSON1_STDIN, stdin: new TextEncoder().encode(script) }); + expect(asBytes).toEqual(asString); + }); + + it("defaults stdin to empty", async () => { + const result = await lint({ args: JSON1_STDIN }); + expect(result.exitCode).toBe(0); + expect(result.stdout).toBe('{"comments":[]}\n'); + }); + + it("runs lints back to back on fresh Instances in one Worker", async () => { + const [first, second] = await Promise.all([ + lint({ args: JSON1_STDIN, stdin: "echo $a\n" }), + lint({ args: JSON1_STDIN, stdin: "echo $b\n" }), + ]); + expect(first.stdout).toContain("a is referenced"); + expect(second.stdout).toContain("b is referenced"); + expect(shellcheck.spawned).toBe(1); + }); + + it("passes the environment through", async () => { + // SHELLCHECK_OPTS is read by ShellCheck itself, so its effect proves env reaches the guest. + const result = await lint({ + args: JSON1_STDIN, + stdin: "echo $x\n", + env: { SHELLCHECK_OPTS: "-e SC2086 -e SC2154" }, + }); + expect(result.exitCode).toBe(0); + expect(result.stdout).toBe('{"comments":[]}\n'); + }); + + it("cannot follow a source directive: nothing is mounted", async () => { + const result = await lint({ + args: ["-x", ...JSON1_STDIN], + stdin: 'source ./lib.sh\necho "$LIB_HOME"\n', + }); + expect(codesOf(result.stdout)).toContain(1091); + }); + + it("fails fast, without hanging, when PWD is set and nothing is mounted", async () => { + // The GHC RTS chdir()s to PWD before main runs; with no preopen to satisfy it the + // program aborts before ShellCheck can write anything to stdout. + const result = await lint({ args: JSON1_STDIN, stdin: "echo $x\n", env: { PWD: "/nowhere" } }); + expect(result.stdout).toBe(""); + expect(result.exitCode).not.toBe(0); + expect(result.stderr).toContain("chdir"); + }); +}); diff --git a/test/package.test.ts b/test/package.test.ts index 61338cf..207ac33 100644 --- a/test/package.test.ts +++ b/test/package.test.ts @@ -1,22 +1,15 @@ import { spawnSync } from "node:child_process"; import { createHash } from "node:crypto"; -import { existsSync, readFileSync } from "node:fs"; +import { existsSync, readdirSync, readFileSync } from "node:fs"; +import { builtinModules } from "node:module"; import path from "node:path"; import { describe, expect, it } from "vitest"; import { SHELLCHECK_VERSION } from "../src/index.js"; -import { - buildInfoHint, - buildInfoPath, - hasBuildInfo, - hasWasm, - repoRoot, - skipHint, - wasmHint, - wasmPath, -} from "./helpers.js"; +import { hasWasm, repoRoot, skipHint, wasmHint, wasmPath } from "./helpers.js"; const packageName = "@vscode-shellcheck/shellcheck-wasm"; -const distIndex = path.join(repoRoot, "dist", "index.js"); +const distDir = path.join(repoRoot, "dist"); +const distIndex = path.join(distDir, "index.js"); const hasDist = existsSync(distIndex); const distHint = "dist/index.js is missing; run `npm run build` first"; const hasLicense = existsSync(path.join(repoRoot, "LICENSE")); @@ -50,9 +43,45 @@ function nodeEval(source: string): string { return result.stdout.trim(); } +function distScripts(dir = distDir): string[] { + return readdirSync(dir, { withFileTypes: true }).flatMap((entry) => { + const full = path.join(dir, entry.name); + if (entry.isDirectory()) return distScripts(full); + return entry.name.endsWith(".js") ? [full] : []; + }); +} + +/** Every module specifier a compiled file loads: static, side-effect, dynamic and require. */ +function specifiersOf(file: string): string[] { + const source = readFileSync(file, "utf8"); + const patterns = [ + /^\s*(?:import|export)\s[^;]*?\sfrom\s*["']([^"']+)["']/gm, + /^\s*import\s*["']([^"']+)["']/gm, + /\bimport\s*\(\s*["']([^"']+)["']\s*\)/g, + /\brequire\s*\(\s*["']([^"']+)["']\s*\)/g, + ]; + return patterns.flatMap((pattern) => [...source.matchAll(pattern)].map((match) => match[1]!)); +} + +/** Files reachable from `entry` through relative imports, and the bare specifiers they use. */ +function importGraph(entry: string): { files: string[]; packages: string[] } { + const files = new Set(); + const packages = new Set(); + const visit = (file: string): void => { + if (files.has(file)) return; + files.add(file); + for (const specifier of specifiersOf(file)) { + if (specifier.startsWith(".")) visit(path.resolve(path.dirname(file), specifier)); + else packages.add(specifier); + } + }; + visit(entry); + return { files: [...files], packages: [...packages] }; +} + describe("package", () => { describe.skipIf(skipHint(!hasWasm, wasmHint))("npm pack", () => { - it("ships the runtime entry points and the artifact", () => { + it("ships the entry points and the artifact, but not build-info.json", () => { const pack = npmPackDryRun(); expect(pack.name).toBe(packageName); const files = pack.files.map((file) => file.path); @@ -61,100 +90,138 @@ describe("package", () => { "dist/shellcheck.wasm", "dist/index.js", "dist/index.d.ts", - "dist/node.js", - "dist/node.d.ts", + "dist/worker.js", + "dist/worker.d.ts", "dist/generated/build-info.js", "package.json", "README.md", ]), ); + expect(files).not.toContain("dist/build-info.json"); + expect(files).not.toContain("dist/node.js"); expect(files.filter((file) => file.endsWith(".test.js") || file.startsWith("test/"))).toEqual( [], ); expect(files.filter((file) => file.startsWith("src/"))).toEqual([]); }); - it.skipIf(skipHint(!hasLicense, "LICENSE is missing; it is added in the docs workstream"))( - "ships the license", - () => { - expect(npmPackDryRun().files.map((file) => file.path)).toContain("LICENSE"); - }, - ); - - it("does not ship build-info.json, which is compiled in", () => { - expect(npmPackDryRun().files.map((file) => file.path)).not.toContain("dist/build-info.json"); + it.skipIf(skipHint(!hasLicense, "LICENSE is missing"))("ships the license", () => { + expect(npmPackDryRun().files.map((file) => file.path)).toContain("LICENSE"); }); }); + it("exports only the documented subpaths", () => { + const manifest = JSON.parse(readFileSync(path.join(repoRoot, "package.json"), "utf8")) as { + exports: Record; + }; + expect(Object.keys(manifest.exports)).toEqual([ + ".", + "./worker", + "./shellcheck.wasm", + "./package.json", + ]); + }); + describe.skipIf(skipHint(!hasDist, distHint))("built output", () => { - it("exposes the documented API from dist/index.js", async () => { + it("loads no Node built-in from any shipped module", () => { + const builtins = new Set(builtinModules); + const offenders = distScripts().flatMap((file) => + specifiersOf(file) + .filter((specifier) => specifier.startsWith("node:") || builtins.has(specifier)) + .map((specifier) => `${path.relative(repoRoot, file)}: ${specifier}`), + ); + expect(offenders).toEqual([]); + expect(specifiersOf(path.join(distDir, "worker.js"))).toContain("./bridge.js"); + }); + + it("keeps the WASI shim out of the caller's entry", () => { + const index = importGraph(distIndex); + expect(index.files.length).toBeGreaterThan(1); + expect(index.packages).toEqual([]); + expect(importGraph(path.join(distDir, "worker.js")).packages).toEqual([ + "@bjorn3/browser_wasi_shim", + ]); + }); + + it("exposes the documented API from dist/index.js and dist/worker.js", async () => { const index = (await import(distIndex)) as typeof import("../src/index.js"); - expect(typeof index.run).toBe("function"); + expect(typeof index.createShellCheck).toBe("function"); expect(index.SHELLCHECK_VERSION).toBe(SHELLCHECK_VERSION); + expect(index.BUILD_INFO.shellcheckVersion).toBe(SHELLCHECK_VERSION); expect(index.wasmUrl).toBeInstanceOf(URL); expect(index.wasmUrl.pathname.endsWith("/dist/shellcheck.wasm")).toBe(true); + const worker = (await import( + path.join(distDir, "worker.js") + )) as typeof import("../src/worker.js"); + expect(typeof worker.startWorker).toBe("function"); }); - it("exposes the documented API from dist/node.js", async () => { - const node = (await import( - path.join(repoRoot, "dist", "node.js") - )) as typeof import("../src/node.js"); - expect(typeof node.run).toBe("function"); - expect(typeof node.loadModule).toBe("function"); - expect(typeof node.createReadOnlyPreopen).toBe("function"); - expect(node.wasmPath).toBe(path.join(repoRoot, "dist", "shellcheck.wasm")); - expect(node.BUILD_INFO.shellcheckVersion).toBe(SHELLCHECK_VERSION); - }); - - it.skipIf(skipHint(!hasBuildInfo, buildInfoHint) || skipHint(!hasWasm, wasmHint))( + it.skipIf(skipHint(!hasWasm, wasmHint))( "compiles in the build info of the artifact", - () => { - const output = nodeEval( - `import { BUILD_INFO } from ${JSON.stringify(packageName)}; - console.log(JSON.stringify(BUILD_INFO));`, - ); - const info = JSON.parse(output) as { sha256: string; size: number }; - expect(info).toEqual(JSON.parse(readFileSync(buildInfoPath, "utf8"))); + async () => { + const { BUILD_INFO } = (await import(distIndex)) as typeof import("../src/index.js"); const wasm = readFileSync(wasmPath); - expect(info.sha256).toBe(createHash("sha256").update(wasm).digest("hex")); - expect(info.size).toBe(wasm.byteLength); + expect(BUILD_INFO.sha256).toBe(createHash("sha256").update(wasm).digest("hex")); + expect(BUILD_INFO.size).toBe(wasm.byteLength); + expect(BUILD_INFO.targetFeatures).toContain("+tail-call"); }, ); - it("resolves the subpath exports by package name", () => { + it("resolves the subpath exports by package name, and nothing else", () => { const resolved = nodeEval( - `console.log(JSON.stringify([ - import.meta.resolve(${JSON.stringify(packageName)}), - import.meta.resolve(${JSON.stringify(`${packageName}/node`)}), - import.meta.resolve(${JSON.stringify(`${packageName}/shellcheck.wasm`)}), - import.meta.resolve(${JSON.stringify(`${packageName}/package.json`)}), - ]))`, + `const resolve = (s) => { try { return import.meta.resolve(s); } catch (e) { return e.code; } }; + console.log(JSON.stringify(${JSON.stringify( + [ + "", + "/worker", + "/shellcheck.wasm", + "/package.json", + "/node", + "/build-info", + "/build-info.json", + ].map((subpath) => packageName + subpath), + )}.map(resolve)));`, ); - const [index, node, wasm, pkg] = JSON.parse(resolved) as string[]; - expect(index).toBe(`file://${path.join(repoRoot, "dist", "index.js")}`); - expect(node).toBe(`file://${path.join(repoRoot, "dist", "node.js")}`); - expect(wasm).toBe(`file://${path.join(repoRoot, "dist", "shellcheck.wasm")}`); - expect(pkg).toBe(`file://${path.join(repoRoot, "package.json")}`); - }); - - it("no longer exports build-info.json or ./build-info", () => { - for (const subpath of ["/build-info.json", "/build-info"]) { - const code = nodeEval( - `try { import.meta.resolve(${JSON.stringify(packageName + subpath)}); console.log("resolved"); } - catch (error) { console.log(error.code); }`, - ); - expect(code, subpath).toBe("ERR_PACKAGE_PATH_NOT_EXPORTED"); - } + expect(JSON.parse(resolved)).toEqual([ + `file://${path.join(distDir, "index.js")}`, + `file://${path.join(distDir, "worker.js")}`, + `file://${path.join(distDir, "shellcheck.wasm")}`, + `file://${path.join(repoRoot, "package.json")}`, + "ERR_PACKAGE_PATH_NOT_EXPORTED", + "ERR_PACKAGE_PATH_NOT_EXPORTED", + "ERR_PACKAGE_PATH_NOT_EXPORTED", + ]); }); it.skipIf(skipHint(!hasWasm, wasmHint))( - "runs end to end through the published entry point", + "lints end to end through the published entry points", () => { + const workerEntry = `import { parentPort } from "node:worker_threads"; + import { startWorker } from "${packageName}/worker"; + startWorker({ + postMessage: (message) => parentPort.postMessage(message), + onMessage: (listener) => parentPort.on("message", listener), + });`; const output = nodeEval( - `import { loadModule, run, SHELLCHECK_VERSION } from ${JSON.stringify(`${packageName}/node`)}; - const result = run(await loadModule(), { args: ["--version"] }); - const line = new TextDecoder().decode(result.stdout).split("\\n")[1]; - console.log(\`\${result.exitCode} \${SHELLCHECK_VERSION} \${line}\`);`, + `import { readFile } from "node:fs/promises"; + import { Worker } from "node:worker_threads"; + import { createShellCheck, SHELLCHECK_VERSION, wasmUrl } from ${JSON.stringify(packageName)}; + const shellcheck = createShellCheck({ + module: WebAssembly.compile(await readFile(wasmUrl)), + createWorker() { + const worker = new Worker(${JSON.stringify(workerEntry)}, { eval: true }); + return { + postMessage: (message) => worker.postMessage(message), + onMessage: (listener) => worker.on("message", listener), + onError: (listener) => worker.on("error", listener), + onExit: (listener) => worker.on("exit", listener), + terminate: () => worker.terminate(), + }; + }, + }); + const result = await shellcheck.lint({ args: ["--version"] }); + await shellcheck.dispose(); + console.log(\`\${result.exitCode} \${SHELLCHECK_VERSION} \${result.stdout.split("\\n")[1]}\`);`, ); expect(output).toBe(`0 ${SHELLCHECK_VERSION} version: ${SHELLCHECK_VERSION.slice(1)}`); }, diff --git a/test/parity.test.ts b/test/parity.test.ts index 91b69d1..dc66afe 100644 --- a/test/parity.test.ts +++ b/test/parity.test.ts @@ -3,24 +3,29 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; import { afterAll, beforeAll, describe, expect, it } from "vitest"; -import { createReadOnlyPreopen, run, type RunResult } from "../src/node.js"; import { + createTestShellCheck, fixtureRoot, hasNative, - hasWasm, - loadTestModule, nativeHint, nativePath, skipHint, - text, - wasmHint, + skipWorkerTests, + type TestShellCheck, } from "./helpers.js"; +import { nodeFileSystem } from "./support/file-systems.js"; + +interface Outcome { + stdout: Buffer; + stderr: Buffer; + exitCode: number; +} interface Scenario { readonly name: string; readonly args: readonly string[]; readonly stdin?: string; - /** Host directory exposed to the guest as `/`; native runs with cwd inside it. */ + /** Host directory the guest sees as `/`; native runs with cwd inside it. */ readonly hostDir?: string; /** Guest working directory (`PWD`), relative to `hostDir`. */ readonly pwd?: string; @@ -243,14 +248,14 @@ const scenarios: readonly Scenario[] = [ }, ]; -describe.skipIf(skipHint(!hasWasm, wasmHint) || skipHint(!hasNative, nativeHint))("parity", () => { - let module: WebAssembly.Module; +describe.skipIf(skipWorkerTests() || skipHint(!hasNative, nativeHint))("parity", () => { + let shellcheck: TestShellCheck; let sandbox: string; let emptyCwd: string; let emptyHome: string; - beforeAll(async () => { - module = await loadTestModule(); + beforeAll(() => { + shellcheck = createTestShellCheck(); sandbox = fs.mkdtempSync(path.join(os.tmpdir(), "shellcheck-wasm-parity-")); emptyCwd = path.join(sandbox, "cwd"); emptyHome = path.join(sandbox, "home"); @@ -258,11 +263,12 @@ describe.skipIf(skipHint(!hasWasm, wasmHint) || skipHint(!hasNative, nativeHint) fs.mkdirSync(emptyHome); }); - afterAll(() => { + afterAll(async () => { + await shellcheck.dispose(); fs.rmSync(sandbox, { recursive: true, force: true }); }); - function runNative(scenario: Scenario): RunResult { + function runNative(scenario: Scenario): Outcome { const result = spawnSync(nativePath, [...scenario.args], { cwd: scenario.hostDir === undefined ? emptyCwd : path.join(scenario.hostDir, scenario.pwd ?? ""), @@ -277,37 +283,38 @@ describe.skipIf(skipHint(!hasWasm, wasmHint) || skipHint(!hasNative, nativeHint) return { stdout: result.stdout, stderr: result.stderr, exitCode: result.status ?? -1 }; } - function runWasm(scenario: Scenario): RunResult { - if (scenario.hostDir === undefined) { - return run(module, { args: scenario.args, stdin: scenario.stdin ?? "" }); - } - const preopen = createReadOnlyPreopen(scenario.hostDir); - try { - return run(module, { - args: scenario.args, - stdin: scenario.stdin ?? "", - env: { PWD: path.posix.join("/", scenario.pwd ?? "") }, - preopens: [preopen], - }); - } finally { - preopen.dispose(); - } + async function runWasm(scenario: Scenario): Promise { + const result = await shellcheck.lint({ + args: scenario.args, + stdin: scenario.stdin ?? "", + ...(scenario.hostDir === undefined + ? {} + : { + env: { PWD: path.posix.join("/", scenario.pwd ?? "") }, + fs: nodeFileSystem(scenario.hostDir), + }), + }); + return { + stdout: Buffer.from(result.stdout), + stderr: Buffer.from(result.stderr), + exitCode: result.exitCode, + }; } for (const scenario of scenarios) { - it(scenario.name, () => { + it(scenario.name, async () => { const native = runNative(scenario); - const wasm = runWasm(scenario); + const wasm = await runWasm(scenario); if (scenario.expectStdout !== undefined) - expect(text(native.stdout)).toContain(scenario.expectStdout); + expect(native.stdout.toString()).toContain(scenario.expectStdout); if (scenario.expectStderr !== undefined) - expect(text(native.stderr)).toContain(scenario.expectStderr); + expect(native.stderr.toString()).toContain(scenario.expectStderr); if (scenario.rejectStdout !== undefined) - expect(text(native.stdout)).not.toContain(scenario.rejectStdout); + expect(native.stdout.toString()).not.toContain(scenario.rejectStdout); - expect(text(wasm.stderr)).toBe(text(native.stderr)); - expect(text(wasm.stdout)).toBe(text(native.stdout)); + expect(wasm.stderr.toString()).toBe(native.stderr.toString()); + expect(wasm.stdout.toString()).toBe(native.stdout.toString()); expect(Buffer.compare(wasm.stdout, native.stdout)).toBe(0); expect(Buffer.compare(wasm.stderr, native.stderr)).toBe(0); expect(wasm.exitCode).toBe(native.exitCode); diff --git a/test/preopen.test.ts b/test/preopen.test.ts index 5d6fc62..e90776a 100644 --- a/test/preopen.test.ts +++ b/test/preopen.test.ts @@ -1,179 +1,170 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; -import { wasi } from "@bjorn3/browser_wasi_shim"; -import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; -import { createReadOnlyPreopen, run, type ReadOnlyPreopen } from "../src/node.js"; +import { wasi, type Fd } from "@bjorn3/browser_wasi_shim"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import type { DirectoryEntry, SyncFileSystem } from "../src/bridge.js"; +import type { FileType } from "../src/index.js"; +import { createReadOnlyPreopen } from "../src/preopen.js"; import { codesOf, + createTestShellCheck, fixtureRoot, - hasWasm, json1Comments, - loadTestModule, - skipHint, - text, - wasmHint, + skipWorkerTests, + type TestShellCheck, } from "./helpers.js"; +import { fileSystemError, memoryFileSystem, nodeFileSystem } from "./support/file-systems.js"; const JSON1_STDIN = ["-f", "json1", "-s", "bash", "-"]; const rcRoot = path.join(fixtureRoot, "rc"); -const openFdCount = (): number => fs.readdirSync("/proc/self/fd").length; -const onLinux = process.platform === "linux"; - -function openFile(preopen: ReadOnlyPreopen, guestPath: string, oflags = 0) { - return preopen.path_open(0, guestPath, oflags, 0n, 0n, 0); +const encoder = new TextEncoder(); +const decoder = new TextDecoder(); + +/** A synchronous in-memory stand-in for the bridge; parent directories are implied. */ +function syncFileSystem(files: Record): SyncFileSystem & { calls: string[] } { + const calls: string[] = []; + const directories = new Map>([["/", new Map()]]); + for (const file of Object.keys(files)) { + let child = file; + let type: FileType = "file"; + while (child !== "/") { + const parent = path.posix.dirname(child); + if (!directories.has(parent)) directories.set(parent, new Map()); + directories.get(parent)!.set(path.posix.basename(child), type); + child = parent; + type = "directory"; + } + } + return { + calls, + stat: (guestPath) => { + calls.push(`stat ${guestPath}`); + const content = files[guestPath]; + if (content !== undefined) return { type: "file", size: content.length, mtime: 1_000 }; + if (directories.has(guestPath)) return { type: "directory", size: 0, mtime: 1_000 }; + return wasi.ERRNO_NOENT; + }, + readFile: (guestPath) => { + calls.push(`readFile ${guestPath}`); + const content = files[guestPath]; + return content === undefined ? wasi.ERRNO_NOENT : encoder.encode(content); + }, + readDirectory: (guestPath): readonly DirectoryEntry[] | number => { + calls.push(`readDirectory ${guestPath}`); + const entries = directories.get(guestPath); + return entries === undefined ? wasi.ERRNO_NOTDIR : [...entries]; + }, + }; } -describe("createReadOnlyPreopen", () => { - const preopens: ReadOnlyPreopen[] = []; - let scratch: string; - let outside: string; +function openAt(dir: Fd, guestPath: string, oflags = 0) { + return dir.path_open(0, guestPath, oflags, 0n, 0n, 0); +} - const track = (preopen: ReadOnlyPreopen): ReadOnlyPreopen => { - preopens.push(preopen); - return preopen; +describe("read-only preopen", () => { + const files = { + "/.shellcheckrc": "disable=SC2086\n", + "/child/lib.sh": 'LIB_HOME="/opt/lib"\n', + "/child/nested/deep.sh": "echo deep\n", }; - beforeAll(() => { - const base = fs.mkdtempSync(path.join(os.tmpdir(), "shellcheck-wasm-preopen-")); - scratch = path.join(base, "root"); - outside = path.join(base, "outside"); - fs.mkdirSync(path.join(scratch, "child"), { recursive: true }); - fs.mkdirSync(outside); - fs.writeFileSync(path.join(outside, "secret.sh"), "SECRET=1\n"); - fs.writeFileSync(path.join(scratch, "child", "inside.sh"), "INSIDE=1\n"); - fs.symlinkSync(path.join(outside, "secret.sh"), path.join(scratch, "child", "escape.sh")); - fs.symlinkSync(outside, path.join(scratch, "child", "escape-dir")); - fs.symlinkSync("inside.sh", path.join(scratch, "child", "alias.sh")); - }); - - afterEach(() => { - for (const preopen of preopens.splice(0)) preopen.dispose(); - }); - - afterAll(() => { - fs.rmSync(path.dirname(scratch), { recursive: true, force: true }); - }); - - it("announces the guest path as its prestat", () => { - const preopen = track(createReadOnlyPreopen(rcRoot, "/work")); - const { ret, prestat } = preopen.fd_prestat_get(); + it("announces itself as guest /", () => { + const { ret, prestat } = createReadOnlyPreopen(syncFileSystem(files)).fd_prestat_get(); expect(ret).toBe(wasi.ERRNO_SUCCESS); - expect(new TextDecoder().decode(prestat?.inner.pr_name)).toBe("/work"); - expect(createReadOnlyPreopen(rcRoot).fd_prestat_get().prestat?.inner.pr_name).toEqual( - new TextEncoder().encode("/"), - ); - }); - - it("rejects a host path that is not a directory", () => { - expect(() => createReadOnlyPreopen(path.join(rcRoot, ".shellcheckrc"))).toThrow( - /not a directory/, - ); - expect(() => createReadOnlyPreopen(path.join(rcRoot, "does-not-exist"))).toThrow(/ENOENT/); + expect(decoder.decode(prestat?.inner.pr_name)).toBe("/"); }); it("opens and reads a file through a nested directory fd", () => { - const preopen = track(createReadOnlyPreopen(rcRoot)); - const dir = openFile(preopen, "child", wasi.OFLAGS_DIRECTORY); + const preopen = createReadOnlyPreopen(syncFileSystem(files)); + const dir = openAt(preopen, "child", wasi.OFLAGS_DIRECTORY); expect(dir.ret).toBe(wasi.ERRNO_SUCCESS); expect(dir.fd_obj?.fd_fdstat_get().fdstat?.fs_filetype).toBe(wasi.FILETYPE_DIRECTORY); - const file = dir.fd_obj!.path_open(0, "lib.sh", 0, 0n, 0n, 0); + const file = openAt(dir.fd_obj!, "lib.sh"); expect(file.ret).toBe(wasi.ERRNO_SUCCESS); const fd = file.fd_obj!; expect(fd.fd_fdstat_get().fdstat?.fs_filetype).toBe(wasi.FILETYPE_REGULAR_FILE); + expect(fd.fd_fdstat_set_flags(0)).toBe(wasi.ERRNO_SUCCESS); - const expected = fs.readFileSync(path.join(rcRoot, "child", "lib.sh")); + const expected = encoder.encode(files["/child/lib.sh"]); const head = fd.fd_read(8); const rest = fd.fd_read(1 << 16); - expect(Buffer.concat([head.data, rest.data])).toEqual(expected); + expect(new Uint8Array([...head.data, ...rest.data])).toEqual(expected); expect(fd.fd_read(16).data).toHaveLength(0); expect(fd.fd_tell().offset).toBe(BigInt(expected.length)); - expect(fd.fd_seek(0n, wasi.WHENCE_SET)).toEqual({ ret: wasi.ERRNO_SUCCESS, offset: 0n }); expect(fd.fd_seek(-3n, wasi.WHENCE_END).offset).toBe(BigInt(expected.length - 3)); - expect(fd.fd_pread(4, 0n).data).toEqual(new Uint8Array(expected.subarray(0, 4))); - expect(fd.fd_filestat_get().filestat?.size).toBe(BigInt(expected.length)); - expect(fd.fd_close()).toBe(wasi.ERRNO_SUCCESS); - expect(fd.fd_read(1).ret).toBe(wasi.ERRNO_BADF); + expect(fd.fd_pread(4, 0n).data).toEqual(expected.subarray(0, 4)); + + const byFd = fd.fd_filestat_get().filestat!; + const byPath = preopen.path_filestat_get(1, "child/lib.sh").filestat!; + expect(byFd.size).toBe(BigInt(expected.length)); + expect(byFd.filetype).toBe(wasi.FILETYPE_REGULAR_FILE); + expect(byFd.ino).toBe(byPath.ino); + expect(byPath.mtim).toBe(1_000_000_000n); }); - it("stats paths with and without following symlinks", () => { - const preopen = track(createReadOnlyPreopen(scratch)); - const followed = preopen.path_filestat_get(1, "child/alias.sh"); - expect(followed.ret).toBe(wasi.ERRNO_SUCCESS); - expect(followed.filestat?.filetype).toBe(wasi.FILETYPE_REGULAR_FILE); - const unfollowed = preopen.path_filestat_get(0, "child/alias.sh"); - expect(unfollowed.filestat?.filetype).toBe(wasi.FILETYPE_SYMBOLIC_LINK); - expect(preopen.path_readlink("child/alias.sh")).toEqual({ - ret: wasi.ERRNO_SUCCESS, - data: "inside.sh", - }); - expect(preopen.path_filestat_get(1, "child").filestat?.filetype).toBe(wasi.FILETYPE_DIRECTORY); - expect(preopen.path_filestat_get(1, "./child/../child/./inside.sh").ret).toBe( - wasi.ERRNO_SUCCESS, - ); - }); - - it("maps host errors to errnos", () => { - const preopen = track(createReadOnlyPreopen(rcRoot)); - expect(openFile(preopen, "missing.sh").ret).toBe(wasi.ERRNO_NOENT); + it("maps lookups to errnos", () => { + const preopen = createReadOnlyPreopen(syncFileSystem(files)); + expect(openAt(preopen, "missing.sh").ret).toBe(wasi.ERRNO_NOENT); expect(preopen.path_filestat_get(1, "child/missing").ret).toBe(wasi.ERRNO_NOENT); - expect(openFile(preopen, "child/lib.sh", wasi.OFLAGS_DIRECTORY).ret).toBe(wasi.ERRNO_NOTDIR); - expect(openFile(preopen, "child/lib.sh/x").ret).toBe(wasi.ERRNO_NOTDIR); - expect(openFile(preopen, "child\0x").ret).toBe(wasi.ERRNO_INVAL); - const dir = openFile(preopen, "child"); - expect(dir.ret).toBe(wasi.ERRNO_SUCCESS); + expect(openAt(preopen, "child/lib.sh", wasi.OFLAGS_DIRECTORY).ret).toBe(wasi.ERRNO_NOTDIR); + expect(openAt(preopen, "child/lib.sh/").ret).toBe(wasi.ERRNO_NOTDIR); + expect(preopen.path_filestat_get(1, "child/lib.sh/").ret).toBe(wasi.ERRNO_NOTDIR); + expect(openAt(preopen, "child\0x").ret).toBe(wasi.ERRNO_INVAL); + expect(preopen.path_filestat_get(1, "child/").filestat?.filetype).toBe(wasi.FILETYPE_DIRECTORY); + expect(preopen.path_filestat_get(1, "./child/../child/./lib.sh").ret).toBe(wasi.ERRNO_SUCCESS); + const dir = openAt(preopen, "child"); expect(dir.fd_obj?.fd_read(16).ret).toBe(wasi.ERRNO_ISDIR); - expect(preopen.path_readlink("child/lib.sh").ret).toBe(wasi.ERRNO_INVAL); + expect(dir.fd_obj?.fd_filestat_get().filestat?.filetype).toBe(wasi.FILETYPE_DIRECTORY); + }); + + it("reports nothing as a symlink", () => { + const preopen = createReadOnlyPreopen(syncFileSystem(files)); + expect(preopen.path_readlink("child/lib.sh")).toEqual({ ret: wasi.ERRNO_INVAL, data: null }); + expect(preopen.path_readlink("child")).toEqual({ ret: wasi.ERRNO_INVAL, data: null }); + expect(preopen.path_readlink("child/missing")).toEqual({ ret: wasi.ERRNO_NOENT, data: null }); }); it("enumerates a directory with . and .. first", () => { - const preopen = track(createReadOnlyPreopen(rcRoot)); - const names: string[] = []; + const preopen = createReadOnlyPreopen(syncFileSystem(files)); + const entries: [string, number][] = []; let cookie = 0n; for (;;) { const { ret, dirent } = preopen.fd_readdir_single(cookie); expect(ret).toBe(wasi.ERRNO_SUCCESS); if (dirent === null) break; - names.push(new TextDecoder().decode(dirent.dir_name)); + entries.push([decoder.decode(dirent.dir_name), dirent.d_type]); cookie = dirent.d_next; } - expect(names.slice(0, 2)).toEqual([".", ".."]); - expect(names.slice(2).toSorted()).toEqual([".shellcheckrc", "child", "nearest"]); - expect(preopen.fd_readdir_single(2n).dirent?.d_type).toBeOneOf([ - wasi.FILETYPE_REGULAR_FILE, - wasi.FILETYPE_DIRECTORY, + expect(entries).toEqual([ + [".", wasi.FILETYPE_DIRECTORY], + ["..", wasi.FILETYPE_DIRECTORY], + [".shellcheckrc", wasi.FILETYPE_REGULAR_FILE], + ["child", wasi.FILETYPE_DIRECTORY], ]); }); - it("refuses paths that leave the root, textually or through a symlink", () => { - const preopen = track(createReadOnlyPreopen(scratch)); - for (const escaping of [ - "..", - "../outside/secret.sh", - "child/../../outside/secret.sh", - "/etc/passwd", - ]) { - expect(openFile(preopen, escaping).ret, escaping).toBe(wasi.ERRNO_NOTCAPABLE); + it("refuses paths that leave the mount without asking the file system", () => { + const mount = syncFileSystem(files); + const preopen = createReadOnlyPreopen(mount); + const child = openAt(preopen, "child").fd_obj!; + mount.calls.length = 0; + for (const escaping of ["..", "../outside/secret.sh", "child/../../outside", "/etc/passwd"]) { + expect(openAt(preopen, escaping).ret, escaping).toBe(wasi.ERRNO_NOTCAPABLE); expect(preopen.path_filestat_get(1, escaping).ret, escaping).toBe(wasi.ERRNO_NOTCAPABLE); + expect(preopen.path_readlink(escaping).ret, escaping).toBe(wasi.ERRNO_NOTCAPABLE); } - for (const viaSymlink of [ - "child/escape.sh", - "child/escape-dir", - "child/escape-dir/secret.sh", - ]) { - expect(openFile(preopen, viaSymlink).ret, viaSymlink).toBe(wasi.ERRNO_NOTCAPABLE); - expect(preopen.path_filestat_get(1, viaSymlink).ret, viaSymlink).toBe(wasi.ERRNO_NOTCAPABLE); - expect(preopen.path_filestat_get(0, viaSymlink).ret, viaSymlink).toBe(wasi.ERRNO_NOTCAPABLE); - expect(preopen.path_readlink(viaSymlink).ret, viaSymlink).toBe(wasi.ERRNO_NOTCAPABLE); - } - expect(openFile(preopen, "child/alias.sh").ret).toBe(wasi.ERRNO_SUCCESS); + expect(openAt(child, "../../child/lib.sh").ret).toBe(wasi.ERRNO_NOTCAPABLE); + expect(mount.calls).toEqual([]); + expect(openAt(child, "../child/./lib.sh").ret).toBe(wasi.ERRNO_SUCCESS); + expect(mount.calls).toEqual(["stat /child/lib.sh", "readFile /child/lib.sh"]); }); it("refuses every mutating operation with EROFS", () => { - const preopen = track(createReadOnlyPreopen(rcRoot)); - expect(openFile(preopen, "new.sh", wasi.OFLAGS_CREAT).ret).toBe(wasi.ERRNO_ROFS); - expect(openFile(preopen, "child/lib.sh", wasi.OFLAGS_TRUNC).ret).toBe(wasi.ERRNO_ROFS); + const preopen = createReadOnlyPreopen(syncFileSystem(files)); + expect(openAt(preopen, "new.sh", wasi.OFLAGS_CREAT).ret).toBe(wasi.ERRNO_ROFS); + expect(openAt(preopen, "child/lib.sh", wasi.OFLAGS_TRUNC).ret).toBe(wasi.ERRNO_ROFS); expect(preopen.path_open(0, "child/lib.sh", 0, BigInt(wasi.RIGHTS_FD_WRITE), 0n, 0).ret).toBe( wasi.ERRNO_ROFS, ); @@ -187,150 +178,130 @@ describe("createReadOnlyPreopen", () => { expect(preopen.fd_filestat_set_size(0n)).toBe(wasi.ERRNO_ROFS); expect(preopen.fd_filestat_set_times(0n, 0n, 0)).toBe(wasi.ERRNO_ROFS); - const file = openFile(preopen, "child/lib.sh").fd_obj!; + const file = openAt(preopen, "child/lib.sh").fd_obj!; expect(file.fd_write(new Uint8Array([1])).ret).toBe(wasi.ERRNO_ROFS); expect(file.fd_pwrite(new Uint8Array([1]), 0n).ret).toBe(wasi.ERRNO_ROFS); expect(file.fd_allocate(0n, 1n)).toBe(wasi.ERRNO_ROFS); expect(file.fd_filestat_set_size(0n)).toBe(wasi.ERRNO_ROFS); expect(file.fd_filestat_set_times(0n, 0n, 0)).toBe(wasi.ERRNO_ROFS); - expect(fs.readFileSync(path.join(rcRoot, "child", "lib.sh"), "utf8")).toContain("lib_greet"); + expect(decoder.decode(file.fd_pread(64, 0n).data)).toBe(files["/child/lib.sh"]); }); +}); - it("caps open files at 256 with ENFILE", () => { - const preopen = track(createReadOnlyPreopen(rcRoot)); - for (let i = 0; i < 256; i += 1) { - expect(openFile(preopen, "child/lib.sh").ret).toBe(wasi.ERRNO_SUCCESS); - } - expect(openFile(preopen, "child/lib.sh").ret).toBe(wasi.ERRNO_NFILE); - preopen.dispose(); - expect(openFile(preopen, "child/lib.sh").ret).toBe(wasi.ERRNO_SUCCESS); - }); +describe.skipIf(skipWorkerTests())("read-only preopen with ShellCheck", () => { + let shellcheck: TestShellCheck; + let scratch: string; - it.skipIf(!onLinux)("dispose closes host descriptors the guest left open", () => { - const before = openFdCount(); - const preopen = createReadOnlyPreopen(rcRoot); - const first = openFile(preopen, "child/lib.sh"); - const second = openFile(preopen, "child/main.sh"); - expect(openFdCount()).toBe(before + 2); - expect(first.fd_obj!.fd_close()).toBe(wasi.ERRNO_SUCCESS); - expect(openFdCount()).toBe(before + 1); - preopen.dispose(); - expect(openFdCount()).toBe(before); - expect(second.fd_obj!.fd_read(1).ret).toBe(wasi.ERRNO_BADF); - preopen.dispose(); - expect(openFdCount()).toBe(before); + beforeAll(() => { + shellcheck = createTestShellCheck(); + scratch = fs.mkdtempSync(path.join(os.tmpdir(), "shellcheck-wasm-preopen-")); + fs.mkdirSync(path.join(scratch, "root", "child"), { recursive: true }); + fs.mkdirSync(path.join(scratch, "outside")); + fs.writeFileSync(path.join(scratch, "outside", "secret.sh"), "SECRET=1\n"); + fs.writeFileSync(path.join(scratch, "root", "child", "inside.sh"), "INSIDE=1\n"); }); - it.skipIf(!onLinux)("closes the host descriptor when fstat fails after open", () => { - const preopen = track(createReadOnlyPreopen(rcRoot)); - const before = openFdCount(); - const fstat = vi.spyOn(fs, "fstatSync").mockImplementationOnce(() => { - throw Object.assign(new Error("injected"), { code: "EIO" }); - }); - try { - expect(openFile(preopen, "child/lib.sh").ret).toBe(wasi.ERRNO_IO); - } finally { - fstat.mockRestore(); - } - expect(openFdCount()).toBe(before); + afterAll(async () => { + await shellcheck.dispose(); + fs.rmSync(scratch, { recursive: true, force: true }); }); - describe.skipIf(skipHint(!hasWasm, wasmHint))("with ShellCheck", () => { - let module: WebAssembly.Module; - - beforeAll(async () => { - module = await loadTestModule(); + it("finds a .shellcheckrc above the guest working directory", async () => { + const withRc = await shellcheck.lint({ + args: JSON1_STDIN, + stdin: "echo $x\n", + env: { PWD: "/child" }, + fs: nodeFileSystem(rcRoot), }); + expect(codesOf(withRc.stdout)).toEqual([2154]); - it("finds a .shellcheckrc above the guest working directory", () => { - const withRc = run(module, { - args: JSON1_STDIN, - stdin: "echo $x\n", - env: { PWD: "/child" }, - preopens: [track(createReadOnlyPreopen(rcRoot))], - }); - expect(codesOf(withRc.stdout)).toEqual([2154]); + const withoutRc = await shellcheck.lint({ args: JSON1_STDIN, stdin: "echo $x\n" }); + expect(codesOf(withoutRc.stdout)).toEqual([2154, 2086]); + }); - const withoutRc = run(module, { args: JSON1_STDIN, stdin: "echo $x\n" }); - expect(codesOf(withoutRc.stdout)).toEqual([2154, 2086]); + it("lets the nearest .shellcheckrc shadow the one above it", async () => { + const result = await shellcheck.lint({ + args: JSON1_STDIN, + stdin: "unused_here=1\necho $PATH\n", + env: { PWD: "/nearest" }, + fs: nodeFileSystem(rcRoot), }); + expect(codesOf(result.stdout)).toEqual([2086]); + }); - it("lets the nearest .shellcheckrc shadow the one above it", () => { - const result = run(module, { - args: JSON1_STDIN, - stdin: "unused_here=1\necho $PATH\n", - env: { PWD: "/nearest" }, - preopens: [track(createReadOnlyPreopen(rcRoot))], - }); - expect(codesOf(result.stdout)).toEqual([2086]); + it("follows source relative to the guest working directory with -x", async () => { + const script = 'source ./lib.sh\necho "$LIB_HOME"\n'; + const followed = await shellcheck.lint({ + args: ["-x", ...JSON1_STDIN], + stdin: script, + env: { PWD: "/child" }, + fs: nodeFileSystem(rcRoot), }); - - it("follows source relative to the guest working directory with -x", () => { - const script = 'source ./lib.sh\necho "$LIB_HOME"\n'; - const followed = run(module, { - args: ["-x", ...JSON1_STDIN], - stdin: script, - env: { PWD: "/child" }, - preopens: [track(createReadOnlyPreopen(rcRoot))], - }); - expect(followed.exitCode).toBe(0); - expect(codesOf(followed.stdout)).toEqual([]); - - const unfollowed = run(module, { - args: JSON1_STDIN, - stdin: script, - env: { PWD: "/child" }, - preopens: [track(createReadOnlyPreopen(rcRoot))], - }); - expect(codesOf(unfollowed.stdout)).toEqual([1091]); + expect(followed.exitCode).toBe(0); + expect(codesOf(followed.stdout)).toEqual([]); + + const unfollowed = await shellcheck.lint({ + args: JSON1_STDIN, + stdin: script, + env: { PWD: "/child" }, + fs: nodeFileSystem(rcRoot), }); + expect(codesOf(unfollowed.stdout)).toEqual([1091]); + }); - it("reports a sourced file behind an escaping symlink as unreadable", () => { - const result = run(module, { - args: ["-x", ...JSON1_STDIN], - stdin: 'source ./escape.sh\necho "$SECRET"\n', - env: { PWD: "/child" }, - preopens: [track(createReadOnlyPreopen(scratch))], - }); - const comments = json1Comments(result.stdout); - expect(comments.map((comment) => comment.code)).toEqual([1091]); - expect(comments[0]?.message).toContain("./escape.sh"); - expect(comments[0]?.message).not.toContain("does not exist"); - expect(text(result.stderr)).toBe(""); - - const control = run(module, { - args: ["-x", ...JSON1_STDIN], - stdin: 'source ./inside.sh\necho "$INSIDE"\n', - env: { PWD: "/child" }, - preopens: [track(createReadOnlyPreopen(scratch))], - }); - expect(codesOf(control.stdout)).toEqual([]); + it("asks the file system about each path at most once per lint", async () => { + const mount = nodeFileSystem(rcRoot); + await shellcheck.lint({ + args: ["-x", ...JSON1_STDIN], + stdin: "source ../lib.sh\nsource ../lib.sh\necho $x\n", + env: { PWD: "/child/nested" }, + fs: mount, }); + expect(mount.calls).toContain("readFile /.shellcheckrc"); + expect(mount.calls).toContain("readFile /child/lib.sh"); + expect(new Set(mount.calls).size).toBe(mount.calls.length); + }); - it("mounts at a guest path other than /", () => { - const result = run(module, { - args: ["-x", ...JSON1_STDIN], - stdin: 'source /work/child/lib.sh\necho "$LIB_HOME"\n', - env: { PWD: "/work/child" }, - preopens: [track(createReadOnlyPreopen(rcRoot, "/work"))], - }); - expect(codesOf(result.stdout)).toEqual([]); + it("never lets .. reach a file outside the mount", async () => { + const mount = nodeFileSystem(path.join(scratch, "root")); + const result = await shellcheck.lint({ + args: ["-x", ...JSON1_STDIN], + stdin: 'source ../../outside/secret.sh\necho "$SECRET"\n', + env: { PWD: "/child" }, + fs: mount, + }); + const comments = json1Comments(result.stdout); + expect(comments.map((comment) => comment.code)).toEqual([1091]); + expect(comments[0]?.message).toContain("../../outside/secret.sh"); + expect(result.stderr).toBe(""); + expect(mount.calls.filter((call) => call.includes("secret"))).toEqual([]); + + const control = await shellcheck.lint({ + args: ["-x", ...JSON1_STDIN], + stdin: 'source ./inside.sh\necho "$INSIDE"\n', + env: { PWD: "/child" }, + fs: nodeFileSystem(path.join(scratch, "root")), }); + expect(codesOf(control.stdout)).toEqual([]); + }); - it.skipIf(!onLinux)("does not leak host descriptors across 50 runs", () => { - const before = openFdCount(); - for (let i = 0; i < 50; i += 1) { - const preopen = createReadOnlyPreopen(rcRoot); - const result = run(module, { - args: ["-x", ...JSON1_STDIN], - stdin: "source ./lib.sh\nlib_greet $USER\n", - env: { PWD: "/child" }, - preopens: [preopen], - }); - preopen.dispose(); - expect(result.exitCode).toBe(0); - } - expect(openFdCount()).toBeLessThanOrEqual(before); + it("reports a sourced file the file system refuses", async () => { + const mount = memoryFileSystem( + { "/work/lib.sh": "LIB=1\n" }, + { + readFile: async () => { + throw fileSystemError("NoPermissions"); + }, + }, + ); + const result = await shellcheck.lint({ + args: ["-x", ...JSON1_STDIN], + stdin: 'source ./lib.sh\necho "$LIB"\n', + env: { PWD: "/work" }, + fs: mount, }); + const comments = json1Comments(result.stdout); + expect(comments[0]?.code).toBe(1091); + expect(comments[0]?.message).toMatch(/permission denied/i); }); }); diff --git a/test/runner.test.ts b/test/runner.test.ts deleted file mode 100644 index 2e66d3b..0000000 --- a/test/runner.test.ts +++ /dev/null @@ -1,86 +0,0 @@ -import { beforeAll, describe, expect, it } from "vitest"; -import { run, SHELLCHECK_VERSION } from "../src/index.js"; -import { codesOf, hasWasm, loadTestModule, skipHint, text, wasmHint } from "./helpers.js"; - -describe.skipIf(skipHint(!hasWasm, wasmHint))("run", () => { - let module: WebAssembly.Module; - - beforeAll(async () => { - module = await loadTestModule(); - }); - - it("reports the version recorded in version.txt", () => { - const result = run(module, { args: ["--version"] }); - expect(result.exitCode).toBe(0); - expect(text(result.stdout)).toContain(`version: ${SHELLCHECK_VERSION.slice(1)}`); - expect(result.stderr).toHaveLength(0); - }); - - it("reports SC2086 as json1 with exit code 1", () => { - const result = run(module, { args: ["-f", "json1", "-s", "bash", "-"], stdin: "echo $x\n" }); - expect(result.exitCode).toBe(1); - expect(codesOf(result.stdout)).toContain(2086); - expect(result.stderr).toHaveLength(0); - }); - - it("exits 0 with an empty comment list for a clean script", () => { - const result = run(module, { args: ["-f", "json1", "-s", "bash", "-"], stdin: 'echo "$1"\n' }); - expect(result.exitCode).toBe(0); - expect(text(result.stdout)).toBe('{"comments":[]}\n'); - }); - - it("rejects an unknown flag on stderr with exit code >= 2", () => { - const result = run(module, { args: ["--no-such-flag"] }); - expect(result.exitCode).toBeGreaterThanOrEqual(2); - expect(text(result.stderr)).toContain("no-such-flag"); - }); - - it("treats string and Uint8Array stdin alike", () => { - const script = "echo $x\n"; - const asString = run(module, { args: ["-f", "json1", "-s", "bash", "-"], stdin: script }); - const asBytes = run(module, { - args: ["-f", "json1", "-s", "bash", "-"], - stdin: new TextEncoder().encode(script), - }); - expect(asBytes).toEqual(asString); - }); - - it("defaults stdin to empty", () => { - const result = run(module, { args: ["-f", "json1", "-s", "bash", "-"] }); - expect(result.exitCode).toBe(0); - expect(text(result.stdout)).toBe('{"comments":[]}\n'); - }); - - it("runs the same Module repeatedly with a fresh Instance each time", () => { - const first = run(module, { args: ["-f", "json1", "-s", "bash", "-"], stdin: "echo $a\n" }); - const second = run(module, { args: ["-f", "json1", "-s", "bash", "-"], stdin: "echo $b\n" }); - expect(first.exitCode).toBe(1); - expect(second.exitCode).toBe(1); - expect(text(first.stdout)).toContain("a is referenced"); - expect(text(second.stdout)).toContain("b is referenced"); - }); - - it("passes the environment through", () => { - // SHELLCHECK_OPTS is read by ShellCheck itself, so its effect proves env reaches the guest. - const result = run(module, { - args: ["-f", "json1", "-s", "bash", "-"], - stdin: "echo $x\n", - env: { SHELLCHECK_OPTS: "-e SC2086 -e SC2154" }, - }); - expect(result.exitCode).toBe(0); - expect(text(result.stdout)).toBe('{"comments":[]}\n'); - }); - - it("fails fast, without hanging, when PWD is outside every preopen", () => { - // The GHC RTS chdir()s to PWD before main runs; with no preopen to satisfy it the - // program aborts before ShellCheck can write anything to stdout. - const result = run(module, { - args: ["-f", "json1", "-s", "bash", "-"], - stdin: "echo $x\n", - env: { PWD: "/nowhere" }, - }); - expect(result.stdout).toHaveLength(0); - expect(result.exitCode).not.toBe(0); - expect(text(result.stderr)).toContain("chdir"); - }); -}); diff --git a/test/scheduler.test.ts b/test/scheduler.test.ts new file mode 100644 index 0000000..a75786f --- /dev/null +++ b/test/scheduler.test.ts @@ -0,0 +1,204 @@ +import { afterEach, describe, expect, it } from "vitest"; +import type { LintRequest, ShellCheckFileSystem } from "../src/index.js"; +import { + codesOf, + createTestShellCheck, + nodeWorkerPort, + skipWorkerTests, + supportWorkerUrl, + type TestShellCheck, +} from "./helpers.js"; +import { memoryFileSystem } from "./support/file-systems.js"; + +const JSON1_STDIN = ["-f", "json1", "-s", "bash", "-"]; +const EMPTY_WASM_MODULE = new Uint8Array([0x00, 0x61, 0x73, 0x6d, 0x01, 0x00, 0x00, 0x00]); + +/** Long enough that ShellCheck is still busy when a short timeout fires. */ +function slowScript(): string { + const lines = ["#!/bin/bash"]; + for (let i = 0; i < 400; i += 1) lines.push(i % 3 ? `echo $v${i}` : `v${i}="value ${i}"`); + return `${lines.join("\n")}\n`; +} + +function deferred(): { promise: Promise; resolve: () => void } { + let resolve!: () => void; + const promise = new Promise((settle) => { + resolve = settle; + }); + return { promise, resolve }; +} + +/** A mount whose root stat waits for `gate`, holding the lint inside the bridge. */ +function gatedFileSystem(gate: Promise, onStat?: () => void): ShellCheckFileSystem { + const inner = memoryFileSystem({ "/lib.sh": "LIB=1\n" }); + return { + ...inner, + stat: async (path) => { + onStat?.(); + await gate; + return inner.stat(path); + }, + }; +} + +const mountedLint = (fs: ShellCheckFileSystem): LintRequest => ({ + args: JSON1_STDIN, + stdin: "echo $x\n", + env: { PWD: "/" }, + fs, +}); + +describe.skipIf(skipWorkerTests())("createShellCheck scheduling", () => { + const instances: TestShellCheck[] = []; + const track = (instance: TestShellCheck): TestShellCheck => { + instances.push(instance); + return instance; + }; + + afterEach(async () => { + await Promise.all(instances.splice(0).map((instance) => instance.dispose())); + }); + + it("starts no Worker before the first lint and then keeps one", async () => { + const shellcheck = track(createTestShellCheck()); + expect(shellcheck.spawned).toBe(0); + await shellcheck.lint({ args: ["--version"] }); + await shellcheck.lint({ args: ["--version"] }); + expect(shellcheck.spawned).toBe(1); + }); + + it("runs lints one at a time, first in first out", async () => { + const shellcheck = track(createTestShellCheck()); + const log: string[] = []; + const logged = (name: string): ShellCheckFileSystem => { + const inner = memoryFileSystem({ "/a.sh": "A=1\n" }); + return { + ...inner, + stat: async (path) => { + log.push(name); + return inner.stat(path); + }, + }; + }; + const names = ["first", "second", "third"]; + const results = await Promise.all( + names.map((name) => shellcheck.lint(mountedLint(logged(name))).then(() => name)), + ); + expect(results).toEqual(names); + const firstIndexes = names.map((name) => log.indexOf(name)); + const lastIndexes = names.map((name) => log.lastIndexOf(name)); + expect(firstIndexes[1]).toBeGreaterThan(lastIndexes[0]!); + expect(firstIndexes[2]).toBeGreaterThan(lastIndexes[1]!); + }); + + it("rejects an already aborted signal without starting a Worker", async () => { + const shellcheck = track(createTestShellCheck()); + const reason = new Error("stale"); + await expect( + shellcheck.lint({ args: ["--version"] }, { signal: AbortSignal.abort(reason) }), + ).rejects.toBe(reason); + expect(shellcheck.spawned).toBe(0); + }); + + it("drops a queued lint when its signal aborts", async () => { + const shellcheck = track(createTestShellCheck()); + const gate = deferred(); + const running = shellcheck.lint(mountedLint(gatedFileSystem(gate.promise))); + const queuedFs = memoryFileSystem({}); + const controller = new AbortController(); + const queued = shellcheck.lint(mountedLint(queuedFs), { signal: controller.signal }); + controller.abort(); + await expect(queued).rejects.toMatchObject({ name: "AbortError" }); + gate.resolve(); + expect((await running).exitCode).toBe(1); + expect(queuedFs.calls).toEqual([]); + expect(shellcheck.spawned).toBe(1); + }); + + it("terminates the Worker when the running lint aborts inside a file-system call", async () => { + const shellcheck = track(createTestShellCheck()); + const controller = new AbortController(); + const never = new Promise(() => {}); + const blocked = shellcheck.lint(mountedLint(gatedFileSystem(never, () => controller.abort())), { + signal: controller.signal, + }); + await expect(blocked).rejects.toMatchObject({ name: "AbortError" }); + const next = await shellcheck.lint({ args: JSON1_STDIN, stdin: "echo $x\n" }); + expect(codesOf(next.stdout)).toContain(2086); + expect(shellcheck.spawned).toBe(2); + }); + + it("terminates the Worker when the running lint times out mid-computation", async () => { + const shellcheck = track(createTestShellCheck()); + await shellcheck.lint({ args: ["--version"] }); + const started = performance.now(); + await expect( + shellcheck.lint( + { args: JSON1_STDIN, stdin: slowScript() }, + { signal: AbortSignal.timeout(300) }, + ), + ).rejects.toMatchObject({ name: "TimeoutError" }); + expect(performance.now() - started).toBeLessThan(2_000); + const next = await shellcheck.lint({ args: JSON1_STDIN, stdin: "echo $x\n" }); + expect(next.exitCode).toBe(1); + expect(shellcheck.spawned).toBe(2); + }); + + it("rejects the running lint when the Worker exits, then starts a new one", async () => { + const shellcheck = track( + createTestShellCheck({ + createWorker: () => nodeWorkerPort(supportWorkerUrl("crash-worker.mjs")), + }), + ); + await expect(shellcheck.lint({ args: ["--test-exit"] })).rejects.toThrow( + "ShellCheck worker exited unexpectedly with code 7", + ); + expect((await shellcheck.lint({ args: ["--version"] })).exitCode).toBe(0); + expect(shellcheck.spawned).toBe(2); + }); + + it("rejects the running lint when the Worker throws, then starts a new one", async () => { + const shellcheck = track( + createTestShellCheck({ + createWorker: () => nodeWorkerPort(supportWorkerUrl("crash-worker.mjs")), + }), + ); + await expect(shellcheck.lint({ args: ["--test-throw"] })).rejects.toThrow( + "ShellCheck worker failed: injected worker failure", + ); + expect((await shellcheck.lint({ args: ["--version"] })).exitCode).toBe(0); + expect(shellcheck.spawned).toBe(2); + }); + + it("rejects a lint the guest cannot run without losing the Worker", async () => { + const shellcheck = track( + createTestShellCheck({ module: WebAssembly.compile(EMPTY_WASM_MODULE) }), + ); + for (let i = 0; i < 2; i += 1) { + await expect(shellcheck.lint({ args: [] })).rejects.toThrow( + /^ShellCheck failed: TypeError: shellcheck.wasm is not a WASI command module/, + ); + } + expect(shellcheck.spawned).toBe(1); + }); + + it("rejects when the module cannot be loaded", async () => { + const failure = new Error("no wasm here"); + const module = Promise.reject(failure); + module.catch(() => {}); + const shellcheck = track(createTestShellCheck({ module })); + await expect(shellcheck.lint({ args: [] })).rejects.toBe(failure); + expect(shellcheck.spawned).toBe(0); + }); + + it("rejects pending and later lints once disposed", async () => { + const shellcheck = createTestShellCheck(); + const never = new Promise(() => {}); + const running = shellcheck.lint(mountedLint(gatedFileSystem(never))); + const queued = shellcheck.lint({ args: ["--version"] }); + await shellcheck.dispose(); + await expect(running).rejects.toThrow("disposed"); + await expect(queued).rejects.toThrow("disposed"); + await expect(shellcheck.lint({ args: ["--version"] })).rejects.toThrow("disposed"); + }); +}); diff --git a/test/support/bridge-worker.mjs b/test/support/bridge-worker.mjs new file mode 100644 index 0000000..9a5e4b2 --- /dev/null +++ b/test/support/bridge-worker.mjs @@ -0,0 +1,30 @@ +import { createHash } from "node:crypto"; +import { parentPort } from "node:worker_threads"; +import { Bridge } from "../../dist/bridge.js"; + +// Speaks the worker protocol without ShellCheck, to exercise the bridge alone: each lint +// arg is " ", and stdout is the JSON list of what the bridge returned for each. +let shared; +parentPort.on("message", (message) => { + if (message.type === "init") { + shared = message.shared; + return; + } + const bridge = new Bridge(shared, (reply) => parentPort.postMessage(reply)); + const results = message.args.map((request) => { + const [op, path] = request.split(" "); + const result = bridge[op](path); + if (typeof result === "number") return { errno: result }; + if (result instanceof Uint8Array) { + const sha256 = createHash("sha256").update(result).digest("hex"); + return { size: result.byteLength, sha256 }; + } + return { value: result }; + }); + parentPort.postMessage({ + type: "done", + stdout: JSON.stringify(results), + stderr: "", + exitCode: 0, + }); +}); diff --git a/test/support/crash-worker.mjs b/test/support/crash-worker.mjs new file mode 100644 index 0000000..243e4a5 --- /dev/null +++ b/test/support/crash-worker.mjs @@ -0,0 +1,16 @@ +import { parentPort } from "node:worker_threads"; +import { startWorker } from "../../dist/worker.js"; + +// Fails on demand before ShellCheck sees the lint: `--test-exit` ends the thread and +// `--test-throw` raises an uncaught exception. +startWorker({ + postMessage: (message) => parentPort.postMessage(message), + onMessage: (listener) => + parentPort.on("message", (message) => { + if (message.type === "lint" && message.args[0] === "--test-exit") process.exit(7); + if (message.type === "lint" && message.args[0] === "--test-throw") { + throw new Error("injected worker failure"); + } + listener(message); + }), +}); diff --git a/test/support/file-systems.ts b/test/support/file-systems.ts new file mode 100644 index 0000000..b04aa96 --- /dev/null +++ b/test/support/file-systems.ts @@ -0,0 +1,132 @@ +import type { Stats } from "node:fs"; +import fs from "node:fs/promises"; +import path from "node:path"; +import type { + FileStat, + FileSystemErrorCode, + FileType, + ShellCheckFileSystem, +} from "../../src/index.js"; + +export interface RecordingFileSystem extends ShellCheckFileSystem { + /** Every call made, as `" "`, in order. */ + readonly calls: string[]; +} + +const CODES: Partial> = { + ENOENT: "FileNotFound", + ENOTDIR: "FileNotADirectory", + EISDIR: "FileIsADirectory", + EACCES: "NoPermissions", + EPERM: "NoPermissions", +}; + +export function fileSystemError(code: FileSystemErrorCode | string, message = code): Error { + return Object.assign(new Error(message), { code }); +} + +function typeOf(stats: Stats): FileType { + if (stats.isFile()) return "file"; + if (stats.isDirectory()) return "directory"; + return "other"; +} + +/** + * What a Node Host would pass as `fs`: `root` seen as guest `/`, with Node errors + * translated to the codes the package understands. No symlink containment; tests only. + */ +export function nodeFileSystem(root: string): RecordingFileSystem { + const calls: string[] = []; + const host = (guestPath: string): string => path.join(root, guestPath); + const translate = async (work: () => Promise): Promise => { + try { + return await work(); + } catch (error) { + const code = CODES[(error as NodeJS.ErrnoException).code ?? ""]; + throw code === undefined ? error : fileSystemError(code, (error as Error).message); + } + }; + return { + calls, + stat: (guestPath) => { + calls.push(`stat ${guestPath}`); + return translate(async (): Promise => { + const stats = await fs.stat(host(guestPath)); + return { type: typeOf(stats), size: stats.size, mtime: stats.mtimeMs }; + }); + }, + readFile: (guestPath) => { + calls.push(`readFile ${guestPath}`); + return translate(() => fs.readFile(host(guestPath))); + }, + readDirectory: (guestPath) => { + calls.push(`readDirectory ${guestPath}`); + return translate(async () => { + const entries = await fs.readdir(host(guestPath), { withFileTypes: true }); + return entries.map((entry): [string, FileType] => [ + entry.name, + entry.isDirectory() ? "directory" : entry.isFile() ? "file" : "other", + ]); + }); + }, + }; +} + +/** + * An in-memory tree from guest paths to contents; parent directories are implied. + * `overrides` replace individual operations, e.g. to throw. + */ +export function memoryFileSystem( + files: Record, + overrides: Partial = {}, +): RecordingFileSystem { + const calls: string[] = []; + const contents = new Map(); + const directories = new Map>([["/", new Map()]]); + for (const [file, content] of Object.entries(files)) { + contents.set(file, typeof content === "string" ? new TextEncoder().encode(content) : content); + let child = file; + let type: FileType = "file"; + while (child !== "/") { + const parent = path.posix.dirname(child); + if (!directories.has(parent)) directories.set(parent, new Map()); + directories.get(parent)!.set(path.posix.basename(child), type); + child = parent; + type = "directory"; + } + } + const missing = (guestPath: string): Error => + fileSystemError("FileNotFound", `${guestPath} not found`); + const base: ShellCheckFileSystem = { + stat: async (guestPath) => { + const data = contents.get(guestPath); + if (data !== undefined) return { type: "file", size: data.byteLength, mtime: 0 }; + if (directories.has(guestPath)) return { type: "directory", size: 0, mtime: 0 }; + throw missing(guestPath); + }, + readFile: async (guestPath) => { + const data = contents.get(guestPath); + if (data !== undefined) return data; + if (directories.has(guestPath)) throw fileSystemError("FileIsADirectory"); + throw missing(guestPath); + }, + readDirectory: async (guestPath) => { + const entries = directories.get(guestPath); + if (entries !== undefined) return [...entries]; + if (contents.has(guestPath)) throw fileSystemError("FileNotADirectory"); + throw missing(guestPath); + }, + }; + const record = + (op: string, fn: (guestPath: string, ...rest: A) => R) => + (guestPath: string, ...rest: A): R => { + calls.push(`${op} ${guestPath}`); + return fn(guestPath, ...rest); + }; + return { + calls, + stat: record("stat", overrides.stat ?? base.stat), + readFile: record("readFile", overrides.readFile ?? base.readFile), + readDirectory: record("readDirectory", overrides.readDirectory ?? base.readDirectory), + }; +} diff --git a/test/support/node-worker.mjs b/test/support/node-worker.mjs new file mode 100644 index 0000000..78643a7 --- /dev/null +++ b/test/support/node-worker.mjs @@ -0,0 +1,7 @@ +import { parentPort } from "node:worker_threads"; +import { startWorker } from "../../dist/worker.js"; + +startWorker({ + postMessage: (message) => parentPort.postMessage(message), + onMessage: (listener) => parentPort.on("message", listener), +}); From a10dd70e0164b3377d2e12a90a042a4fddad318a Mon Sep 17 00:00:00 2001 From: timonwong Date: Wed, 23 Sep 2026 09:24:35 +0000 Subject: [PATCH 3/6] test: benchmark lint latency against 0.1.1 and native ShellCheck The bridge adds a round trip to the caller's thread for every file ShellCheck touches, so the redesign has to show it costs no more than 10% over the 0.1.1 node:fs runner. `npm run bench` installs 0.1.1 into .cache/bench, runs both in worker_threads Workers on generated 23, 307 and 1503-line scripts with a .shellcheckrc and a sourced file, and reports native ShellCheck and a `+RTS -A64m` run as data points. --- package.json | 1 + scripts/bench.mjs | 359 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 360 insertions(+) create mode 100644 scripts/bench.mjs diff --git a/package.json b/package.json index 62b2d0e..89a9649 100644 --- a/package.json +++ b/package.json @@ -36,6 +36,7 @@ "build": "npm run gen:version && npm run gen:build-info && tsc -p tsconfig.build.json", "build:wasm": "scripts/build-wasm.sh", "fetch:native": "scripts/fetch-native-shellcheck.sh", + "bench": "node scripts/bench.mjs", "lint": "oxlint", "fmt": "oxfmt", "fmt:check": "oxfmt --check", diff --git a/scripts/bench.mjs b/scripts/bench.mjs new file mode 100644 index 0000000..b1dfb13 --- /dev/null +++ b/scripts/bench.mjs @@ -0,0 +1,359 @@ +// Compares per-lint latency of this checkout against the published 0.1.1 runner and native +// ShellCheck. Not shipped. Needs `npm run build`, dist/shellcheck.wasm and `npm run fetch:native`. +// +// node scripts/bench.mjs # N=20 timed runs per size after WARM=3 warmups +// N=5 SIZES=small node scripts/bench.mjs +// +// Every wasm variant runs in a worker_threads Worker driven from this thread, so all of them +// pay the same round trip. Exits 1 when the new runner is more than 10% slower than 0.1.1. +import { execFileSync, spawnSync } from "node:child_process"; +import { createHash } from "node:crypto"; +import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; +import fs from "node:fs/promises"; +import path from "node:path"; +import { fileURLToPath, pathToFileURL } from "node:url"; +import { Worker } from "node:worker_threads"; + +const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); +const cacheDir = path.join(repoRoot, ".cache", "bench"); +const nativePath = + process.env.SHELLCHECK_NATIVE ?? path.join(repoRoot, ".cache", "native", "shellcheck"); +const OLD_VERSION = "0.1.1"; +const N = Number(process.env.N ?? 20); +const WARM = Number(process.env.WARM ?? 3); +const SIZES = (process.env.SIZES ?? "small,medium,large").split(","); +const GATE = 0.1; +const ARGS = ["-f", "json1", "-s", "bash", "-"]; +const RTS_ARGS = ["+RTS", "-A64m", "-RTS", ...ARGS]; +const PWD = "/src"; + +// Varied function blocks, each triggering several ShellCheck warnings; the same generator as +// the 2026-09 runtime spike, so the sizes come out at 28 / 308 / 1508 lines. +const blocks = [ + (i) => ` +process_files_${i}() { + local UNUSED_VAR_${i}="never read" + target=$1 + echo Processing $target + for f in $(ls /tmp/dir_${i}); do + cp $f /backup/$target + done + echo "$UNDECLARED_${i}" +} +`, + (i) => ` +compute_sum_${i}() { + declare TOTAL_${i}=0 + args="$*" + cd /var/data_${i} + result=\`expr $args + 1\` + [ $result == "1" ] && echo yes + echo $HOME/$args + return $UNSET_RET_${i} +} +`, + (i) => ` +sync_remote_${i}() { + HOST_${i}=example.com + RSYNC_OPTS_${i}="-az --delete" + rsync $RSYNC_OPTS_${i} ./src $HOST_${i}:/dst + if [ $? -ne 0 ]; then + echo "fail" > /dev/stderr + fi + echo "\${MISSING_${i}}" +} +`, + (i) => ` +parse_config_${i}() { + local cfg=$1 + KEY_${i}=$(grep key $cfg | cut -d= -f2) + DEAD_${i}="unused" + while read line; do + echo $line | grep -q $KEY_${i} + done < $cfg + eval "echo \\$OTHER_${i}" + echo $NOPE_${i} +} +`, + (i) => ` +build_pkg_${i}() { + VERSION_${i}=1.0.${i} + ARCHIVE_${i}="pkg-\${VERSION_${i}}.tar" + tar cf $ARCHIVE_${i} $SRC_DIR_${i} + test -e $ARCHIVE_${i} || exit 1 + echo done +} +`, +]; + +function script(minLines) { + const out = ["#!/bin/bash", "# generated bench fixture", "source ./lib.sh"]; + for (let i = 0; out.join("\n").split("\n").length < minLines; i += 1) { + out.push(blocks[i % blocks.length](i).trim(), ""); + } + return `${out.join("\n")}\n`; +} + +const scripts = { small: script(18), medium: script(300), large: script(1500) }; + +function prepareMount() { + const root = path.join(cacheDir, "mount"); + mkdirSync(path.join(root, "src"), { recursive: true }); + writeFileSync(path.join(root, ".shellcheckrc"), "external-sources=true\ndisable=SC2164\n"); + writeFileSync( + path.join(root, "src", "lib.sh"), + '#!/bin/bash\nLIB_HOME="/opt/lib"\nlib_log() { echo "$LIB_HOME: $*"; }\n', + ); + return root; +} + +function installOld() { + const dir = path.join(cacheDir, `old-${OLD_VERSION}`); + const entry = path.join(dir, "old-worker.mjs"); + mkdirSync(dir, { recursive: true }); + // Without a package.json of its own, the entry would resolve the package name to this + // checkout through self-reference instead of to the installed 0.1.1. + writeFileSync(path.join(dir, "package.json"), '{ "private": true, "type": "module" }\n'); + if (!existsSync(path.join(dir, "node_modules", "@vscode-shellcheck", "shellcheck-wasm"))) { + execFileSync( + "npm", + [ + "install", + "--no-save", + "--no-package-lock", + "--prefix", + dir, + `@vscode-shellcheck/shellcheck-wasm@${OLD_VERSION}`, + ], + { stdio: "inherit" }, + ); + } + writeFileSync( + entry, + `import { parentPort } from "node:worker_threads"; +import { createReadOnlyPreopen, loadModule, run } from "@vscode-shellcheck/shellcheck-wasm/node"; + +const module = await loadModule(); +const decoder = new TextDecoder(); +parentPort.on("message", ({ args, stdin, env, root }) => { + const preopen = createReadOnlyPreopen(root); + try { + const result = run(module, { args, stdin, env, preopens: [preopen] }); + parentPort.postMessage({ + stdout: decoder.decode(result.stdout), + stderr: decoder.decode(result.stderr), + exitCode: result.exitCode, + }); + } finally { + preopen.dispose(); + } +}); +parentPort.postMessage("ready"); +`, + ); + const wasm = path.join( + dir, + "node_modules", + "@vscode-shellcheck", + "shellcheck-wasm", + "dist", + "shellcheck.wasm", + ); + return { entry, wasm }; +} + +/** The 0.1.1 runner in a Worker; one request in flight at a time. */ +async function startOld(entry) { + const worker = new Worker(entry); + await new Promise((resolve, reject) => { + worker.once("message", resolve); + worker.once("error", reject); + }); + return { + lint: (request) => + new Promise((resolve, reject) => { + worker.once("error", reject); + worker.once("message", (result) => { + worker.off("error", reject); + resolve(result); + }); + worker.postMessage(request); + }), + close: () => worker.terminate(), + }; +} + +const CODES = { + ENOENT: "FileNotFound", + ENOTDIR: "FileNotADirectory", + EISDIR: "FileIsADirectory", + EACCES: "NoPermissions", + EPERM: "NoPermissions", +}; + +/** A node:fs/promises file system over `root`, counting calls. */ +function nodeFileSystem(root, counter) { + const host = (guestPath) => path.join(root, guestPath); + const translate = async (work) => { + counter.calls += 1; + try { + return await work(); + } catch (error) { + const code = CODES[error.code]; + throw code === undefined ? error : Object.assign(new Error(error.message), { code }); + } + }; + return { + stat: (guestPath) => + translate(async () => { + const stats = await fs.stat(host(guestPath)); + const type = stats.isFile() ? "file" : stats.isDirectory() ? "directory" : "other"; + return { type, size: stats.size, mtime: stats.mtimeMs }; + }), + readFile: (guestPath) => translate(() => fs.readFile(host(guestPath))), + readDirectory: (guestPath) => + translate(async () => + (await fs.readdir(host(guestPath), { withFileTypes: true })).map((entry) => [ + entry.name, + entry.isDirectory() ? "directory" : entry.isFile() ? "file" : "other", + ]), + ), + }; +} + +async function startNew(module) { + const { createShellCheck } = await import(pathToFileURL(path.join(repoRoot, "dist", "index.js"))); + const workerUrl = pathToFileURL(path.join(repoRoot, "test", "support", "node-worker.mjs")); + const shellcheck = createShellCheck({ + module, + createWorker() { + const worker = new Worker(workerUrl); + return { + postMessage: (message) => worker.postMessage(message), + onMessage: (listener) => worker.on("message", listener), + onError: (listener) => worker.on("error", listener), + onExit: (listener) => worker.on("exit", listener), + terminate: () => worker.terminate(), + }; + }, + }); + return shellcheck; +} + +function runNative(stdin, root, home) { + const result = spawnSync(nativePath, ARGS, { + cwd: path.join(root, PWD), + input: stdin, + env: { PATH: process.env.PATH ?? "", HOME: home, LANG: "C.UTF-8" }, + maxBuffer: 64 * 1024 * 1024, + }); + if (result.error) throw result.error; + return { + stdout: result.stdout.toString(), + stderr: result.stderr.toString(), + exitCode: result.status, + }; +} + +const median = (values) => { + const sorted = values.toSorted((a, b) => a - b); + const mid = sorted.length >> 1; + return sorted.length % 2 ? sorted[mid] : (sorted[mid - 1] + sorted[mid]) / 2; +}; +const ms = (value) => value.toFixed(1); +const sha = (bytes) => createHash("sha256").update(bytes).digest("hex").slice(0, 12); + +async function main() { + for (const required of [path.join(repoRoot, "dist", "index.js"), nativePath]) { + if (!existsSync(required)) + throw new Error(`${required} is missing; see the header of this script`); + } + const root = prepareMount(); + const home = path.join(cacheDir, "home"); + mkdirSync(home, { recursive: true }); + const old = installOld(); + + const newWasm = readFileSync(path.join(repoRoot, "dist", "shellcheck.wasm")); + const oldWasm = readFileSync(old.wasm); + console.log( + `# node ${process.version}, N=${N}, warmup=${WARM}; artifact new ${sha(newWasm)} old ${sha(oldWasm)}`, + ); + + const oldRunner = await startOld(old.entry); + const module = WebAssembly.compile(newWasm); + const newRunner = await startNew(module); + // Its own Worker: a 64 MiB nursery left behind in a shared one would slow (b) down. + const rtsRunner = await startNew(module); + const counter = { calls: 0 }; + const env = { PWD }; + + const variants = { + old: (stdin) => oldRunner.lint({ args: ARGS, stdin, env, root }), + new: (stdin) => newRunner.lint({ args: ARGS, stdin, env, fs: nodeFileSystem(root, counter) }), + rts: (stdin) => + rtsRunner.lint({ args: RTS_ARGS, stdin, env, fs: nodeFileSystem(root, counter) }), + native: async (stdin) => runNative(stdin, root, home), + }; + + const rows = []; + let failed = false; + for (const size of SIZES) { + const stdin = scripts[size]; + const lines = stdin.split("\n").length - 1; + const reference = await variants.native(stdin); + const notes = []; + for (const [name, lint] of Object.entries(variants)) { + const result = await lint(stdin); + if (result.stdout !== reference.stdout || result.exitCode !== reference.exitCode) { + notes.push( + `${name} differs from native (exit ${result.exitCode}): ${result.stderr.slice(0, 200)}`, + ); + } + } + counter.calls = 0; + await variants.new(stdin); + const bridgeCalls = counter.calls; + + const times = Object.fromEntries(Object.keys(variants).map((name) => [name, []])); + // The gated pair is interleaved so drift in machine load hits both alike; (c) and (d) + // get passes of their own so a 64 MiB nursery or a native process cannot perturb it. + for (const group of [["old", "new"], ["rts"], ["native"]]) { + for (let i = 0; i < WARM + N; i += 1) { + for (const name of group) { + const started = performance.now(); + await variants[name](stdin); + if (i >= WARM) times[name].push(performance.now() - started); + } + } + } + const med = Object.fromEntries( + Object.entries(times).map(([name, list]) => [name, median(list)]), + ); + const delta = med.new / med.old - 1; + if (delta > GATE) failed = true; + rows.push({ size, lines, med, delta, bridgeCalls, notes }); + console.log(` done ${size}`); + } + await oldRunner.close(); + await newRunner.dispose(); + await rtsRunner.dispose(); + + console.log( + "\n| script | lines | (a) 0.1.1 | (b) new | (b)/(a) | (c) new +RTS -A64m | (d) native | (b)/(d) | fs calls/lint |", + ); + console.log("|---|---|---|---|---|---|---|---|---|"); + for (const { size, lines, med, delta, bridgeCalls } of rows) { + const sign = delta >= 0 ? "+" : ""; + console.log( + `| ${size} | ${lines} | ${ms(med.old)} | ${ms(med.new)} | ${sign}${(delta * 100).toFixed(1)}% | ${ms(med.rts)} | ${ms(med.native)} | ${(med.new / med.native).toFixed(2)}x | ${bridgeCalls} |`, + ); + } + for (const { size, notes } of rows) + for (const note of notes) console.log(`note (${size}): ${note}`); + console.log( + `\nmedian ms per lint; gate: (b) at most ${GATE * 100}% slower than (a): ${failed ? "FAIL" : "pass"}`, + ); + process.exitCode = failed ? 1 : 0; +} + +await main(); From 97a6d153d307185283b39551c92849157712339d Mon Sep 17 00:00:00 2001 From: timonwong Date: Wed, 23 Sep 2026 09:24:35 +0000 Subject: [PATCH 4/6] ci(release): publish prereleases under the next dist-tag A version with a prerelease suffix (0.2.0-next.0) would otherwise become `latest` on npm and a regular GitHub Release. Publish it with `--tag next` and mark its GitHub Release as a prerelease; stable versions publish as before. --- .github/workflows/release.yml | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 4442d13..32df852 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -120,14 +120,34 @@ jobs: - run: npm ci - run: npm run build + # A prerelease version (x.y.z-tag.n) goes to the `next` dist-tag so `latest` keeps + # pointing at the last stable release. + - name: Classify the version + id: channel + run: | + if [[ "${GITHUB_REF_NAME#v}" == *-* ]]; then + echo "prerelease=true" >> "$GITHUB_OUTPUT" + else + echo "prerelease=false" >> "$GITHUB_OUTPUT" + fi + # Trusted Publishing: npm exchanges the job's OIDC token for publish rights, so no # NODE_AUTH_TOKEN is set anywhere in this workflow. - - run: npm publish --provenance --access public + - if: steps.channel.outputs.prerelease == 'false' + run: npm publish --provenance --access public + + - if: steps.channel.outputs.prerelease == 'true' + run: npm publish --provenance --access public --tag next + # build-info.json is not in the npm tarball (its contents are compiled into BUILD_INFO) + # but stays a release asset next to the artifact it describes. - name: Create GitHub Release env: GH_TOKEN: ${{ github.token }} + PRERELEASE: ${{ steps.channel.outputs.prerelease }} run: | + flags=(--verify-tag --generate-notes) + if [ "$PRERELEASE" = "true" ]; then flags+=(--prerelease); fi gh release create "$GITHUB_REF_NAME" \ dist/shellcheck.wasm dist/shellcheck.wasm.sha256 dist/build-info.json \ - --verify-tag --generate-notes + "${flags[@]}" From ac73c3dc2825e0fd658005c30fcc9d3b8e1ce5bf Mon Sep 17 00:00:00 2001 From: timonwong Date: Wed, 23 Sep 2026 09:24:35 +0000 Subject: [PATCH 5/6] docs: document the worker API and record the file-system decision Rewrite the README around createShellCheck, the Worker entry and the ShellCheckFileSystem contract, including the web requirement of cross-origin isolation for SharedArrayBuffer. Add ADR 0006 (the package is file-system agnostic), amend ADR 0005 now that the Worker and bridge live in the package while policy stays in the host, and note in ADR 0003 that the evidence against a reactor build measured a third-party build rather than the reactor model. --- AGENTS.md | 47 ++++--- CONTEXT.md | 18 ++- README.md | 131 +++++++++++++----- docs/adr/0003-wasi-command-module.md | 4 + docs/adr/0005-runner-without-host-policy.md | 8 +- docs/adr/0006-file-system-agnostic-package.md | 20 +++ 6 files changed, 172 insertions(+), 56 deletions(-) create mode 100644 docs/adr/0006-file-system-agnostic-package.md diff --git a/AGENTS.md b/AGENTS.md index a41098f..1a05ff7 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,7 +1,8 @@ # shellcheck-wasm npm package `@vscode-shellcheck/shellcheck-wasm`: ShellCheck compiled to a WASI command module -plus a synchronous TypeScript runner. Built in GitHub Actions from pinned inputs; the artifact is +plus a TypeScript runner that lints in a Host-supplied Worker and reads files only through the +Host's async file system. Built in GitHub Actions from pinned inputs; the artifact is never committed. Vocabulary: `CONTEXT.md` (use its terms). Decisions: `docs/adr/`. Plans: `docs/plans/`. ## Layout @@ -9,9 +10,13 @@ never committed. Vocabulary: `CONTEXT.md` (use its terms). Decisions: `docs/adr/ - `buildtools/wasm/` — everything the Docker build reads: `Dockerfile`, pins (`version.txt`, `shellcheck-src.sha256`, `ghc-wasm-meta.txt`), `cabal.project`, `build.sh` (every step after the sources are unpacked), the tail-call gate script. -- `src/generated/` — written by `npm run build` from `buildtools/wasm/version.txt`; gitignored. -- `dist/` — compiled JS plus `shellcheck.wasm`, `shellcheck.wasm.sha256`, `build-info.json`; gitignored. -- `.cache/native/` — native ShellCheck for the parity suite; gitignored. +- `src/generated/` — written by `npm run build` from `buildtools/wasm/version.txt` and + `dist/build-info.json`; gitignored. +- `dist/` — compiled JS plus `shellcheck.wasm`, `shellcheck.wasm.sha256`, `build-info.json`; + gitignored. `build-info.json` is not packed (it is compiled into `BUILD_INFO`). +- `test/support/` — test-only Worker entries and `ShellCheckFileSystem` adapters. +- `.cache/native/` — native ShellCheck for the parity suite; `.cache/bench/` — `npm run bench` + scratch; gitignored. ## Working locally @@ -19,16 +24,20 @@ Scripts are defined in `package.json`; there is no Docker on the dev machine, so is CI-only. 1. `npm ci` -2. Get an artifact into `dist/`: download `shellcheck.wasm` and `build-info.json` from a GitHub - Release, or use the dev stand-in, the upstream 0.11.0 command module (no tail calls, fine for - runner and parity tests, never for a release): - `mkdir -p dist && curl -fL -o dist/shellcheck.wasm https://raw.githubusercontent.com/wasilibs/go-shellcheck/24025c1590296bcce8e494624e8c3561740a32a4/internal/wasm/shellcheck.wasm` +2. Get an artifact into `dist/`: download `shellcheck.wasm`, `shellcheck.wasm.sha256` and + `build-info.json` from a GitHub Release for the ShellCheck version in `version.txt`. + `npm run build` refuses a `build-info.json` whose sha256 does not match the artifact. 3. `npm run fetch:native` 4. `npm run build && npm run lint && npm run fmt:check && npm test` -Tests print `[skip] …` and skip when `dist/shellcheck.wasm`, the native binary, -`dist/build-info.json` or `LICENSE` is missing; with `CI=1` the same conditions fail -(`test/helpers.ts`). With the stand-in, only the `build-info.json` skip remains. +Tests that start a Worker load `dist/worker.js`, so rebuild after changing `src/`; they fail +when `dist/` is older than `src/`. Tests print `[skip] …` and skip when `dist/shellcheck.wasm`, +`dist/worker.js`, the native binary or `LICENSE` is missing; with `CI=1` the same conditions +fail (`test/helpers.ts`). + +`npm run bench` compares per-lint latency with the published 0.1.1 runner and native +ShellCheck (`scripts/bench.mjs`, several minutes); it exits 1 when the new runner is more than +10% slower than 0.1.1. ## Invariants @@ -39,10 +48,15 @@ Tests print `[skip] …` and skip when `dist/shellcheck.wasm`, the native binary - The tail-call gate runs on the pre-`wasm-opt` linker output because `wasm-opt --enable-tail-call` rewrites `target_features`. `-mtail-call` stays in `WASM_CFLAGS`: ghc-wasm-meta dropped it from its defaults in 2025-09, and GHC only emits `return_call` when the flag is baked in. -- Guest `PWD` must be inside a preopen or unset. The GHC RTS chdir()s to it at init; on failure - the run exits 1 with empty stdout and `hs_init_ghc: chdir(...) failed` on stderr. -- The runner stays free of threads, cancellation and filesystem policy; those are host - concerns (ADR 0005). +- Guest `PWD` must name a directory in the lint's `fs`, or be unset. The GHC RTS chdir()s to it + at init; on failure the run exits 1 with empty stdout and `hs_init_ghc: chdir(...) failed` on + stderr. +- No shipped module imports `node:*` or a bare Node built-in, and files reach the guest only + through the Host's `ShellCheckFileSystem` (ADR 0006; guarded by `test/package.test.ts`). The + caller-side entry `.` does not load the WASI shim; only `./worker` does. +- The package provides the Worker protocol, bridge, FIFO queue and `AbortSignal` cancellation; + creating Workers, what to mount, scheduling policy and watchdog durations stay in the Host + (ADR 0005). - `wasm32-wasi-cabal` is the wrapper ghc-wasm-meta ships (cabal 3.14.x). The wrapper breaks with cabal 3.16; keep the shipped one. @@ -65,7 +79,8 @@ The Dockerfile is validated by review and by `ci.yml`, which builds the artifact `npm publish --provenance` through npm Trusted Publishing (no token secret; npm-side setup is package `@vscode-shellcheck/shellcheck-wasm`, repo `vscode-shellcheck/shellcheck-wasm`, workflow `release.yml`), then creates the GitHub Release with `shellcheck.wasm`, `.sha256` - and `build-info.json`. + and `build-info.json`. A prerelease version (`x.y.z-next.n`) is published with + `--tag next` and its GitHub Release is marked as a prerelease; `latest` is untouched. `bump-shellcheck.yml` runs daily and opens `chore(wasm): bump ShellCheck to ` PRs. PRs opened with `github.token` get no CI run; set the `BUMP_PR_TOKEN` secret (PAT or App token) so they do. diff --git a/CONTEXT.md b/CONTEXT.md index 529d5a5..d9d0044 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -16,7 +16,7 @@ _Avoid_: tool version, upstream version The npm semver of this package. Bumps independently of the ShellCheck version so wrapper fixes can ship without a new ShellCheck release. **Build info**: -The metadata describing how a given Artifact was built: its ShellCheck version, toolchain versions, compiler flags, target features, digest and size. Produced together with the Artifact and exposed to Hosts as a typed value. +The metadata describing how a given Artifact was built: its ShellCheck version, toolchain versions, compiler flags, target features, digest and size. Produced together with the Artifact as `build-info.json` and compiled into the package as `BUILD_INFO`. _Avoid_: build metadata, manifest **Command module**: @@ -24,16 +24,24 @@ The WASI execution model the Artifact uses: exports `_start`, reads argv/stdin, _Avoid_: reactor, JSFFI module **Runner**: -The thin JavaScript layer in this package that instantiates the Artifact with a WASI host and runs it once with given args/stdin, returning stdout/stderr/exit code. Contains no threading, cancellation or filesystem policy; those belong to the Host. +The JavaScript layer in this package (`createShellCheck` plus the Worker side, `startWorker`) that runs the Artifact once per lint on a fresh `WebAssembly.Instance` inside a Worker, one lint at a time in call order, and returns stdout/stderr/exit code. It aborts a lint when told to but holds no policy about when; that belongs to the Host. _Avoid_: wrapper, FFI, binding **Host**: -The consumer that embeds the Runner (e.g. the vscode-shellcheck extension). Owns Worker isolation, watchdogs, cancellation and which directories the Artifact may see. +The consumer that embeds the Runner (e.g. the vscode-shellcheck extension). Creates the Worker, decides what each lint may see through its File system, and owns scheduling policy and watchdog durations. _Avoid_: client, consumer, caller +**File system**: +The async `ShellCheckFileSystem` (`stat`, `readFile`, `readDirectory`) a Host passes with a lint, run on the Host's thread. Paths are guest paths such as `/dir/.shellcheckrc`; symlink containment is its job. +_Avoid_: backend, VFS + +**Bridge**: +The SharedArrayBuffer channel that lets the Artifact's synchronous WASI calls in the Worker wait on the File system's async answers on the Host's thread. +_Avoid_: RPC, proxy + **Preopen**: -A directory the Host exposes to the Artifact through WASI. ShellCheck needs one to discover `.shellcheckrc` and follow `source` directives; without one it can only lint stdin. -_Avoid_: mount, sandbox +The read-only directory tree the Artifact sees at guest `/` during one lint, served lazily from the File system through the Bridge. ShellCheck needs one to discover `.shellcheckrc` and follow `source` directives; a lint without a File system has none and can only lint stdin. +_Avoid_: sandbox **Parity**: The property that, for the same args, stdin, environment and filesystem, the Artifact's stdout, stderr and exit code are byte-identical to the same ShellCheck version's native binary. The acceptance bar for every Artifact. diff --git a/README.md b/README.md index 375c870..5084f88 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,16 @@ # @vscode-shellcheck/shellcheck-wasm -[ShellCheck](https://www.shellcheck.net/) compiled to a WASI command module, plus a minimal -runner. This package exists primarily as the WebAssembly runtime of the -[vscode-shellcheck](https://github.com/vscode-shellcheck/vscode-shellcheck) extension; other Node.js -hosts can use it the same way. Given the same args, stdin, environment and visible files, output -is byte-identical to the native `shellcheck` binary of the same version. The module uses wasm tail -calls and requires Node.js 22 or later. +[ShellCheck](https://www.shellcheck.net/) compiled to a WASI command module, plus a runner that +lints in a Worker you provide and reads files only through a file system you provide. This +package exists primarily as the WebAssembly runtime of the +[vscode-shellcheck](https://github.com/vscode-shellcheck/vscode-shellcheck) extension; other +JavaScript hosts can use it the same way. Given the same args, stdin, environment and visible +files, output is byte-identical to the native `shellcheck` binary of the same version. The module +uses wasm tail calls and requires Node.js 22 or later, or a browser with tail calls and +`SharedArrayBuffer`. + +No module in the package imports `node:*`: the host supplies the Worker and, per lint, the files +ShellCheck may read. ## Install @@ -15,54 +20,112 @@ npm install @vscode-shellcheck/shellcheck-wasm ## Usage +The Worker entry is a file the host ships. In Node: + ```ts -import { loadModule, createReadOnlyPreopen, run } from "@vscode-shellcheck/shellcheck-wasm/node"; +// shellcheck-worker.ts +import { parentPort } from "node:worker_threads"; +import { startWorker } from "@vscode-shellcheck/shellcheck-wasm/worker"; + +startWorker({ + postMessage: (message) => parentPort!.postMessage(message), + onMessage: (listener) => parentPort!.on("message", listener), +}); +``` -const module = await loadModule(); // compile once, reuse for every run +On the thread that lints: -const preopen = createReadOnlyPreopen("/path/to/workspace"); // visible to ShellCheck as "/" -try { - const result = run(module, { +```ts +import { readFile } from "node:fs/promises"; +import { Worker } from "node:worker_threads"; +import { createShellCheck, wasmUrl } from "@vscode-shellcheck/shellcheck-wasm"; + +const shellcheck = createShellCheck({ + module: WebAssembly.compile(await readFile(wasmUrl)), // compiled once, sent to each Worker + createWorker() { + const worker = new Worker(new URL("./shellcheck-worker.js", import.meta.url)); + return { + postMessage: (message) => worker.postMessage(message), + onMessage: (listener) => worker.on("message", listener), + onError: (listener) => worker.on("error", listener), + onExit: (listener) => worker.on("exit", listener), + terminate: () => worker.terminate(), + }; + }, +}); + +const result = await shellcheck.lint( + { args: ["-f", "json1", "-s", "bash", "-"], stdin: script, - env: { PWD: "/sub/dir" }, // guest path of the script's directory, inside the preopen - preopens: [preopen], - }); - const { comments } = JSON.parse(new TextDecoder().decode(result.stdout)); -} finally { - preopen.dispose(); -} + env: { PWD: "/sub/dir" }, // guest path of the script's directory + fs: workspaceFileSystem, // seen by ShellCheck as "/"; see below + }, + { signal: AbortSignal.timeout(10_000) }, +); +const { comments } = JSON.parse(result.stdout); + +await shellcheck.dispose(); // when done with the instance ``` `exitCode` is ShellCheck's own: `0` no findings, `1` findings, `2` or higher for invalid input. -`PWD` must be a path inside one of the preopens, or unset. +A lint rejects only when it is aborted, the instance is disposed, or the Worker or the guest +fails. + +`createShellCheck` starts its Worker on the first lint and runs lints one at a time, in call +order. Aborting a queued lint removes it; aborting the running lint terminates the Worker, and +the next lint starts a new one, as it does after the Worker exits or throws. Timeouts, +priorities and dropping stale lints are up to the host, through the `signal` it passes. + +In a web Worker, `onMessage` unwraps the event +(`self.addEventListener("message", (event) => listener(event.data))`), and on the creating side +`onExit` can be omitted. Browsers only provide `SharedArrayBuffer` to +[cross-origin isolated](https://developer.mozilla.org/docs/Web/API/Window/crossOriginIsolated) +pages; `WebAssembly.compileStreaming(fetch(wasmUrl))` loads the module there. + +## Files + +```ts +type FileType = "file" | "directory" | "other"; + +interface ShellCheckFileSystem { + stat(path: string): Promise<{ type: FileType; size: number; mtime: number }>; + readFile(path: string): Promise; + readDirectory(path: string): Promise>; +} +``` -The runner is synchronous: it blocks the calling thread until ShellCheck exits, with no timeout, -cancellation or output limit. Hosts that need any of those run it in a Worker and own the limits. +The `fs` of a lint is mounted read-only at guest `/`. ShellCheck needs it to find +`.shellcheckrc` above `PWD` and to follow `source` directives; without `fs` it sees no files at +all and can only lint stdin, and `PWD` must then be left unset. The methods run on the thread +that called `lint()` while the Worker waits, and every `path` is a normalized guest path such as +`/` or `/sub/dir/.shellcheckrc`. Results are cached for the rest of the lint, never across lints. -`createReadOnlyPreopen` refuses guest paths that resolve outside its directory, including through -symlinks. The check runs against the path as it is when ShellCheck asks, so a process that rewrites -symlinks in that directory during a run can race it. Expose only directories that no untrusted -process writes to while ShellCheck runs. +Signal failures by throwing an error whose `code` is `FileNotFound`, `FileNotADirectory`, +`FileIsADirectory`, `NoPermissions` or `Unavailable` (the `vscode.FileSystemError` codes); +anything else reaches ShellCheck as an I/O error. The package never writes, and a guest path that +climbs out of `/` with `..` is refused before `fs` sees it. Symlinks are resolved however `fs` +resolves them: a host that must keep ShellCheck inside a directory has to keep its `fs` from +following links out of it. ## Entry points -| Subpath | Provides | -| ------------------- | ----------------------------------------------------------------------------------------------------------------- | -| `.` | `run`, `wasmUrl`, `SHELLCHECK_VERSION`, `RunOptions`, `RunResult`; no Node imports | -| `./node` | The above plus `loadModule`, `createReadOnlyPreopen`, `readBuildInfo`, `wasmPath`, `ReadOnlyPreopen`, `BuildInfo` | -| `./build-info` | `readBuildInfo`, `BuildInfo`: toolchain and feature metadata for the bundled module, without the runner | -| `./shellcheck.wasm` | The compiled ShellCheck module | -| `./package.json` | The package manifest | +| Subpath | Provides | +| ------------------- | -------------------------------------------------------------------------------------------- | +| `.` | `createShellCheck`, `wasmUrl`, `SHELLCHECK_VERSION`, `BUILD_INFO` and types; no WASI shim | +| `./worker` | `startWorker`, `ParentPort`: the Worker side | +| `./shellcheck.wasm` | The compiled ShellCheck module | +| `./package.json` | The package manifest | Bundlers that relocate modules can copy the artifact from `require.resolve("@vscode-shellcheck/shellcheck-wasm/shellcheck.wasm")` and pass their own -`WebAssembly.Module` to `run`. +`WebAssembly.Module` as `module`. ## Versioning The package follows its own semver, independent of the bundled ShellCheck release exposed as -`SHELLCHECK_VERSION`; `readBuildInfo()` describes the build that produced the artifact. +`SHELLCHECK_VERSION`; `BUILD_INFO` describes the build that produced the artifact. Prereleases +are published under the `next` dist-tag. ## License diff --git a/docs/adr/0003-wasi-command-module.md b/docs/adr/0003-wasi-command-module.md index 2afde41..f808ade 100644 --- a/docs/adr/0003-wasi-command-module.md +++ b/docs/adr/0003-wasi-command-module.md @@ -4,3 +4,7 @@ status: accepted # Ship a WASI command module, not a reactor The artifact exports `_start` and is run once per `WebAssembly.Instance`, like a CLI. A reactor with JSFFI exports (as in the `shellcheck-wasm` npm package) would avoid re-initialising the GHC RTS per lint, but its output schema differs from `shellcheck -f json1`, it cannot discover `.shellcheckrc` through the normal path, and re-entry bugs poison the instance. Byte-identical parity with native ShellCheck matters more than the instantiate cost (~20 ms; the lint itself dominates). + +## Note 2026-09-23 + +The measurements that had a reactor build losing on medium and large scripts came from the third-party `shellcheck-wasm` npm package, built with GHC 9.10, without tail calls or `wasm-opt`, and loaded with `hs_init` instead of `_initialize`. They measure that build, not the reactor model. A JSFFI reactor whose `SystemInterface` calls an async file system directly would need neither the SharedArrayBuffer bridge of ADR 0006 nor cross-origin isolation in browsers; re-evaluate it when adding web support, weighing that it means maintaining a fork of ShellCheck's `shellcheck.hs` and re-establishing parity on every ShellCheck upgrade. diff --git a/docs/adr/0005-runner-without-host-policy.md b/docs/adr/0005-runner-without-host-policy.md index 4df9850..a9248f9 100644 --- a/docs/adr/0005-runner-without-host-policy.md +++ b/docs/adr/0005-runner-without-host-policy.md @@ -1,6 +1,12 @@ --- -status: accepted +status: accepted, amended 2026-09-23 (see ADR 0006) --- # The runner runs the artifact; the host owns isolation and filesystem policy The package exports a thin runner (args, stdin, env, WASI fds in → stdout, stderr, exit code out) built on `@bjorn3/browser_wasi_shim`, and nothing more. Worker threads, watchdogs, cancellation and read-only filesystem containment stay in the host (vscode-shellcheck), because those are security and UX decisions specific to each embedder and would otherwise be duplicated or fought over. `node:wasi` was rejected because it segfaults with preopens on Node 22 and leaks fds on Node 24. + +## Amendment 2026-09-23 + +The Worker protocol, the synchronous file-system bridge and the read-only preopen now live in the package (ADR 0006). Once files come from an async API such as `vscode.workspace.fs`, the guest can only read them through a bridge that blocks the Worker while the host's thread answers, and both ends of that bridge have to agree on one protocol, so they ship together. The package therefore runs one lint at a time in a Worker, in call order, and aborts a lint when the host's `AbortSignal` fires. + +Policy stays in the host: how the Worker is created (the package never starts one itself), what each lint may see (the `fs` it passes, including containment of symlinks), scheduling beyond first in, first out, such as putting the active editor first or dropping stale lints, and how long a lint may run before the host aborts it. diff --git a/docs/adr/0006-file-system-agnostic-package.md b/docs/adr/0006-file-system-agnostic-package.md new file mode 100644 index 0000000..bbf5680 --- /dev/null +++ b/docs/adr/0006-file-system-agnostic-package.md @@ -0,0 +1,20 @@ +--- +status: accepted +--- +# The package is file-system agnostic + +vscode-shellcheck will lint documents of any URI scheme (remote, virtual) by reading them through `vscode.workspace.fs`, and later run in VS Code for the Web. Neither works if the package reads files with `node:fs`, so no shipped entry imports `node:*`, and the only way the artifact sees a file is a `ShellCheckFileSystem` the host passes with each lint: async `stat`, `readFile` and `readDirectory` on guest paths, run on the thread that called `lint()`. + +ShellCheck is a WASI command module and calls the file system synchronously from inside `_start`. The package bridges the two: the guest runs in a Worker the host creates; on a WASI call the Worker posts a request and blocks in `Atomics.wait` on a SharedArrayBuffer allocated once per Worker; the host's thread runs the async call, writes the answer into the buffer (in 1 MiB chunks) and wakes it. Answers, failures included, are cached for the rest of the lint because ShellCheck probes the same rc paths repeatedly. The preopen built on the bridge is read-only by construction, since the interface has no write operation; `..` is folded lexically and may not climb above guest `/`. Symlinks are whatever the file system makes of them: containing them is the backend's job, and the realpath checks of the `node:fs` preopen are gone with it. For the same reason build info is compiled into the package as `BUILD_INFO` rather than read from `build-info.json` at run time. + +## Considered options + +- `ms-vscode.wasm-wasi-core`: rejected. Dormant since its 1.0.1 release in 2024, it accepts no custom imports, cannot mount a directory read-only, services every syscall on the extension host's main thread, and would be a hard dependency on another extension. +- `@vscode/sync-api-client` / `@vscode/sync-api-service`: rejected. A generic JSON RPC over the same SharedArrayBuffer mechanism, with an extra round trip per `readFile` (size first, then content) and far more surface than three read operations need. +- Keep `node:fs` in the Worker: rejected. Only `file:` documents could be linted and the package could never run on the web. + +## Consequences + +- Browsers only provide `SharedArrayBuffer` to cross-origin isolated pages. Node and VS Code desktop always have it. +- Every uncached file-system call costs a round trip to the host's thread, and that thread must keep its event loop running while a lint that reads files is in progress. A typical lint with a `.shellcheckrc` and one sourced file makes 8 of them. The Worker polls for up to 1 ms before sleeping on each, because a guest that sleeps for every answer runs measurably slower after waking; `npm run bench` holds the result to within 10% of the 0.1.1 `node:fs` runner. +- A host exposing a directory other processes can write to decides for itself how to treat symlinks inside it. From 2618a418d6b9f5aba882da554f391be9cc26e651 Mon Sep 17 00:00:00 2001 From: timonwong Date: Wed, 23 Sep 2026 09:24:35 +0000 Subject: [PATCH 6/6] chore: release 0.2.0-next.0 --- package-lock.json | 4 ++-- package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index 5624e81..d371836 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@vscode-shellcheck/shellcheck-wasm", - "version": "0.1.1", + "version": "0.2.0-next.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@vscode-shellcheck/shellcheck-wasm", - "version": "0.1.1", + "version": "0.2.0-next.0", "license": "GPL-3.0-or-later", "dependencies": { "@bjorn3/browser_wasi_shim": "^0.4.2" diff --git a/package.json b/package.json index 89a9649..bba39e5 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@vscode-shellcheck/shellcheck-wasm", - "version": "0.1.1", + "version": "0.2.0-next.0", "description": "ShellCheck compiled to WebAssembly (WASI command module) with a Worker-based runner", "license": "GPL-3.0-or-later", "repository": {