From e9826f1c6c7dcd312ee7f188cb5c800d9ec16083 Mon Sep 17 00:00:00 2001 From: Turadg Aleahmad Date: Thu, 19 Feb 2026 10:55:01 -0800 Subject: [PATCH 01/22] feat(bundle-source): add opt-in chrome trace profiling --- packages/bundle-source/README.md | 35 ++++ packages/bundle-source/src/endo.js | 193 ++++++++++-------- packages/bundle-source/src/exports.d.ts | 1 + packages/bundle-source/src/profile.js | 145 +++++++++++++ packages/bundle-source/src/script.js | 172 ++++++++++------ packages/bundle-source/src/types.ts | 21 ++ packages/bundle-source/src/zip-base64.js | 157 +++++++++----- packages/bundle-source/test/profiling.test.js | 36 ++++ 8 files changed, 566 insertions(+), 194 deletions(-) create mode 100644 packages/bundle-source/src/profile.js create mode 100644 packages/bundle-source/test/profiling.test.js diff --git a/packages/bundle-source/README.md b/packages/bundle-source/README.md index 28af0b4274..de9c67f52d 100644 --- a/packages/bundle-source/README.md +++ b/packages/bundle-source/README.md @@ -120,6 +120,41 @@ map for every physical module. It is not yet quite clever enough to collect source maps for sources that do not exist. +## Profiling + +`bundle-source` can emit Chrome trace files for performance analysis. +This works for programmatic usage and CLI usage, including builds in larger +repos like `agoric-sdk`. + +Enable with environment variables: + +```console +ENDO_BUNDLE_SOURCE_PROFILE=1 \ +ENDO_BUNDLE_SOURCE_PROFILE_DIR=/tmp/bs-profiles \ +yarn bundle-source app.js > /tmp/app-bundle.json +``` + +Each bundle call writes one `*.trace.json` file. Open these in Chrome tracing +tools or convert for Speedscope. + +You can also control profiling in code: + +```js +await bundleSource('program.js', { + profile: { + enabled: true, + traceDir: '/tmp/bs-profiles', + // or traceFile: '/tmp/specific.trace.json' + }, +}); +``` + +Environment variables: +- `ENDO_BUNDLE_SOURCE_PROFILE`: enable profiling when truthy (`1`, `true`, `yes`, `on`) +- `ENDO_BUNDLE_SOURCE_PROFILE_DIR`: output directory for generated trace files +- `ENDO_BUNDLE_SOURCE_PROFILE_FILE`: explicit output file for a single run +- `ENDO_BUNDLE_SOURCE_PROFILE_STDERR`: if truthy, prints each generated trace path to stderr + ## `moduleFormat` explanations diff --git a/packages/bundle-source/src/endo.js b/packages/bundle-source/src/endo.js index 793fb758c6..243f337d30 100644 --- a/packages/bundle-source/src/endo.js +++ b/packages/bundle-source/src/endo.js @@ -22,7 +22,7 @@ const textDecoder = new TextDecoder(); */ export const makeBundlingKit = ( io, - { cacheSourceMaps, elideComments, noTransforms, commonDependencies }, + { cacheSourceMaps, elideComments, noTransforms, commonDependencies, profiler }, ) => { const { pathResolve, userInfo, computeSha512, platform, env } = io; if (noTransforms && elideComments) { @@ -50,63 +50,71 @@ export const makeBundlingKit = ( sourceMap, { sha512, compartment: packageLocation, module: moduleSpecifier }, ) => { - const location = new URL(moduleSpecifier, packageLocation).href; - const locationSha512 = computeSha512(location); - const locationSha512Head = locationSha512.slice(0, 2); - const locationSha512Tail = locationSha512.slice(2); - const sha512Head = sha512.slice(0, 2); - const sha512Tail = sha512.slice(2); - const sourceMapTrackerDirectory = pathResolve( - sourceMapsTrackerDirectory, - locationSha512Head, - ); - const sourceMapTrackerPath = pathResolve( - sourceMapTrackerDirectory, - locationSha512Tail, - ); - const sourceMapDirectory = pathResolve( - sourceMapsCacheDirectory, - sha512Head, - ); - const sourceMapPath = pathResolve( - sourceMapDirectory, - `${sha512Tail}.map.json`, + const endWriteSourceMap = profiler?.startSpan( + 'bundleSource.writeSourceMap', + { moduleSpecifier, packageLocation }, ); + try { + const location = new URL(moduleSpecifier, packageLocation).href; + const locationSha512 = computeSha512(location); + const locationSha512Head = locationSha512.slice(0, 2); + const locationSha512Tail = locationSha512.slice(2); + const sha512Head = sha512.slice(0, 2); + const sha512Tail = sha512.slice(2); + const sourceMapTrackerDirectory = pathResolve( + sourceMapsTrackerDirectory, + locationSha512Head, + ); + const sourceMapTrackerPath = pathResolve( + sourceMapTrackerDirectory, + locationSha512Tail, + ); + const sourceMapDirectory = pathResolve( + sourceMapsCacheDirectory, + sha512Head, + ); + const sourceMapPath = pathResolve( + sourceMapDirectory, + `${sha512Tail}.map.json`, + ); - await fs.promises - .readFile(sourceMapTrackerPath, 'utf-8') - .then(async oldSha512 => { - oldSha512 = oldSha512.trim(); - if (oldSha512 === sha512) { - return; - } - const oldSha512Head = oldSha512.slice(0, 2); - const oldSha512Tail = oldSha512.slice(2); - const oldSourceMapDirectory = pathResolve( - sourceMapsCacheDirectory, - oldSha512Head, - ); - const oldSourceMapPath = pathResolve( - oldSourceMapDirectory, - `${oldSha512Tail}.map.json`, - ); - await fs.promises.unlink(oldSourceMapPath); - const entries = await fs.promises.readdir(oldSourceMapDirectory); - if (entries.length === 0) { - await fs.promises.rmdir(oldSourceMapDirectory); - } - }) - .catch(error => { - if (error.code !== 'ENOENT') { - throw error; - } - }); + await fs.promises + .readFile(sourceMapTrackerPath, 'utf-8') + .then(async oldSha512 => { + oldSha512 = oldSha512.trim(); + if (oldSha512 === sha512) { + return; + } + const oldSha512Head = oldSha512.slice(0, 2); + const oldSha512Tail = oldSha512.slice(2); + const oldSourceMapDirectory = pathResolve( + sourceMapsCacheDirectory, + oldSha512Head, + ); + const oldSourceMapPath = pathResolve( + oldSourceMapDirectory, + `${oldSha512Tail}.map.json`, + ); + await fs.promises.unlink(oldSourceMapPath); + const entries = await fs.promises.readdir(oldSourceMapDirectory); + if (entries.length === 0) { + await fs.promises.rmdir(oldSourceMapDirectory); + } + }) + .catch(error => { + if (error.code !== 'ENOENT') { + throw error; + } + }); - await fs.promises.mkdir(sourceMapDirectory, { recursive: true }); - await fs.promises.writeFile(sourceMapPath, sourceMap); + await fs.promises.mkdir(sourceMapDirectory, { recursive: true }); + await fs.promises.writeFile(sourceMapPath, sourceMap); - await fs.promises.mkdir(sourceMapTrackerDirectory, { recursive: true }); - await fs.promises.writeFile(sourceMapTrackerPath, sha512); + await fs.promises.mkdir(sourceMapTrackerDirectory, { recursive: true }); + await fs.promises.writeFile(sourceMapTrackerPath, sha512); + } finally { + endWriteSourceMap?.(); + } }; } @@ -124,6 +132,11 @@ export const makeBundlingKit = ( location, sourceMap, ) => { + const endTransformModule = profiler?.startSpan('bundleSource.transformModule', { + parser, + specifier, + location, + }); if (!['mjs', 'cjs'].includes(parser)) { throw Error(`Parser ${parser} not supported in evadeEvalCensor`); } @@ -131,18 +144,22 @@ export const makeBundlingKit = ( const source = textDecoder.decode(sourceBytes); const priorSourceMap = typeof sourceMap === 'string' ? sourceMap : undefined; - const { code: object, map } = await evadeCensor(source, { - sourceType: babelSourceType, - sourceMap: priorSourceMap, - sourceUrl: new URL(specifier, location).href, - elideComments, - }); - const objectBytes = textEncoder.encode(object); - return { - bytes: objectBytes, - parser, - sourceMap: map ? JSON.stringify(map) : undefined, - }; + try { + const { code: object, map } = await evadeCensor(source, { + sourceType: babelSourceType, + sourceMap: priorSourceMap, + sourceUrl: new URL(specifier, location).href, + elideComments, + }); + const objectBytes = textEncoder.encode(object); + return { + bytes: objectBytes, + parser, + sourceMap: map ? JSON.stringify(map) : undefined, + }; + } finally { + endTransformModule?.(); + } }; /** @type {ParserForLanguageLike} */ @@ -199,18 +216,26 @@ export const makeBundlingKit = ( packageLocation, options = undefined, ) { + const endTypeErasure = profiler?.startSpan('bundleSource.typeErase', { + parser: 'mts', + specifier, + }); const sourceText = textDecoder.decode(sourceBytes); const { code: objectText } = transformSync(sourceText, { mode: 'strip-only', }); const objectBytes = textEncoder.encode(objectText); - return parserForLanguage.mjs.parse( - objectBytes, - specifier, - moduleLocation, - packageLocation, - options, - ); + try { + return parserForLanguage.mjs.parse( + objectBytes, + specifier, + moduleLocation, + packageLocation, + options, + ); + } finally { + endTypeErasure?.(); + } }, heuristicImports: false, synchronous: false, @@ -225,18 +250,26 @@ export const makeBundlingKit = ( packageLocation, options = undefined, ) { + const endTypeErasure = profiler?.startSpan('bundleSource.typeErase', { + parser: 'cts', + specifier, + }); const sourceText = textDecoder.decode(sourceBytes); const { code: objectText } = transformSync(sourceText, { mode: 'strip-only', }); const objectBytes = textEncoder.encode(objectText); - return parserForLanguage.cjs.parse( - objectBytes, - specifier, - moduleLocation, - packageLocation, - options, - ); + try { + return parserForLanguage.cjs.parse( + objectBytes, + specifier, + moduleLocation, + packageLocation, + options, + ); + } finally { + endTypeErasure?.(); + } }, heuristicImports: true, synchronous: false, diff --git a/packages/bundle-source/src/exports.d.ts b/packages/bundle-source/src/exports.d.ts index b274ab6a3b..b51bdb2ed4 100644 --- a/packages/bundle-source/src/exports.d.ts +++ b/packages/bundle-source/src/exports.d.ts @@ -1,4 +1,5 @@ export type { + BundleProfilingOptions, BundleOptions, BundleSource, BundleSourceResult, diff --git a/packages/bundle-source/src/profile.js b/packages/bundle-source/src/profile.js new file mode 100644 index 0000000000..f815f6c602 --- /dev/null +++ b/packages/bundle-source/src/profile.js @@ -0,0 +1,145 @@ +// @ts-check + +import fs from 'fs'; +import os from 'os'; +import path from 'path'; +import { performance } from 'perf_hooks'; + +/** @import {BundleProfilingOptions} from './types.js' */ + +let nextTraceFileId = 0; + +const truthy = new Set(['1', 'true', 'yes', 'on']); + +/** + * @param {string | undefined} value + * @returns {boolean} + */ +const parseBoolean = value => { + if (value === undefined) { + return false; + } + return truthy.has(value.toLowerCase()); +}; + +/** + * @param {string} moduleFormat + * @returns {'script' | 'zip'} + */ +const classifyModuleFormat = moduleFormat => + moduleFormat === 'endoZipBase64' ? 'zip' : 'script'; + +/** + * @param {object} options + * @param {string} options.moduleFormat + * @param {string} options.startFilename + * @param {number} [options.pid] + * @param {Record} [options.env] + * @param {BundleProfilingOptions | undefined} [options.profile] + */ +export const makeBundleProfiler = ({ + moduleFormat, + startFilename, + pid = process.pid, + env = process.env, + profile = undefined, +}) => { + const enabled = + profile?.enabled !== undefined + ? profile.enabled + : parseBoolean(env.ENDO_BUNDLE_SOURCE_PROFILE); + const logToStderr = parseBoolean(env.ENDO_BUNDLE_SOURCE_PROFILE_STDERR); + + if (!enabled) { + const noop = () => {}; + return { + enabled, + startSpan: (_name, _args = undefined) => noop, + async flush(_args = undefined) {}, + }; + } + + const traceFile = + profile?.traceFile || env.ENDO_BUNDLE_SOURCE_PROFILE_FILE || undefined; + const traceDir = + profile?.traceDir || + env.ENDO_BUNDLE_SOURCE_PROFILE_DIR || + path.join(os.tmpdir(), 'endo-bundle-source-profiles'); + const phase = classifyModuleFormat(moduleFormat); + + const tracePath = + traceFile || + path.join( + traceDir, + `bundle-source-${phase}-${pid}-${Date.now()}-${nextTraceFileId++}.trace.json`, + ); + + /** @type {Array>} */ + const traceEvents = []; + const zeroMs = performance.now(); + + /** + * @param {number} ms + * @returns {number} + */ + const toMicros = ms => Math.round(ms * 1000); + + /** + * @param {string} name + * @param {Record | undefined} args + */ + const startSpan = (name, args = undefined) => { + const startMs = performance.now() - zeroMs; + return extraArgs => { + const endMs = performance.now() - zeroMs; + const payload = extraArgs ? { ...args, ...extraArgs } : args; + traceEvents.push( + payload + ? { + name, + cat: 'bundle-source', + ph: 'X', + ts: toMicros(startMs), + dur: toMicros(endMs - startMs), + pid, + tid: 0, + args: payload, + } + : { + name, + cat: 'bundle-source', + ph: 'X', + ts: toMicros(startMs), + dur: toMicros(endMs - startMs), + pid, + tid: 0, + }, + ); + }; + }; + + const bundleStart = startSpan('bundleSource.total', { + moduleFormat, + startFilename, + }); + + return { + enabled, + startSpan, + /** + * @param {Record | undefined} [result] + */ + async flush(result = undefined) { + bundleStart(result); + await fs.promises.mkdir(path.dirname(tracePath), { recursive: true }); + const trace = { + traceEvents, + displayTimeUnit: 'ms', + }; + await fs.promises.writeFile(tracePath, JSON.stringify(trace, null, 2)); + if (logToStderr) { + process.stderr.write(`bundle-source profile trace: ${tracePath}\n`); + } + }, + }; +}; diff --git a/packages/bundle-source/src/script.js b/packages/bundle-source/src/script.js index b144d766ec..459ef80b00 100644 --- a/packages/bundle-source/src/script.js +++ b/packages/bundle-source/src/script.js @@ -12,6 +12,7 @@ import { makeFunctor } from '@endo/compartment-mapper/functor.js'; import { makeReadPowers } from '@endo/compartment-mapper/node-powers.js'; import { makeBundlingKit } from './endo.js'; +import { makeBundleProfiler } from './profile.js'; /** @import {BundleScriptModuleFormat, BundleScriptOptions, BundlingKitIO, SharedPowers} from './types.js' */ @@ -36,6 +37,7 @@ export async function bundleScript( elideComments = false, conditions = [], commonDependencies, + profile, } = options; const powers = /** @type {typeof readPowers & SharedPowers} */ ({ @@ -51,79 +53,121 @@ export async function bundleScript( } = powers; const entry = url.pathToFileURL(pathResolve(startFilename)); + const profiler = makeBundleProfiler({ + moduleFormat, + startFilename, + env, + profile, + }); - const { - sourceMapHook, - sourceMapJobs, - moduleTransforms, - parserForLanguage, - workspaceLanguageForExtension, - workspaceCommonjsLanguageForExtension, - workspaceModuleLanguageForExtension, - } = makeBundlingKit( - /** @type {BundlingKitIO} */ ({ - pathResolve, - userInfo, - platform, - env, - computeSha512, - }), - { - cacheSourceMaps, - noTransforms, - elideComments, - commonDependencies, - dev, - }, - ); - const parserForLanguageForFunctor = parserForLanguage; + let phaseStatus = 'ok'; + let phaseError; + try { + const endMakeBundlingKit = profiler.startSpan('bundleSource.makeBundlingKit'); + const { + sourceMapHook, + sourceMapJobs, + moduleTransforms, + parserForLanguage, + workspaceLanguageForExtension, + workspaceCommonjsLanguageForExtension, + workspaceModuleLanguageForExtension, + } = (() => { + try { + return makeBundlingKit( + /** @type {BundlingKitIO} */ ({ + pathResolve, + userInfo, + platform, + env, + computeSha512, + }), + { + cacheSourceMaps, + noTransforms, + elideComments, + commonDependencies, + dev, + profiler, + }, + ); + } finally { + endMakeBundlingKit(); + } + })(); + const parserForLanguageForFunctor = parserForLanguage; - let source = await makeFunctor(powers, entry.href, { - // For backward-compatibility, the nestedEvaluate and getExport formats - // may implicitly include devDependencies of the entry module's package, - // but this courtesy will not be extended to any future bundle formats. - dev: - dev || moduleFormat === 'nestedEvaluate' || moduleFormat === 'getExport', - conditions: new Set(conditions), - commonDependencies, - parserForLanguage: parserForLanguageForFunctor, - workspaceLanguageForExtension, - workspaceCommonjsLanguageForExtension, - workspaceModuleLanguageForExtension, - moduleTransforms, - sourceMapHook, - useEvaluate: moduleFormat === 'nestedEvaluate', - sourceUrlPrefix: '/bundled-source/.../', - // For backward-compatibility, the nestedEvaluate and getExport formats - // also may implicitly reach for the require function in lexical context - // to import CommonJS modules, as if they were CommonJS modules themselves. - // This default will not extend to any future bundle formats, which will - // be obliged to choose inject an exit import hook explicitly. - format: - moduleFormat === 'nestedEvaluate' || moduleFormat === 'getExport' - ? 'cjs' - : undefined, - }); + const endMakeFunctor = profiler.startSpan('bundleSource.makeFunctor'); + let source; + try { + source = await makeFunctor(powers, entry.href, { + // For backward-compatibility, the nestedEvaluate and getExport formats + // may implicitly include devDependencies of the entry module's package, + // but this courtesy will not be extended to any future bundle formats. + dev: + dev || + moduleFormat === 'nestedEvaluate' || + moduleFormat === 'getExport', + conditions: new Set(conditions), + commonDependencies, + parserForLanguage: parserForLanguageForFunctor, + workspaceLanguageForExtension, + workspaceCommonjsLanguageForExtension, + workspaceModuleLanguageForExtension, + moduleTransforms, + sourceMapHook, + useEvaluate: moduleFormat === 'nestedEvaluate', + sourceUrlPrefix: '/bundled-source/.../', + // For backward-compatibility, the nestedEvaluate and getExport formats + // also may implicitly reach for the require function in lexical context + // to import CommonJS modules, as if they were CommonJS modules themselves. + // This default will not extend to any future bundle formats, which will + // be obliged to choose inject an exit import hook explicitly. + format: + moduleFormat === 'nestedEvaluate' || moduleFormat === 'getExport' + ? 'cjs' + : undefined, + }); + } finally { + endMakeFunctor(); + } - if (moduleFormat === 'endoScript') { - source = `(${source})()`; - } - if (moduleFormat === 'nestedEvaluate') { - source = `\ + if (moduleFormat === 'endoScript') { + source = `(${source})()`; + } + if (moduleFormat === 'nestedEvaluate') { + source = `\ (sourceUrlPrefix) => (${source})({ sourceUrlPrefix, evaluate: typeof nestedEvaluate === 'function' ? nestedEvaluate : undefined, require: typeof require === 'function' ? require : undefined, }) `; - } + } - await Promise.all(sourceMapJobs); + const endSourceMapJobs = profiler.startSpan('bundleSource.sourceMapJobs'); + try { + await Promise.all(sourceMapJobs); + } finally { + endSourceMapJobs(); + } - return harden({ - moduleFormat, - source, - // TODO - sourceMap: '', - }); + return harden({ + moduleFormat, + source, + // TODO + sourceMap: '', + }); + } catch (error) { + phaseStatus = 'error'; + phaseError = + error instanceof Error ? `${error.name}: ${error.message}` : `${error}`; + throw error; + } finally { + await profiler.flush( + phaseError + ? { status: phaseStatus, error: phaseError } + : { status: phaseStatus }, + ); + } } diff --git a/packages/bundle-source/src/types.ts b/packages/bundle-source/src/types.ts index 57d6db448e..9f2f35b9be 100644 --- a/packages/bundle-source/src/types.ts +++ b/packages/bundle-source/src/types.ts @@ -140,6 +140,7 @@ export interface BundleScriptOptions { elideComments?: boolean | undefined; conditions?: string[] | undefined; commonDependencies?: Record | undefined; + profile?: BundleProfilingOptions | undefined; } export interface BundleZipBase64Options extends BundleScriptOptions { @@ -174,6 +175,15 @@ export interface BundlingKitOptions { noTransforms: boolean; commonDependencies?: Record | undefined; dev?: boolean | undefined; + profiler?: + | { + enabled: boolean; + startSpan: ( + name: string, + args?: Record | undefined, + ) => (args?: Record | undefined) => void; + } + | undefined; } export interface BundlingKit { @@ -277,6 +287,11 @@ export type BundleOptions = { * common dependencies for the entry package. */ commonDependencies?: Record | undefined; + /** + * - enables instrumentation spans and trace emission suitable for Chrome + * tracing or speedscope conversion. + */ + profile?: BundleProfilingOptions | undefined; } & (T extends 'endoZipBase64' ? { /** @@ -289,6 +304,12 @@ export type BundleOptions = { } : {}); +export interface BundleProfilingOptions { + enabled?: boolean | undefined; + traceFile?: string | undefined; + traceDir?: string | undefined; +} + export type ReadFn = (location: string) => Promise; /** diff --git a/packages/bundle-source/src/zip-base64.js b/packages/bundle-source/src/zip-base64.js index 1a9b37e212..9ce01b64dd 100644 --- a/packages/bundle-source/src/zip-base64.js +++ b/packages/bundle-source/src/zip-base64.js @@ -14,6 +14,7 @@ import { encodeBase64 } from '@endo/base64'; import { makeReadPowers } from '@endo/compartment-mapper/node-powers.js'; import { makeBundlingKit } from './endo.js'; +import { makeBundleProfiler } from './profile.js'; /** @import {BundleZipBase64Options, BundlingKitIO, SharedPowers} from './types.js' */ @@ -37,6 +38,7 @@ export async function bundleZipBase64( conditions = [], commonDependencies, importHook, + profile, } = options; const powers = /** @type {typeof readPowers & SharedPowers} */ ({ ...readPowers, @@ -51,57 +53,112 @@ export async function bundleZipBase64( } = powers; const entry = url.pathToFileURL(pathResolve(startFilename)); - - const { - sourceMapHook, - sourceMapJobs, - moduleTransforms, - parserForLanguage, - workspaceLanguageForExtension, - workspaceCommonjsLanguageForExtension, - workspaceModuleLanguageForExtension, - } = makeBundlingKit( - /** @type {BundlingKitIO} */ ({ - pathResolve, - userInfo, - platform, - env, - computeSha512, - }), - { - cacheSourceMaps, - noTransforms, - elideComments, - commonDependencies, - }, - ); - const importHookForArchive = importHook; - - const compartmentMap = await mapNodeModules(powers, entry.href, { - dev, - conditions: new Set(conditions), - commonDependencies, - workspaceLanguageForExtension, - workspaceCommonjsLanguageForExtension, - workspaceModuleLanguageForExtension, + const profiler = makeBundleProfiler({ + moduleFormat: 'endoZipBase64', + startFilename, + env, + profile, }); - const { bytes, sha512 } = await makeAndHashArchiveFromMap( - powers, - compartmentMap, - { - parserForLanguage, - moduleTransforms, + let phaseStatus = 'ok'; + let phaseError; + try { + const endMakeBundlingKit = profiler.startSpan('bundleSource.makeBundlingKit'); + const { sourceMapHook, - importHook: importHookForArchive, - }, - ); - assert(sha512); - await Promise.all(sourceMapJobs); - const endoZipBase64 = encodeBase64(bytes); - return harden({ - moduleFormat: /** @type {const} */ ('endoZipBase64'), - endoZipBase64, - endoZipBase64Sha512: sha512, - }); + sourceMapJobs, + moduleTransforms, + parserForLanguage, + workspaceLanguageForExtension, + workspaceCommonjsLanguageForExtension, + workspaceModuleLanguageForExtension, + } = (() => { + try { + return makeBundlingKit( + /** @type {BundlingKitIO} */ ({ + pathResolve, + userInfo, + platform, + env, + computeSha512, + }), + { + cacheSourceMaps, + noTransforms, + elideComments, + commonDependencies, + profiler, + }, + ); + } finally { + endMakeBundlingKit(); + } + })(); + const importHookForArchive = importHook; + + const endMapNodeModules = profiler.startSpan('bundleSource.mapNodeModules'); + let compartmentMap; + try { + compartmentMap = await mapNodeModules(powers, entry.href, { + dev, + conditions: new Set(conditions), + commonDependencies, + workspaceLanguageForExtension, + workspaceCommonjsLanguageForExtension, + workspaceModuleLanguageForExtension, + }); + } finally { + endMapNodeModules(); + } + + const endMakeArchive = profiler.startSpan('bundleSource.makeAndHashArchiveFromMap'); + let bytes; + let sha512; + try { + ({ bytes, sha512 } = await makeAndHashArchiveFromMap( + powers, + compartmentMap, + { + parserForLanguage, + moduleTransforms, + sourceMapHook, + importHook: importHookForArchive, + }, + )); + } finally { + endMakeArchive(); + } + assert(sha512); + + const endSourceMapJobs = profiler.startSpan('bundleSource.sourceMapJobs'); + try { + await Promise.all(sourceMapJobs); + } finally { + endSourceMapJobs(); + } + + const endEncodeBase64 = profiler.startSpan('bundleSource.encodeBase64'); + let endoZipBase64; + try { + endoZipBase64 = encodeBase64(bytes); + } finally { + endEncodeBase64(); + } + return harden({ + moduleFormat: /** @type {const} */ ('endoZipBase64'), + endoZipBase64, + endoZipBase64Sha512: sha512, + }); + } catch (error) { + phaseStatus = 'error'; + phaseError = + error instanceof Error ? `${error.name}: ${error.message}` : `${error}`; + throw error; + } finally { + await profiler.flush( + phaseError + ? { status: phaseStatus, error: phaseError } + : { status: phaseStatus }, + ); + } } diff --git a/packages/bundle-source/test/profiling.test.js b/packages/bundle-source/test/profiling.test.js new file mode 100644 index 0000000000..ad579ae7cd --- /dev/null +++ b/packages/bundle-source/test/profiling.test.js @@ -0,0 +1,36 @@ +import os from 'os'; +import path from 'path'; +import fs from 'fs/promises'; +import url from 'url'; +import test from '@endo/ses-ava/prepare-endo.js'; + +import bundleSource from '../src/index.js'; + +test('bundle-source profiling writes a chrome trace file', async t => { + const traceDir = await fs.mkdtemp(path.join(os.tmpdir(), 'bundle-prof-')); + t.teardown(() => fs.rm(traceDir, { recursive: true, force: true })); + + const entryPath = url.fileURLToPath( + new URL('../demo/meaning.js', import.meta.url), + ); + + await bundleSource(entryPath, { + format: 'endoZipBase64', + profile: { + enabled: true, + traceDir, + }, + }); + + const files = await fs.readdir(traceDir); + t.true(files.length > 0); + + const tracePath = path.join(traceDir, files[0]); + const traceText = await fs.readFile(tracePath, 'utf-8'); + const trace = JSON.parse(traceText); + t.true(Array.isArray(trace.traceEvents)); + + const names = new Set(trace.traceEvents.map(({ name }) => name)); + t.true(names.has('bundleSource.total')); + t.true(names.has('bundleSource.mapNodeModules')); +}); From 5e66892497943bbe68b04d7df480f095c8108c01 Mon Sep 17 00:00:00 2001 From: Turadg Aleahmad Date: Thu, 19 Feb 2026 10:56:01 -0800 Subject: [PATCH 02/22] feat(bundle-source): add trace merge and summary tool --- packages/bundle-source/README.md | 11 + packages/bundle-source/package.json | 1 + packages/bundle-source/tools/trace-merge.js | 262 ++++++++++++++++++++ 3 files changed, 274 insertions(+) create mode 100644 packages/bundle-source/tools/trace-merge.js diff --git a/packages/bundle-source/README.md b/packages/bundle-source/README.md index de9c67f52d..dbddbf55e2 100644 --- a/packages/bundle-source/README.md +++ b/packages/bundle-source/README.md @@ -155,6 +155,17 @@ Environment variables: - `ENDO_BUNDLE_SOURCE_PROFILE_FILE`: explicit output file for a single run - `ENDO_BUNDLE_SOURCE_PROFILE_STDERR`: if truthy, prints each generated trace path to stderr +Merge and summarize many profile traces: + +```console +yarn workspace @endo/bundle-source trace:merge -- /tmp/bs-profiles +``` + +This generates: +- `merged.trace.json` for trace viewers. +- `summary.json` with aggregate span statistics. +- `summary.md` with a top spans table by total duration. + ## `moduleFormat` explanations diff --git a/packages/bundle-source/package.json b/packages/bundle-source/package.json index 43357dbca9..f790f800a4 100644 --- a/packages/bundle-source/package.json +++ b/packages/bundle-source/package.json @@ -14,6 +14,7 @@ "scripts": { "build": "exit 0", "test": "ava", + "trace:merge": "node tools/trace-merge.js", "test:xs": "exit 0", "lint-fix": "eslint --fix '**/*.js'", "lint": "yarn lint:types && yarn lint:eslint", diff --git a/packages/bundle-source/tools/trace-merge.js b/packages/bundle-source/tools/trace-merge.js new file mode 100644 index 0000000000..6de92791d7 --- /dev/null +++ b/packages/bundle-source/tools/trace-merge.js @@ -0,0 +1,262 @@ +#!/usr/bin/env node +// @ts-check +/* global process */ + +import fs from 'fs/promises'; +import path from 'path'; +import { parseArgs } from 'util'; + +const options = /** @type {const} */ ({ + 'out-trace': { type: 'string' }, + 'out-summary': { type: 'string' }, + 'out-markdown': { type: 'string' }, + top: { type: 'string' }, + stacked: { type: 'boolean' }, +}); + +const usage = `\ +Usage: + node tools/trace-merge.js [--out-trace merged.trace.json] [--out-summary summary.json] [--out-markdown summary.md] [--top 30] [--stacked] ... + +Examples: + node tools/trace-merge.js /tmp/bs-profiles + node tools/trace-merge.js --out-markdown /tmp/summary.md /tmp/bs-profiles /tmp/other.trace.json +`; + +/** + * @param {string} value + * @returns {number} + */ +const toInt = value => { + const n = Number.parseInt(value, 10); + if (!Number.isFinite(n) || n <= 0) { + throw new Error(`Expected positive integer, got: ${value}`); + } + return n; +}; + +/** + * @param {string} filePath + * @returns {Promise} + */ +const exists = async filePath => { + try { + await fs.stat(filePath); + return true; + } catch { + return false; + } +}; + +/** + * @param {string} root + * @returns {Promise} + */ +const findTraceFiles = async root => { + const stat = await fs.stat(root); + if (stat.isFile()) { + return root.endsWith('.trace.json') ? [root] : []; + } + if (!stat.isDirectory()) { + return []; + } + /** @type {string[]} */ + const found = []; + const queue = [root]; + while (queue.length > 0) { + const dir = queue.shift(); + if (!dir) { + continue; + } + // eslint-disable-next-line no-await-in-loop + const entries = await fs.readdir(dir, { withFileTypes: true }); + for (const entry of entries) { + const filePath = path.join(dir, entry.name); + if (entry.isDirectory()) { + queue.push(filePath); + } else if (entry.isFile() && entry.name.endsWith('.trace.json')) { + found.push(filePath); + } + } + } + return found; +}; + +/** + * @param {number[]} sorted + * @param {number} p + * @returns {number} + */ +const percentile = (sorted, p) => { + if (sorted.length === 0) { + return 0; + } + if (sorted.length === 1) { + return sorted[0]; + } + const idx = (p / 100) * (sorted.length - 1); + const lo = Math.floor(idx); + const hi = Math.ceil(idx); + if (lo === hi) { + return sorted[lo]; + } + const frac = idx - lo; + return sorted[lo] * (1 - frac) + sorted[hi] * frac; +}; + +/** + * @param {number} micros + * @returns {string} + */ +const microsToMsText = micros => (micros / 1000).toFixed(3); + +/** + * @param {Array>} events + * @param {number} top + */ +const summarize = (events, top) => { + /** @type {Map} */ + const durationsByName = new Map(); + for (const event of events) { + if (event.ph !== 'X' || typeof event.name !== 'string') { + continue; + } + const dur = typeof event.dur === 'number' ? event.dur : undefined; + if (dur === undefined) { + continue; + } + const bucket = durationsByName.get(event.name); + if (bucket) { + bucket.push(dur); + } else { + durationsByName.set(event.name, [dur]); + } + } + + const rows = [...durationsByName.entries()].map(([name, durations]) => { + durations.sort((a, b) => a - b); + const total = durations.reduce((sum, value) => sum + value, 0); + return { + name, + count: durations.length, + totalUs: total, + avgUs: total / durations.length, + minUs: durations[0], + maxUs: durations[durations.length - 1], + p50Us: percentile(durations, 50), + p95Us: percentile(durations, 95), + }; + }); + rows.sort((a, b) => b.totalUs - a.totalUs); + return rows.slice(0, top); +}; + +/** + * @param {ReturnType} rows + * @returns {string} + */ +const summarizeMarkdown = rows => { + const header = [ + '| Span | Count | Total ms | Avg ms | P50 ms | P95 ms | Max ms |', + '| --- | ---: | ---: | ---: | ---: | ---: | ---: |', + ]; + const body = rows.map(row => + [ + `| ${row.name}`, + `${row.count}`, + `${microsToMsText(row.totalUs)}`, + `${microsToMsText(row.avgUs)}`, + `${microsToMsText(row.p50Us)}`, + `${microsToMsText(row.p95Us)}`, + `${microsToMsText(row.maxUs)} |`, + ].join(' | '), + ); + return `${header.join('\n')}\n${body.join('\n')}\n`; +}; + +const main = async () => { + const { + values: { + 'out-trace': outTrace = 'merged.trace.json', + 'out-summary': outSummary = 'summary.json', + 'out-markdown': outMarkdown = 'summary.md', + top: topRaw = '30', + stacked = true, + }, + positionals, + } = parseArgs({ options, allowPositionals: true }); + + if (positionals.length === 0) { + throw new Error(usage); + } + const top = toInt(topRaw); + + /** @type {string[]} */ + const traceFiles = []; + for (const input of positionals) { + // eslint-disable-next-line no-await-in-loop + if (!(await exists(input))) { + throw new Error(`Input does not exist: ${input}`); + } + // eslint-disable-next-line no-await-in-loop + const files = await findTraceFiles(input); + traceFiles.push(...files); + } + + if (traceFiles.length === 0) { + throw new Error(`No *.trace.json files found in inputs.\n\n${usage}`); + } + + /** @type {Array>} */ + const mergedEvents = []; + let offsetUs = 0; + for (const filePath of traceFiles.sort()) { + // eslint-disable-next-line no-await-in-loop + const text = await fs.readFile(filePath, 'utf-8'); + /** @type {{traceEvents?: Array>}} */ + const trace = JSON.parse(text); + const events = trace.traceEvents || []; + let maxEndUs = 0; + for (const event of events) { + const copy = { ...event, args: { ...(event.args || {}), source: filePath } }; + if (stacked) { + if (typeof copy.ts === 'number') { + copy.ts += offsetUs; + } + } + const ts = typeof copy.ts === 'number' ? copy.ts : 0; + const dur = typeof copy.dur === 'number' ? copy.dur : 0; + maxEndUs = Math.max(maxEndUs, ts + dur); + mergedEvents.push(copy); + } + if (stacked) { + offsetUs = maxEndUs + 1000; + } + } + + const summaryTop = summarize(mergedEvents, top); + const summary = { + generatedAt: new Date().toISOString(), + traceFileCount: traceFiles.length, + eventCount: mergedEvents.length, + topSpansByTotalDuration: summaryTop, + traceFiles, + }; + + await fs.writeFile( + outTrace, + JSON.stringify({ traceEvents: mergedEvents, displayTimeUnit: 'ms' }, null, 2), + ); + await fs.writeFile(outSummary, JSON.stringify(summary, null, 2)); + await fs.writeFile(outMarkdown, summarizeMarkdown(summaryTop)); + + process.stdout.write(`Merged ${traceFiles.length} trace files\n`); + process.stdout.write(`Wrote merged trace: ${outTrace}\n`); + process.stdout.write(`Wrote summary JSON: ${outSummary}\n`); + process.stdout.write(`Wrote summary Markdown: ${outMarkdown}\n`); +}; + +main().catch(error => { + process.stderr.write(`${error instanceof Error ? error.stack : error}\n`); + process.exit(1); +}); From 3b3a129c6bb72933b2f05ec9be78a4c4ef2ecbf1 Mon Sep 17 00:00:00 2001 From: Turadg Aleahmad Date: Thu, 19 Feb 2026 11:41:18 -0800 Subject: [PATCH 03/22] feat(bundle-source): add agoric source-spec registry bundling tool --- packages/bundle-source/README.md | 17 + packages/bundle-source/package.json | 1 + .../tools/bundle-agoric-source-specs.js | 292 ++++++++++++++++++ 3 files changed, 310 insertions(+) create mode 100644 packages/bundle-source/tools/bundle-agoric-source-specs.js diff --git a/packages/bundle-source/README.md b/packages/bundle-source/README.md index dbddbf55e2..c1ad69328a 100644 --- a/packages/bundle-source/README.md +++ b/packages/bundle-source/README.md @@ -166,6 +166,23 @@ This generates: - `summary.json` with aggregate span statistics. - `summary.md` with a top spans table by total duration. +Bundle all `source-spec-registry.js` entries from `agoric-sdk` using the +current checkout's `bundle-source`: + +```console +ENDO_BUNDLE_SOURCE_PROFILE=1 \ +ENDO_BUNDLE_SOURCE_PROFILE_DIR=/tmp/agoric-bundle-traces \ +yarn workspace @endo/bundle-source bundle:agoric-source-specs -- \ + --agoric-sdk-root /opt/agoric/agoric-sdk \ + --out-dir /tmp/agoric-source-spec-bundles +``` + +Then summarize profiling data: + +```console +yarn workspace @endo/bundle-source trace:merge -- /tmp/agoric-bundle-traces +``` + ## `moduleFormat` explanations diff --git a/packages/bundle-source/package.json b/packages/bundle-source/package.json index f790f800a4..406cde459d 100644 --- a/packages/bundle-source/package.json +++ b/packages/bundle-source/package.json @@ -14,6 +14,7 @@ "scripts": { "build": "exit 0", "test": "ava", + "bundle:agoric-source-specs": "node tools/bundle-agoric-source-specs.js", "trace:merge": "node tools/trace-merge.js", "test:xs": "exit 0", "lint-fix": "eslint --fix '**/*.js'", diff --git a/packages/bundle-source/tools/bundle-agoric-source-specs.js b/packages/bundle-source/tools/bundle-agoric-source-specs.js new file mode 100644 index 0000000000..2263df2f77 --- /dev/null +++ b/packages/bundle-source/tools/bundle-agoric-source-specs.js @@ -0,0 +1,292 @@ +#!/usr/bin/env node +// @ts-check +/* global process */ + +import fs from 'fs/promises'; +import path from 'path'; +import { pathToFileURL } from 'url'; +import { parseArgs } from 'util'; + +const options = /** @type {const} */ ({ + 'agoric-sdk-root': { type: 'string' }, + 'out-dir': { type: 'string' }, + format: { type: 'string' }, + condition: { type: 'string', multiple: true }, + 'dry-run': { type: 'boolean' }, + verbose: { type: 'boolean' }, +}); + +const usage = `\ +Usage: + node tools/bundle-agoric-source-specs.js [--agoric-sdk-root ] [--out-dir ] [--format ] [--condition ]... [--dry-run] [--verbose] + +Defaults: + --agoric-sdk-root /opt/agoric/agoric-sdk + --out-dir ./agoric-source-spec-bundles + --format endoZipBase64 +`; + +/** + * @param {string} filePath + * @returns {Promise} + */ +const exists = async filePath => { + try { + await fs.stat(filePath); + return true; + } catch { + return false; + } +}; + +/** + * @param {string} root + * @returns {Promise} + */ +const findSourceSpecRegistryFiles = async root => { + /** @type {string[]} */ + const found = []; + const queue = [root]; + while (queue.length > 0) { + const dir = queue.shift(); + if (!dir) { + continue; + } + // eslint-disable-next-line no-await-in-loop + const entries = await fs.readdir(dir, { withFileTypes: true }); + for (const entry of entries) { + const fullPath = path.join(dir, entry.name); + if (entry.isDirectory()) { + queue.push(fullPath); + } else if (entry.isFile() && entry.name === 'source-spec-registry.js') { + found.push(fullPath); + } + } + } + return found.sort(); +}; + +/** + * @param {unknown} value + * @returns {Record | undefined} + */ +const toRecord = value => { + if ( + value && + typeof value === 'object' && + !Array.isArray(value) && + Object.getPrototypeOf(value) === Object.prototype + ) { + return /** @type {Record} */ (value); + } + return undefined; +}; + +/** + * @param {string} filePath + * @param {Record} moduleNs + * @returns {Array<{ + * registryFile: string; + * registryExport: string; + * key: string; + * bundleName: string; + * sourceSpec: string; + * packagePath?: string; + * }>} + */ +const collectSpecsFromModule = (filePath, moduleNs) => { + /** @type {Array<{ + * registryFile: string; + * registryExport: string; + * key: string; + * bundleName: string; + * sourceSpec: string; + * packagePath?: string; + * }>} */ + const collected = []; + for (const [exportName, exportValue] of Object.entries(moduleNs)) { + const maybeRegistry = toRecord(exportValue); + if (!maybeRegistry) { + continue; + } + for (const [key, descriptorMaybe] of Object.entries(maybeRegistry)) { + const descriptor = toRecord(descriptorMaybe); + if (!descriptor) { + continue; + } + if (typeof descriptor.sourceSpec !== 'string') { + continue; + } + const bundleName = + typeof descriptor.bundleName === 'string' + ? descriptor.bundleName + : key; + const packagePath = + typeof descriptor.packagePath === 'string' + ? descriptor.packagePath + : undefined; + collected.push({ + registryFile: filePath, + registryExport: exportName, + key, + bundleName, + sourceSpec: descriptor.sourceSpec, + packagePath, + }); + } + } + return collected; +}; + +/** + * @param {string} text + * @returns {string} + */ +const sanitizeName = text => + text.replace(/[^a-zA-Z0-9_.-]+/g, '-').replace(/^-+|-+$/g, ''); + +const main = async () => { + const { + values: { + 'agoric-sdk-root': agoricSdkRoot = '/opt/agoric/agoric-sdk', + 'out-dir': outDir = path.resolve(process.cwd(), 'agoric-source-spec-bundles'), + format = 'endoZipBase64', + condition: conditions = [], + 'dry-run': dryRun = false, + verbose = false, + }, + positionals, + } = parseArgs({ options, allowPositionals: true }); + + if (positionals.length > 0) { + throw new Error(`Unexpected arguments: ${positionals.join(' ')}\n\n${usage}`); + } + + if (!(await exists(agoricSdkRoot))) { + throw new Error(`agoric-sdk root does not exist: ${agoricSdkRoot}`); + } + + const registryFiles = await findSourceSpecRegistryFiles(agoricSdkRoot); + if (registryFiles.length === 0) { + throw new Error( + `No source-spec-registry.js files found under: ${agoricSdkRoot}`, + ); + } + + /** @type {Array<{ + * registryFile: string; + * registryExport: string; + * key: string; + * bundleName: string; + * sourceSpec: string; + * packagePath?: string; + * }>} */ + const allSpecs = []; + for (const registryFile of registryFiles) { + // eslint-disable-next-line no-await-in-loop + const moduleNs = await import(pathToFileURL(registryFile).href); + allSpecs.push(...collectSpecsFromModule(registryFile, moduleNs)); + } + + if (allSpecs.length === 0) { + throw new Error(`No bundle source specs discovered.\n\n${usage}`); + } + + /** @type {Map} */ + const deduped = new Map(); + for (const spec of allSpecs) { + const dedupeKey = `${spec.sourceSpec}::${spec.bundleName}`; + if (!deduped.has(dedupeKey)) { + deduped.set(dedupeKey, spec); + } + } + const specs = [...deduped.values()]; + + if (!dryRun) { + await fs.mkdir(outDir, { recursive: true }); + } + + /** @type {typeof import('../src/index.js').default | undefined} */ + let bundleSourceFn = undefined; + if (!dryRun) { + ({ default: bundleSourceFn } = await import('../src/index.js')); + } + + /** @type {Array<{ + * id: string; + * bundleFile: string; + * sourceSpec: string; + * bundleName: string; + * registryFile: string; + * registryExport: string; + * key: string; + * packagePath?: string; + * }>} */ + const manifestEntries = []; + + for (let index = 0; index < specs.length; index += 1) { + const spec = specs[index]; + const registryPackage = path.basename(path.dirname(spec.registryFile)); + const id = sanitizeName(`${registryPackage}-${spec.bundleName}-${spec.key}`); + const bundleFile = `${id}.bundle.json`; + if (verbose || dryRun) { + process.stdout.write( + `[${index + 1}/${specs.length}] ${spec.sourceSpec} -> ${bundleFile}\n`, + ); + } + if (!dryRun) { + if (!bundleSourceFn) { + throw new Error('bundleSource is not available'); + } + // eslint-disable-next-line no-await-in-loop + const bundle = await bundleSourceFn(spec.sourceSpec, { + format: /** @type {import('../src/types.js').ModuleFormat} */ (format), + conditions, + }); + // eslint-disable-next-line no-await-in-loop + await fs.writeFile( + path.join(outDir, bundleFile), + `${JSON.stringify(bundle)}\n`, + ); + } + manifestEntries.push({ + id, + bundleFile, + sourceSpec: spec.sourceSpec, + bundleName: spec.bundleName, + registryFile: spec.registryFile, + registryExport: spec.registryExport, + key: spec.key, + packagePath: spec.packagePath, + }); + } + + const manifest = { + generatedAt: new Date().toISOString(), + agoricSdkRoot, + outDir, + format, + conditions, + registryFileCount: registryFiles.length, + entryCount: manifestEntries.length, + dryRun, + entries: manifestEntries, + }; + + const manifestPath = path.join(outDir, 'manifest.json'); + if (!dryRun) { + await fs.writeFile(manifestPath, `${JSON.stringify(manifest, null, 2)}\n`); + } + + process.stdout.write( + `${dryRun ? 'Discovered' : 'Bundled'} ${manifest.entryCount} source specs from ${manifest.registryFileCount} registries\n`, + ); + if (!dryRun) { + process.stdout.write(`Wrote manifest: ${manifestPath}\n`); + } +}; + +main().catch(error => { + process.stderr.write(`${error instanceof Error ? error.stack : error}\n`); + process.exit(1); +}); From db1cc4805e7937e09bb626fa2ff92b059d7406dd Mon Sep 17 00:00:00 2001 From: Turadg Aleahmad Date: Thu, 19 Feb 2026 11:46:45 -0800 Subject: [PATCH 04/22] feat(bundle-source): add profile-agoric-bundling tool --- packages/bundle-source/README.md | 17 +- packages/bundle-source/package.json | 2 +- .../tools/bundle-agoric-source-specs.js | 292 ------------ .../tools/profile-agoric-bundling.mts | 427 ++++++++++++++++++ 4 files changed, 434 insertions(+), 304 deletions(-) delete mode 100644 packages/bundle-source/tools/bundle-agoric-source-specs.js create mode 100755 packages/bundle-source/tools/profile-agoric-bundling.mts diff --git a/packages/bundle-source/README.md b/packages/bundle-source/README.md index c1ad69328a..8431e041d3 100644 --- a/packages/bundle-source/README.md +++ b/packages/bundle-source/README.md @@ -166,22 +166,17 @@ This generates: - `summary.json` with aggregate span statistics. - `summary.md` with a top spans table by total duration. -Bundle all `source-spec-registry.js` entries from `agoric-sdk` using the -current checkout's `bundle-source`: +Profile bundling all `source-spec-registry.js` entries from an `agoric-sdk` +checkout using the current checkout's `bundle-source`: ```console -ENDO_BUNDLE_SOURCE_PROFILE=1 \ -ENDO_BUNDLE_SOURCE_PROFILE_DIR=/tmp/agoric-bundle-traces \ -yarn workspace @endo/bundle-source bundle:agoric-source-specs -- \ +yarn workspace @endo/bundle-source profile:agoric-bundling -- \ --agoric-sdk-root /opt/agoric/agoric-sdk \ - --out-dir /tmp/agoric-source-spec-bundles + --out-dir /tmp/profile-agoric-bundling ``` -Then summarize profiling data: - -```console -yarn workspace @endo/bundle-source trace:merge -- /tmp/agoric-bundle-traces -``` +The tool writes bundles, raw traces, merged trace, and summary files to +`--out-dir`, and prints a top-spans summary table at the end. ## `moduleFormat` explanations diff --git a/packages/bundle-source/package.json b/packages/bundle-source/package.json index 406cde459d..fe49d2926e 100644 --- a/packages/bundle-source/package.json +++ b/packages/bundle-source/package.json @@ -14,7 +14,7 @@ "scripts": { "build": "exit 0", "test": "ava", - "bundle:agoric-source-specs": "node tools/bundle-agoric-source-specs.js", + "profile:agoric-bundling": "tools/profile-agoric-bundling.mts", "trace:merge": "node tools/trace-merge.js", "test:xs": "exit 0", "lint-fix": "eslint --fix '**/*.js'", diff --git a/packages/bundle-source/tools/bundle-agoric-source-specs.js b/packages/bundle-source/tools/bundle-agoric-source-specs.js deleted file mode 100644 index 2263df2f77..0000000000 --- a/packages/bundle-source/tools/bundle-agoric-source-specs.js +++ /dev/null @@ -1,292 +0,0 @@ -#!/usr/bin/env node -// @ts-check -/* global process */ - -import fs from 'fs/promises'; -import path from 'path'; -import { pathToFileURL } from 'url'; -import { parseArgs } from 'util'; - -const options = /** @type {const} */ ({ - 'agoric-sdk-root': { type: 'string' }, - 'out-dir': { type: 'string' }, - format: { type: 'string' }, - condition: { type: 'string', multiple: true }, - 'dry-run': { type: 'boolean' }, - verbose: { type: 'boolean' }, -}); - -const usage = `\ -Usage: - node tools/bundle-agoric-source-specs.js [--agoric-sdk-root ] [--out-dir ] [--format ] [--condition ]... [--dry-run] [--verbose] - -Defaults: - --agoric-sdk-root /opt/agoric/agoric-sdk - --out-dir ./agoric-source-spec-bundles - --format endoZipBase64 -`; - -/** - * @param {string} filePath - * @returns {Promise} - */ -const exists = async filePath => { - try { - await fs.stat(filePath); - return true; - } catch { - return false; - } -}; - -/** - * @param {string} root - * @returns {Promise} - */ -const findSourceSpecRegistryFiles = async root => { - /** @type {string[]} */ - const found = []; - const queue = [root]; - while (queue.length > 0) { - const dir = queue.shift(); - if (!dir) { - continue; - } - // eslint-disable-next-line no-await-in-loop - const entries = await fs.readdir(dir, { withFileTypes: true }); - for (const entry of entries) { - const fullPath = path.join(dir, entry.name); - if (entry.isDirectory()) { - queue.push(fullPath); - } else if (entry.isFile() && entry.name === 'source-spec-registry.js') { - found.push(fullPath); - } - } - } - return found.sort(); -}; - -/** - * @param {unknown} value - * @returns {Record | undefined} - */ -const toRecord = value => { - if ( - value && - typeof value === 'object' && - !Array.isArray(value) && - Object.getPrototypeOf(value) === Object.prototype - ) { - return /** @type {Record} */ (value); - } - return undefined; -}; - -/** - * @param {string} filePath - * @param {Record} moduleNs - * @returns {Array<{ - * registryFile: string; - * registryExport: string; - * key: string; - * bundleName: string; - * sourceSpec: string; - * packagePath?: string; - * }>} - */ -const collectSpecsFromModule = (filePath, moduleNs) => { - /** @type {Array<{ - * registryFile: string; - * registryExport: string; - * key: string; - * bundleName: string; - * sourceSpec: string; - * packagePath?: string; - * }>} */ - const collected = []; - for (const [exportName, exportValue] of Object.entries(moduleNs)) { - const maybeRegistry = toRecord(exportValue); - if (!maybeRegistry) { - continue; - } - for (const [key, descriptorMaybe] of Object.entries(maybeRegistry)) { - const descriptor = toRecord(descriptorMaybe); - if (!descriptor) { - continue; - } - if (typeof descriptor.sourceSpec !== 'string') { - continue; - } - const bundleName = - typeof descriptor.bundleName === 'string' - ? descriptor.bundleName - : key; - const packagePath = - typeof descriptor.packagePath === 'string' - ? descriptor.packagePath - : undefined; - collected.push({ - registryFile: filePath, - registryExport: exportName, - key, - bundleName, - sourceSpec: descriptor.sourceSpec, - packagePath, - }); - } - } - return collected; -}; - -/** - * @param {string} text - * @returns {string} - */ -const sanitizeName = text => - text.replace(/[^a-zA-Z0-9_.-]+/g, '-').replace(/^-+|-+$/g, ''); - -const main = async () => { - const { - values: { - 'agoric-sdk-root': agoricSdkRoot = '/opt/agoric/agoric-sdk', - 'out-dir': outDir = path.resolve(process.cwd(), 'agoric-source-spec-bundles'), - format = 'endoZipBase64', - condition: conditions = [], - 'dry-run': dryRun = false, - verbose = false, - }, - positionals, - } = parseArgs({ options, allowPositionals: true }); - - if (positionals.length > 0) { - throw new Error(`Unexpected arguments: ${positionals.join(' ')}\n\n${usage}`); - } - - if (!(await exists(agoricSdkRoot))) { - throw new Error(`agoric-sdk root does not exist: ${agoricSdkRoot}`); - } - - const registryFiles = await findSourceSpecRegistryFiles(agoricSdkRoot); - if (registryFiles.length === 0) { - throw new Error( - `No source-spec-registry.js files found under: ${agoricSdkRoot}`, - ); - } - - /** @type {Array<{ - * registryFile: string; - * registryExport: string; - * key: string; - * bundleName: string; - * sourceSpec: string; - * packagePath?: string; - * }>} */ - const allSpecs = []; - for (const registryFile of registryFiles) { - // eslint-disable-next-line no-await-in-loop - const moduleNs = await import(pathToFileURL(registryFile).href); - allSpecs.push(...collectSpecsFromModule(registryFile, moduleNs)); - } - - if (allSpecs.length === 0) { - throw new Error(`No bundle source specs discovered.\n\n${usage}`); - } - - /** @type {Map} */ - const deduped = new Map(); - for (const spec of allSpecs) { - const dedupeKey = `${spec.sourceSpec}::${spec.bundleName}`; - if (!deduped.has(dedupeKey)) { - deduped.set(dedupeKey, spec); - } - } - const specs = [...deduped.values()]; - - if (!dryRun) { - await fs.mkdir(outDir, { recursive: true }); - } - - /** @type {typeof import('../src/index.js').default | undefined} */ - let bundleSourceFn = undefined; - if (!dryRun) { - ({ default: bundleSourceFn } = await import('../src/index.js')); - } - - /** @type {Array<{ - * id: string; - * bundleFile: string; - * sourceSpec: string; - * bundleName: string; - * registryFile: string; - * registryExport: string; - * key: string; - * packagePath?: string; - * }>} */ - const manifestEntries = []; - - for (let index = 0; index < specs.length; index += 1) { - const spec = specs[index]; - const registryPackage = path.basename(path.dirname(spec.registryFile)); - const id = sanitizeName(`${registryPackage}-${spec.bundleName}-${spec.key}`); - const bundleFile = `${id}.bundle.json`; - if (verbose || dryRun) { - process.stdout.write( - `[${index + 1}/${specs.length}] ${spec.sourceSpec} -> ${bundleFile}\n`, - ); - } - if (!dryRun) { - if (!bundleSourceFn) { - throw new Error('bundleSource is not available'); - } - // eslint-disable-next-line no-await-in-loop - const bundle = await bundleSourceFn(spec.sourceSpec, { - format: /** @type {import('../src/types.js').ModuleFormat} */ (format), - conditions, - }); - // eslint-disable-next-line no-await-in-loop - await fs.writeFile( - path.join(outDir, bundleFile), - `${JSON.stringify(bundle)}\n`, - ); - } - manifestEntries.push({ - id, - bundleFile, - sourceSpec: spec.sourceSpec, - bundleName: spec.bundleName, - registryFile: spec.registryFile, - registryExport: spec.registryExport, - key: spec.key, - packagePath: spec.packagePath, - }); - } - - const manifest = { - generatedAt: new Date().toISOString(), - agoricSdkRoot, - outDir, - format, - conditions, - registryFileCount: registryFiles.length, - entryCount: manifestEntries.length, - dryRun, - entries: manifestEntries, - }; - - const manifestPath = path.join(outDir, 'manifest.json'); - if (!dryRun) { - await fs.writeFile(manifestPath, `${JSON.stringify(manifest, null, 2)}\n`); - } - - process.stdout.write( - `${dryRun ? 'Discovered' : 'Bundled'} ${manifest.entryCount} source specs from ${manifest.registryFileCount} registries\n`, - ); - if (!dryRun) { - process.stdout.write(`Wrote manifest: ${manifestPath}\n`); - } -}; - -main().catch(error => { - process.stderr.write(`${error instanceof Error ? error.stack : error}\n`); - process.exit(1); -}); diff --git a/packages/bundle-source/tools/profile-agoric-bundling.mts b/packages/bundle-source/tools/profile-agoric-bundling.mts new file mode 100755 index 0000000000..6bec07ae10 --- /dev/null +++ b/packages/bundle-source/tools/profile-agoric-bundling.mts @@ -0,0 +1,427 @@ +#!/usr/bin/env node --experimental-strip-types +/* global process */ +import '@endo/init'; + +import fs from 'fs/promises'; +import os from 'os'; +import path from 'path'; +import { pathToFileURL } from 'url'; +import { parseArgs } from 'util'; + +import bundleSource from '../src/index.js'; + +const options = { + 'agoric-sdk-root': { type: 'string' }, + 'out-dir': { type: 'string' }, + format: { type: 'string' }, + condition: { type: 'string', multiple: true }, + top: { type: 'string' }, + verbose: { type: 'boolean' }, +} as const; + +const usage = `\ +Usage: + node --experimental-strip-types tools/profile-agoric-bundling.mts [--agoric-sdk-root ] [--out-dir ] [--format ] [--condition ]... [--top 30] [--verbose] + +Defaults: + --agoric-sdk-root /opt/agoric/agoric-sdk + --out-dir /tmp/profile-agoric-bundling- + --format endoZipBase64 + --top 30 +`; + +const toInt = (value: string): number => { + const n = Number.parseInt(value, 10); + if (!Number.isFinite(n) || n <= 0) { + throw new Error(`Expected positive integer, got: ${value}`); + } + return n; +}; + +const exists = async (filePath: string): Promise => { + try { + await fs.stat(filePath); + return true; + } catch { + return false; + } +}; + +const findSourceSpecRegistryFiles = async (root: string): Promise => { + const found: string[] = []; + const queue = [root]; + while (queue.length > 0) { + const dir = queue.shift(); + if (!dir) { + continue; + } + // eslint-disable-next-line no-await-in-loop + const entries = await fs.readdir(dir, { withFileTypes: true }); + for (const entry of entries) { + const fullPath = path.join(dir, entry.name); + if (entry.isDirectory()) { + queue.push(fullPath); + } else if (entry.isFile() && entry.name === 'source-spec-registry.js') { + found.push(fullPath); + } + } + } + return found.sort(); +}; + +const toRecord = (value: unknown): Record | undefined => { + if ( + value && + typeof value === 'object' && + !Array.isArray(value) && + Object.getPrototypeOf(value) === Object.prototype + ) { + return value as Record; + } + return undefined; +}; + +type SourceSpec = { + registryFile: string; + registryExport: string; + key: string; + bundleName: string; + sourceSpec: string; + packagePath?: string; +}; + +const collectSpecsFromModule = ( + filePath: string, + moduleNs: Record, +): SourceSpec[] => { + const collected: SourceSpec[] = []; + for (const [exportName, exportValue] of Object.entries(moduleNs)) { + const maybeRegistry = toRecord(exportValue); + if (!maybeRegistry) { + continue; + } + for (const [key, descriptorMaybe] of Object.entries(maybeRegistry)) { + const descriptor = toRecord(descriptorMaybe); + if (!descriptor || typeof descriptor.sourceSpec !== 'string') { + continue; + } + collected.push({ + registryFile: filePath, + registryExport: exportName, + key, + bundleName: + typeof descriptor.bundleName === 'string' ? descriptor.bundleName : key, + sourceSpec: descriptor.sourceSpec, + packagePath: + typeof descriptor.packagePath === 'string' + ? descriptor.packagePath + : undefined, + }); + } + } + return collected; +}; + +const sanitizeName = (text: string): string => + text.replace(/[^a-zA-Z0-9_.-]+/g, '-').replace(/^-+|-+$/g, ''); + +const findTraceFiles = async (root: string): Promise => { + const found: string[] = []; + const queue = [root]; + while (queue.length > 0) { + const dir = queue.shift(); + if (!dir) { + continue; + } + // eslint-disable-next-line no-await-in-loop + const entries = await fs.readdir(dir, { withFileTypes: true }); + for (const entry of entries) { + const filePath = path.join(dir, entry.name); + if (entry.isDirectory()) { + queue.push(filePath); + } else if (entry.isFile() && entry.name.endsWith('.trace.json')) { + found.push(filePath); + } + } + } + return found.sort(); +}; + +const percentile = (sorted: number[], p: number): number => { + if (sorted.length === 0) { + return 0; + } + if (sorted.length === 1) { + return sorted[0]; + } + const idx = (p / 100) * (sorted.length - 1); + const lo = Math.floor(idx); + const hi = Math.ceil(idx); + if (lo === hi) { + return sorted[lo]; + } + const frac = idx - lo; + return sorted[lo] * (1 - frac) + sorted[hi] * frac; +}; + +const microsToMsText = (micros: number): string => (micros / 1000).toFixed(3); + +type SummaryRow = { + name: string; + count: number; + totalUs: number; + avgUs: number; + maxUs: number; + p50Us: number; + p95Us: number; +}; + +const summarizeEvents = ( + events: Array>, + top: number, +): SummaryRow[] => { + const durationsByName = new Map(); + for (const event of events) { + if (event.ph !== 'X' || typeof event.name !== 'string') { + continue; + } + const dur = typeof event.dur === 'number' ? event.dur : undefined; + if (dur === undefined) { + continue; + } + const bucket = durationsByName.get(event.name); + if (bucket) { + bucket.push(dur); + } else { + durationsByName.set(event.name, [dur]); + } + } + + const rows: SummaryRow[] = [...durationsByName.entries()].map( + ([name, durations]) => { + durations.sort((a, b) => a - b); + const total = durations.reduce((sum, value) => sum + value, 0); + return { + name, + count: durations.length, + totalUs: total, + avgUs: total / durations.length, + maxUs: durations[durations.length - 1], + p50Us: percentile(durations, 50), + p95Us: percentile(durations, 95), + }; + }, + ); + rows.sort((a, b) => b.totalUs - a.totalUs); + return rows.slice(0, top); +}; + +const summarizeMarkdown = (rows: SummaryRow[]): string => { + const header = [ + '| Span | Count | Total ms | Avg ms | P50 ms | P95 ms | Max ms |', + '| --- | ---: | ---: | ---: | ---: | ---: | ---: |', + ]; + const body = rows.map(row => + [ + `| ${row.name}`, + `${row.count}`, + `${microsToMsText(row.totalUs)}`, + `${microsToMsText(row.avgUs)}`, + `${microsToMsText(row.p50Us)}`, + `${microsToMsText(row.p95Us)}`, + `${microsToMsText(row.maxUs)} |`, + ].join(' | '), + ); + return `${header.join('\n')}\n${body.join('\n')}\n`; +}; + +const mergeTraceFiles = async ( + traceFiles: string[], +): Promise>> => { + const mergedEvents: Array> = []; + let offsetUs = 0; + for (const filePath of traceFiles) { + // eslint-disable-next-line no-await-in-loop + const text = await fs.readFile(filePath, 'utf-8'); + const trace = JSON.parse(text) as { traceEvents?: Array> }; + const events = trace.traceEvents || []; + let maxEndUs = 0; + for (const event of events) { + const copy = { + ...event, + args: { ...(event.args || {}), source: filePath }, + }; + if (typeof copy.ts === 'number') { + copy.ts += offsetUs; + } + const ts = typeof copy.ts === 'number' ? copy.ts : 0; + const dur = typeof copy.dur === 'number' ? copy.dur : 0; + maxEndUs = Math.max(maxEndUs, ts + dur); + mergedEvents.push(copy); + } + offsetUs = maxEndUs + 1000; + } + return mergedEvents; +}; + +const main = async () => { + const timestamp = new Date().toISOString().replace(/[:.]/g, '-'); + const defaultOutDir = path.join( + os.tmpdir(), + `profile-agoric-bundling-${timestamp}`, + ); + const { + values: { + 'agoric-sdk-root': agoricSdkRoot = '/opt/agoric/agoric-sdk', + 'out-dir': outDir = defaultOutDir, + format = 'endoZipBase64', + condition: conditions = [], + top: topRaw = '30', + verbose = false, + }, + positionals, + } = parseArgs({ options, allowPositionals: true }); + if (positionals.length > 0) { + throw new Error(`Unexpected arguments: ${positionals.join(' ')}\n\n${usage}`); + } + const top = toInt(topRaw); + + if (!(await exists(agoricSdkRoot))) { + throw new Error(`agoric-sdk root does not exist: ${agoricSdkRoot}`); + } + + const bundlesDir = path.join(outDir, 'bundles'); + const tracesDir = path.join(outDir, 'traces'); + await fs.mkdir(bundlesDir, { recursive: true }); + await fs.mkdir(tracesDir, { recursive: true }); + + const registryFiles = await findSourceSpecRegistryFiles(agoricSdkRoot); + if (registryFiles.length === 0) { + throw new Error( + `No source-spec-registry.js files found under: ${agoricSdkRoot}`, + ); + } + + const allSpecs: SourceSpec[] = []; + for (const registryFile of registryFiles) { + // eslint-disable-next-line no-await-in-loop + const moduleNs = (await import(pathToFileURL(registryFile).href)) as Record< + string, + unknown + >; + allSpecs.push(...collectSpecsFromModule(registryFile, moduleNs)); + } + if (allSpecs.length === 0) { + throw new Error(`No bundle source specs discovered.\n\n${usage}`); + } + + const deduped = new Map(); + for (const spec of allSpecs) { + const dedupeKey = `${spec.sourceSpec}::${spec.bundleName}`; + if (!deduped.has(dedupeKey)) { + deduped.set(dedupeKey, spec); + } + } + const specs = [...deduped.values()]; + + const manifestEntries: Array<{ + id: string; + bundleFile: string; + sourceSpec: string; + bundleName: string; + registryFile: string; + registryExport: string; + key: string; + packagePath?: string; + }> = []; + + for (let index = 0; index < specs.length; index += 1) { + const spec = specs[index]; + const registryPackage = path.basename(path.dirname(spec.registryFile)); + const id = sanitizeName(`${registryPackage}-${spec.bundleName}-${spec.key}`); + const bundleFile = `${id}.bundle.json`; + if (verbose) { + process.stdout.write( + `[${index + 1}/${specs.length}] ${spec.sourceSpec} -> ${bundleFile}\n`, + ); + } + + // eslint-disable-next-line no-await-in-loop + const bundle = await bundleSource(spec.sourceSpec, { + format: format as import('../src/types.js').ModuleFormat, + conditions, + profile: { + enabled: true, + traceDir: tracesDir, + }, + }); + // eslint-disable-next-line no-await-in-loop + await fs.writeFile(path.join(bundlesDir, bundleFile), `${JSON.stringify(bundle)}\n`); + + manifestEntries.push({ + id, + bundleFile, + sourceSpec: spec.sourceSpec, + bundleName: spec.bundleName, + registryFile: spec.registryFile, + registryExport: spec.registryExport, + key: spec.key, + packagePath: spec.packagePath, + }); + } + + const traceFiles = await findTraceFiles(tracesDir); + const mergedEvents = await mergeTraceFiles(traceFiles); + const topRows = summarizeEvents(mergedEvents, top); + + const manifest = { + generatedAt: new Date().toISOString(), + agoricSdkRoot, + outDir, + bundlesDir, + tracesDir, + format, + conditions, + registryFileCount: registryFiles.length, + entryCount: manifestEntries.length, + traceFileCount: traceFiles.length, + entries: manifestEntries, + }; + + const summary = { + generatedAt: new Date().toISOString(), + traceFileCount: traceFiles.length, + eventCount: mergedEvents.length, + topSpansByTotalDuration: topRows, + traceFiles, + }; + + const mergedTracePath = path.join(outDir, 'merged.trace.json'); + const summaryPath = path.join(outDir, 'summary.json'); + const summaryMdPath = path.join(outDir, 'summary.md'); + const manifestPath = path.join(outDir, 'manifest.json'); + + await fs.writeFile( + mergedTracePath, + JSON.stringify({ traceEvents: mergedEvents, displayTimeUnit: 'ms' }, null, 2), + ); + await fs.writeFile(summaryPath, JSON.stringify(summary, null, 2)); + await fs.writeFile(summaryMdPath, summarizeMarkdown(topRows)); + await fs.writeFile(manifestPath, `${JSON.stringify(manifest, null, 2)}\n`); + + process.stdout.write( + `Bundled ${manifest.entryCount} source specs from ${manifest.registryFileCount} registries\n`, + ); + process.stdout.write(`Output directory: ${outDir}\n`); + process.stdout.write(`Manifest: ${manifestPath}\n`); + process.stdout.write(`Merged trace: ${mergedTracePath}\n`); + process.stdout.write(`Summary JSON: ${summaryPath}\n`); + process.stdout.write(`Summary Markdown: ${summaryMdPath}\n\n`); + process.stdout.write('Top spans by total duration:\n'); + process.stdout.write(summarizeMarkdown(topRows)); +}; + +main().catch(error => { + process.stderr.write(`${error instanceof Error ? error.stack : error}\n`); + process.exit(1); +}); From d765c58704431823683500e616389eb7af3d7417 Mon Sep 17 00:00:00 2001 From: Turadg Aleahmad Date: Thu, 19 Feb 2026 11:53:34 -0800 Subject: [PATCH 05/22] chore(bundle-source): render profile summary with console.table --- .../tools/profile-agoric-bundling.mts | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/packages/bundle-source/tools/profile-agoric-bundling.mts b/packages/bundle-source/tools/profile-agoric-bundling.mts index 6bec07ae10..23ae2466c9 100755 --- a/packages/bundle-source/tools/profile-agoric-bundling.mts +++ b/packages/bundle-source/tools/profile-agoric-bundling.mts @@ -235,6 +235,21 @@ const summarizeMarkdown = (rows: SummaryRow[]): string => { return `${header.join('\n')}\n${body.join('\n')}\n`; }; +const summarizeConsoleRows = (rows: SummaryRow[]) => + Object.fromEntries( + rows.map(row => [ + row.name, + { + count: row.count, + totalMs: Number(microsToMsText(row.totalUs)), + avgMs: Number(microsToMsText(row.avgUs)), + p50Ms: Number(microsToMsText(row.p50Us)), + p95Ms: Number(microsToMsText(row.p95Us)), + maxMs: Number(microsToMsText(row.maxUs)), + }, + ]), + ); + const mergeTraceFiles = async ( traceFiles: string[], ): Promise>> => { @@ -418,7 +433,7 @@ const main = async () => { process.stdout.write(`Summary JSON: ${summaryPath}\n`); process.stdout.write(`Summary Markdown: ${summaryMdPath}\n\n`); process.stdout.write('Top spans by total duration:\n'); - process.stdout.write(summarizeMarkdown(topRows)); + console.table(summarizeConsoleRows(topRows)); }; main().catch(error => { From f9dcde557190a0a1d8f3cca6324b3b09399b4865 Mon Sep 17 00:00:00 2001 From: Turadg Aleahmad Date: Thu, 19 Feb 2026 12:00:43 -0800 Subject: [PATCH 06/22] feat(profiling): add compartment-mapper phase spans --- packages/bundle-source/src/script.js | 5 +- packages/bundle-source/src/zip-base64.js | 2 + .../compartment-mapper/src/archive-lite.js | 70 +++++++- .../compartment-mapper/src/node-modules.js | 163 ++++++++++++------ .../compartment-mapper/src/types/external.ts | 19 +- 5 files changed, 202 insertions(+), 57 deletions(-) diff --git a/packages/bundle-source/src/script.js b/packages/bundle-source/src/script.js index 459ef80b00..9c2616e0d5 100644 --- a/packages/bundle-source/src/script.js +++ b/packages/bundle-source/src/script.js @@ -109,8 +109,9 @@ export async function bundleScript( moduleFormat === 'nestedEvaluate' || moduleFormat === 'getExport', conditions: new Set(conditions), - commonDependencies, - parserForLanguage: parserForLanguageForFunctor, + commonDependencies, + profileStartSpan: profiler.startSpan, + parserForLanguage: parserForLanguageForFunctor, workspaceLanguageForExtension, workspaceCommonjsLanguageForExtension, workspaceModuleLanguageForExtension, diff --git a/packages/bundle-source/src/zip-base64.js b/packages/bundle-source/src/zip-base64.js index 9ce01b64dd..4e512b7c15 100644 --- a/packages/bundle-source/src/zip-base64.js +++ b/packages/bundle-source/src/zip-base64.js @@ -103,6 +103,7 @@ export async function bundleZipBase64( dev, conditions: new Set(conditions), commonDependencies, + profileStartSpan: profiler.startSpan, workspaceLanguageForExtension, workspaceCommonjsLanguageForExtension, workspaceModuleLanguageForExtension, @@ -123,6 +124,7 @@ export async function bundleZipBase64( moduleTransforms, sourceMapHook, importHook: importHookForArchive, + profileStartSpan: profiler.startSpan, }, )); } finally { diff --git a/packages/compartment-mapper/src/archive-lite.js b/packages/compartment-mapper/src/archive-lite.js index 0a41fc20b4..0876bd908e 100644 --- a/packages/compartment-mapper/src/archive-lite.js +++ b/packages/compartment-mapper/src/archive-lite.js @@ -73,6 +73,8 @@ const { assign, create, freeze, keys } = Object; */ const addSourcesToArchive = async (archive, sources) => { await null; + let moduleCount = 0; + let byteCount = 0; for (const compartment of keys(sources).sort()) { const modules = sources[compartment]; for (const specifier of keys(modules).sort()) { @@ -82,10 +84,13 @@ const addSourcesToArchive = async (archive, sources) => { if (bytes !== undefined) { // eslint-disable-next-line no-await-in-loop await archive.write(path, bytes); + moduleCount += 1; + byteCount += bytes.length; } } } } + return { moduleCount, byteCount }; }; /** @@ -196,11 +201,16 @@ const digestFromMap = async (powers, compartmentMap, options = {}) => { sourceMapHook = undefined, parserForLanguage: parserForLanguageOption = {}, log: _log = noop, + profileStartSpan = undefined, } = options; + const endSetupParserForLanguage = profileStartSpan?.( + 'compartmentMapper.archiveLite.setupParserForLanguage', + ); const parserForLanguage = freeze( assign(create(null), parserForLanguageOption), ); + endSetupParserForLanguage?.(); const { read, computeSha512 } = unpackReadPowers(powers); @@ -212,12 +222,19 @@ const digestFromMap = async (powers, compartmentMap, options = {}) => { /** @type {Sources} */ const sources = Object.create(null); + const endExitImportHookMaker = profileStartSpan?.( + 'compartmentMapper.archiveLite.exitModuleImportHookMaker', + ); const consolidatedExitModuleImportHook = exitModuleImportHookMaker({ modules: exitModules, exitModuleImportHook, entryCompartmentName, }); + endExitImportHookMaker?.(); + const endMakeImportHook = profileStartSpan?.( + 'compartmentMapper.archiveLite.makeImportHookMaker', + ); const makeImportHook = makeImportHookMaker(read, entryCompartmentName, { sources, compartmentDescriptors: compartments, @@ -229,8 +246,10 @@ const digestFromMap = async (powers, compartmentMap, options = {}) => { importHook: consolidatedExitModuleImportHook, sourceMapHook, }); + endMakeImportHook?.(); // Induce importHook to record all the necessary modules to import the given module specifier. + const endLink = profileStartSpan?.('compartmentMapper.archiveLite.link'); const { compartment, attenuatorsCompartment } = link(compartmentMap, { resolve, makeImportHook, @@ -238,21 +257,38 @@ const digestFromMap = async (powers, compartmentMap, options = {}) => { parserForLanguage, archiveOnly: true, }); + endLink?.(); + + const endLoadEntry = profileStartSpan?.( + 'compartmentMapper.archiveLite.compartment.loadEntry', + ); await compartment.load(entryModuleSpecifier); + endLoadEntry?.(); if (policy) { // retain all attenuators. + const endLoadAttenuators = profileStartSpan?.( + 'compartmentMapper.archiveLite.compartment.loadAttenuators', + ); await Promise.all( detectAttenuators(policy).map(attenuatorSpecifier => attenuatorsCompartment.load(attenuatorSpecifier), ), ); + endLoadAttenuators?.(); } + const endMakeArchiveCompartmentMap = profileStartSpan?.( + 'compartmentMapper.archiveLite.makeArchiveCompartmentMap', + ); const { archiveCompartmentMap, archiveSources } = makeArchiveCompartmentMap( compartmentMap, sources, ); + endMakeArchiveCompartmentMap?.(); + const endEncodeCompartmentMap = profileStartSpan?.( + 'compartmentMapper.archiveLite.encodeCompartmentMap', + ); const archiveCompartmentMapText = JSON.stringify( archiveCompartmentMap, (key, value) => (key.startsWith('_') ? undefined : value), @@ -261,14 +297,25 @@ const digestFromMap = async (powers, compartmentMap, options = {}) => { const archiveCompartmentMapBytes = textEncoder.encode( archiveCompartmentMapText, ); + endEncodeCompartmentMap?.({ + bytes: archiveCompartmentMapBytes.length, + }); if (captureSourceLocation !== undefined) { + const endCaptureSourceLocations = profileStartSpan?.( + 'compartmentMapper.archiveLite.captureSourceLocations', + ); captureSourceLocations(archiveSources, captureSourceLocation); + endCaptureSourceLocations?.(); } let archiveSha512; if (computeSha512 !== undefined) { + const endHashCompartmentMap = profileStartSpan?.( + 'compartmentMapper.archiveLite.hashCompartmentMap', + ); archiveSha512 = computeSha512(archiveCompartmentMapBytes); + endHashCompartmentMap?.(); } return { @@ -289,16 +336,37 @@ export const makeAndHashArchiveFromMap = async ( compartmentMap, options, ) => { + const { profileStartSpan = undefined } = options || {}; + const endDigestFromMap = profileStartSpan?.( + 'compartmentMapper.archiveLite.digestFromMap', + ); const { compartmentMapBytes, sources, sha512 } = await digestFromMap( powers, compartmentMap, options, ); + endDigestFromMap?.(); + const endWriteZipCreate = profileStartSpan?.( + 'compartmentMapper.archiveLite.writeZip.create', + ); const archive = writeZip(); + endWriteZipCreate?.(); + const endWriteCompartmentMap = profileStartSpan?.( + 'compartmentMapper.archiveLite.writeZip.compartmentMap', + ); await archive.write('compartment-map.json', compartmentMapBytes); - await addSourcesToArchive(archive, sources); + endWriteCompartmentMap?.({ bytes: compartmentMapBytes.length }); + const endAddSources = profileStartSpan?.( + 'compartmentMapper.archiveLite.writeZip.sources', + ); + const { moduleCount, byteCount } = await addSourcesToArchive(archive, sources); + endAddSources?.({ moduleCount, byteCount }); + const endZipSnapshot = profileStartSpan?.( + 'compartmentMapper.archiveLite.writeZip.snapshot', + ); const bytes = await archive.snapshot(); + endZipSnapshot?.({ bytes: bytes.length }); return { bytes, ...(sha512 !== undefined && { sha512 }) }; }; diff --git a/packages/compartment-mapper/src/node-modules.js b/packages/compartment-mapper/src/node-modules.js index 7918f48753..769993380e 100644 --- a/packages/compartment-mapper/src/node-modules.js +++ b/packages/compartment-mapper/src/node-modules.js @@ -1357,6 +1357,7 @@ export const compartmentMapForNodeModules_ = async ( policy, strict = false, log = noop, + profileStartSpan = undefined, unknownCanonicalNameHook, packageDataHook, packageDependenciesHook, @@ -1388,19 +1389,28 @@ export const compartmentMapForNodeModules_ = async ( // dev is only set for the entry package, and implied by the development // condition. - const { graph, readDescriptor } = await graphPackages( - maybeRead, - canonical, - entryPackageLocation, - conditions, - packageDescriptor, - dev || (conditions && conditions.has('development')), - commonDependencies, - languageOptions, - strict, - logicalPathGraph, - { log, policy, packageDependenciesHook }, + const endGraphPackages = profileStartSpan?.( + 'compartmentMapper.nodeModules.graphPackages', ); + let graph; + let readDescriptor; + try { + ({ graph, readDescriptor } = await graphPackages( + maybeRead, + canonical, + entryPackageLocation, + conditions, + packageDescriptor, + dev || (conditions && conditions.has('development')), + commonDependencies, + languageOptions, + strict, + logicalPathGraph, + { log, policy, packageDependenciesHook }, + )); + } finally { + endGraphPackages?.(); + } // Graph additional package locations that are not reachable from the entry's // dependency tree (e.g., a project root package when the entry is a tool @@ -1460,40 +1470,54 @@ export const compartmentMapForNodeModules_ = async ( */ const canonicalNameMap = new Map(); + const endFinalizeGraph = profileStartSpan?.( + 'compartmentMapper.nodeModules.finalizeGraph', + ); const finalGraph = finalizeGraph( graph, logicalPathGraph, entryPackageLocation, canonicalNameMap, ); + endFinalizeGraph?.(); // if policy exists, cross-reference the policy "resources" against the list // of known canonical names and fire the `unknownCanonicalName` hook for each // unknown resource, if found if (policy) { - const canonicalNames = new Set(canonicalNameMap.keys()); - const issues = validatePolicyResources(canonicalNames, policy) ?? []; - // Call default handler first if policy exists - for (const { message, canonicalName, path, suggestion } of issues) { - const hookInput = { - canonicalName, - message, - path, - log, - }; - if (suggestion) { - hookInput.suggestion = suggestion; - } - defaultUnknownCanonicalNameHandler(hookInput); - // Then call user-provided hook if it exists - if (unknownCanonicalNameHook) { - unknownCanonicalNameHook(hookInput); + const endValidatePolicy = profileStartSpan?.( + 'compartmentMapper.nodeModules.validatePolicyResources', + ); + try { + const canonicalNames = new Set(canonicalNameMap.keys()); + const issues = validatePolicyResources(canonicalNames, policy) ?? []; + // Call default handler first if policy exists + for (const { message, canonicalName, path, suggestion } of issues) { + const hookInput = { + canonicalName, + message, + path, + log, + }; + if (suggestion) { + hookInput.suggestion = suggestion; + } + defaultUnknownCanonicalNameHandler(hookInput); + // Then call user-provided hook if it exists + if (unknownCanonicalNameHook) { + unknownCanonicalNameHook(hookInput); + } } + } finally { + endValidatePolicy?.(); } } // Fire packageData hook with all package data before translateGraph if (packageDataHook) { + const endPackageDataHook = profileStartSpan?.( + 'compartmentMapper.nodeModules.packageDataHook', + ); const packageData = /** @type {Map} */ ( new Map( @@ -1512,8 +1536,12 @@ export const compartmentMapForNodeModules_ = async ( packageData, log, }); + endPackageDataHook?.({ packages: packageData.size }); } + const endTranslateGraph = profileStartSpan?.( + 'compartmentMapper.nodeModules.translateGraph', + ); const compartmentMap = translateGraph( entryPackageLocation, entryModuleSpecifier, @@ -1521,6 +1549,9 @@ export const compartmentMapForNodeModules_ = async ( conditions, { policy, log, packageDependenciesHook }, ); + endTranslateGraph?.({ + compartmentCount: keys(compartmentMap.compartments).length, + }); return compartmentMap; }; @@ -1543,6 +1574,7 @@ export const mapNodeModules = async ( tags = new Set(), conditions = tags, log = noop, + profileStartSpan = undefined, unknownCanonicalNameHook, packageDataHook, packageDependenciesHook, @@ -1550,36 +1582,63 @@ export const mapNodeModules = async ( ...otherOptions } = {}, ) => { - const { - packageLocation, - packageDescriptorText, - packageDescriptorLocation, - moduleSpecifier, - } = await search(readPowers, moduleLocation, { log }); + const endSearch = profileStartSpan?.('compartmentMapper.nodeModules.search'); + let packageLocation; + let packageDescriptorText; + let packageDescriptorLocation; + let moduleSpecifier; + let searchSpanArgs; + try { + ({ + packageLocation, + packageDescriptorText, + packageDescriptorLocation, + moduleSpecifier, + } = await search(readPowers, moduleLocation, { log })); + searchSpanArgs = { packageLocation, moduleSpecifier }; + } finally { + endSearch?.(searchSpanArgs); + } - const packageDescriptor = /** @type {typeof parseLocatedJson} */ ( - parseLocatedJson - )(packageDescriptorText, packageDescriptorLocation); + const endParseDescriptor = profileStartSpan?.( + 'compartmentMapper.nodeModules.parsePackageDescriptor', + ); + let packageDescriptor; + try { + packageDescriptor = /** @type {typeof parseLocatedJson} */ ( + parseLocatedJson + )(packageDescriptorText, packageDescriptorLocation); + } finally { + endParseDescriptor?.(); + } assertPackageDescriptor(packageDescriptor); assertPackageDescriptorHasName(packageDescriptor, packageDescriptorLocation); assertFileUrlString(packageLocation); - return compartmentMapForNodeModules_( - readPowers, - packageLocation, - conditions, - packageDescriptor, - moduleSpecifier, - { - log, - policy, - unknownCanonicalNameHook, - packageDependenciesHook, - packageDataHook, - ...otherOptions, - }, + const endCompartmentMapForNodeModules = profileStartSpan?.( + 'compartmentMapper.nodeModules.compartmentMapForNodeModules', ); + try { + return compartmentMapForNodeModules_( + readPowers, + packageLocation, + conditions, + packageDescriptor, + moduleSpecifier, + { + log, + policy, + profileStartSpan, + unknownCanonicalNameHook, + packageDependenciesHook, + packageDataHook, + ...otherOptions, + }, + ); + } finally { + endCompartmentMapForNodeModules?.(); + } }; /** diff --git a/packages/compartment-mapper/src/types/external.ts b/packages/compartment-mapper/src/types/external.ts index 6062b0ea75..7fb365f2a4 100644 --- a/packages/compartment-mapper/src/types/external.ts +++ b/packages/compartment-mapper/src/types/external.ts @@ -191,13 +191,27 @@ export interface LogOptions { log?: LogFn | undefined; } +/** + * Options having an optional profiling span hook. + */ +export interface ProfilingOptions { + /** + * Starts a profiling span and returns an end function. + */ + profileStartSpan?: ( + name: string, + args?: Record, + ) => (args?: Record) => void; +} + /** * Options for `mapNodeModules()` */ export type MapNodeModulesOptions = MapNodeModulesOptionsOmitPolicy & PolicyOption & MapNodeModulesHookOptions & - LogOptions; + LogOptions & + ProfilingOptions; type MapNodeModulesOptionsOmitPolicy = Partial<{ /** @deprecated renamed `conditions` to be consistent with Node.js */ @@ -349,7 +363,8 @@ export type ArchiveLiteOptions = SyncOrAsyncArchiveOptions & ImportingOptions & ExitModuleImportHookOption & LinkingOptions & - LogOptions; + LogOptions & + ProfilingOptions; export type SyncArchiveLiteOptions = SyncOrAsyncArchiveOptions & SyncModuleTransformsOption & From 79b91d30246c7abd086ffec1030487723026dfd0 Mon Sep 17 00:00:00 2001 From: Turadg Aleahmad Date: Thu, 19 Feb 2026 12:10:02 -0800 Subject: [PATCH 07/22] feat(profiling): add compression and babel stage spans --- packages/bundle-source/src/endo.js | 1 + packages/bundle-source/src/zip-base64.js | 5 +- .../compartment-mapper/src/archive-lite.js | 8 +- .../compartment-mapper/src/import-hook.js | 81 +++++++++++++------ .../src/parse-archive-mjs.js | 11 ++- packages/compartment-mapper/src/parse-mjs.js | 8 +- .../compartment-mapper/src/types/external.ts | 4 + .../compartment-mapper/src/types/internal.ts | 8 +- packages/evasive-transform/src/index.js | 19 ++++- packages/module-source/src/module-source.js | 3 + .../module-source/src/transform-source.js | 17 ++++ 11 files changed, 134 insertions(+), 31 deletions(-) diff --git a/packages/bundle-source/src/endo.js b/packages/bundle-source/src/endo.js index 243f337d30..1fc2178b62 100644 --- a/packages/bundle-source/src/endo.js +++ b/packages/bundle-source/src/endo.js @@ -150,6 +150,7 @@ export const makeBundlingKit = ( sourceMap: priorSourceMap, sourceUrl: new URL(specifier, location).href, elideComments, + profileStartSpan: profiler?.startSpan, }); const objectBytes = textEncoder.encode(object); return { diff --git a/packages/bundle-source/src/zip-base64.js b/packages/bundle-source/src/zip-base64.js index 4e512b7c15..4e2cb61aae 100644 --- a/packages/bundle-source/src/zip-base64.js +++ b/packages/bundle-source/src/zip-base64.js @@ -144,7 +144,10 @@ export async function bundleZipBase64( try { endoZipBase64 = encodeBase64(bytes); } finally { - endEncodeBase64(); + endEncodeBase64({ + inputBytes: bytes.length, + outputBytes: endoZipBase64?.length, + }); } return harden({ moduleFormat: /** @type {const} */ ('endoZipBase64'), diff --git a/packages/compartment-mapper/src/archive-lite.js b/packages/compartment-mapper/src/archive-lite.js index 0876bd908e..6760f6e2f6 100644 --- a/packages/compartment-mapper/src/archive-lite.js +++ b/packages/compartment-mapper/src/archive-lite.js @@ -245,6 +245,7 @@ const digestFromMap = async (powers, compartmentMap, options = {}) => { entryModuleSpecifier, importHook: consolidatedExitModuleImportHook, sourceMapHook, + profileStartSpan, }); endMakeImportHook?.(); @@ -366,7 +367,12 @@ export const makeAndHashArchiveFromMap = async ( 'compartmentMapper.archiveLite.writeZip.snapshot', ); const bytes = await archive.snapshot(); - endZipSnapshot?.({ bytes: bytes.length }); + endZipSnapshot?.({ + bytes: bytes.length, + sourceBytes: byteCount, + sourceModuleCount: moduleCount, + compartmentMapBytes: compartmentMapBytes.length, + }); return { bytes, ...(sha512 !== undefined && { sha512 }) }; }; diff --git a/packages/compartment-mapper/src/import-hook.js b/packages/compartment-mapper/src/import-hook.js index fb3c040af3..1f70536071 100644 --- a/packages/compartment-mapper/src/import-hook.js +++ b/packages/compartment-mapper/src/import-hook.js @@ -361,6 +361,7 @@ function* chooseModuleDescriptor( archiveOnly, sourceMapHook, moduleSourceHook, + profileStartSpan = undefined, strictlyRequiredForCompartment, log = noop, }, @@ -420,32 +421,62 @@ function* chooseModuleDescriptor( // "next" values must have type assertions for narrowing because we have // multiple yielded types - const moduleBytes = /** @type {Uint8Array|undefined} */ ( - yield maybeRead(moduleLocation) + const endReadModuleBytes = profileStartSpan?.( + 'compartmentMapper.importHook.readModuleBytes', + { moduleLocation }, ); + let moduleBytes; + try { + moduleBytes = /** @type {Uint8Array|undefined} */ ( + yield maybeRead(moduleLocation) + ); + } finally { + endReadModuleBytes?.({ bytes: moduleBytes?.length }); + } if (moduleBytes !== undefined) { /** @type {string | undefined} */ let sourceMap; // must be narrowed - const envelope = /** @type {ParseResult} */ ( - yield parse( - moduleBytes, + const endParseModule = profileStartSpan?.( + 'compartmentMapper.importHook.parseModule', + { candidateSpecifier, moduleLocation, - packageLocation, - { - readPowers, - archiveOnly, - sourceMapHook: - sourceMapHook && - (nextSourceMapObject => { - sourceMap = JSON.stringify(nextSourceMapObject); - }), - compartmentDescriptor, - }, - ) + }, ); + let envelope; + try { + envelope = /** @type {ParseResult} */ ( + yield parse( + moduleBytes, + candidateSpecifier, + moduleLocation, + packageLocation, + { + readPowers, + archiveOnly, + sourceMapHook: + sourceMapHook && + (nextSourceMapObject => { + sourceMap = JSON.stringify(nextSourceMapObject); + }), + compartmentDescriptor, + profileStartSpan, + }, + ) + ); + } finally { + endParseModule?.( + envelope + ? { + parser: envelope.parser, + inputBytes: moduleBytes.length, + outputBytes: envelope.bytes.length, + } + : { inputBytes: moduleBytes.length }, + ); + } const { parser, bytes: transformedBytes, @@ -589,6 +620,7 @@ export const makeImportHookMaker = ( entryModuleSpecifier, importHook: exitModuleImportHook = undefined, moduleSourceHook, + profileStartSpan = undefined, log = noop, }, ) => { @@ -718,13 +750,14 @@ export const makeImportHookMaker = ( moduleSpecifier, packageLocation, packageSources, - readPowers, - archiveOnly, - sourceMapHook, - moduleSourceHook, - strictlyRequiredForCompartment, - log, - }, + readPowers, + archiveOnly, + sourceMapHook, + moduleSourceHook, + profileStartSpan, + strictlyRequiredForCompartment, + log, + }, { maybeRead, parse, shouldDeferError }, ); diff --git a/packages/compartment-mapper/src/parse-archive-mjs.js b/packages/compartment-mapper/src/parse-archive-mjs.js index 487705660b..ef6d87cf08 100644 --- a/packages/compartment-mapper/src/parse-archive-mjs.js +++ b/packages/compartment-mapper/src/parse-archive-mjs.js @@ -20,14 +20,23 @@ export const parseArchiveMjs = ( _packageLocation, options = {}, ) => { - const { sourceMap, sourceMapHook } = options; + const { sourceMap, sourceMapHook, profileStartSpan } = options; const source = textDecoder.decode(bytes); + const endModuleSource = profileStartSpan?.( + 'compartmentMapper.parseArchiveMjs.moduleSource', + ); const record = new ModuleSource(source, { sourceMap, sourceMapUrl: sourceUrl, sourceMapHook, + profileStartSpan, }); + endModuleSource?.(); + const endStringify = profileStartSpan?.( + 'compartmentMapper.parseArchiveMjs.stringifyRecord', + ); const pre = textEncoder.encode(JSON.stringify(record)); + endStringify?.(); return { parser: 'pre-mjs-json', bytes: pre, diff --git a/packages/compartment-mapper/src/parse-mjs.js b/packages/compartment-mapper/src/parse-mjs.js index 8548933748..443a37db21 100644 --- a/packages/compartment-mapper/src/parse-mjs.js +++ b/packages/compartment-mapper/src/parse-mjs.js @@ -14,14 +14,20 @@ export const parseMjs = ( _packageLocation, options = {}, ) => { - const { sourceMap, sourceMapHook, archiveOnly = false } = options; + const { sourceMap, sourceMapHook, archiveOnly = false, profileStartSpan } = + options; const source = textDecoder.decode(bytes); + const endModuleSource = profileStartSpan?.( + 'compartmentMapper.parseMjs.moduleSource', + ); const record = new ModuleSource(source, { sourceUrl: archiveOnly ? undefined : sourceUrl, sourceMap, sourceMapUrl: sourceUrl, sourceMapHook, + profileStartSpan, }); + endModuleSource?.(); return { parser: 'mjs', bytes, diff --git a/packages/compartment-mapper/src/types/external.ts b/packages/compartment-mapper/src/types/external.ts index 7fb365f2a4..3614a787e6 100644 --- a/packages/compartment-mapper/src/types/external.ts +++ b/packages/compartment-mapper/src/types/external.ts @@ -796,6 +796,10 @@ type ParseArguments = [ sourceMapUrl: string | undefined; readPowers: ReadFn | ReadPowers | undefined; compartmentDescriptor: CompartmentDescriptor | undefined; + profileStartSpan: ( + name: string, + args?: Record, + ) => (endArgs?: Record) => void; }> & ArchiveOnlyOption, ]; diff --git a/packages/compartment-mapper/src/types/internal.ts b/packages/compartment-mapper/src/types/internal.ts index f75af06dc1..76eedf9f1d 100644 --- a/packages/compartment-mapper/src/types/internal.ts +++ b/packages/compartment-mapper/src/types/internal.ts @@ -34,6 +34,7 @@ import type { ModuleTransforms, ParseFn, ParserForLanguage, + ProfilingOptions, PolicyOption, SearchSuffixesOption, SourceMapHook, @@ -96,7 +97,8 @@ export type MakeImportHookMakersOptions = { SearchSuffixesOption & ArchiveOnlyOption & SourceMapHookOption & - LogOptions; + LogOptions & + ProfilingOptions; export type MakeImportHookMakerOptions = MakeImportHookMakersOptions & ExitModuleImportHookOption; @@ -158,6 +160,10 @@ export type ChooseModuleDescriptorParams = { */ sourceMapHook?: SourceMapHook | undefined; moduleSourceHook?: ModuleSourceHook | undefined; + profileStartSpan?: ( + name: string, + args?: Record, + ) => (args?: Record) => void; strictlyRequiredForCompartment: StrictlyRequiredFn; } & ComputeSha512Option & diff --git a/packages/evasive-transform/src/index.js b/packages/evasive-transform/src/index.js index 337ae91665..edb4014fda 100644 --- a/packages/evasive-transform/src/index.js +++ b/packages/evasive-transform/src/index.js @@ -25,6 +25,7 @@ import { generate } from './generate.js'; comment contents, preserving code positions within each line * @property {(path: import('@babel/traverse').NodePath) => void} [customVisitor] - A visitor function to be called on each node, in addition to the standard transforms. Receives the same path argument as a normal Babel visitor. * @property {boolean | undefined} [useLocationUnmap] - deprecated, vestigial + * @property {(name: string, args?: Record) => (args?: Record) => void} [profileStartSpan] - Optional profiling span hook * @public */ @@ -71,24 +72,38 @@ export function evadeCensorSync(source, options) { elideComments = false, onlyComments = false, customVisitor, + profileStartSpan, } = options || {}; // Parse the rolled-up chunk with Babel. // We are prepared for different module systems. + const endParse = profileStartSpan?.('evasiveTransform.babel.parse', { + sourceType, + }); const ast = parseAst(source, { sourceType, }); + endParse?.(); + const endTraverse = profileStartSpan?.('evasiveTransform.babel.traverse', { + elideComments, + }); transformAst(ast, { elideComments, onlyComments, customVisitor }); + endTraverse?.(); + const endGenerate = profileStartSpan?.('evasiveTransform.babel.generate'); if (sourceUrl) { - return generate(ast, { + const generated = generate(ast, { source, sourceUrl, ...(sourceMap !== undefined && { sourceMap }), }); + endGenerate?.(); + return generated; } - return generate(ast, { source }); + const generated = generate(ast, { source }); + endGenerate?.(); + return generated; } /** diff --git a/packages/module-source/src/module-source.js b/packages/module-source/src/module-source.js index 91253706d3..fe1ceb02e1 100644 --- a/packages/module-source/src/module-source.js +++ b/packages/module-source/src/module-source.js @@ -53,6 +53,7 @@ const analyzeModule = makeModuleAnalyzer(); * @property {string | undefined} [sourceMap] * @property {string | undefined} [sourceMapUrl] * @property {SourceMapHook | undefined} [sourceMapHook] + * @property {(name: string, args?: Record) => (args?: Record) => void} [profileStartSpan] */ // XXX implements import('ses').PrecompiledModuleSource but adding @@ -75,6 +76,7 @@ export function ModuleSource(source, opts = {}) { if (typeof opts === 'string') { opts = { sourceUrl: opts }; } + const endAnalyze = opts.profileStartSpan?.('moduleSource.analyzeModule'); const { imports, functorSource, @@ -85,6 +87,7 @@ export function ModuleSource(source, opts = {}) { needsImport, needsImportMeta, } = analyzeModule(source, opts); + endAnalyze?.(); this.imports = freeze([...keys(imports)]); this.exports = freeze( [ diff --git a/packages/module-source/src/transform-source.js b/packages/module-source/src/transform-source.js index 69be45b8d5..dc88ae55f5 100644 --- a/packages/module-source/src/transform-source.js +++ b/packages/module-source/src/transform-source.js @@ -55,13 +55,21 @@ export const makeTransformSource = (makeModulePlugins, babel = null) => { const { sourceUrl, sourceMapUrl, sourceType, sourceMap, sourceMapHook } = sourceOptions; + const { profileStartSpan } = sourceOptions; + const endParse = profileStartSpan?.('moduleSource.babel.parse', { + sourceType, + }); const ast = babelParse(source, { sourceType, tokens: true, createParenthesizedExpressions: true, }); + endParse?.(); + const endAnalyzeTraverse = profileStartSpan?.( + 'moduleSource.babel.traverseAnalyze', + ); // Each pass needs its own wrapper because `NodePath.get` caches paths // keyed on the `parent` node; reusing the same wrapper across passes // would let stale state from the analyze pass leak into transform. @@ -72,6 +80,10 @@ export const makeTransformSource = (makeModulePlugins, babel = null) => { undefined, makeHubParentPath(ast), ); + endAnalyzeTraverse?.(); + const endTransformTraverse = profileStartSpan?.( + 'moduleSource.babel.traverseTransform', + ); traverseBabel( ast, visitorFromPlugin(transformPlugin), @@ -79,9 +91,13 @@ export const makeTransformSource = (makeModulePlugins, babel = null) => { undefined, makeHubParentPath(ast), ); + endTransformTraverse?.(); const sourceMaps = sourceOptions.sourceMapHook !== undefined; + const endGenerate = profileStartSpan?.('moduleSource.babel.generate', { + sourceMaps, + }); const { code: transformedSource, map: transformedSourceMap } = generateBabel( ast, @@ -96,6 +112,7 @@ export const makeTransformSource = (makeModulePlugins, babel = null) => { }, source, ); + endGenerate?.(); if (sourceMaps) { sourceMapHook(transformedSourceMap, { From 897cfb9751f1fec672d2c8b0380eabf1d5c48349 Mon Sep 17 00:00:00 2001 From: Turadg Aleahmad Date: Thu, 19 Feb 2026 12:10:16 -0800 Subject: [PATCH 08/22] fix(profiling): await node-modules parent span timing --- packages/compartment-mapper/src/node-modules.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/compartment-mapper/src/node-modules.js b/packages/compartment-mapper/src/node-modules.js index 769993380e..8f717e8646 100644 --- a/packages/compartment-mapper/src/node-modules.js +++ b/packages/compartment-mapper/src/node-modules.js @@ -1620,7 +1620,7 @@ export const mapNodeModules = async ( 'compartmentMapper.nodeModules.compartmentMapForNodeModules', ); try { - return compartmentMapForNodeModules_( + return await compartmentMapForNodeModules_( readPowers, packageLocation, conditions, From 49bc42a2fab47938a543eaf03866731a02b774b4 Mon Sep 17 00:00:00 2001 From: Turadg Aleahmad Date: Thu, 19 Feb 2026 12:16:28 -0800 Subject: [PATCH 09/22] feat(profiling): add critical-path metrics and fix module-source span propagation --- .../tools/profile-agoric-bundling.mts | 47 ++++++++++++++++-- packages/bundle-source/tools/trace-merge.js | 48 +++++++++++++++++-- .../module-source/src/transform-analyze.js | 3 +- 3 files changed, 91 insertions(+), 7 deletions(-) diff --git a/packages/bundle-source/tools/profile-agoric-bundling.mts b/packages/bundle-source/tools/profile-agoric-bundling.mts index 23ae2466c9..92c83efbf1 100755 --- a/packages/bundle-source/tools/profile-agoric-bundling.mts +++ b/packages/bundle-source/tools/profile-agoric-bundling.mts @@ -170,23 +170,50 @@ type SummaryRow = { name: string; count: number; totalUs: number; + criticalPathUs: number; + overlapFactor: number; avgUs: number; maxUs: number; p50Us: number; p95Us: number; }; +const unionDuration = (intervals: Array<[number, number]>): number => { + if (intervals.length === 0) { + return 0; + } + intervals.sort((a, b) => a[0] - b[0]); + let total = 0; + let [curStart, curEnd] = intervals[0]; + for (let i = 1; i < intervals.length; i += 1) { + const [start, end] = intervals[i]; + if (start <= curEnd) { + if (end > curEnd) { + curEnd = end; + } + } else { + total += curEnd - curStart; + curStart = start; + curEnd = end; + } + } + total += curEnd - curStart; + return total; +}; + const summarizeEvents = ( events: Array>, top: number, ): SummaryRow[] => { const durationsByName = new Map(); + const intervalsByName = new Map>(); for (const event of events) { if (event.ph !== 'X' || typeof event.name !== 'string') { continue; } const dur = typeof event.dur === 'number' ? event.dur : undefined; - if (dur === undefined) { + const ts = typeof event.ts === 'number' ? event.ts : undefined; + if (dur === undefined || ts === undefined) { continue; } const bucket = durationsByName.get(event.name); @@ -195,16 +222,26 @@ const summarizeEvents = ( } else { durationsByName.set(event.name, [dur]); } + const interval: [number, number] = [ts, ts + dur]; + const intervals = intervalsByName.get(event.name); + if (intervals) { + intervals.push(interval); + } else { + intervalsByName.set(event.name, [interval]); + } } const rows: SummaryRow[] = [...durationsByName.entries()].map( ([name, durations]) => { durations.sort((a, b) => a - b); const total = durations.reduce((sum, value) => sum + value, 0); + const criticalPathUs = unionDuration([...(intervalsByName.get(name) || [])]); return { name, count: durations.length, totalUs: total, + criticalPathUs, + overlapFactor: criticalPathUs > 0 ? total / criticalPathUs : 0, avgUs: total / durations.length, maxUs: durations[durations.length - 1], p50Us: percentile(durations, 50), @@ -218,14 +255,16 @@ const summarizeEvents = ( const summarizeMarkdown = (rows: SummaryRow[]): string => { const header = [ - '| Span | Count | Total ms | Avg ms | P50 ms | P95 ms | Max ms |', - '| --- | ---: | ---: | ---: | ---: | ---: | ---: |', + '| Span | Count | Total ms | Critical ms | Overlap x | Avg ms | P50 ms | P95 ms | Max ms |', + '| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |', ]; const body = rows.map(row => [ `| ${row.name}`, `${row.count}`, `${microsToMsText(row.totalUs)}`, + `${microsToMsText(row.criticalPathUs)}`, + `${row.overlapFactor.toFixed(2)}`, `${microsToMsText(row.avgUs)}`, `${microsToMsText(row.p50Us)}`, `${microsToMsText(row.p95Us)}`, @@ -242,6 +281,8 @@ const summarizeConsoleRows = (rows: SummaryRow[]) => { count: row.count, totalMs: Number(microsToMsText(row.totalUs)), + criticalMs: Number(microsToMsText(row.criticalPathUs)), + overlapX: Number(row.overlapFactor.toFixed(2)), avgMs: Number(microsToMsText(row.avgUs)), p50Ms: Number(microsToMsText(row.p50Us)), p95Ms: Number(microsToMsText(row.p95Us)), diff --git a/packages/bundle-source/tools/trace-merge.js b/packages/bundle-source/tools/trace-merge.js index 6de92791d7..834ade6708 100644 --- a/packages/bundle-source/tools/trace-merge.js +++ b/packages/bundle-source/tools/trace-merge.js @@ -110,6 +110,33 @@ const percentile = (sorted, p) => { */ const microsToMsText = micros => (micros / 1000).toFixed(3); +/** + * @param {Array<[number, number]>} intervals + * @returns {number} + */ +const unionDuration = intervals => { + if (intervals.length === 0) { + return 0; + } + intervals.sort((a, b) => a[0] - b[0]); + let total = 0; + let [curStart, curEnd] = intervals[0]; + for (let i = 1; i < intervals.length; i += 1) { + const [start, end] = intervals[i]; + if (start <= curEnd) { + if (end > curEnd) { + curEnd = end; + } + } else { + total += curEnd - curStart; + curStart = start; + curEnd = end; + } + } + total += curEnd - curStart; + return total; +}; + /** * @param {Array>} events * @param {number} top @@ -117,12 +144,15 @@ const microsToMsText = micros => (micros / 1000).toFixed(3); const summarize = (events, top) => { /** @type {Map} */ const durationsByName = new Map(); + /** @type {Map>} */ + const intervalsByName = new Map(); for (const event of events) { if (event.ph !== 'X' || typeof event.name !== 'string') { continue; } const dur = typeof event.dur === 'number' ? event.dur : undefined; - if (dur === undefined) { + const ts = typeof event.ts === 'number' ? event.ts : undefined; + if (dur === undefined || ts === undefined) { continue; } const bucket = durationsByName.get(event.name); @@ -131,15 +161,25 @@ const summarize = (events, top) => { } else { durationsByName.set(event.name, [dur]); } + const intervals = intervalsByName.get(event.name); + const interval = /** @type {[number, number]} */ ([ts, ts + dur]); + if (intervals) { + intervals.push(interval); + } else { + intervalsByName.set(event.name, [interval]); + } } const rows = [...durationsByName.entries()].map(([name, durations]) => { durations.sort((a, b) => a - b); const total = durations.reduce((sum, value) => sum + value, 0); + const criticalPathUs = unionDuration([...(intervalsByName.get(name) || [])]); return { name, count: durations.length, totalUs: total, + criticalPathUs, + overlapFactor: criticalPathUs > 0 ? total / criticalPathUs : 0, avgUs: total / durations.length, minUs: durations[0], maxUs: durations[durations.length - 1], @@ -157,14 +197,16 @@ const summarize = (events, top) => { */ const summarizeMarkdown = rows => { const header = [ - '| Span | Count | Total ms | Avg ms | P50 ms | P95 ms | Max ms |', - '| --- | ---: | ---: | ---: | ---: | ---: | ---: |', + '| Span | Count | Total ms | Critical ms | Overlap x | Avg ms | P50 ms | P95 ms | Max ms |', + '| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |', ]; const body = rows.map(row => [ `| ${row.name}`, `${row.count}`, `${microsToMsText(row.totalUs)}`, + `${microsToMsText(row.criticalPathUs)}`, + `${row.overlapFactor.toFixed(2)}`, `${microsToMsText(row.avgUs)}`, `${microsToMsText(row.p50Us)}`, `${microsToMsText(row.p95Us)}`, diff --git a/packages/module-source/src/transform-analyze.js b/packages/module-source/src/transform-analyze.js index 9f918ee6a9..a99f650159 100644 --- a/packages/module-source/src/transform-analyze.js +++ b/packages/module-source/src/transform-analyze.js @@ -16,7 +16,7 @@ const makeCreateStaticRecord = transformSource => */ function createStaticRecord( moduleSource, - { sourceUrl, sourceMapUrl, sourceMap, sourceMapHook } = {}, + { sourceUrl, sourceMapUrl, sourceMap, sourceMapHook, profileStartSpan } = {}, ) { // Transform the Module source code. const sourceOptions = { @@ -24,6 +24,7 @@ const makeCreateStaticRecord = transformSource => sourceMap, sourceMapUrl, sourceMapHook, + profileStartSpan, sourceType: 'module', // exportNames of variables that are only initialized and used, but // never assigned to. From 4b7d4f647df3bc3b124c7eb95d4cef8b8bbfadbb Mon Sep 17 00:00:00 2001 From: Turadg Aleahmad Date: Thu, 19 Feb 2026 15:03:52 -0800 Subject: [PATCH 10/22] perf(bundle-source): speed up multi-entry bundling and add profiling detail --- packages/bundle-source/src/endo.js | 20 +- packages/bundle-source/src/types.ts | 7 +- packages/bundle-source/src/zip-base64.js | 96 +++++++- .../tools/profile-agoric-bundling.mts | 169 +++++++++++++- packages/bundle-source/tools/trace-merge.js | 160 ++++++++++++- .../compartment-mapper/src/archive-lite.js | 47 +++- .../compartment-mapper/src/generic-graph.js | 18 ++ .../compartment-mapper/src/import-hook.js | 66 +++++- packages/compartment-mapper/src/map-parser.js | 59 ++++- .../compartment-mapper/src/node-modules.js | 9 +- .../compartment-mapper/src/node-powers.js | 94 +++++--- .../src/parse-archive-mjs.js | 46 +++- .../compartment-mapper/src/types/powers.ts | 7 +- .../test/node-powers.test.js | 78 +++++++ .../test/parse-archive-mjs.test.js | 61 +++++ packages/evasive-transform/src/index.js | 67 ++++++ .../test/evade-censor.test.js | 35 +++ packages/zip/package.json | 3 + packages/zip/src/format-writer.js | 55 ++++- packages/zip/src/types.js | 4 +- packages/zip/src/writer.js | 43 +++- packages/zip/tools/benchmark-writer.mjs | 210 ++++++++++++++++++ 22 files changed, 1257 insertions(+), 97 deletions(-) create mode 100644 packages/compartment-mapper/test/node-powers.test.js create mode 100644 packages/compartment-mapper/test/parse-archive-mjs.test.js create mode 100644 packages/zip/tools/benchmark-writer.mjs diff --git a/packages/bundle-source/src/endo.js b/packages/bundle-source/src/endo.js index 1fc2178b62..8351ecd744 100644 --- a/packages/bundle-source/src/endo.js +++ b/packages/bundle-source/src/endo.js @@ -33,8 +33,8 @@ export const makeBundlingKit = ( /** @type {Set>} */ const sourceMapJobs = new Set(); - /** @type {(sourceMap: string, sourceDescriptor: SourceMapDescriptor) => Promise} */ - let writeSourceMap = async () => {}; + /** @type {((sourceMap: string, sourceDescriptor: SourceMapDescriptor) => Promise) | undefined} */ + let writeSourceMap; if (cacheSourceMaps) { if (!computeSha512) { throw new Error('computeSha512 is required when cacheSourceMaps is true'); @@ -144,6 +144,8 @@ export const makeBundlingKit = ( const source = textDecoder.decode(sourceBytes); const priorSourceMap = typeof sourceMap === 'string' ? sourceMap : undefined; + /** @type {number | undefined} */ + let outputBytes; try { const { code: object, map } = await evadeCensor(source, { sourceType: babelSourceType, @@ -153,13 +155,17 @@ export const makeBundlingKit = ( profileStartSpan: profiler?.startSpan, }); const objectBytes = textEncoder.encode(object); + outputBytes = objectBytes.length; return { bytes: objectBytes, parser, sourceMap: map ? JSON.stringify(map) : undefined, }; } finally { - endTransformModule?.(); + endTransformModule?.({ + inputBytes: sourceBytes.length, + outputBytes, + }); } }; @@ -279,9 +285,11 @@ export const makeBundlingKit = ( parserForLanguage = { ...parserForLanguage, mts: mtsParser, cts: ctsParser }; /** @type {BundlingKit['sourceMapHook']} */ - const sourceMapHook = (sourceMap, sourceDescriptor) => { - sourceMapJobs.add(writeSourceMap(sourceMap, sourceDescriptor)); - }; + const sourceMapHook = + writeSourceMap && + ((sourceMap, sourceDescriptor) => { + sourceMapJobs.add(writeSourceMap(sourceMap, sourceDescriptor)); + }); const workspaceLanguageForExtension = { mts: 'mts', cts: 'cts' }; const workspaceModuleLanguageForExtension = { ts: 'mts' }; diff --git a/packages/bundle-source/src/types.ts b/packages/bundle-source/src/types.ts index 9f2f35b9be..fdf9cdae6b 100644 --- a/packages/bundle-source/src/types.ts +++ b/packages/bundle-source/src/types.ts @@ -187,10 +187,9 @@ export interface BundlingKitOptions { } export interface BundlingKit { - sourceMapHook: ( - sourceMap: string, - sourceDescriptor: SourceMapDescriptor, - ) => void; + sourceMapHook?: + | ((sourceMap: string, sourceDescriptor: SourceMapDescriptor) => void) + | undefined; sourceMapJobs: Set>; moduleTransforms: ModuleTransformsLike; parserForLanguage: ParserForLanguageLike; diff --git a/packages/bundle-source/src/zip-base64.js b/packages/bundle-source/src/zip-base64.js index 4e2cb61aae..e04dd69268 100644 --- a/packages/bundle-source/src/zip-base64.js +++ b/packages/bundle-source/src/zip-base64.js @@ -19,6 +19,49 @@ import { makeBundleProfiler } from './profile.js'; /** @import {BundleZipBase64Options, BundlingKitIO, SharedPowers} from './types.js' */ const readPowers = makeReadPowers({ fs, url, crypto }); +const DEFAULT_READ_CACHE_MAX_BYTES = 64 * 1024 * 1024; +const configuredReadCacheMaxBytes = Number.parseInt( + process.env.ENDO_BUNDLE_SOURCE_READ_CACHE_MAX_BYTES || + `${DEFAULT_READ_CACHE_MAX_BYTES}`, + 10, +); +const readCacheMaxBytes = + Number.isFinite(configuredReadCacheMaxBytes) && configuredReadCacheMaxBytes >= 0 + ? configuredReadCacheMaxBytes + : DEFAULT_READ_CACHE_MAX_BYTES; +/** @type {Map} */ +const cachedReads = new Map(); +/** @type {Map>} */ +const pendingReads = new Map(); +let cachedReadBytes = 0; + +/** + * @param {string} location + * @param {Uint8Array | undefined} bytes + */ +const cacheReadValue = (location, bytes) => { + const prior = cachedReads.get(location); + if (prior !== undefined) { + cachedReadBytes -= prior.length; + } + + cachedReads.set(location, bytes); + if (bytes !== undefined) { + cachedReadBytes += bytes.length; + } + + while (cachedReadBytes > readCacheMaxBytes && cachedReads.size > 0) { + const oldestKey = cachedReads.keys().next().value; + if (oldestKey === undefined) { + break; + } + const value = cachedReads.get(oldestKey); + cachedReads.delete(oldestKey); + if (value !== undefined) { + cachedReadBytes -= value.length; + } + } +}; /** * @param {string} startFilename @@ -63,6 +106,55 @@ export async function bundleZipBase64( let phaseStatus = 'ok'; let phaseError; try { + const maybeRead = async location => { + if (readCacheMaxBytes === 0) { + return powers.maybeRead(location); + } + const hit = cachedReads.has(location); + if (hit) { + const endCacheHit = profiler.startSpan('bundleSource.readCache.hit'); + try { + return cachedReads.get(location); + } finally { + endCacheHit(); + } + } + + let pending = pendingReads.get(location); + if (pending !== undefined) { + const endPending = profiler.startSpan('bundleSource.readCache.pending'); + try { + return pending; + } finally { + endPending(); + } + } + + const endCacheMiss = profiler.startSpan('bundleSource.readCache.miss'); + pending = powers.maybeRead(location).then(bytes => { + cacheReadValue(location, bytes); + pendingReads.delete(location); + return bytes; + }, error => { + pendingReads.delete(location); + throw error; + }); + pendingReads.set(location, pending); + try { + return await pending; + } finally { + endCacheMiss({ + cacheEntries: cachedReads.size, + cacheBytes: cachedReadBytes, + }); + } + }; + + const cachedPowers = /** @type {typeof powers} */ ({ + ...powers, + maybeRead, + }); + const endMakeBundlingKit = profiler.startSpan('bundleSource.makeBundlingKit'); const { sourceMapHook, @@ -99,7 +191,7 @@ export async function bundleZipBase64( const endMapNodeModules = profiler.startSpan('bundleSource.mapNodeModules'); let compartmentMap; try { - compartmentMap = await mapNodeModules(powers, entry.href, { + compartmentMap = await mapNodeModules(cachedPowers, entry.href, { dev, conditions: new Set(conditions), commonDependencies, @@ -117,7 +209,7 @@ export async function bundleZipBase64( let sha512; try { ({ bytes, sha512 } = await makeAndHashArchiveFromMap( - powers, + cachedPowers, compartmentMap, { parserForLanguage, diff --git a/packages/bundle-source/tools/profile-agoric-bundling.mts b/packages/bundle-source/tools/profile-agoric-bundling.mts index 92c83efbf1..3681a54206 100755 --- a/packages/bundle-source/tools/profile-agoric-bundling.mts +++ b/packages/bundle-source/tools/profile-agoric-bundling.mts @@ -178,6 +178,28 @@ type SummaryRow = { p95Us: number; }; +type DerivedMetrics = { + bundlesProcessed: number; + modulesParsed: number; + modulesTransformed: number; + fastPathHitCount: number; + fastPathMissCount: number; + fastPathHitRate: number; + bytesRead: number; + bytesTransformed: number; + bytesArchived: number; + msPerBundle: number; + msPerModuleParsed: number; + msPerModuleTransformed: number; + usPerKBRead: number; +}; + +const FAST_PATH_FOCUS_SPANS = [ + 'evasiveTransform.fastPath.scan', + 'evasiveTransform.fastPath.hit', + 'evasiveTransform.fastPath.miss', +] as const; + const unionDuration = (intervals: Array<[number, number]>): number => { if (intervals.length === 0) { return 0; @@ -203,7 +225,6 @@ const unionDuration = (intervals: Array<[number, number]>): number => { const summarizeEvents = ( events: Array>, - top: number, ): SummaryRow[] => { const durationsByName = new Map(); const intervalsByName = new Map>(); @@ -250,10 +271,25 @@ const summarizeEvents = ( }, ); rows.sort((a, b) => b.totalUs - a.totalUs); - return rows.slice(0, top); + return rows; }; -const summarizeMarkdown = (rows: SummaryRow[]): string => { +const zeroRow = (name: string): SummaryRow => ({ + name, + count: 0, + totalUs: 0, + criticalPathUs: 0, + overlapFactor: 0, + avgUs: 0, + maxUs: 0, + p50Us: 0, + p95Us: 0, +}); + +const summarizeMarkdown = ( + rows: SummaryRow[], + focusRows: SummaryRow[] = [], +): string => { const header = [ '| Span | Count | Total ms | Critical ms | Overlap x | Avg ms | P50 ms | P95 ms | Max ms |', '| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |', @@ -271,7 +307,24 @@ const summarizeMarkdown = (rows: SummaryRow[]): string => { `${microsToMsText(row.maxUs)} |`, ].join(' | '), ); - return `${header.join('\n')}\n${body.join('\n')}\n`; + const main = `${header.join('\n')}\n${body.join('\n')}\n`; + if (focusRows.length === 0) { + return main; + } + const focusBody = focusRows.map(row => + [ + `| ${row.name}`, + `${row.count}`, + `${microsToMsText(row.totalUs)}`, + `${microsToMsText(row.criticalPathUs)}`, + `${row.overlapFactor.toFixed(2)}`, + `${microsToMsText(row.avgUs)}`, + `${microsToMsText(row.p50Us)}`, + `${microsToMsText(row.p95Us)}`, + `${microsToMsText(row.maxUs)} |`, + ].join(' | '), + ); + return `${main}\nFocus spans:\n\n${header.join('\n')}\n${focusBody.join('\n')}\n`; }; const summarizeConsoleRows = (rows: SummaryRow[]) => @@ -291,6 +344,87 @@ const summarizeConsoleRows = (rows: SummaryRow[]) => ]), ); +const sumNumericArgBySpan = ( + events: Array>, + spanName: string, + argName: string, +): number => { + let total = 0; + for (const event of events) { + if (event.ph !== 'X' || event.name !== spanName) { + continue; + } + const args = event.args as Record | undefined; + const value = args?.[argName]; + if (typeof value === 'number' && Number.isFinite(value)) { + total += value; + } + } + return total; +}; + +const makeDerivedMetrics = ( + rows: SummaryRow[], + focusRows: SummaryRow[], + events: Array>, +): DerivedMetrics => { + const rowByName = new Map(rows.map(row => [row.name, row])); + const focusByName = new Map(focusRows.map(row => [row.name, row])); + const bundlesProcessed = rowByName.get('bundleSource.total')?.count || 0; + const modulesParsed = + rowByName.get('compartmentMapper.importHook.parseModule')?.count || 0; + const modulesTransformed = rowByName.get('bundleSource.transformModule')?.count || 0; + const fastPathHitCount = + focusByName.get('evasiveTransform.fastPath.hit')?.count || 0; + const fastPathMissCount = + focusByName.get('evasiveTransform.fastPath.miss')?.count || 0; + const fastPathTotal = fastPathHitCount + fastPathMissCount; + const fastPathHitRate = fastPathTotal > 0 ? fastPathHitCount / fastPathTotal : 0; + + const bytesRead = sumNumericArgBySpan( + events, + 'compartmentMapper.importHook.readModuleBytes', + 'bytes', + ); + const bytesTransformed = sumNumericArgBySpan( + events, + 'bundleSource.transformModule', + 'outputBytes', + ); + const bytesArchived = sumNumericArgBySpan( + events, + 'compartmentMapper.archiveLite.writeZip.snapshot', + 'bytes', + ); + + const totalBundleMs = (rowByName.get('bundleSource.total')?.totalUs || 0) / 1000; + const totalParseMs = + (rowByName.get('compartmentMapper.importHook.parseModule')?.totalUs || 0) / + 1000; + const totalTransformMs = + (rowByName.get('bundleSource.transformModule')?.totalUs || 0) / 1000; + const totalReadUs = + rowByName.get('compartmentMapper.importHook.readModuleBytes')?.totalUs || 0; + const readKB = bytesRead / 1024; + + return { + bundlesProcessed, + modulesParsed, + modulesTransformed, + fastPathHitCount, + fastPathMissCount, + fastPathHitRate, + bytesRead, + bytesTransformed, + bytesArchived, + msPerBundle: bundlesProcessed > 0 ? totalBundleMs / bundlesProcessed : 0, + msPerModuleParsed: modulesParsed > 0 ? totalParseMs / modulesParsed : 0, + msPerModuleTransformed: + modulesTransformed > 0 ? totalTransformMs / modulesTransformed : 0, + usPerKBRead: readKB > 0 ? totalReadUs / readKB : 0, + }; +}; + const mergeTraceFiles = async ( traceFiles: string[], ): Promise>> => { @@ -428,7 +562,13 @@ const main = async () => { const traceFiles = await findTraceFiles(tracesDir); const mergedEvents = await mergeTraceFiles(traceFiles); - const topRows = summarizeEvents(mergedEvents, top); + const allRows = summarizeEvents(mergedEvents); + const topRows = allRows.slice(0, top); + const rowsByName = new Map(allRows.map(row => [row.name, row])); + const focusRows = FAST_PATH_FOCUS_SPANS.map( + name => rowsByName.get(name) || zeroRow(name), + ); + const derivedMetrics = makeDerivedMetrics(allRows, focusRows, mergedEvents); const manifest = { generatedAt: new Date().toISOString(), @@ -449,6 +589,8 @@ const main = async () => { traceFileCount: traceFiles.length, eventCount: mergedEvents.length, topSpansByTotalDuration: topRows, + focusSpans: focusRows, + derivedMetrics, traceFiles, }; @@ -462,7 +604,7 @@ const main = async () => { JSON.stringify({ traceEvents: mergedEvents, displayTimeUnit: 'ms' }, null, 2), ); await fs.writeFile(summaryPath, JSON.stringify(summary, null, 2)); - await fs.writeFile(summaryMdPath, summarizeMarkdown(topRows)); + await fs.writeFile(summaryMdPath, summarizeMarkdown(topRows, focusRows)); await fs.writeFile(manifestPath, `${JSON.stringify(manifest, null, 2)}\n`); process.stdout.write( @@ -475,6 +617,21 @@ const main = async () => { process.stdout.write(`Summary Markdown: ${summaryMdPath}\n\n`); process.stdout.write('Top spans by total duration:\n'); console.table(summarizeConsoleRows(topRows)); + process.stdout.write('\nFocus spans:\n'); + console.table(summarizeConsoleRows(focusRows)); + process.stdout.write('\nDerived metrics:\n'); + console.table({ + metrics: { + ...derivedMetrics, + fastPathHitRate: Number((derivedMetrics.fastPathHitRate * 100).toFixed(2)), + msPerBundle: Number(derivedMetrics.msPerBundle.toFixed(3)), + msPerModuleParsed: Number(derivedMetrics.msPerModuleParsed.toFixed(3)), + msPerModuleTransformed: Number( + derivedMetrics.msPerModuleTransformed.toFixed(3), + ), + usPerKBRead: Number(derivedMetrics.usPerKBRead.toFixed(3)), + }, + }); }; main().catch(error => { diff --git a/packages/bundle-source/tools/trace-merge.js b/packages/bundle-source/tools/trace-merge.js index 834ade6708..be56f6cb10 100644 --- a/packages/bundle-source/tools/trace-merge.js +++ b/packages/bundle-source/tools/trace-merge.js @@ -23,6 +23,36 @@ Examples: node tools/trace-merge.js --out-markdown /tmp/summary.md /tmp/bs-profiles /tmp/other.trace.json `; +const FOCUS_SPANS = [ + 'evasiveTransform.fastPath.scan', + 'evasiveTransform.fastPath.hit', + 'evasiveTransform.fastPath.miss', + 'bundleSource.readCache.hit', + 'bundleSource.readCache.miss', + 'bundleSource.readCache.pending', +]; + +/** + * @param {Array>} events + * @param {string} spanName + * @param {string} argName + * @returns {number} + */ +const sumNumericArgBySpan = (events, spanName, argName) => { + let total = 0; + for (const event of events) { + if (event.ph !== 'X' || event.name !== spanName) { + continue; + } + const args = /** @type {Record | undefined} */ (event.args); + const value = args && args[argName]; + if (typeof value === 'number' && Number.isFinite(value)) { + total += value; + } + } + return total; +}; + /** * @param {string} value * @returns {number} @@ -191,11 +221,33 @@ const summarize = (events, top) => { return rows.slice(0, top); }; +/** + * @param {ReturnType} rows + * @returns {Map[number]>} + */ +const rowsByName = rows => new Map(rows.map(row => [row.name, row])); + +/** + * @param {string} name + */ +const zeroRow = name => ({ + name, + count: 0, + totalUs: 0, + criticalPathUs: 0, + overlapFactor: 0, + avgUs: 0, + minUs: 0, + maxUs: 0, + p50Us: 0, + p95Us: 0, +}); + /** * @param {ReturnType} rows * @returns {string} */ -const summarizeMarkdown = rows => { +const summarizeMarkdown = (rows, focusRows = []) => { const header = [ '| Span | Count | Total ms | Critical ms | Overlap x | Avg ms | P50 ms | P95 ms | Max ms |', '| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |', @@ -213,7 +265,99 @@ const summarizeMarkdown = rows => { `${microsToMsText(row.maxUs)} |`, ].join(' | '), ); - return `${header.join('\n')}\n${body.join('\n')}\n`; + const main = `${header.join('\n')}\n${body.join('\n')}\n`; + if (focusRows.length === 0) { + return main; + } + const focusBody = focusRows.map(row => + [ + `| ${row.name}`, + `${row.count}`, + `${microsToMsText(row.totalUs)}`, + `${microsToMsText(row.criticalPathUs)}`, + `${row.overlapFactor.toFixed(2)}`, + `${microsToMsText(row.avgUs)}`, + `${microsToMsText(row.p50Us)}`, + `${microsToMsText(row.p95Us)}`, + `${microsToMsText(row.maxUs)} |`, + ].join(' | '), + ); + return `${main}\nFocus spans:\n\n${header.join('\n')}\n${focusBody.join('\n')}\n`; +}; + +/** + * @param {ReturnType} allRows + * @param {ReturnType} focusSpans + * @param {Array>} events + */ +const makeDerivedMetrics = (allRows, focusSpans, events) => { + const allByName = rowsByName(allRows); + const focusByName = rowsByName(focusSpans); + const bundlesProcessed = allByName.get('bundleSource.total')?.count || 0; + const modulesParsed = + allByName.get('compartmentMapper.importHook.parseModule')?.count || 0; + const modulesTransformed = allByName.get('bundleSource.transformModule')?.count || 0; + const fastPathHitCount = + focusByName.get('evasiveTransform.fastPath.hit')?.count || 0; + const fastPathMissCount = + focusByName.get('evasiveTransform.fastPath.miss')?.count || 0; + const fastPathTotal = fastPathHitCount + fastPathMissCount; + const fastPathHitRate = fastPathTotal > 0 ? fastPathHitCount / fastPathTotal : 0; + const readCacheHitCount = focusByName.get('bundleSource.readCache.hit')?.count || 0; + const readCacheMissCount = + focusByName.get('bundleSource.readCache.miss')?.count || 0; + const readCachePendingCount = + focusByName.get('bundleSource.readCache.pending')?.count || 0; + const readCacheLookups = readCacheHitCount + readCacheMissCount; + const readCacheHitRate = + readCacheLookups > 0 ? readCacheHitCount / readCacheLookups : 0; + + const bytesRead = sumNumericArgBySpan( + events, + 'compartmentMapper.importHook.readModuleBytes', + 'bytes', + ); + const bytesTransformed = sumNumericArgBySpan( + events, + 'bundleSource.transformModule', + 'outputBytes', + ); + const bytesArchived = sumNumericArgBySpan( + events, + 'compartmentMapper.archiveLite.writeZip.snapshot', + 'bytes', + ); + + const totalBundleMs = (allByName.get('bundleSource.total')?.totalUs || 0) / 1000; + const totalParseMs = + (allByName.get('compartmentMapper.importHook.parseModule')?.totalUs || 0) / + 1000; + const totalTransformMs = + (allByName.get('bundleSource.transformModule')?.totalUs || 0) / 1000; + const totalReadUs = + allByName.get('compartmentMapper.importHook.readModuleBytes')?.totalUs || 0; + const readKB = bytesRead / 1024; + + return { + bundlesProcessed, + modulesParsed, + modulesTransformed, + fastPathHitCount, + fastPathMissCount, + fastPathHitRate, + readCacheHitCount, + readCacheMissCount, + readCachePendingCount, + readCacheHitRate, + bytesRead, + bytesTransformed, + bytesArchived, + msPerBundle: bundlesProcessed > 0 ? totalBundleMs / bundlesProcessed : 0, + msPerModuleParsed: modulesParsed > 0 ? totalParseMs / modulesParsed : 0, + msPerModuleTransformed: + modulesTransformed > 0 ? totalTransformMs / modulesTransformed : 0, + usPerKBRead: readKB > 0 ? totalReadUs / readKB : 0, + }; }; const main = async () => { @@ -276,12 +420,20 @@ const main = async () => { } } - const summaryTop = summarize(mergedEvents, top); + const allRows = summarize(mergedEvents, Number.MAX_SAFE_INTEGER); + const summaryTop = allRows.slice(0, top); + const focusLookup = rowsByName(allRows); + const focusSpans = FOCUS_SPANS.map( + name => focusLookup.get(name) || zeroRow(name), + ); + const derivedMetrics = makeDerivedMetrics(allRows, focusSpans, mergedEvents); const summary = { generatedAt: new Date().toISOString(), traceFileCount: traceFiles.length, eventCount: mergedEvents.length, topSpansByTotalDuration: summaryTop, + focusSpans, + derivedMetrics, traceFiles, }; @@ -290,7 +442,7 @@ const main = async () => { JSON.stringify({ traceEvents: mergedEvents, displayTimeUnit: 'ms' }, null, 2), ); await fs.writeFile(outSummary, JSON.stringify(summary, null, 2)); - await fs.writeFile(outMarkdown, summarizeMarkdown(summaryTop)); + await fs.writeFile(outMarkdown, summarizeMarkdown(summaryTop, focusSpans)); process.stdout.write(`Merged ${traceFiles.length} trace files\n`); process.stdout.write(`Wrote merged trace: ${outTrace}\n`); diff --git a/packages/compartment-mapper/src/archive-lite.js b/packages/compartment-mapper/src/archive-lite.js index 6760f6e2f6..c1fdb3ed76 100644 --- a/packages/compartment-mapper/src/archive-lite.js +++ b/packages/compartment-mapper/src/archive-lite.js @@ -70,26 +70,53 @@ const { assign, create, freeze, keys } = Object; /** * @param {ArchiveWriter} archive * @param {Sources} sources + * @param {(name: string, args?: Record) => (args?: Record) => void} [profileStartSpan] */ -const addSourcesToArchive = async (archive, sources) => { - await null; +const addSourcesToArchive = async (archive, sources, profileStartSpan = undefined) => { let moduleCount = 0; let byteCount = 0; - for (const compartment of keys(sources).sort()) { + const endSortCompartments = profileStartSpan?.( + 'compartmentMapper.archiveLite.writeZip.sources.sortCompartments', + ); + const sortedCompartments = keys(sources).sort(); + endSortCompartments?.({ compartmentCount: sortedCompartments.length }); + + let asyncWriteCount = 0; + const endWriteModules = profileStartSpan?.( + 'compartmentMapper.archiveLite.writeZip.sources.writeModules', + ); + for (const compartment of sortedCompartments) { const modules = sources[compartment]; - for (const specifier of keys(modules).sort()) { + const sortedSpecifiers = keys(modules).sort(); + for (const specifier of sortedSpecifiers) { if ('location' in modules[specifier]) { const { bytes, location } = modules[specifier]; const path = `${compartment}/${location}`; if (bytes !== undefined) { - // eslint-disable-next-line no-await-in-loop - await archive.write(path, bytes); + const maybeWrite = archive.write(path, bytes); + if ( + maybeWrite && + typeof maybeWrite === 'object' && + 'then' in maybeWrite && + typeof maybeWrite.then === 'function' + ) { + asyncWriteCount += 1; + // Preserve deterministic write order for truly async writers. + // eslint-disable-next-line no-await-in-loop + await maybeWrite; + } moduleCount += 1; byteCount += bytes.length; } } } } + endWriteModules?.({ + moduleCount, + byteCount, + asyncWriteCount, + syncWriteCount: moduleCount - asyncWriteCount, + }); return { moduleCount, byteCount }; }; @@ -351,7 +378,7 @@ export const makeAndHashArchiveFromMap = async ( const endWriteZipCreate = profileStartSpan?.( 'compartmentMapper.archiveLite.writeZip.create', ); - const archive = writeZip(); + const archive = writeZip({ profileStartSpan }); endWriteZipCreate?.(); const endWriteCompartmentMap = profileStartSpan?.( 'compartmentMapper.archiveLite.writeZip.compartmentMap', @@ -361,7 +388,11 @@ export const makeAndHashArchiveFromMap = async ( const endAddSources = profileStartSpan?.( 'compartmentMapper.archiveLite.writeZip.sources', ); - const { moduleCount, byteCount } = await addSourcesToArchive(archive, sources); + const { moduleCount, byteCount } = await addSourcesToArchive( + archive, + sources, + profileStartSpan, + ); endAddSources?.({ moduleCount, byteCount }); const endZipSnapshot = profileStartSpan?.( 'compartmentMapper.archiveLite.writeZip.snapshot', diff --git a/packages/compartment-mapper/src/generic-graph.js b/packages/compartment-mapper/src/generic-graph.js index 33032f25fd..226c03add0 100644 --- a/packages/compartment-mapper/src/generic-graph.js +++ b/packages/compartment-mapper/src/generic-graph.js @@ -324,3 +324,21 @@ export const makeShortestPath = graph => { }; return shortestPath; }; + +/** + * Returns a function for shortest-path lookups from one fixed source. + * Computes Dijkstra traversal context once and reuses it for all targets. + * + * @template [T=string] + * @param {GenericGraph} graph Graph to use + * @param {NoInfer} source Source node for all path lookups + */ +export const makeShortestPathFromSource = (graph, source) => { + const context = dijkstra(graph, source); + /** + * @param {NoInfer} target Target node + * @returns {[T, T, ...T[]]} + */ + const shortestPath = target => getPath(context, source, target); + return shortestPath; +}; diff --git a/packages/compartment-mapper/src/import-hook.js b/packages/compartment-mapper/src/import-hook.js index 1f70536071..5874483087 100644 --- a/packages/compartment-mapper/src/import-hook.js +++ b/packages/compartment-mapper/src/import-hook.js @@ -108,11 +108,9 @@ const nodejsConventionSearchSuffixes = [ // LOAD_AS_FILE(X) '.js', '.json', - '.node', // LOAD_INDEX(X) '/index.js', '/index.json', - '/index.node', ]; /** @@ -259,9 +257,18 @@ export const exitModuleImportHookMaker = ({ * `moduleSpecifier` itself) */ const nominateCandidates = (moduleSpecifier, searchSuffixes) => { - // Collate candidate locations for the moduleSpecifier, - // to support Node.js conventions and similar. + // Collate candidate locations for the moduleSpecifier. + // Apply suffix expansion only when the specifier does not already + // include an explicit extension. const candidates = [moduleSpecifier]; + const endsWithSlash = moduleSpecifier.endsWith('/'); + const lastSlash = moduleSpecifier.lastIndexOf('/'); + const leaf = + lastSlash >= 0 ? moduleSpecifier.slice(lastSlash + 1) : moduleSpecifier; + const hasExplicitExtension = leaf.includes('.'); + if (!endsWithSlash && hasExplicitExtension && moduleSpecifier !== '.') { + return candidates; + } for (const candidateSuffix of searchSuffixes) { candidates.push(`${moduleSpecifier}${candidateSuffix}`); } @@ -485,6 +492,15 @@ function* chooseModuleDescriptor( // Facilitate a redirect if the returned record has a different // module specifier than the requested one. + const endAssembleRecord = profileStartSpan?.( + 'compartmentMapper.parseModule.assembleRecord', + { + moduleSpecifier, + candidateSpecifier, + moduleLocation, + parser, + }, + ); if (candidateSpecifier !== moduleSpecifier) { moduleDescriptors[moduleSpecifier] = { retained: true, @@ -499,8 +515,13 @@ function* chooseModuleDescriptor( specifier: candidateSpecifier, importMeta: { url: moduleLocation }, }; + endAssembleRecord?.(); let sha512; + const endHash = profileStartSpan?.('compartmentMapper.parseModule.hash', { + candidateSpecifier, + moduleLocation, + }); if (computeSha512 !== undefined) { sha512 = computeSha512(transformedBytes); @@ -513,6 +534,7 @@ function* chooseModuleDescriptor( }); } } + endHash?.({ hashed: sha512 !== undefined }); const packageRelativeLocation = moduleLocation.slice( packageLocation.length, @@ -539,11 +561,20 @@ function* chooseModuleDescriptor( ); if (!shouldDeferError(parser)) { + const endStrictlyRequired = profileStartSpan?.( + 'compartmentMapper.parseModule.collectImports', + { + candidateSpecifier, + moduleSpecifier, + moduleLocation, + }, + ); for (const importSpecifier of getImportsFromRecord(record)) { strictlyRequiredForCompartment(packageLocation).add( resolve(importSpecifier, moduleSpecifier), ); } + endStrictlyRequired?.(); } return record; @@ -624,6 +655,29 @@ export const makeImportHookMaker = ( log = noop, }, ) => { + const { maybeRead } = unpackReadPowers(readPowers); + /** @type {Map>} */ + const maybeReadCache = new Map(); + /** + * Cache both hits and misses for module reads during a mapping run. + * This avoids repeated filesystem probes for the same candidate path. + * + * @param {string} location + * @returns {Promise} + */ + const cachedMaybeRead = location => { + const cached = maybeReadCache.get(location); + if (cached !== undefined) { + return cached; + } + const pending = Promise.resolve(maybeRead(location)); + maybeReadCache.set(location, pending); + pending.catch(() => { + maybeReadCache.delete(location); + }); + return pending; + }; + // Set of specifiers for modules (scoped to compartment) whose parser is not // using heuristics to determine imports. /** @type {Map>} compartment name ->* module specifier */ @@ -734,8 +788,6 @@ export const makeImportHookMaker = ( ); } - const { maybeRead } = unpackReadPowers(readPowers); - const candidates = nominateCandidates(moduleSpecifier, searchSuffixes); const record = await asyncTrampoline( @@ -758,7 +810,7 @@ export const makeImportHookMaker = ( strictlyRequiredForCompartment, log, }, - { maybeRead, parse, shouldDeferError }, + { maybeRead: cachedMaybeRead, parse, shouldDeferError }, ); if (record) { diff --git a/packages/compartment-mapper/src/map-parser.js b/packages/compartment-mapper/src/map-parser.js index fe1ff93ac4..44e676b63d 100644 --- a/packages/compartment-mapper/src/map-parser.js +++ b/packages/compartment-mapper/src/map-parser.js @@ -115,17 +115,32 @@ function* getParserGenerator( transforms, } = config; - let language = resolveLanguage( - specifier, - location, - languageForExtension, - languageForModuleSpecifier, + const { profileStartSpan = undefined } = options || {}; + let language = ''; + const extension = parseExtension(location); + const endLanguageSelect = profileStartSpan?.( + 'compartmentMapper.parseModule.selectLanguage', + { specifier, location, extension }, ); + try { + language = resolveLanguage( + specifier, + location, + languageForExtension, + languageForModuleSpecifier, + ); + } finally { + endLanguageSelect?.({ selectedLanguage: language }); + } /** @type {string | undefined} */ let sourceMap; if (has(transforms, language)) { + const endTransform = profileStartSpan?.( + 'compartmentMapper.parseModule.transform', + { specifier, location, language }, + ); try { ({ bytes, @@ -140,23 +155,49 @@ function* getParserGenerator( sourceMap, }, )); + endTransform?.({ + parser: language, + outputBytes: bytes.length, + hasSourceMap: sourceMap !== undefined, + }); } catch (err) { + endTransform?.(); throw Error( `Error transforming ${q(language)} source in ${q(location)}: ${/** @type {Error} */ (err).message}`, { cause: err }, ); } } + const endParserLookup = profileStartSpan?.( + 'compartmentMapper.parseModule.lookupParser', + { specifier, location, language }, + ); if (!has(parserForLanguage, language)) { + endParserLookup?.(); throw Error( `Cannot parse module ${specifier} at ${location}, no parser configured for the language ${language}`, ); } const { parse } = parserForLanguage[language]; - return parse(bytes, specifier, location, packageLocation, { - sourceMap, - ...options, - }); + endParserLookup?.(); + + const endParserExecute = profileStartSpan?.( + 'compartmentMapper.parseModule.executeParser', + { + specifier, + location, + language, + inputBytes: bytes.length, + }, + ); + try { + return parse(bytes, specifier, location, packageLocation, { + sourceMap, + ...options, + }); + } finally { + endParserExecute?.({ parser: language }); + } } /** diff --git a/packages/compartment-mapper/src/node-modules.js b/packages/compartment-mapper/src/node-modules.js index 8f717e8646..e655df7d42 100644 --- a/packages/compartment-mapper/src/node-modules.js +++ b/packages/compartment-mapper/src/node-modules.js @@ -26,7 +26,7 @@ import { import { dependencyAllowedByPolicy, makePackagePolicy } from './policy.js'; import { unpackReadPowers } from './powers.js'; import { search, searchDescriptor } from './search.js'; -import { GenericGraph, makeShortestPath } from './generic-graph.js'; +import { GenericGraph, makeShortestPathFromSource } from './generic-graph.js'; /** * @import { @@ -1202,7 +1202,10 @@ const finalizeGraph = ( entryPackageLocation, canonicalNameMap, ) => { - const shortestPath = makeShortestPath(logicalPathGraph); + const shortestPathFromEntry = makeShortestPathFromSource( + logicalPathGraph, + entryPackageLocation, + ); // neither the entry package nor the attenuators compartment have a path; omit const { @@ -1241,7 +1244,7 @@ const finalizeGraph = ( ); for (const [location, node] of subgraphEntries) { - const shortestLogicalPath = shortestPath(entryPackageLocation, location); + const shortestLogicalPath = shortestPathFromEntry(location); // the first element will always be the root package location; this is omitted from the path. shortestLogicalPath.shift(); diff --git a/packages/compartment-mapper/src/node-powers.js b/packages/compartment-mapper/src/node-powers.js index 9b253f9a52..ac25ab65fb 100644 --- a/packages/compartment-mapper/src/node-powers.js +++ b/packages/compartment-mapper/src/node-powers.js @@ -70,6 +70,7 @@ const fakeIsAbsolute = () => false; * @param {UrlInterface} [args.url] * @param {CryptoInterface} [args.crypto] * @param {PathInterface} [args.path] + * @param {number} [args.maxConcurrentReads] * @returns {MaybeReadPowers} */ const makeReadPowersSloppy = ({ @@ -77,6 +78,7 @@ const makeReadPowersSloppy = ({ url = undefined, crypto = undefined, path = undefined, + maxConcurrentReads = 32, }) => { const fileURLToPath = url === undefined ? fakeFileURLToPath : url.fileURLToPath; @@ -84,26 +86,45 @@ const makeReadPowersSloppy = ({ url === undefined ? fakePathToFileURL : url.pathToFileURL; const isAbsolute = path === undefined ? fakeIsAbsolute : path.isAbsolute; - let readMutex = Promise.resolve(undefined); + const readConcurrencyLimit = + Number.isInteger(maxConcurrentReads) && maxConcurrentReads > 0 + ? maxConcurrentReads + : 1; + let activeReadCount = 0; + /** @type {Array<(value: undefined) => void>} */ + const pendingReadQueue = []; + /** @type {Map>} */ + const canonicalMemo = new Map(); + + const acquireReadSlot = async () => { + if (activeReadCount < readConcurrencyLimit) { + activeReadCount += 1; + return; + } + await new Promise(resolve => { + pendingReadQueue.push(resolve); + }); + activeReadCount += 1; + }; + + const releaseReadSlot = () => { + activeReadCount -= 1; + const next = pendingReadQueue.shift(); + if (next) { + next(undefined); + } + }; /** * @type {ReadFn} */ const read = async location => { - const promise = readMutex; - let release = Function.prototype; - readMutex = new Promise(resolve => { - release = resolve; - }); - await promise; - + await acquireReadSlot(); const filepath = fileURLToPath(location); try { - // We await here to ensure that we release the mutex only after - // completing the read. return await fs.promises.readFile(filepath); } finally { - release(undefined); + releaseReadSlot(); } }; @@ -140,23 +161,31 @@ const makeReadPowersSloppy = ({ * * @type {CanonicalFn} */ - const canonical = async location => { - await null; - try { - if (location.endsWith('/')) { - const realPath = await fs.promises.realpath( - fileURLToPath(location).replace(/\/$/, ''), - ); - return /** @type {FileUrlString} */ ( - `${pathToFileURL(realPath).href}/` - ); - } else { - const realPath = await fs.promises.realpath(fileURLToPath(location)); - return /** @type {FileUrlString} */ (pathToFileURL(realPath).href); - } - } catch { - return location; + const canonical = location => { + const pending = canonicalMemo.get(location); + if (pending !== undefined) { + return pending; } + const promise = (async () => { + await null; + try { + if (location.endsWith('/')) { + const realPath = await fs.promises.realpath( + fileURLToPath(location).replace(/\/$/, ''), + ); + return /** @type {FileUrlString} */ ( + `${pathToFileURL(realPath).href}/` + ); + } else { + const realPath = await fs.promises.realpath(fileURLToPath(location)); + return /** @type {FileUrlString} */ (pathToFileURL(realPath).href); + } + } catch { + return location; + } + })(); + canonicalMemo.set(location, promise); + return promise; }; /** @type {HashFn | undefined} */ @@ -188,6 +217,7 @@ const makeReadPowersSloppy = ({ * @param {UrlInterface} [args.url] * @param {CryptoInterface} [args.crypto] * @param {PathInterface} [args.path] + * @param {number} [args.maxConcurrentReads] * @returns {ReadNowPowers} */ export const makeReadNowPowers = ({ @@ -195,8 +225,15 @@ export const makeReadNowPowers = ({ url = undefined, crypto = undefined, path = undefined, + maxConcurrentReads = 32, }) => { - const powers = makeReadPowersSloppy({ fs, url, crypto, path }); + const powers = makeReadPowersSloppy({ + fs, + url, + crypto, + path, + maxConcurrentReads, + }); const fileURLToPath = powers.fileURLToPath || fakeFileURLToPath; const isAbsolute = powers.isAbsolute || fakeIsAbsolute; @@ -258,6 +295,7 @@ const makeWritePowersSloppy = ({ fs, url = undefined }) => { * @param {FsInterface} args.fs * @param {UrlInterface} args.url * @param {CryptoInterface} [args.crypto] + * @param {number} [args.maxConcurrentReads] */ export const makeReadPowers = makeReadPowersSloppy; diff --git a/packages/compartment-mapper/src/parse-archive-mjs.js b/packages/compartment-mapper/src/parse-archive-mjs.js index ef6d87cf08..949db10e66 100644 --- a/packages/compartment-mapper/src/parse-archive-mjs.js +++ b/packages/compartment-mapper/src/parse-archive-mjs.js @@ -11,6 +11,10 @@ import { ModuleSource } from '@endo/module-source'; const textEncoder = new TextEncoder(); const textDecoder = new TextDecoder(); +/** @type {Map>>} */ +const parseArchiveMjsCache = new Map(); +const MAX_PARSE_ARCHIVE_MJS_CACHE_ENTRIES = 20_000; +let parseArchiveMjsCacheEntries = 0; /** @type {ParseFn} */ export const parseArchiveMjs = ( @@ -21,7 +25,29 @@ export const parseArchiveMjs = ( options = {}, ) => { const { sourceMap, sourceMapHook, profileStartSpan } = options; + const canUseCache = sourceMapHook === undefined; const source = textDecoder.decode(bytes); + const sourceMapKey = + sourceMap === undefined + ? '' + : typeof sourceMap === 'string' + ? sourceMap + : JSON.stringify(sourceMap); + const cacheKey = `${source}\n//# sourceMappingURL=${sourceMapKey}`; + if (canUseCache) { + const byLocation = parseArchiveMjsCache.get(sourceUrl); + const cached = byLocation?.get(cacheKey); + if (cached !== undefined) { + profileStartSpan?.('compartmentMapper.parseArchiveMjs.cache.hit')?.(); + return cached; + } + profileStartSpan?.('compartmentMapper.parseArchiveMjs.cache.miss')?.(); + } else { + profileStartSpan?.('compartmentMapper.parseArchiveMjs.cache.bypass')?.({ + hasSourceMap: sourceMap !== undefined, + hasSourceMapHook: sourceMapHook !== undefined, + }); + } const endModuleSource = profileStartSpan?.( 'compartmentMapper.parseArchiveMjs.moduleSource', ); @@ -37,11 +63,29 @@ export const parseArchiveMjs = ( ); const pre = textEncoder.encode(JSON.stringify(record)); endStringify?.(); - return { + const result = { parser: 'pre-mjs-json', bytes: pre, record, }; + if (canUseCache) { + let byLocation = parseArchiveMjsCache.get(sourceUrl); + if (byLocation === undefined) { + byLocation = new Map(); + parseArchiveMjsCache.set(sourceUrl, byLocation); + } + if (!byLocation.has(cacheKey)) { + parseArchiveMjsCacheEntries += 1; + if (parseArchiveMjsCacheEntries > MAX_PARSE_ARCHIVE_MJS_CACHE_ENTRIES) { + parseArchiveMjsCache.clear(); + parseArchiveMjsCacheEntries = 0; + byLocation = new Map(); + parseArchiveMjsCache.set(sourceUrl, byLocation); + } + } + byLocation.set(cacheKey, result); + } + return result; }; /** @type {import('./types.js').ParserImplementation} */ diff --git a/packages/compartment-mapper/src/types/powers.ts b/packages/compartment-mapper/src/types/powers.ts index f779ce6d39..32c21c0693 100644 --- a/packages/compartment-mapper/src/types/powers.ts +++ b/packages/compartment-mapper/src/types/powers.ts @@ -136,14 +136,17 @@ export type WritePowers = { write: WriteFn; }; -export type WriteFn = (location: string, bytes: Uint8Array) => Promise; +export type WriteFn = ( + location: string, + bytes: Uint8Array, +) => void | Promise; export type ArchiveWriter = { write: WriteFn; snapshot: SnapshotFn; }; -export type SnapshotFn = () => Promise; +export type SnapshotFn = () => Uint8Array | Promise; // #endregion // #region execute diff --git a/packages/compartment-mapper/test/node-powers.test.js b/packages/compartment-mapper/test/node-powers.test.js new file mode 100644 index 0000000000..0617227ce1 --- /dev/null +++ b/packages/compartment-mapper/test/node-powers.test.js @@ -0,0 +1,78 @@ +import test from 'ava'; + +import url from 'url'; + +import { makeReadPowers } from '../src/node-powers.js'; + +/** + * @param {number} ms + */ +const sleep = ms => new Promise(resolve => setTimeout(resolve, ms)); + +test('makeReadPowers limits concurrent reads with maxConcurrentReads', async t => { + let activeReads = 0; + let maxObservedReads = 0; + + const fakeFs = { + promises: { + /** + * @param {string} filePath + */ + readFile: async filePath => { + activeReads += 1; + maxObservedReads = Math.max(maxObservedReads, activeReads); + await sleep(20); + activeReads -= 1; + return Buffer.from(filePath); + }, + }, + }; + + const { read } = makeReadPowers({ + fs: /** @type {any} */ (fakeFs), + url, + maxConcurrentReads: 2, + }); + + const a = url.pathToFileURL('/tmp/a.js').href; + const b = url.pathToFileURL('/tmp/b.js').href; + const c = url.pathToFileURL('/tmp/c.js').href; + + await Promise.all([read(a), read(b), read(c)]); + + t.is(maxObservedReads, 2); +}); + +test('makeReadPowers canonical memoizes realpath lookups per location', async t => { + const calls = []; + const fakeFs = { + promises: { + /** + * @param {string} filePath + */ + realpath: async filePath => { + calls.push(filePath); + return filePath; + }, + /** + * @param {string} filePath + */ + readFile: async filePath => Buffer.from(filePath), + }, + }; + + const { canonical } = makeReadPowers({ + fs: /** @type {any} */ (fakeFs), + url, + }); + + const file = url.pathToFileURL('/tmp/file.js').href; + const dir = url.pathToFileURL('/tmp/pkg').href + '/'; + + const [a, b] = await Promise.all([canonical(file), canonical(file)]); + const [c, d] = await Promise.all([canonical(dir), canonical(dir)]); + + t.is(a, b); + t.is(c, d); + t.deepEqual(calls, ['/tmp/file.js', '/tmp/pkg']); +}); diff --git a/packages/compartment-mapper/test/parse-archive-mjs.test.js b/packages/compartment-mapper/test/parse-archive-mjs.test.js new file mode 100644 index 0000000000..339d727805 --- /dev/null +++ b/packages/compartment-mapper/test/parse-archive-mjs.test.js @@ -0,0 +1,61 @@ +import test from 'ava'; + +import { parseArchiveMjs } from '../src/parse-archive-mjs.js'; + +const encoder = new TextEncoder(); + +test('parseArchiveMjs caches results for identical bytes and source URL', t => { + const bytes = encoder.encode('export const value = 1;'); + const first = parseArchiveMjs( + bytes, + './mod.js', + 'file:///tmp/mod.js', + 'file:///tmp/', + ); + const second = parseArchiveMjs( + bytes, + './mod.js', + 'file:///tmp/mod.js', + 'file:///tmp/', + ); + + t.is(first, second); +}); + +test('parseArchiveMjs cache key includes source URL', t => { + const bytes = encoder.encode('export const value = 2;'); + const first = parseArchiveMjs( + bytes, + './mod.js', + 'file:///tmp/a.js', + 'file:///tmp/', + ); + const second = parseArchiveMjs( + bytes, + './mod.js', + 'file:///tmp/b.js', + 'file:///tmp/', + ); + + t.not(first, second); +}); + +test('parseArchiveMjs bypasses cache when source maps are requested', t => { + const bytes = encoder.encode('export const value = 3;'); + const first = parseArchiveMjs( + bytes, + './mod.js', + 'file:///tmp/mod.js', + 'file:///tmp/', + { sourceMapHook: () => {} }, + ); + const second = parseArchiveMjs( + bytes, + './mod.js', + 'file:///tmp/mod.js', + 'file:///tmp/', + { sourceMapHook: () => {} }, + ); + + t.not(first, second); +}); diff --git a/packages/evasive-transform/src/index.js b/packages/evasive-transform/src/index.js index edb4014fda..7b62a93a13 100644 --- a/packages/evasive-transform/src/index.js +++ b/packages/evasive-transform/src/index.js @@ -13,6 +13,54 @@ import { transformAst } from './transform-ast.js'; import { parseAst } from './parse-ast.js'; import { generate } from './generate.js'; +/** + * @param {string} source + * @param {boolean} elideComments + * @returns {boolean} + */ +const shouldRunTransform = (source, elideComments) => { + if (elideComments) { + return true; + } + // Fast path: if none of the risky comment payload tokens appear anywhere in + // the source, the transform cannot change semantics-relevant content. + return ( + source.includes('import(') || + source.includes('') + ); +}; + +/** + * Create a lightweight identity source map when we skip parsing. + * + * @param {string} source + * @param {string|undefined} sourceUrl + * @param {string|object|undefined} sourceMap + */ +const makeFastPathMap = (source, sourceUrl, sourceMap) => { + if (sourceMap && typeof sourceMap === 'object') { + return sourceMap; + } + if (typeof sourceMap === 'string') { + try { + return JSON.parse(sourceMap); + } catch { + // Fall through to an identity map if provided source map is malformed. + } + } + if (!sourceUrl) { + return undefined; + } + return { + version: 3, + names: [], + sources: [sourceUrl], + sourcesContent: [source], + mappings: '', + }; +}; + /** * Options for {@link evadeCensorSync} * @@ -75,6 +123,25 @@ export function evadeCensorSync(source, options) { profileStartSpan, } = options || {}; + const endFastPathScan = profileStartSpan?.('evasiveTransform.fastPath.scan', { + sourceType, + inputChars: source.length, + elideComments, + }); + const fastPathHit = !shouldRunTransform(source, elideComments); + endFastPathScan?.({ fastPathHit }); + if (fastPathHit) { + const endFastPathHit = profileStartSpan?.('evasiveTransform.fastPath.hit'); + const map = makeFastPathMap(source, sourceUrl, sourceMap); + endFastPathHit?.({ hasMap: map !== undefined }); + return { + code: source, + map, + }; + } + const endFastPathMiss = profileStartSpan?.('evasiveTransform.fastPath.miss'); + endFastPathMiss?.(); + // Parse the rolled-up chunk with Babel. // We are prepared for different module systems. const endParse = profileStartSpan?.('evasiveTransform.babel.parse', { diff --git a/packages/evasive-transform/test/evade-censor.test.js b/packages/evasive-transform/test/evade-censor.test.js index 6903715388..41dc15961c 100644 --- a/packages/evasive-transform/test/evade-censor.test.js +++ b/packages/evasive-transform/test/evade-censor.test.js @@ -359,3 +359,38 @@ test('evadeCensor() - x-->y transform preserves meaning', async t => { t.is(originalResult, transformedResult); t.is(transformedResult, 'ok'); }); + +test('evadeCensor() - fast path for source without comment markers', async t => { + const source = `const answer = 42;\nexport { answer };`; + const { code, map } = evadeCensorSync(source, { sourceType: 'module' }); + t.is(code, source); + t.is(map, undefined); +}); + +test('evadeCensor() - fast path returns source map when sourceUrl is provided', async t => { + const source = `const answer = 42;\nexport { answer };`; + const sourceUrl = 'fast-path.js'; + const { code, map } = evadeCensorSync(source, { + sourceType: 'module', + sourceUrl, + }); + t.is(code, source); + t.truthy(map); + t.deepEqual(map.sources, [sourceUrl]); +}); + +test('evadeCensor() - fast path can skip despite ordinary comments', async t => { + const source = `// hello\nconst answer = 42; /* ordinary */\nexport { answer };`; + const { code, map } = evadeCensorSync(source, { sourceType: 'module' }); + t.is(code, source); + t.is(map, undefined); +}); + +test('evadeCensor() - elideComments still forces transform on fast-path source', async t => { + const source = `// hello\nconst answer = 42;\nexport { answer };`; + const { code } = evadeCensorSync(source, { + sourceType: 'module', + elideComments: true, + }); + t.not(code, source); +}); diff --git a/packages/zip/package.json b/packages/zip/package.json index e9010f3d3a..1460beaa09 100644 --- a/packages/zip/package.json +++ b/packages/zip/package.json @@ -28,6 +28,9 @@ }, "scripts": { "build": "exit 0", + "bench:writer": "node tools/benchmark-writer.mjs", + "prepack": "tsc --build tsconfig.build.json", + "postpack": "git clean -fX \"*.d.ts*\" \"*.d.cts*\" \"*.d.mts*\" \"*.tsbuildinfo\"", "lint": "yarn lint:types && yarn lint:eslint", "lint-fix": "eslint --fix .", "lint:eslint": "eslint .", diff --git a/packages/zip/src/format-writer.js b/packages/zip/src/format-writer.js index 4c16d79f4d..0087cf418e 100644 --- a/packages/zip/src/format-writer.js +++ b/packages/zip/src/format-writer.js @@ -70,7 +70,7 @@ function writeFile(writer, file) { writer.write(signature.LOCAL_FILE_HEADER); const headerStart = writer.index; // Version needed to extract - writer.writeUint16(10, true); + writer.writeUint16(file.versionNeeded, true); writer.writeUint16(file.bitFlag, true); writer.writeUint16(file.compressionMethod, true); writeDosDateTime(writer, file.date); @@ -144,24 +144,43 @@ function writeEndOfCentralDirectoryRecord( * @param {BufferWriter} writer * @param {Array} records * @param {string} comment + * @param {(name: string, args?: Record) => (args?: Record) => void} [profileStartSpan] */ -export function writeZipRecords(writer, records, comment = '') { +export function writeZipRecords( + writer, + records, + comment = '', + profileStartSpan = undefined, +) { // Write records with local headers. + const endLocalFiles = profileStartSpan?.('zip.formatWriter.writeLocalFiles', { + entryCount: records.length, + }); const locators = []; for (let i = 0; i < records.length; i += 1) { locators.push(writeFile(writer, records[i])); } + endLocalFiles?.(); // writeCentralDirectory + const endCentralDirectory = profileStartSpan?.( + 'zip.formatWriter.writeCentralDirectory', + { entryCount: records.length }, + ); const centralDirectoryStart = writer.index; for (let i = 0; i < locators.length; i += 1) { writeCentralFileHeader(writer, records[i], locators[i]); } const centralDirectoryLength = writer.index - centralDirectoryStart; + endCentralDirectory?.({ centralDirectoryLength }); const commentBytes = textEncoder.encode(comment); // Write central directory end. + const endCentralDirectoryEnd = profileStartSpan?.( + 'zip.formatWriter.writeCentralDirectoryEnd', + { entryCount: records.length }, + ); writeEndOfCentralDirectoryRecord( writer, records.length, @@ -169,6 +188,7 @@ export function writeZipRecords(writer, records, comment = '') { centralDirectoryLength, commentBytes, ); + endCentralDirectoryEnd?.({ commentBytes: commentBytes.length }); } /** @@ -235,7 +255,7 @@ function makeFileRecord(file) { centralName: file.name, madeBy: UNIX, version: UNIX_VERSION, - versionNeeded: 0, // TODO this is probably too lax. + versionNeeded: 10, bitFlag: 0, compressionMethod: compression.STORE, date: file.date, @@ -254,11 +274,30 @@ function makeFileRecord(file) { * @param {BufferWriter} writer * @param {Array} files * @param {string} comment + * @param {(name: string, args?: Record) => (args?: Record) => void} [profileStartSpan] */ -export function writeZip(writer, files, comment = '') { - const encodedFiles = files.map(encodeFile); - const compressedFiles = encodedFiles.map(compressFileWithStore); +export function writeZip( + writer, + files, + comment = '', + profileStartSpan = undefined, +) { + // Build file records in one pass to reduce allocation churn when writing + // large archives. + const endBuildRecords = profileStartSpan?.( + 'zip.formatWriter.buildFileRecords', + { entryCount: files.length }, + ); + /** @type {Array} */ + const fileRecords = []; + let totalContentBytes = 0; + for (let i = 0; i < files.length; i += 1) { + const encoded = encodeFile(files[i]); + const compressed = compressFileWithStore(encoded); + totalContentBytes += compressed.content.length; + fileRecords.push(makeFileRecord(compressed)); + } + endBuildRecords?.({ entryCount: fileRecords.length, totalContentBytes }); // TODO collate directoryRecords from file bases. - const fileRecords = compressedFiles.map(makeFileRecord); - writeZipRecords(writer, fileRecords, comment); + writeZipRecords(writer, fileRecords, comment, profileStartSpan); } diff --git a/packages/zip/src/types.js b/packages/zip/src/types.js index c34fe3d7dc..a13e88e98e 100644 --- a/packages/zip/src/types.js +++ b/packages/zip/src/types.js @@ -67,10 +67,10 @@ export {}; * @callback WriteFn * @param {string} name * @param {Uint8Array} bytes - * @returns {Promise} + * @returns {void | Promise} */ /** * @callback SnapshotFn - * @returns {Promise} + * @returns {Uint8Array | Promise} */ diff --git a/packages/zip/src/writer.js b/packages/zip/src/writer.js index 0f99b5e01e..9f60287364 100644 --- a/packages/zip/src/writer.js +++ b/packages/zip/src/writer.js @@ -3,17 +3,41 @@ import { BufferWriter } from './buffer-writer.js'; import { writeZip as writeZipFormat } from './format-writer.js'; +const LOCAL_FILE_HEADER_FIXED_BYTES = 30; +const CENTRAL_FILE_HEADER_FIXED_BYTES = 46; +const CENTRAL_DIRECTORY_END_FIXED_BYTES = 22; + +/** + * @param {Array} files + * @returns {number} + */ +const estimateZipSize = files => { + let total = CENTRAL_DIRECTORY_END_FIXED_BYTES; + for (let i = 0; i < files.length; i += 1) { + const file = files[i]; + // Names/comments are expected ASCII path segments; this estimate may be low + // for non-ASCII but BufferWriter will grow if needed. + const nameLength = file.name.length; + const commentLength = file.comment.length; + total += LOCAL_FILE_HEADER_FIXED_BYTES + nameLength + file.content.length; + total += CENTRAL_FILE_HEADER_FIXED_BYTES + nameLength + commentLength; + } + return total; +}; + export class ZipWriter { /** * @param {{ * date: Date, + * profileStartSpan?: (name: string, args?: Record) => (args?: Record) => void, * }} options */ constructor(options = { date: new Date() }) { - const { date } = options; + const { date, profileStartSpan = undefined } = options; /** type {Map} */ this.files = new Map(); this.date = date; + this.profileStartSpan = profileStartSpan; } /** @@ -43,22 +67,27 @@ export class ZipWriter { * @returns {Uint8Array} */ snapshot() { - const writer = new BufferWriter(); - writeZipFormat(writer, Array.from(this.files.values())); + const files = Array.from(this.files.values()); + const writer = new BufferWriter(estimateZipSize(files)); + writeZipFormat(writer, files, '', this.profileStartSpan); return writer.subarray(); } } /** + * @param {{ + * date?: Date, + * profileStartSpan?: (name: string, args?: Record) => (args?: Record) => void, + * }} [options] * @returns {import('./types.js').ArchiveWriter} */ -export const writeZip = () => { - const writer = new ZipWriter(); +export const writeZip = (options = {}) => { + const writer = new ZipWriter({ date: new Date(), ...options }); /** @type {import('./types.js').WriteFn} */ - const write = async (path, data) => { + const write = (path, data) => { writer.write(path, data); }; /** @type {import('./types.js').SnapshotFn} */ - const snapshot = async () => writer.snapshot(); + const snapshot = () => writer.snapshot(); return { write, snapshot }; }; diff --git a/packages/zip/tools/benchmark-writer.mjs b/packages/zip/tools/benchmark-writer.mjs new file mode 100644 index 0000000000..a2f3c7dbc7 --- /dev/null +++ b/packages/zip/tools/benchmark-writer.mjs @@ -0,0 +1,210 @@ +#!/usr/bin/env node +/* global process */ + +import { parseArgs } from 'util'; +import { ZipWriter } from '../src/writer.js'; + +const options = { + entries: { type: 'string' }, + 'total-bytes': { type: 'string' }, + iterations: { type: 'string' }, + warmup: { type: 'string' }, +}; + +const toPositiveInt = (value, name) => { + const n = Number.parseInt(value, 10); + if (!Number.isFinite(n) || n <= 0) { + throw new Error(`Expected positive integer for --${name}, got: ${value}`); + } + return n; +}; + +/** + * @param {number} seed + */ +const makeRng = seed => { + let state = seed >>> 0; + return () => { + state ^= state << 13; + state ^= state >>> 17; + state ^= state << 5; + return state >>> 0; + }; +}; + +/** + * @param {number} entries + * @param {number} totalBytes + */ +const makeInputFiles = (entries, totalBytes) => { + const files = []; + const rng = makeRng(0xdeadbeef); + const base = Math.max(64, Math.floor(totalBytes / entries)); + const contents = new Map(); + let planned = 0; + + for (let i = 0; i < entries; i += 1) { + const jitter = (rng() % (base + 1)) - Math.floor(base / 3); + const size = Math.max(16, base + jitter); + planned += size; + if (!contents.has(size)) { + const bytes = new Uint8Array(size); + for (let j = 0; j < size; j += 1) { + bytes[j] = (j + i) & 0xff; + } + contents.set(size, bytes); + } + const bucket = String(i % 128).padStart(3, '0'); + const name = `compartment-${bucket}/module-${String(i).padStart(5, '0')}.js`; + files.push({ + name, + content: contents.get(size), + }); + } + + // Adjust the final file size so total source bytes match requested total. + const delta = totalBytes - planned; + if (delta !== 0 && files.length > 0) { + const last = files[files.length - 1]; + const newSize = Math.max(16, last.content.length + delta); + const adjusted = new Uint8Array(newSize); + adjusted.set(last.content.subarray(0, Math.min(last.content.length, newSize))); + for (let i = last.content.length; i < newSize; i += 1) { + adjusted[i] = i & 0xff; + } + last.content = adjusted; + } + + return files; +}; + +const median = values => { + if (values.length === 0) { + return 0; + } + const sorted = [...values].sort((a, b) => a - b); + const mid = Math.floor(sorted.length / 2); + if (sorted.length % 2 === 1) { + return sorted[mid]; + } + return (sorted[mid - 1] + sorted[mid]) / 2; +}; + +const ms = nanos => Number(nanos) / 1e6; + +const main = () => { + const { + values: { + entries: entriesRaw = '4244', + 'total-bytes': totalBytesRaw = '27417643', + iterations: iterationsRaw = '15', + warmup: warmupRaw = '3', + }, + } = parseArgs({ options }); + + const entries = toPositiveInt(entriesRaw, 'entries'); + const totalBytes = toPositiveInt(totalBytesRaw, 'total-bytes'); + const iterations = toPositiveInt(iterationsRaw, 'iterations'); + const warmup = toPositiveInt(warmupRaw, 'warmup'); + + const files = makeInputFiles(entries, totalBytes); + const totalInputBytes = files.reduce((sum, file) => sum + file.content.length, 0); + + /** @type {Record} */ + const spanSamples = Object.create(null); + /** @type {number[]} */ + const writeMs = []; + /** @type {number[]} */ + const snapshotMs = []; + /** @type {number[]} */ + const totalMs = []; + /** @type {number[]} */ + const zipSizes = []; + + const runOnce = () => { + const spansNs = new Map(); + const profileStartSpan = name => { + const start = process.hrtime.bigint(); + return () => { + const end = process.hrtime.bigint(); + const elapsed = end - start; + const prior = spansNs.get(name) || 0n; + spansNs.set(name, prior + elapsed); + }; + }; + + const totalStart = process.hrtime.bigint(); + const writer = new ZipWriter({ date: new Date(0), profileStartSpan }); + + const writeStart = process.hrtime.bigint(); + for (let i = 0; i < files.length; i += 1) { + const file = files[i]; + writer.write(file.name, file.content); + } + const writeEnd = process.hrtime.bigint(); + + const snapshotStart = process.hrtime.bigint(); + const bytes = writer.snapshot(); + const snapshotEnd = process.hrtime.bigint(); + const totalEnd = process.hrtime.bigint(); + + return { + writeMs: ms(writeEnd - writeStart), + snapshotMs: ms(snapshotEnd - snapshotStart), + totalMs: ms(totalEnd - totalStart), + zipSize: bytes.length, + spansNs, + }; + }; + + for (let i = 0; i < warmup; i += 1) { + runOnce(); + } + + for (let i = 0; i < iterations; i += 1) { + const sample = runOnce(); + writeMs.push(sample.writeMs); + snapshotMs.push(sample.snapshotMs); + totalMs.push(sample.totalMs); + zipSizes.push(sample.zipSize); + for (const [name, elapsed] of sample.spansNs.entries()) { + if (!spanSamples[name]) { + spanSamples[name] = []; + } + spanSamples[name].push(ms(elapsed)); + } + } + + const spanRows = Object.entries(spanSamples) + .map(([name, values]) => ({ + name, + avgMs: values.reduce((sum, value) => sum + value, 0) / values.length, + p50Ms: median(values), + })) + .sort((a, b) => b.avgMs - a.avgMs); + + process.stdout.write( + [ + `zip writer benchmark`, + `entries=${entries}`, + `inputBytes=${totalInputBytes}`, + `iterations=${iterations}`, + `warmup=${warmup}`, + `zipBytes(avg)=${Math.round( + zipSizes.reduce((sum, value) => sum + value, 0) / zipSizes.length, + )}`, + `totalMs avg=${(totalMs.reduce((a, b) => a + b, 0) / totalMs.length).toFixed(3)} p50=${median(totalMs).toFixed(3)}`, + `writeMs avg=${(writeMs.reduce((a, b) => a + b, 0) / writeMs.length).toFixed(3)} p50=${median(writeMs).toFixed(3)}`, + `snapshotMs avg=${(snapshotMs.reduce((a, b) => a + b, 0) / snapshotMs.length).toFixed(3)} p50=${median(snapshotMs).toFixed(3)}`, + '', + 'snapshot span breakdown (avg / p50 ms):', + ...spanRows.map( + row => + ` ${row.name}: ${row.avgMs.toFixed(3)} / ${row.p50Ms.toFixed(3)}`, + ), + '', + ].join('\n'), + ); +}; + +main(); From b32c6c3fcece47be37ae01e07f0b78f36bcd92e6 Mon Sep 17 00:00:00 2001 From: Turadg Aleahmad Date: Tue, 9 Jun 2026 14:02:52 -0700 Subject: [PATCH 11/22] fixup! feat(bundle-source): add opt-in chrome trace profiling --- packages/bundle-source/README.md | 12 ++++--- packages/bundle-source/src/profile.js | 45 ++++++++++++++++----------- 2 files changed, 34 insertions(+), 23 deletions(-) diff --git a/packages/bundle-source/README.md b/packages/bundle-source/README.md index 8431e041d3..486ec8ff13 100644 --- a/packages/bundle-source/README.md +++ b/packages/bundle-source/README.md @@ -155,19 +155,21 @@ Environment variables: - `ENDO_BUNDLE_SOURCE_PROFILE_FILE`: explicit output file for a single run - `ENDO_BUNDLE_SOURCE_PROFILE_STDERR`: if truthy, prints each generated trace path to stderr -Merge and summarize many profile traces: +Merge and summarize Chrome trace files: ```console yarn workspace @endo/bundle-source trace:merge -- /tmp/bs-profiles ``` +This utility is not specific to `bundle-source`; it accepts trace files and +directories containing `*.trace.json` files from any compatible producer. This generates: - `merged.trace.json` for trace viewers. - `summary.json` with aggregate span statistics. - `summary.md` with a top spans table by total duration. -Profile bundling all `source-spec-registry.js` entries from an `agoric-sdk` -checkout using the current checkout's `bundle-source`: +For Agoric SDK bundle profiling, profile every `source-spec-registry.js` entry +from an `agoric-sdk` checkout using the current checkout's `bundle-source`: ```console yarn workspace @endo/bundle-source profile:agoric-bundling -- \ @@ -175,8 +177,8 @@ yarn workspace @endo/bundle-source profile:agoric-bundling -- \ --out-dir /tmp/profile-agoric-bundling ``` -The tool writes bundles, raw traces, merged trace, and summary files to -`--out-dir`, and prints a top-spans summary table at the end. +This Agoric-specific helper writes bundles, raw traces, merged trace, and +summary files to `--out-dir`, and prints a top-spans summary table at the end. ## `moduleFormat` explanations diff --git a/packages/bundle-source/src/profile.js b/packages/bundle-source/src/profile.js index f815f6c602..06af06b83d 100644 --- a/packages/bundle-source/src/profile.js +++ b/packages/bundle-source/src/profile.js @@ -1,4 +1,5 @@ // @ts-check +/* global process */ import fs from 'fs'; import os from 'os'; @@ -7,20 +8,29 @@ import { performance } from 'perf_hooks'; /** @import {BundleProfilingOptions} from './types.js' */ +/** + * @typedef {object} BundleProfiler + * @property {boolean} enabled + * @property {(name: string, args?: Record) => (args?: Record) => void} startSpan + * @property {(result?: Record) => Promise} flush + */ + let nextTraceFileId = 0; const truthy = new Set(['1', 'true', 'yes', 'on']); +const noop = () => {}; /** * @param {string | undefined} value * @returns {boolean} */ -const parseBoolean = value => { - if (value === undefined) { - return false; - } - return truthy.has(value.toLowerCase()); -}; +const parseBoolean = value => truthy.has(value?.toLowerCase() ?? ''); + +/** + * @param {number} ms + * @returns {number} + */ +const toMicros = ms => Math.round(ms * 1000); /** * @param {string} moduleFormat @@ -36,6 +46,7 @@ const classifyModuleFormat = moduleFormat => * @param {number} [options.pid] * @param {Record} [options.env] * @param {BundleProfilingOptions | undefined} [options.profile] + * @returns {BundleProfiler} */ export const makeBundleProfiler = ({ moduleFormat, @@ -45,17 +56,16 @@ export const makeBundleProfiler = ({ profile = undefined, }) => { const enabled = - profile?.enabled !== undefined - ? profile.enabled - : parseBoolean(env.ENDO_BUNDLE_SOURCE_PROFILE); + profile?.enabled ?? parseBoolean(env.ENDO_BUNDLE_SOURCE_PROFILE); const logToStderr = parseBoolean(env.ENDO_BUNDLE_SOURCE_PROFILE_STDERR); if (!enabled) { - const noop = () => {}; return { enabled, startSpan: (_name, _args = undefined) => noop, - async flush(_args = undefined) {}, + async flush(_args = undefined) { + return undefined; + }, }; } @@ -71,20 +81,19 @@ export const makeBundleProfiler = ({ traceFile || path.join( traceDir, - `bundle-source-${phase}-${pid}-${Date.now()}-${nextTraceFileId++}.trace.json`, + `bundle-source-${phase}-${pid}-${Date.now()}-${nextTraceFileId}.trace.json`, ); + nextTraceFileId += 1; /** @type {Array>} */ const traceEvents = []; const zeroMs = performance.now(); /** - * @param {number} ms - * @returns {number} - */ - const toMicros = ms => Math.round(ms * 1000); - - /** + * Start a Chrome Trace Event "complete event" span. The generated event uses + * the trace-event field names (`ph`, `ts`, `dur`, `pid`, `tid`) described by + * https://chromium.googlesource.com/catapult/+/HEAD/tracing/README.md . + * * @param {string} name * @param {Record | undefined} args */ From 351080d2c3ff95f2ab1a138b52e12d4f4341faa2 Mon Sep 17 00:00:00 2001 From: Turadg Aleahmad Date: Tue, 9 Jun 2026 14:02:57 -0700 Subject: [PATCH 12/22] fixup! feat(bundle-source): add trace merge and summary tool --- packages/bundle-source/tools/trace-merge.js | 108 ++++++++++++-------- 1 file changed, 63 insertions(+), 45 deletions(-) diff --git a/packages/bundle-source/tools/trace-merge.js b/packages/bundle-source/tools/trace-merge.js index be56f6cb10..e2dd4559c9 100644 --- a/packages/bundle-source/tools/trace-merge.js +++ b/packages/bundle-source/tools/trace-merge.js @@ -6,6 +6,12 @@ import fs from 'fs/promises'; import path from 'path'; import { parseArgs } from 'util'; +/** + * Merge Chrome trace-event files and write aggregate duration summaries. This + * accepts any `*.trace.json` file with a `traceEvents` array, not just traces + * produced by `@endo/bundle-source`. + */ + const options = /** @type {const} */ ({ 'out-trace': { type: 'string' }, 'out-summary': { type: 'string' }, @@ -41,13 +47,14 @@ const FOCUS_SPANS = [ const sumNumericArgBySpan = (events, spanName, argName) => { let total = 0; for (const event of events) { - if (event.ph !== 'X' || event.name !== spanName) { - continue; - } - const args = /** @type {Record | undefined} */ (event.args); - const value = args && args[argName]; - if (typeof value === 'number' && Number.isFinite(value)) { - total += value; + if (event.ph === 'X' && event.name === spanName) { + const args = /** @type {Record | undefined} */ ( + event.args + ); + const value = args && args[argName]; + if (typeof value === 'number' && Number.isFinite(value)) { + total += value; + } } } return total; @@ -95,17 +102,16 @@ const findTraceFiles = async root => { const queue = [root]; while (queue.length > 0) { const dir = queue.shift(); - if (!dir) { - continue; - } - // eslint-disable-next-line no-await-in-loop - const entries = await fs.readdir(dir, { withFileTypes: true }); - for (const entry of entries) { - const filePath = path.join(dir, entry.name); - if (entry.isDirectory()) { - queue.push(filePath); - } else if (entry.isFile() && entry.name.endsWith('.trace.json')) { - found.push(filePath); + if (dir) { + // eslint-disable-next-line no-await-in-loop + const entries = await fs.readdir(dir, { withFileTypes: true }); + for (const entry of entries) { + const filePath = path.join(dir, entry.name); + if (entry.isDirectory()) { + queue.push(filePath); + } else if (entry.isFile() && entry.name.endsWith('.trace.json')) { + found.push(filePath); + } } } } @@ -177,33 +183,33 @@ const summarize = (events, top) => { /** @type {Map>} */ const intervalsByName = new Map(); for (const event of events) { - if (event.ph !== 'X' || typeof event.name !== 'string') { - continue; - } - const dur = typeof event.dur === 'number' ? event.dur : undefined; - const ts = typeof event.ts === 'number' ? event.ts : undefined; - if (dur === undefined || ts === undefined) { - continue; - } - const bucket = durationsByName.get(event.name); - if (bucket) { - bucket.push(dur); - } else { - durationsByName.set(event.name, [dur]); - } - const intervals = intervalsByName.get(event.name); - const interval = /** @type {[number, number]} */ ([ts, ts + dur]); - if (intervals) { - intervals.push(interval); - } else { - intervalsByName.set(event.name, [interval]); + if (event.ph === 'X' && typeof event.name === 'string') { + const dur = typeof event.dur === 'number' ? event.dur : undefined; + const ts = typeof event.ts === 'number' ? event.ts : undefined; + if (dur !== undefined && ts !== undefined) { + const bucket = durationsByName.get(event.name); + if (bucket) { + bucket.push(dur); + } else { + durationsByName.set(event.name, [dur]); + } + const intervals = intervalsByName.get(event.name); + const interval = /** @type {[number, number]} */ ([ts, ts + dur]); + if (intervals) { + intervals.push(interval); + } else { + intervalsByName.set(event.name, [interval]); + } + } } } const rows = [...durationsByName.entries()].map(([name, durations]) => { durations.sort((a, b) => a - b); const total = durations.reduce((sum, value) => sum + value, 0); - const criticalPathUs = unionDuration([...(intervalsByName.get(name) || [])]); + const criticalPathUs = unionDuration([ + ...(intervalsByName.get(name) || []), + ]); return { name, count: durations.length, @@ -245,6 +251,7 @@ const zeroRow = name => ({ /** * @param {ReturnType} rows + * @param {ReturnType} [focusRows] * @returns {string} */ const summarizeMarkdown = (rows, focusRows = []) => { @@ -296,14 +303,17 @@ const makeDerivedMetrics = (allRows, focusSpans, events) => { const bundlesProcessed = allByName.get('bundleSource.total')?.count || 0; const modulesParsed = allByName.get('compartmentMapper.importHook.parseModule')?.count || 0; - const modulesTransformed = allByName.get('bundleSource.transformModule')?.count || 0; + const modulesTransformed = + allByName.get('bundleSource.transformModule')?.count || 0; const fastPathHitCount = focusByName.get('evasiveTransform.fastPath.hit')?.count || 0; const fastPathMissCount = focusByName.get('evasiveTransform.fastPath.miss')?.count || 0; const fastPathTotal = fastPathHitCount + fastPathMissCount; - const fastPathHitRate = fastPathTotal > 0 ? fastPathHitCount / fastPathTotal : 0; - const readCacheHitCount = focusByName.get('bundleSource.readCache.hit')?.count || 0; + const fastPathHitRate = + fastPathTotal > 0 ? fastPathHitCount / fastPathTotal : 0; + const readCacheHitCount = + focusByName.get('bundleSource.readCache.hit')?.count || 0; const readCacheMissCount = focusByName.get('bundleSource.readCache.miss')?.count || 0; const readCachePendingCount = @@ -328,7 +338,8 @@ const makeDerivedMetrics = (allRows, focusSpans, events) => { 'bytes', ); - const totalBundleMs = (allByName.get('bundleSource.total')?.totalUs || 0) / 1000; + const totalBundleMs = + (allByName.get('bundleSource.total')?.totalUs || 0) / 1000; const totalParseMs = (allByName.get('compartmentMapper.importHook.parseModule')?.totalUs || 0) / 1000; @@ -404,7 +415,10 @@ const main = async () => { const events = trace.traceEvents || []; let maxEndUs = 0; for (const event of events) { - const copy = { ...event, args: { ...(event.args || {}), source: filePath } }; + const copy = { + ...event, + args: { ...(event.args || {}), source: filePath }, + }; if (stacked) { if (typeof copy.ts === 'number') { copy.ts += offsetUs; @@ -439,7 +453,11 @@ const main = async () => { await fs.writeFile( outTrace, - JSON.stringify({ traceEvents: mergedEvents, displayTimeUnit: 'ms' }, null, 2), + JSON.stringify( + { traceEvents: mergedEvents, displayTimeUnit: 'ms' }, + null, + 2, + ), ); await fs.writeFile(outSummary, JSON.stringify(summary, null, 2)); await fs.writeFile(outMarkdown, summarizeMarkdown(summaryTop, focusSpans)); From 24d64547cc9cfa56768ae0690e5e2fe2ddab0881 Mon Sep 17 00:00:00 2001 From: Turadg Aleahmad Date: Tue, 9 Jun 2026 14:03:01 -0700 Subject: [PATCH 13/22] fixup! feat(bundle-source): add profile-agoric-bundling tool --- .../tools/profile-agoric-bundling.mts | 50 +++++++++++++++---- 1 file changed, 39 insertions(+), 11 deletions(-) diff --git a/packages/bundle-source/tools/profile-agoric-bundling.mts b/packages/bundle-source/tools/profile-agoric-bundling.mts index 3681a54206..7a533e8aaf 100755 --- a/packages/bundle-source/tools/profile-agoric-bundling.mts +++ b/packages/bundle-source/tools/profile-agoric-bundling.mts @@ -10,6 +10,12 @@ import { parseArgs } from 'util'; import bundleSource from '../src/index.js'; +/** + * Profile `bundleSource` over every Agoric SDK `source-spec-registry.js` entry. + * This is a fixture-driven helper for comparing Endo bundling performance + * against Agoric SDK workloads, not a general trace merger. + */ + const options = { 'agoric-sdk-root': { type: 'string' }, 'out-dir': { type: 'string' }, @@ -110,7 +116,9 @@ const collectSpecsFromModule = ( registryExport: exportName, key, bundleName: - typeof descriptor.bundleName === 'string' ? descriptor.bundleName : key, + typeof descriptor.bundleName === 'string' + ? descriptor.bundleName + : key, sourceSpec: descriptor.sourceSpec, packagePath: typeof descriptor.packagePath === 'string' @@ -256,7 +264,9 @@ const summarizeEvents = ( ([name, durations]) => { durations.sort((a, b) => a - b); const total = durations.reduce((sum, value) => sum + value, 0); - const criticalPathUs = unionDuration([...(intervalsByName.get(name) || [])]); + const criticalPathUs = unionDuration([ + ...(intervalsByName.get(name) || []), + ]); return { name, count: durations.length, @@ -373,13 +383,15 @@ const makeDerivedMetrics = ( const bundlesProcessed = rowByName.get('bundleSource.total')?.count || 0; const modulesParsed = rowByName.get('compartmentMapper.importHook.parseModule')?.count || 0; - const modulesTransformed = rowByName.get('bundleSource.transformModule')?.count || 0; + const modulesTransformed = + rowByName.get('bundleSource.transformModule')?.count || 0; const fastPathHitCount = focusByName.get('evasiveTransform.fastPath.hit')?.count || 0; const fastPathMissCount = focusByName.get('evasiveTransform.fastPath.miss')?.count || 0; const fastPathTotal = fastPathHitCount + fastPathMissCount; - const fastPathHitRate = fastPathTotal > 0 ? fastPathHitCount / fastPathTotal : 0; + const fastPathHitRate = + fastPathTotal > 0 ? fastPathHitCount / fastPathTotal : 0; const bytesRead = sumNumericArgBySpan( events, @@ -397,7 +409,8 @@ const makeDerivedMetrics = ( 'bytes', ); - const totalBundleMs = (rowByName.get('bundleSource.total')?.totalUs || 0) / 1000; + const totalBundleMs = + (rowByName.get('bundleSource.total')?.totalUs || 0) / 1000; const totalParseMs = (rowByName.get('compartmentMapper.importHook.parseModule')?.totalUs || 0) / 1000; @@ -433,7 +446,9 @@ const mergeTraceFiles = async ( for (const filePath of traceFiles) { // eslint-disable-next-line no-await-in-loop const text = await fs.readFile(filePath, 'utf-8'); - const trace = JSON.parse(text) as { traceEvents?: Array> }; + const trace = JSON.parse(text) as { + traceEvents?: Array>; + }; const events = trace.traceEvents || []; let maxEndUs = 0; for (const event of events) { @@ -472,7 +487,9 @@ const main = async () => { positionals, } = parseArgs({ options, allowPositionals: true }); if (positionals.length > 0) { - throw new Error(`Unexpected arguments: ${positionals.join(' ')}\n\n${usage}`); + throw new Error( + `Unexpected arguments: ${positionals.join(' ')}\n\n${usage}`, + ); } const top = toInt(topRaw); @@ -528,7 +545,9 @@ const main = async () => { for (let index = 0; index < specs.length; index += 1) { const spec = specs[index]; const registryPackage = path.basename(path.dirname(spec.registryFile)); - const id = sanitizeName(`${registryPackage}-${spec.bundleName}-${spec.key}`); + const id = sanitizeName( + `${registryPackage}-${spec.bundleName}-${spec.key}`, + ); const bundleFile = `${id}.bundle.json`; if (verbose) { process.stdout.write( @@ -546,7 +565,10 @@ const main = async () => { }, }); // eslint-disable-next-line no-await-in-loop - await fs.writeFile(path.join(bundlesDir, bundleFile), `${JSON.stringify(bundle)}\n`); + await fs.writeFile( + path.join(bundlesDir, bundleFile), + `${JSON.stringify(bundle)}\n`, + ); manifestEntries.push({ id, @@ -601,7 +623,11 @@ const main = async () => { await fs.writeFile( mergedTracePath, - JSON.stringify({ traceEvents: mergedEvents, displayTimeUnit: 'ms' }, null, 2), + JSON.stringify( + { traceEvents: mergedEvents, displayTimeUnit: 'ms' }, + null, + 2, + ), ); await fs.writeFile(summaryPath, JSON.stringify(summary, null, 2)); await fs.writeFile(summaryMdPath, summarizeMarkdown(topRows, focusRows)); @@ -623,7 +649,9 @@ const main = async () => { console.table({ metrics: { ...derivedMetrics, - fastPathHitRate: Number((derivedMetrics.fastPathHitRate * 100).toFixed(2)), + fastPathHitRate: Number( + (derivedMetrics.fastPathHitRate * 100).toFixed(2), + ), msPerBundle: Number(derivedMetrics.msPerBundle.toFixed(3)), msPerModuleParsed: Number(derivedMetrics.msPerModuleParsed.toFixed(3)), msPerModuleTransformed: Number( From c3f71b980a4c69565111ab9bd5e8e215e84cab37 Mon Sep 17 00:00:00 2001 From: Turadg Aleahmad Date: Tue, 9 Jun 2026 14:03:08 -0700 Subject: [PATCH 14/22] fixup! feat(profiling): add compartment-mapper phase spans --- .../compartment-mapper/src/import-hook.js | 42 ++++++++++--------- .../compartment-mapper/src/types/external.ts | 2 +- .../compartment-mapper/src/types/internal.ts | 2 +- 3 files changed, 25 insertions(+), 21 deletions(-) diff --git a/packages/compartment-mapper/src/import-hook.js b/packages/compartment-mapper/src/import-hook.js index 5874483087..c76a6b61ed 100644 --- a/packages/compartment-mapper/src/import-hook.js +++ b/packages/compartment-mapper/src/import-hook.js @@ -108,6 +108,8 @@ const nodejsConventionSearchSuffixes = [ // LOAD_AS_FILE(X) '.js', '.json', + // Native Node.js addons are intentionally omitted; compartments cannot load + // native modules from archived or virtual module sources. // LOAD_INDEX(X) '/index.js', '/index.json', @@ -259,14 +261,13 @@ export const exitModuleImportHookMaker = ({ const nominateCandidates = (moduleSpecifier, searchSuffixes) => { // Collate candidate locations for the moduleSpecifier. // Apply suffix expansion only when the specifier does not already - // include an explicit extension. + // include one of the file suffixes we would otherwise search. const candidates = [moduleSpecifier]; const endsWithSlash = moduleSpecifier.endsWith('/'); - const lastSlash = moduleSpecifier.lastIndexOf('/'); - const leaf = - lastSlash >= 0 ? moduleSpecifier.slice(lastSlash + 1) : moduleSpecifier; - const hasExplicitExtension = leaf.includes('.'); - if (!endsWithSlash && hasExplicitExtension && moduleSpecifier !== '.') { + const hasExplicitExtension = searchSuffixes + .filter(suffix => !suffix.startsWith('/')) + .some(suffix => moduleSpecifier.endsWith(suffix)); + if (!endsWithSlash && hasExplicitExtension) { return candidates; } for (const candidateSuffix of searchSuffixes) { @@ -432,14 +433,16 @@ function* chooseModuleDescriptor( 'compartmentMapper.importHook.readModuleBytes', { moduleLocation }, ); - let moduleBytes; + /** @type {{ value?: Uint8Array }} */ + const moduleBytesCell = {}; try { - moduleBytes = /** @type {Uint8Array|undefined} */ ( + moduleBytesCell.value = /** @type {Uint8Array|undefined} */ ( yield maybeRead(moduleLocation) ); } finally { - endReadModuleBytes?.({ bytes: moduleBytes?.length }); + endReadModuleBytes?.({ bytes: moduleBytesCell.value?.length }); } + const { value: moduleBytes } = moduleBytesCell; if (moduleBytes !== undefined) { /** @type {string | undefined} */ @@ -659,8 +662,9 @@ export const makeImportHookMaker = ( /** @type {Map>} */ const maybeReadCache = new Map(); /** - * Cache both hits and misses for module reads during a mapping run. - * This avoids repeated filesystem probes for the same candidate path. + * Cache successful module-read promises during a mapping run, including + * promises that resolve to `undefined` for missing paths. Rejected reads are + * removed so transient filesystem errors can be retried by later candidates. * * @param {string} location * @returns {Promise} @@ -802,14 +806,14 @@ export const makeImportHookMaker = ( moduleSpecifier, packageLocation, packageSources, - readPowers, - archiveOnly, - sourceMapHook, - moduleSourceHook, - profileStartSpan, - strictlyRequiredForCompartment, - log, - }, + readPowers, + archiveOnly, + sourceMapHook, + moduleSourceHook, + profileStartSpan, + strictlyRequiredForCompartment, + log, + }, { maybeRead: cachedMaybeRead, parse, shouldDeferError }, ); diff --git a/packages/compartment-mapper/src/types/external.ts b/packages/compartment-mapper/src/types/external.ts index 3614a787e6..ba4739ad0e 100644 --- a/packages/compartment-mapper/src/types/external.ts +++ b/packages/compartment-mapper/src/types/external.ts @@ -201,7 +201,7 @@ export interface ProfilingOptions { profileStartSpan?: ( name: string, args?: Record, - ) => (args?: Record) => void; + ) => (endArgs?: Record) => void; } /** diff --git a/packages/compartment-mapper/src/types/internal.ts b/packages/compartment-mapper/src/types/internal.ts index 76eedf9f1d..f56fe29e28 100644 --- a/packages/compartment-mapper/src/types/internal.ts +++ b/packages/compartment-mapper/src/types/internal.ts @@ -163,7 +163,7 @@ export type ChooseModuleDescriptorParams = { profileStartSpan?: ( name: string, args?: Record, - ) => (args?: Record) => void; + ) => (endArgs?: Record) => void; strictlyRequiredForCompartment: StrictlyRequiredFn; } & ComputeSha512Option & From ac7614262d3afade60758f34d4fc1f5af8d20cb8 Mon Sep 17 00:00:00 2001 From: Turadg Aleahmad Date: Tue, 9 Jun 2026 14:03:13 -0700 Subject: [PATCH 15/22] fixup! feat(profiling): add compression and babel stage spans --- .../compartment-mapper/src/archive-lite.js | 25 +++++++------------ packages/evasive-transform/src/index.js | 13 +++++----- .../src/transform-comment.js | 8 ++++++ 3 files changed, 23 insertions(+), 23 deletions(-) diff --git a/packages/compartment-mapper/src/archive-lite.js b/packages/compartment-mapper/src/archive-lite.js index c1fdb3ed76..bccbec62d8 100644 --- a/packages/compartment-mapper/src/archive-lite.js +++ b/packages/compartment-mapper/src/archive-lite.js @@ -72,7 +72,11 @@ const { assign, create, freeze, keys } = Object; * @param {Sources} sources * @param {(name: string, args?: Record) => (args?: Record) => void} [profileStartSpan] */ -const addSourcesToArchive = async (archive, sources, profileStartSpan = undefined) => { +const addSourcesToArchive = async ( + archive, + sources, + profileStartSpan = undefined, +) => { let moduleCount = 0; let byteCount = 0; const endSortCompartments = profileStartSpan?.( @@ -81,7 +85,7 @@ const addSourcesToArchive = async (archive, sources, profileStartSpan = undefine const sortedCompartments = keys(sources).sort(); endSortCompartments?.({ compartmentCount: sortedCompartments.length }); - let asyncWriteCount = 0; + await null; const endWriteModules = profileStartSpan?.( 'compartmentMapper.archiveLite.writeZip.sources.writeModules', ); @@ -93,18 +97,9 @@ const addSourcesToArchive = async (archive, sources, profileStartSpan = undefine const { bytes, location } = modules[specifier]; const path = `${compartment}/${location}`; if (bytes !== undefined) { - const maybeWrite = archive.write(path, bytes); - if ( - maybeWrite && - typeof maybeWrite === 'object' && - 'then' in maybeWrite && - typeof maybeWrite.then === 'function' - ) { - asyncWriteCount += 1; - // Preserve deterministic write order for truly async writers. - // eslint-disable-next-line no-await-in-loop - await maybeWrite; - } + // Preserve deterministic write order. + // eslint-disable-next-line no-await-in-loop + await archive.write(path, bytes); moduleCount += 1; byteCount += bytes.length; } @@ -114,8 +109,6 @@ const addSourcesToArchive = async (archive, sources, profileStartSpan = undefine endWriteModules?.({ moduleCount, byteCount, - asyncWriteCount, - syncWriteCount: moduleCount - asyncWriteCount, }); return { moduleCount, byteCount }; }; diff --git a/packages/evasive-transform/src/index.js b/packages/evasive-transform/src/index.js index 7b62a93a13..306d439d44 100644 --- a/packages/evasive-transform/src/index.js +++ b/packages/evasive-transform/src/index.js @@ -12,8 +12,13 @@ import { transformAst } from './transform-ast.js'; import { parseAst } from './parse-ast.js'; import { generate } from './generate.js'; +import { evadeRegexp } from './transform-comment.js'; /** + * Efficiently classify source text as might-need-transformation versus + * definitely-does-not, avoiding Babel work whenever no transformable comment + * substring is present. + * * @param {string} source * @param {boolean} elideComments * @returns {boolean} @@ -22,13 +27,7 @@ const shouldRunTransform = (source, elideComments) => { if (elideComments) { return true; } - // Fast path: if none of the risky comment payload tokens appear anywhere in - // the source, the transform cannot change semantics-relevant content. - return ( - source.includes('import(') || - source.includes('') - ); + return source.search(evadeRegexp) !== -1; }; /** diff --git a/packages/evasive-transform/src/transform-comment.js b/packages/evasive-transform/src/transform-comment.js index 813d5cb647..603c04999d 100644 --- a/packages/evasive-transform/src/transform-comment.js +++ b/packages/evasive-transform/src/transform-comment.js @@ -20,6 +20,14 @@ const HTML_COMMENT_START_RE = new RegExp(`${'<'}!--`, 'g'); */ const HTML_COMMENT_END_RE = new RegExp(`--${'>'}`, 'g'); +/** + * Matches comment substrings that `evadeComment` would rewrite. + */ +export const evadeRegexp = new RegExp( + `${IMPORT_RE.source}|${HTML_COMMENT_START_RE.source}|${HTML_COMMENT_END_RE.source}`, + 's', +); + /** * Rewrites a Comment Node to avoid triggering SES restrictions. * From 0947c78fc233bdc50d0f50230825578bd246fb7b Mon Sep 17 00:00:00 2001 From: Turadg Aleahmad Date: Tue, 9 Jun 2026 14:03:18 -0700 Subject: [PATCH 16/22] fixup! perf(bundle-source): speed up multi-entry bundling and add profiling detail --- packages/bundle-source/src/zip-base64.js | 111 ++++++++++-------- .../src/parse-archive-mjs.js | 56 ++++++--- .../test/node-powers.test.js | 9 +- .../test/parse-archive-mjs.test.js | 19 +++ packages/zip/src/writer.js | 11 +- 5 files changed, 139 insertions(+), 67 deletions(-) diff --git a/packages/bundle-source/src/zip-base64.js b/packages/bundle-source/src/zip-base64.js index e04dd69268..d076452fdf 100644 --- a/packages/bundle-source/src/zip-base64.js +++ b/packages/bundle-source/src/zip-base64.js @@ -26,42 +26,10 @@ const configuredReadCacheMaxBytes = Number.parseInt( 10, ); const readCacheMaxBytes = - Number.isFinite(configuredReadCacheMaxBytes) && configuredReadCacheMaxBytes >= 0 + Number.isFinite(configuredReadCacheMaxBytes) && + configuredReadCacheMaxBytes >= 0 ? configuredReadCacheMaxBytes : DEFAULT_READ_CACHE_MAX_BYTES; -/** @type {Map} */ -const cachedReads = new Map(); -/** @type {Map>} */ -const pendingReads = new Map(); -let cachedReadBytes = 0; - -/** - * @param {string} location - * @param {Uint8Array | undefined} bytes - */ -const cacheReadValue = (location, bytes) => { - const prior = cachedReads.get(location); - if (prior !== undefined) { - cachedReadBytes -= prior.length; - } - - cachedReads.set(location, bytes); - if (bytes !== undefined) { - cachedReadBytes += bytes.length; - } - - while (cachedReadBytes > readCacheMaxBytes && cachedReads.size > 0) { - const oldestKey = cachedReads.keys().next().value; - if (oldestKey === undefined) { - break; - } - const value = cachedReads.get(oldestKey); - cachedReads.delete(oldestKey); - if (value !== undefined) { - cachedReadBytes -= value.length; - } - } -}; /** * @param {string} startFilename @@ -106,11 +74,47 @@ export async function bundleZipBase64( let phaseStatus = 'ok'; let phaseError; try { - const maybeRead = async location => { - if (readCacheMaxBytes === 0) { - return powers.maybeRead(location); + /** @type {Map} */ + const cachedReads = new Map(); + /** @type {Map>} */ + const pendingReads = new Map(); + let cachedReadBytes = 0; + + /** + * Stores one completed read result in this bundle operation's cache and + * evicts oldest byte-bearing entries until the configured byte budget holds. + * + * @param {string} location + * @param {Uint8Array | undefined} bytes + */ + const cacheReadValue = (location, bytes) => { + const prior = cachedReads.get(location); + if (prior !== undefined) { + cachedReadBytes -= prior.length; + } + + cachedReads.set(location, bytes); + if (bytes !== undefined) { + cachedReadBytes += bytes.length; } - const hit = cachedReads.has(location); + + if (cachedReadBytes <= readCacheMaxBytes) { + return; + } + + for (const [oldestKey, value] of cachedReads) { + cachedReads.delete(oldestKey); + if (value !== undefined) { + cachedReadBytes -= value.length; + } + if (cachedReadBytes <= readCacheMaxBytes) { + return; + } + } + }; + + const maybeRead = async location => { + const hit = readCacheMaxBytes > 0 && cachedReads.has(location); if (hit) { const endCacheHit = profiler.startSpan('bundleSource.readCache.hit'); try { @@ -131,14 +135,19 @@ export async function bundleZipBase64( } const endCacheMiss = profiler.startSpan('bundleSource.readCache.miss'); - pending = powers.maybeRead(location).then(bytes => { - cacheReadValue(location, bytes); - pendingReads.delete(location); - return bytes; - }, error => { - pendingReads.delete(location); - throw error; - }); + pending = powers.maybeRead(location).then( + bytes => { + if (readCacheMaxBytes > 0) { + cacheReadValue(location, bytes); + } + pendingReads.delete(location); + return bytes; + }, + error => { + pendingReads.delete(location); + throw error; + }, + ); pendingReads.set(location, pending); try { return await pending; @@ -155,7 +164,9 @@ export async function bundleZipBase64( maybeRead, }); - const endMakeBundlingKit = profiler.startSpan('bundleSource.makeBundlingKit'); + const endMakeBundlingKit = profiler.startSpan( + 'bundleSource.makeBundlingKit', + ); const { sourceMapHook, sourceMapJobs, @@ -204,7 +215,9 @@ export async function bundleZipBase64( endMapNodeModules(); } - const endMakeArchive = profiler.startSpan('bundleSource.makeAndHashArchiveFromMap'); + const endMakeArchive = profiler.startSpan( + 'bundleSource.makeAndHashArchiveFromMap', + ); let bytes; let sha512; try { @@ -232,7 +245,7 @@ export async function bundleZipBase64( } const endEncodeBase64 = profiler.startSpan('bundleSource.encodeBase64'); - let endoZipBase64; + let endoZipBase64 = ''; try { endoZipBase64 = encodeBase64(bytes); } finally { diff --git a/packages/compartment-mapper/src/parse-archive-mjs.js b/packages/compartment-mapper/src/parse-archive-mjs.js index 949db10e66..1eb64b1cf7 100644 --- a/packages/compartment-mapper/src/parse-archive-mjs.js +++ b/packages/compartment-mapper/src/parse-archive-mjs.js @@ -11,11 +11,39 @@ import { ModuleSource } from '@endo/module-source'; const textEncoder = new TextEncoder(); const textDecoder = new TextDecoder(); -/** @type {Map>>} */ +/** @type {Map>>>} */ const parseArchiveMjsCache = new Map(); const MAX_PARSE_ARCHIVE_MJS_CACHE_ENTRIES = 20_000; let parseArchiveMjsCacheEntries = 0; +/** + * @param {string | object | undefined} sourceMap + * @returns {string | undefined} + */ +const getSourceMapCacheKey = sourceMap => { + if (sourceMap === undefined) { + return undefined; + } + if (typeof sourceMap === 'string') { + return sourceMap; + } + return JSON.stringify(sourceMap); +}; + +const evictOldestParseArchiveMjsLocation = () => { + const oldest = parseArchiveMjsCache.entries().next().value; + if (oldest === undefined) { + return; + } + const [sourceUrl, bySource] = oldest; + let evictedEntries = 0; + for (const bySourceMap of bySource.values()) { + evictedEntries += bySourceMap.size; + } + parseArchiveMjsCache.delete(sourceUrl); + parseArchiveMjsCacheEntries -= evictedEntries; +}; + /** @type {ParseFn} */ export const parseArchiveMjs = ( bytes, @@ -27,16 +55,10 @@ export const parseArchiveMjs = ( const { sourceMap, sourceMapHook, profileStartSpan } = options; const canUseCache = sourceMapHook === undefined; const source = textDecoder.decode(bytes); - const sourceMapKey = - sourceMap === undefined - ? '' - : typeof sourceMap === 'string' - ? sourceMap - : JSON.stringify(sourceMap); - const cacheKey = `${source}\n//# sourceMappingURL=${sourceMapKey}`; + const sourceMapKey = getSourceMapCacheKey(sourceMap); if (canUseCache) { const byLocation = parseArchiveMjsCache.get(sourceUrl); - const cached = byLocation?.get(cacheKey); + const cached = byLocation?.get(source)?.get(sourceMapKey); if (cached !== undefined) { profileStartSpan?.('compartmentMapper.parseArchiveMjs.cache.hit')?.(); return cached; @@ -74,16 +96,20 @@ export const parseArchiveMjs = ( byLocation = new Map(); parseArchiveMjsCache.set(sourceUrl, byLocation); } - if (!byLocation.has(cacheKey)) { + let bySource = byLocation.get(source); + if (bySource === undefined) { + bySource = new Map(); + byLocation.set(source, bySource); + } + if (!bySource.has(sourceMapKey)) { + bySource.set(sourceMapKey, result); parseArchiveMjsCacheEntries += 1; if (parseArchiveMjsCacheEntries > MAX_PARSE_ARCHIVE_MJS_CACHE_ENTRIES) { - parseArchiveMjsCache.clear(); - parseArchiveMjsCacheEntries = 0; - byLocation = new Map(); - parseArchiveMjsCache.set(sourceUrl, byLocation); + evictOldestParseArchiveMjsLocation(); } + } else { + bySource.set(sourceMapKey, result); } - byLocation.set(cacheKey, result); } return result; }; diff --git a/packages/compartment-mapper/test/node-powers.test.js b/packages/compartment-mapper/test/node-powers.test.js index 0617227ce1..5e1b7dda3d 100644 --- a/packages/compartment-mapper/test/node-powers.test.js +++ b/packages/compartment-mapper/test/node-powers.test.js @@ -1,4 +1,5 @@ import test from 'ava'; +/* global Buffer, setTimeout */ import url from 'url'; @@ -66,8 +67,12 @@ test('makeReadPowers canonical memoizes realpath lookups per location', async t url, }); - const file = url.pathToFileURL('/tmp/file.js').href; - const dir = url.pathToFileURL('/tmp/pkg').href + '/'; + const file = /** @type {`file://${string}`} */ ( + url.pathToFileURL('/tmp/file.js').href + ); + const dir = /** @type {`file://${string}`} */ ( + `${url.pathToFileURL('/tmp/pkg').href}/` + ); const [a, b] = await Promise.all([canonical(file), canonical(file)]); const [c, d] = await Promise.all([canonical(dir), canonical(dir)]); diff --git a/packages/compartment-mapper/test/parse-archive-mjs.test.js b/packages/compartment-mapper/test/parse-archive-mjs.test.js index 339d727805..02e0a57b48 100644 --- a/packages/compartment-mapper/test/parse-archive-mjs.test.js +++ b/packages/compartment-mapper/test/parse-archive-mjs.test.js @@ -40,6 +40,25 @@ test('parseArchiveMjs cache key includes source URL', t => { t.not(first, second); }); +test('parseArchiveMjs cache key separates source from source map', t => { + const first = parseArchiveMjs( + encoder.encode('export const value = 4;\n//# sourceMappingURL='), + './mod.js', + 'file:///tmp/collision.js', + 'file:///tmp/', + { sourceMap: '' }, + ); + const second = parseArchiveMjs( + encoder.encode('export const value = 4;'), + './mod.js', + 'file:///tmp/collision.js', + 'file:///tmp/', + { sourceMap: '\n//# sourceMappingURL=' }, + ); + + t.not(first, second); +}); + test('parseArchiveMjs bypasses cache when source maps are requested', t => { const bytes = encoder.encode('export const value = 3;'); const first = parseArchiveMjs( diff --git a/packages/zip/src/writer.js b/packages/zip/src/writer.js index 9f60287364..f041d0e84c 100644 --- a/packages/zip/src/writer.js +++ b/packages/zip/src/writer.js @@ -1,8 +1,17 @@ // @ts-check +/** + * @typedef {{ + * name: string, + * mode: number, + * date?: Date, + * content: Uint8Array, + * comment: string, + * }} ZFile + */ + import { BufferWriter } from './buffer-writer.js'; import { writeZip as writeZipFormat } from './format-writer.js'; - const LOCAL_FILE_HEADER_FIXED_BYTES = 30; const CENTRAL_FILE_HEADER_FIXED_BYTES = 46; const CENTRAL_DIRECTORY_END_FIXED_BYTES = 22; From 087a8c4bb0a8a8e74c54c27eacb6fedbffe1bafa Mon Sep 17 00:00:00 2001 From: Turadg Aleahmad Date: Tue, 9 Jun 2026 14:09:45 -0700 Subject: [PATCH 17/22] fixup! feat(bundle-source): add opt-in chrome trace profiling --- packages/bundle-source/src/endo.js | 21 +++++++++++++++------ packages/bundle-source/src/script.js | 10 ++++++---- 2 files changed, 21 insertions(+), 10 deletions(-) diff --git a/packages/bundle-source/src/endo.js b/packages/bundle-source/src/endo.js index 8351ecd744..d638951259 100644 --- a/packages/bundle-source/src/endo.js +++ b/packages/bundle-source/src/endo.js @@ -22,7 +22,13 @@ const textDecoder = new TextDecoder(); */ export const makeBundlingKit = ( io, - { cacheSourceMaps, elideComments, noTransforms, commonDependencies, profiler }, + { + cacheSourceMaps, + elideComments, + noTransforms, + commonDependencies, + profiler, + }, ) => { const { pathResolve, userInfo, computeSha512, platform, env } = io; if (noTransforms && elideComments) { @@ -132,11 +138,14 @@ export const makeBundlingKit = ( location, sourceMap, ) => { - const endTransformModule = profiler?.startSpan('bundleSource.transformModule', { - parser, - specifier, - location, - }); + const endTransformModule = profiler?.startSpan( + 'bundleSource.transformModule', + { + parser, + specifier, + location, + }, + ); if (!['mjs', 'cjs'].includes(parser)) { throw Error(`Parser ${parser} not supported in evadeEvalCensor`); } diff --git a/packages/bundle-source/src/script.js b/packages/bundle-source/src/script.js index 9c2616e0d5..0b55ebe75b 100644 --- a/packages/bundle-source/src/script.js +++ b/packages/bundle-source/src/script.js @@ -63,7 +63,9 @@ export async function bundleScript( let phaseStatus = 'ok'; let phaseError; try { - const endMakeBundlingKit = profiler.startSpan('bundleSource.makeBundlingKit'); + const endMakeBundlingKit = profiler.startSpan( + 'bundleSource.makeBundlingKit', + ); const { sourceMapHook, sourceMapJobs, @@ -109,9 +111,9 @@ export async function bundleScript( moduleFormat === 'nestedEvaluate' || moduleFormat === 'getExport', conditions: new Set(conditions), - commonDependencies, - profileStartSpan: profiler.startSpan, - parserForLanguage: parserForLanguageForFunctor, + commonDependencies, + profileStartSpan: profiler.startSpan, + parserForLanguage: parserForLanguageForFunctor, workspaceLanguageForExtension, workspaceCommonjsLanguageForExtension, workspaceModuleLanguageForExtension, From a4a1f095182f37bcc78ffc07f4c63d7b8b682eae Mon Sep 17 00:00:00 2001 From: Turadg Aleahmad Date: Tue, 9 Jun 2026 14:10:18 -0700 Subject: [PATCH 18/22] fixup! perf(bundle-source): speed up multi-entry bundling and add profiling detail --- packages/compartment-mapper/src/generic-graph.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/compartment-mapper/src/generic-graph.js b/packages/compartment-mapper/src/generic-graph.js index 226c03add0..73b0b10360 100644 --- a/packages/compartment-mapper/src/generic-graph.js +++ b/packages/compartment-mapper/src/generic-graph.js @@ -329,7 +329,7 @@ export const makeShortestPath = graph => { * Returns a function for shortest-path lookups from one fixed source. * Computes Dijkstra traversal context once and reuses it for all targets. * - * @template [T=string] + * @template {GenericGraphNode} [T=string] * @param {GenericGraph} graph Graph to use * @param {NoInfer} source Source node for all path lookups */ From 722a684cf9ea8cd6c96e63c1f08e28a482fc7de0 Mon Sep 17 00:00:00 2001 From: Turadg Aleahmad Date: Tue, 9 Jun 2026 14:11:09 -0700 Subject: [PATCH 19/22] fixup! feat(bundle-source): add opt-in chrome trace profiling --- packages/bundle-source/src/types.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/bundle-source/src/types.ts b/packages/bundle-source/src/types.ts index fdf9cdae6b..7dc00e2c4c 100644 --- a/packages/bundle-source/src/types.ts +++ b/packages/bundle-source/src/types.ts @@ -181,7 +181,7 @@ export interface BundlingKitOptions { startSpan: ( name: string, args?: Record | undefined, - ) => (args?: Record | undefined) => void; + ) => (endArgs?: Record | undefined) => void; } | undefined; } From 3a448fca73c2bb7137f97a0acfc6fc50e2c73a9e Mon Sep 17 00:00:00 2001 From: Turadg Aleahmad Date: Tue, 9 Jun 2026 14:11:13 -0700 Subject: [PATCH 20/22] fixup! perf(bundle-source): speed up multi-entry bundling and add profiling detail --- packages/zip/src/writer.js | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/zip/src/writer.js b/packages/zip/src/writer.js index f041d0e84c..37f5674a50 100644 --- a/packages/zip/src/writer.js +++ b/packages/zip/src/writer.js @@ -12,6 +12,7 @@ import { BufferWriter } from './buffer-writer.js'; import { writeZip as writeZipFormat } from './format-writer.js'; + const LOCAL_FILE_HEADER_FIXED_BYTES = 30; const CENTRAL_FILE_HEADER_FIXED_BYTES = 46; const CENTRAL_DIRECTORY_END_FIXED_BYTES = 22; From f2f1e57d1f458a038a8a7af232e605e58691ebdb Mon Sep 17 00:00:00 2001 From: Turadg Aleahmad Date: Tue, 9 Jun 2026 14:23:52 -0700 Subject: [PATCH 21/22] fixup! perf(bundle-source): speed up multi-entry bundling and add profiling detail --- packages/compartment-mapper/src/parse-mjs.js | 8 ++++++-- packages/module-source/src/transform-analyze.js | 8 +++++++- packages/zip/tools/benchmark-writer.mjs | 9 +++++++-- 3 files changed, 20 insertions(+), 5 deletions(-) diff --git a/packages/compartment-mapper/src/parse-mjs.js b/packages/compartment-mapper/src/parse-mjs.js index 443a37db21..31e2bc794d 100644 --- a/packages/compartment-mapper/src/parse-mjs.js +++ b/packages/compartment-mapper/src/parse-mjs.js @@ -14,8 +14,12 @@ export const parseMjs = ( _packageLocation, options = {}, ) => { - const { sourceMap, sourceMapHook, archiveOnly = false, profileStartSpan } = - options; + const { + sourceMap, + sourceMapHook, + archiveOnly = false, + profileStartSpan, + } = options; const source = textDecoder.decode(bytes); const endModuleSource = profileStartSpan?.( 'compartmentMapper.parseMjs.moduleSource', diff --git a/packages/module-source/src/transform-analyze.js b/packages/module-source/src/transform-analyze.js index a99f650159..424c445903 100644 --- a/packages/module-source/src/transform-analyze.js +++ b/packages/module-source/src/transform-analyze.js @@ -16,7 +16,13 @@ const makeCreateStaticRecord = transformSource => */ function createStaticRecord( moduleSource, - { sourceUrl, sourceMapUrl, sourceMap, sourceMapHook, profileStartSpan } = {}, + { + sourceUrl, + sourceMapUrl, + sourceMap, + sourceMapHook, + profileStartSpan, + } = {}, ) { // Transform the Module source code. const sourceOptions = { diff --git a/packages/zip/tools/benchmark-writer.mjs b/packages/zip/tools/benchmark-writer.mjs index a2f3c7dbc7..5fd1ab1a86 100644 --- a/packages/zip/tools/benchmark-writer.mjs +++ b/packages/zip/tools/benchmark-writer.mjs @@ -68,7 +68,9 @@ const makeInputFiles = (entries, totalBytes) => { const last = files[files.length - 1]; const newSize = Math.max(16, last.content.length + delta); const adjusted = new Uint8Array(newSize); - adjusted.set(last.content.subarray(0, Math.min(last.content.length, newSize))); + adjusted.set( + last.content.subarray(0, Math.min(last.content.length, newSize)), + ); for (let i = last.content.length; i < newSize; i += 1) { adjusted[i] = i & 0xff; } @@ -108,7 +110,10 @@ const main = () => { const warmup = toPositiveInt(warmupRaw, 'warmup'); const files = makeInputFiles(entries, totalBytes); - const totalInputBytes = files.reduce((sum, file) => sum + file.content.length, 0); + const totalInputBytes = files.reduce( + (sum, file) => sum + file.content.length, + 0, + ); /** @type {Record} */ const spanSamples = Object.create(null); From a5ef3f0857d76e55ed264210a5a227c2c15e9109 Mon Sep 17 00:00:00 2001 From: Turadg Aleahmad Date: Tue, 9 Jun 2026 14:42:45 -0700 Subject: [PATCH 22/22] fixup! perf(bundle-source): speed up multi-entry bundling and add profiling detail --- packages/bundle-source/package.json | 2 +- packages/bundle-source/tools/profile-agoric-bundling.mts | 2 +- packages/bundle-source/tools/trace-merge.js | 2 +- packages/zip/src/writer.js | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/bundle-source/package.json b/packages/bundle-source/package.json index fe49d2926e..26a12494b9 100644 --- a/packages/bundle-source/package.json +++ b/packages/bundle-source/package.json @@ -14,7 +14,7 @@ "scripts": { "build": "exit 0", "test": "ava", - "profile:agoric-bundling": "tools/profile-agoric-bundling.mts", + "profile:agoric-bundling": "node --experimental-strip-types tools/profile-agoric-bundling.mts", "trace:merge": "node tools/trace-merge.js", "test:xs": "exit 0", "lint-fix": "eslint --fix '**/*.js'", diff --git a/packages/bundle-source/tools/profile-agoric-bundling.mts b/packages/bundle-source/tools/profile-agoric-bundling.mts index 7a533e8aaf..341c89b30a 100755 --- a/packages/bundle-source/tools/profile-agoric-bundling.mts +++ b/packages/bundle-source/tools/profile-agoric-bundling.mts @@ -1,4 +1,4 @@ -#!/usr/bin/env node --experimental-strip-types +#!/usr/bin/env -S node --experimental-strip-types /* global process */ import '@endo/init'; diff --git a/packages/bundle-source/tools/trace-merge.js b/packages/bundle-source/tools/trace-merge.js index e2dd4559c9..5e7df367fe 100644 --- a/packages/bundle-source/tools/trace-merge.js +++ b/packages/bundle-source/tools/trace-merge.js @@ -378,7 +378,7 @@ const main = async () => { 'out-summary': outSummary = 'summary.json', 'out-markdown': outMarkdown = 'summary.md', top: topRaw = '30', - stacked = true, + stacked = false, }, positionals, } = parseArgs({ options, allowPositionals: true }); diff --git a/packages/zip/src/writer.js b/packages/zip/src/writer.js index 37f5674a50..e2a0ef3ba8 100644 --- a/packages/zip/src/writer.js +++ b/packages/zip/src/writer.js @@ -44,7 +44,7 @@ export class ZipWriter { */ constructor(options = { date: new Date() }) { const { date, profileStartSpan = undefined } = options; - /** type {Map} */ + /** @type {Map} */ this.files = new Map(); this.date = date; this.profileStartSpan = profileStartSpan;