feat(compartment-mapper): add Babel-based CJS parser with import() support

- Remove `extends SourceMapHookDetails` from `TransformSourceParams`;
  `source` is only available at the `sourceMapHook` call site, not in
  the options/state bag. Inline the optional `sourceUrl`/`sourceMapUrl`
  fields directly.
- Add `allowHidden?: boolean` to `TransformSourceParams` to match
  runtime usage in `babel-plugin.js`, removing the `@ts-expect-error`.
- Make `SourceMapObject.file` optional per Source Map v3 spec.
- Use full tuple `[name, false, undefined]` for `hoistedDecls.push()`
  instead of short `[name]`, removing the `@ts-expect-error`.
- Remove stale `@ts-expect-error` directives for `@babel/generator`
  options now recognized by `@types/babel__generator`.

Author: boneskull

packages/compartment-mapper/import-parsers.js

@@ -1,4 +1,7 @@
 // eslint-disable-next-line import/export -- just types
 export * from './src/types-external.js';
 
-export { defaultParserForLanguage } from './src/import-parsers.js';
+export {
+  defaultParserForLanguage,
+  parserForLanguageWithCjsBabel,
+} from './src/import-parsers.js';

packages/compartment-mapper/src/import-parsers.js

@@ -11,9 +11,12 @@ import parserText from './parse-text.js';
 import parserBytes from './parse-bytes.js';
 import parserCjs from './parse-cjs.js';
 import parserMjs from './parse-mjs.js';
+import parserCjsBabel from './parse-cjs-babel.js';
+
+const { freeze } = Object;
 
 /** @satisfies {Readonly<ParserForLanguage>} */
-export const defaultParserForLanguage = Object.freeze(
+export const defaultParserForLanguage = freeze(
   /** @type {const} */ ({
     mjs: parserMjs,
     cjs: parserCjs,
@@ -22,3 +25,11 @@ export const defaultParserForLanguage = Object.freeze(
     bytes: parserBytes,
   }),
 );
+
+/** @satisfies {Readonly<ParserForLanguage>} */
+export const parserForLanguageWithCjsBabel = freeze(
+  /** @type {const} */ ({
+    ...defaultParserForLanguage,
+    cjs: parserCjsBabel,
+  }),
+);

packages/compartment-mapper/src/parse-cjs-babel.js

@@ -0,0 +1,94 @@
+/* eslint-disable no-underscore-dangle */
+/**
+ * Provides language behavior (parser) for importing CommonJS as a virtual
+ * module source, using Babel AST analysis instead of the character-level lexer.
+ *
+ * Drop-in replacement for {@link parse-cjs.js}. Consumers opt in via the
+ * pre-built parser map:
+ *
+ * ```js
+ * import { parserForLanguageWithCjsBabel } from '@endo/compartment-mapper/import-parsers.js';
+ *
+ * await importLocation(readPowers, entryUrl, {
+ *   parserForLanguage: parserForLanguageWithCjsBabel,
+ * });
+ * ```
+ *
+ * @module
+ */
+
+/**
+ * @import {ParseFn} from './types.js'
+ * @import {ParserImplementation} from './types.js'
+ */
+
+import { CjsModuleSource } from '@endo/module-source';
+import { wrap, getModulePaths } from './parse-cjs-shared-export-wrapper.js';
+
+const textDecoder = new TextDecoder();
+
+const { freeze } = Object;
+
+/** @type {ParseFn} */
+export const parseCjsBabel = (
+  bytes,
+  _specifier,
+  location,
+  _packageLocation,
+  { readPowers } = {},
+) => {
+  const source = textDecoder.decode(bytes);
+  const cjsRecord = new CjsModuleSource(source, { sourceUrl: location });
+  const { filename, dirname } = getModulePaths(readPowers, location);
+
+  /**
+   * @param {object} moduleEnvironmentRecord
+   * @param {Compartment} compartment
+   * @param {Record<string, string>} resolvedImports
+   */
+  const execute = (moduleEnvironmentRecord, compartment, resolvedImports) => {
+    const functor = compartment.evaluate(cjsRecord.cjsFunctor);
+
+    const wrapResult = wrap({
+      moduleEnvironmentRecord,
+      compartment,
+      resolvedImports,
+      location,
+      readPowers,
+    });
+
+    const args = [
+      wrapResult.require,
+      wrapResult.moduleExports,
+      wrapResult.module,
+      filename,
+      dirname,
+    ];
+
+    if (cjsRecord.__needsImport__ && wrapResult.importFn) {
+      args.push(wrapResult.importFn);
+    }
+
+    functor.call(wrapResult.moduleExports, ...args);
+
+    wrapResult.afterExecute();
+  };
+
+  return {
+    parser: 'cjs',
+    bytes,
+    record: freeze({
+      imports: cjsRecord.imports,
+      exports: cjsRecord.exports,
+      reexports: cjsRecord.reexports,
+      execute,
+    }),
+  };
+};
+
+/** @type {ParserImplementation} */
+export default {
+  parse: parseCjsBabel,
+  heuristicImports: true,
+  synchronous: true,
+};

packages/compartment-mapper/src/parse-cjs-shared-export-wrapper.js

@@ -83,10 +83,11 @@ export const getModulePaths = (readPowers, location) => {
  * @param {string} in.location
  * @param {ReadFn | ReadPowers | undefined} in.readPowers
  * @returns {{
- *   module: { exports: any },
- *   moduleExports: any,
- *   afterExecute: Function,
- *   require: Function,
+ *   module: { exports: unknown },
+ *   moduleExports: unknown,
+ *   afterExecute: () => void,
+ *   require: (specifier: string) => unknown,
+ *   importFn: (specifier: string) => Promise<unknown>,
  * }}
  */
 export const wrap = ({
@@ -204,6 +205,15 @@ export const wrap = ({
 
   freeze(require);
 
+  /** @param {string} importSpecifier */
+  const importFn = async importSpecifier => {
+    const specifier = has(resolvedImports, importSpecifier)
+      ? resolvedImports[importSpecifier]
+      : importSpecifier;
+    return compartment.import(specifier);
+  };
+  freeze(importFn);
+
   const afterExecute = () => {
     const finalExports = module.exports; // in case it's a getter, only call it once
     const exportsHaveBeenOverwritten = finalExports !== originalExports;
@@ -231,5 +241,6 @@ export const wrap = ({
     moduleExports: originalExports,
     afterExecute,
     require,
+    importFn,
   };
 };

packages/compartment-mapper/test/cjs-compat.test.js

@@ -6,9 +6,14 @@ import 'ses';
 import test from 'ava';
 import path from 'path';
 import { scaffold } from './scaffold.js';
+import {
+  defaultParserForLanguage,
+  parserForLanguageWithCjsBabel,
+} from '../src/import-parsers.js';
 
 /**
  * @import {FixtureAssertionFn} from './test.types.js';
+ * @import {ThirdPartyStaticModuleInterface} from 'ses'
  */
 
 const fixture = new URL(
@@ -19,9 +24,13 @@ const fixtureDirname = new URL(
   'fixtures-cjs-compat/node_modules/app/dirname.js',
   import.meta.url,
 ).toString();
+const fixtureDynamicImport = new URL(
+  'fixtures-cjs-compat/node_modules/dynamic-import/index.js',
+  import.meta.url,
+).toString();
 
 const q = JSON.stringify;
-
+const { freeze } = Object;
 /**
  * @type {FixtureAssertionFn<{requireResolvePaths: string[]}>}
  */
@@ -70,50 +79,94 @@ const assertFixture = (t, { namespace, testCategoryHint }) => {
 
 const fixtureAssertionCount = 2;
 
-scaffold(
-  'fixtures-cjs-compat',
-  test,
-  fixture,
-  assertFixture,
-  fixtureAssertionCount,
-);
+const parsersForLanguage = {
+  default: defaultParserForLanguage,
+  babel: parserForLanguageWithCjsBabel,
+};
+
+for (const [name, parserForLanguage] of Object.entries(parsersForLanguage)) {
+  scaffold(
+    `fixtures-cjs-compat-${name}`,
+    test,
+    fixture,
+    assertFixture,
+    fixtureAssertionCount,
+    {
+      parserForLanguage,
+    },
+  );
 
-// Exit module errors are also deferred
-scaffold(
-  'fixtures-cjs-compat-exit-module',
-  test,
-  fixture,
-  assertFixture,
-  fixtureAssertionCount,
-  {
-    additionalOptions: {
-      importHook: async specifier => {
-        throw Error(`${q(specifier)} is NOT an exit module.`);
+  // Exit module errors are also deferred
+  scaffold(
+    `fixtures-cjs-compat-exit-module-${name}`,
+    test,
+    fixture,
+    assertFixture,
+    fixtureAssertionCount,
+    {
+      additionalOptions: {
+        importHook: async specifier => {
+          throw Error(`${q(specifier)} is NOT an exit module.`);
+        },
       },
+      parserForLanguage,
     },
-  },
-);
+  );
 
-scaffold(
-  'fixtures-cjs-compat-__dirname',
-  test,
-  fixtureDirname,
-  (t, { namespace, testCategoryHint }) => {
-    if (testCategoryHint === 'Location') {
-      const { __filename, __dirname } = namespace;
-      t.is(__filename, path.join(__dirname, '/dirname.js'));
-      t.assert(!__dirname.startsWith('file://'));
-      t.notRegex(
-        __dirname,
-        /[\\/]$/,
-        'Expected __dirname to NOT have a trailing slash',
-      );
-    } else {
-      const { __filename, __dirname } = namespace;
-      t.is(__dirname, null);
-      t.is(__filename, null);
-      t.pass();
-    }
-  },
-  3,
-);
+  scaffold(
+    `fixtures-cjs-compat-__dirname-${name}`,
+    test,
+    fixtureDirname,
+    (t, { namespace, testCategoryHint }) => {
+      if (testCategoryHint === 'Location') {
+        const { __filename, __dirname } = namespace;
+        t.is(__filename, path.join(__dirname, '/dirname.js'));
+        t.assert(!__dirname.startsWith('file://'));
+        t.notRegex(
+          __dirname,
+          /[\\/]$/,
+          'Expected __dirname to NOT have a trailing slash',
+        );
+      } else {
+        const { __filename, __dirname } = namespace;
+        t.is(__dirname, null);
+        t.is(__filename, null);
+        t.pass();
+      }
+    },
+    3,
+    {
+      parserForLanguage,
+    },
+  );
+
+  scaffold(
+    `fixtures-cjs-compat-dynamic-import-${name}`,
+    test,
+    fixtureDynamicImport,
+    async (t, { namespace }) => {
+      const { namespace: dynamicNamespace } =
+        // @ts-expect-error - untyped
+        await namespace.dynamicImport('a');
+      t.is(dynamicNamespace.foo, 'foo');
+    },
+    1,
+    {
+      // NOTE: this should fail with parse-cjs, but not parse-cjs-babel
+      knownFailure: name === 'default',
+      parserForLanguage,
+      additionalOptions: {
+        importHook: async () => {
+          /** @type {ThirdPartyStaticModuleInterface} */
+          return freeze({
+            imports: [],
+            exports: ['foo'],
+            execute: moduleExports => {
+              moduleExports.foo = 'foo';
+            },
+          });
+        },
+      },
+    },
+  );
+}

packages/compartment-mapper/test/dynamic-import-esm.test.js

@@ -0,0 +1,24 @@
+import 'ses';
+
+import test from 'ava';
+import { scaffold } from './scaffold.js';
+
+const fixture = new URL(
+  'fixtures-dynamic-import-esm/node_modules/app/index.js',
+  import.meta.url,
+).toString();
+
+scaffold(
+  'fixtures-dynamic-import-esm',
+  test,
+  fixture,
+  async (t, { namespace }) => {
+    // @ts-expect-error - untyped
+    const foo = await namespace.getFoo();
+    t.is(foo, 'foo');
+  },
+  1,
+  {
+    knownArchiveFailure: true,
+  },
+);