create Babel-based AST parser for CommonJS sources

.changeset/famous-seals-pick.md

@@ -0,0 +1,5 @@
+---
+'@endo/compartment-mapper': minor
+---
+
+Expose Babel-based CJS parser, `parse-cjs-babel`. Expose shared functionality for wrapping CJS functors with `__dirname`, `__filename`, etc. Add support for dynamic `import()` (`parse-cjs-babel` only).

.changeset/huge-mammals-give.md

@@ -0,0 +1,9 @@
+---
+'@endo/compartment-mapper': patch
+---
+
+Fixes `CompartmentDescriptor` so that it is generic on the `PackagePolicy`; externally-defined `ParseFn`s can now refer to the specific contents of a custom `PackagePolicy` present in a `CompartmentDescriptor`.
+
+Introduces `ParseSourceMapHook`; differentiated from `@endo/module-source`'s `SourceMapHook`.
+
+Fixes type of `PolicyItem`; eliminates confusion between `void` (no extra union members) and `any` (`SomePackagePolicy`).

.changeset/open-mammals-scream.md

@@ -0,0 +1,5 @@
+---
+'@endo/module-source': minor
+---
+
+Exposes AST-based parser for CJS, as well as an `analyzeCjs` function from the `analyzer.js` subpath export.

packages/compartment-mapper/cjs.js

@@ -0,0 +1,4 @@
+// eslint-disable-next-line import/export
+export * from './src/types-external.js';
+
+export * from './src/cjs.js';

packages/compartment-mapper/import-parsers.js

@@ -1,4 +1,7 @@
 // eslint-disable-next-line import/export -- just types
 export * from './src/types-external.js';
 
-export { defaultParserForLanguage } from './src/import-parsers.js';
+export {
+  defaultParserForLanguage,
+  parserForLanguageWithCjsBabel,
+} from './src/import-parsers.js';

packages/compartment-mapper/package.json

@@ -23,6 +23,7 @@
   "main": "./index.js",
   "exports": {
     ".": "./index.js",
+    "./cjs.js": "./cjs.js",
     "./import.js": "./import.js",
     "./import-lite.js": "./import-lite.js",
     "./import-parsers.js": "./import-parsers.js",
@@ -45,6 +46,7 @@
     "./script-lite.js": "./script-lite.js",
     "./node-powers.js": "./node-powers.js",
     "./node-modules.js": "./node-modules.js",
+    "./policy.js": "./policy.js",
     "./package.json": "./package.json"
   },
   "scripts": {

packages/compartment-mapper/src/cjs.js

@@ -0,0 +1,79 @@
+/* eslint-disable no-underscore-dangle */
+/**
+ * Provides {@link buildCjsExecuteRecord}, a function that converts a
+ * {@link CjsModuleSourceRecord} into a {@link FinalStaticModuleType}.
+ *
+ * For use with `@endo/parser-pipeline`'s `createComposedParser()`.
+ *
+ * @module
+ */
+
+import { getModulePaths, wrap } from './parse-cjs-shared-export-wrapper.js';
+
+/**
+ * @import {CjsModuleSourceRecord} from '@endo/module-source'
+ * @import {ReadFn, ReadPowers} from './types.js'
+ * @import {FinalStaticModuleType} from 'ses'
+ */
+
+const { freeze } = Object;
+
+/**
+ * Converts a {@link CjsModuleSourceRecord} (which has a `cjsFunctor` string)
+ * into a `FinalStaticModuleType`-compatible record (which has an `execute`
+ * function). This is the bridge between the composed-pipeline CJS analysis and
+ * the compartment-mapper execution model.
+ *
+ * Used by both {@link parseCjsBabel} (single-shot parser) and
+ * {@link createCjsExecParser} (composed-pipeline parser) so the execution
+ * logic lives in exactly one place.
+ *
+ * @param {CjsModuleSourceRecord} cjsRecord
+ * @param {string} location
+ * @param {ReadFn | ReadPowers | undefined} readPowers
+ * @returns {FinalStaticModuleType}
+ */
+
+export const buildCjsExecuteRecord = (cjsRecord, location, readPowers) => {
+  const { filename, dirname } = getModulePaths(readPowers, location);
+
+  /**
+   * @param {object} moduleEnvironmentRecord
+   * @param {Compartment} compartment
+   * @param {Record<string, string>} resolvedImports
+   */
+  const execute = (moduleEnvironmentRecord, compartment, resolvedImports) => {
+    const functor = compartment.evaluate(cjsRecord.cjsFunctor);
+
+    const wrapResult = wrap({
+      moduleEnvironmentRecord,
+      compartment,
+      resolvedImports,
+      location,
+      readPowers,
+    });
+
+    const args = [
+      wrapResult.require,
+      wrapResult.moduleExports,
+      wrapResult.module,
+      filename,
+      dirname,
+    ];
+
+    if (cjsRecord.__needsImport__ && wrapResult.importFn) {
+      args.push(wrapResult.importFn);
+    }
+
+    functor.call(wrapResult.moduleExports, ...args);
+
+    wrapResult.afterExecute();
+  };
+
+  return freeze({
+    imports: cjsRecord.imports,
+    exports: cjsRecord.exports,
+    reexports: cjsRecord.reexports,
+    execute,
+  });
+};

packages/compartment-mapper/src/import-parsers.js

@@ -11,9 +11,12 @@ import parserText from './parse-text.js';
 import parserBytes from './parse-bytes.js';
 import parserCjs from './parse-cjs.js';
 import parserMjs from './parse-mjs.js';
+import parserCjsBabel from './parse-cjs-babel.js';
+
+const { freeze } = Object;
 
 /** @satisfies {Readonly<ParserForLanguage>} */
-export const defaultParserForLanguage = Object.freeze(
+export const defaultParserForLanguage = freeze(
   /** @type {const} */ ({
     mjs: parserMjs,
     cjs: parserCjs,
@@ -22,3 +25,11 @@ export const defaultParserForLanguage = Object.freeze(
     bytes: parserBytes,
   }),
 );
+
+/** @satisfies {Readonly<ParserForLanguage>} */
+export const parserForLanguageWithCjsBabel = freeze(
+  /** @type {const} */ ({
+    ...defaultParserForLanguage,
+    cjs: parserCjsBabel,
+  }),
+);

packages/compartment-mapper/src/parse-cjs-babel.js

@@ -0,0 +1,52 @@
+/* eslint-disable no-underscore-dangle */
+/**
+ * Provides language behavior (parser) for importing CommonJS as a virtual
+ * module source, using Babel AST analysis instead of the character-level lexer.
+ *
+ * Drop-in replacement for {@link parse-cjs.js}. Consumers opt in via the
+ * pre-built parser map:
+ *
+ * ```js
+ * import { parserForLanguageWithCjsBabel } from '@endo/compartment-mapper/import-parsers.js';
+ *
+ * await importLocation(readPowers, entryUrl, {
+ *   parserForLanguage: parserForLanguageWithCjsBabel,
+ * });
+ * ```
+ *
+ * @module
+ */
+
+/**
+ * @import {ParseFn, ParserImplementation} from './types.js'
+ */
+
+import { CjsModuleSource } from '@endo/module-source';
+import { buildCjsExecuteRecord } from './cjs.js';
+
+const textDecoder = new TextDecoder();
+
+/** @type {ParseFn} */
+export const parseCjsBabel = (
+  bytes,
+  _specifier,
+  location,
+  _packageLocation,
+  { readPowers } = {},
+) => {
+  const source = textDecoder.decode(bytes);
+  const cjsRecord = new CjsModuleSource(source, { sourceUrl: location });
+
+  return {
+    parser: 'cjs',
+    bytes,
+    record: buildCjsExecuteRecord(cjsRecord, location, readPowers),
+  };
+};
+
+/** @type {ParserImplementation} */
+export default {
+  parse: parseCjsBabel,
+  heuristicImports: true,
+  synchronous: true,
+};

packages/compartment-mapper/src/parse-cjs-shared-export-wrapper.js

@@ -83,10 +83,11 @@ export const getModulePaths = (readPowers, location) => {
  * @param {string} in.location
  * @param {ReadFn | ReadPowers | undefined} in.readPowers
  * @returns {{
- *   module: { exports: any },
- *   moduleExports: any,
- *   afterExecute: Function,
- *   require: Function,
+ *   module: { exports: unknown },
+ *   moduleExports: unknown,
+ *   afterExecute: () => void,
+ *   require: (specifier: string) => unknown,
+ *   importFn: (specifier: string) => Promise<unknown>,
  * }}
  */
 export const wrap = ({
@@ -204,6 +205,15 @@ export const wrap = ({
 
   freeze(require);
 
+  /** @param {string} importSpecifier */
+  const importFn = async importSpecifier => {
+    const specifier = has(resolvedImports, importSpecifier)
+      ? resolvedImports[importSpecifier]
+      : importSpecifier;
+    return compartment.import(specifier);
+  };
+  freeze(importFn);
+
   const afterExecute = () => {
     const finalExports = module.exports; // in case it's a getter, only call it once
     const exportsHaveBeenOverwritten = finalExports !== originalExports;
@@ -231,5 +241,6 @@ export const wrap = ({
     moduleExports: originalExports,
     afterExecute,
     require,
+    importFn,
   };
 };

packages/compartment-mapper/src/types/compartment-map-schema.ts

@@ -14,7 +14,7 @@ import type {
 } from '../policy-format.js';
 import type { CanonicalName } from './canonical-name.js';
 import type { FileUrlString } from './external.js';
-import type { SomePackagePolicy } from './policy-schema.js';
+import type { PackagePolicy, SomePackagePolicy } from './policy-schema.js';
 import type { PatternDescriptor } from './pattern-replacement.js';
 import type { LiteralUnion } from './typescript.js';
 
@@ -122,23 +122,24 @@ export interface PackageCompartmentDescriptor
  * one for a given library or application `package.json`.
  */
 export interface CompartmentDescriptor<
-  T extends ModuleConfiguration = ModuleConfiguration,
-  U extends string = string,
+  TModuleConfiguration extends ModuleConfiguration = ModuleConfiguration,
+  TCompartmentName extends string = string,
+  TPackagePolicy extends SomePackagePolicy = SomePackagePolicy,
 > {
-  label: CanonicalName<U>;
+  label: CanonicalName<TCompartmentName>;
   /**
    * the name of the originating package suitable for constructing a sourceURL
    * prefix that will match it to files in a developer workspace.
    */
   name: string;
-  modules: Record<string, T>;
+  modules: Record<string, TModuleConfiguration>;
   scopes?: Record<string, ScopeDescriptor>;
   /** language for extension */
   parsers?: LanguageForExtension;
   /** language for module specifier */
   types?: LanguageForModuleSpecifier;
   /** policy specific to compartment */
-  policy?: SomePackagePolicy;
+  policy?: TPackagePolicy;
 
   location: string;
   /**
@@ -154,9 +155,32 @@ export interface CompartmentDescriptor<
   retained?: true;
 }
 
+/**
+ * Any {@link CompartmentDescriptor}
+ */
+export type SomeCompartmentDescriptor = CompartmentDescriptor<any, any, any>;
+
+/**
+ * Any {@link CompartmentDescriptor} with a non-nullish
+ * {@link CompartmentDescriptor.policy} property
+ */
+export type SomeCompartmentDescriptorWithPolicy =
+  CompartmentDescriptorWithPolicy<any, any, any>;
+
+/**
+ * A {@link CompartmentDescriptor} with a non-nullish
+ * {@link CompartmentDescriptor.policy} property
+ */
 export type CompartmentDescriptorWithPolicy<
-  T extends ModuleConfiguration = ModuleConfiguration,
-> = Omit<CompartmentDescriptor<T>, 'policy'> & { policy: SomePackagePolicy };
+  TModuleConfiguration extends ModuleConfiguration = ModuleConfiguration,
+  TCompartmentName extends string = string,
+  TPackagePolicy extends SomePackagePolicy = SomePackagePolicy,
+> = Omit<
+  CompartmentDescriptor<TModuleConfiguration, TCompartmentName, TPackagePolicy>,
+  'policy'
+> & {
+  policy: TPackagePolicy;
+};
 
 /**
  * A compartment descriptor digested by `digestCompartmentMap()`