From 2b680ef22f31272fe96c6257b1c6d623c9ad0c02 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 26 Oct 2026 13:46:13 +0000 Subject: [PATCH] refactor(metadata-protocol): move the stored-overlay row reads out of protocol.ts (#25467 cut 16) Move-only. The bodies of readActiveOverlayRows, storedOverlayEntries, namesWithOwnStoredRow and readItemActiveRows move into the new protocol-stored-overlay-reads.ts (host shape B: a live engine getter and protocol-bound closures for findEnvironmentRows and convertStoredItem; namesWithOwnStoredRow reads no member, so its method calls it directly, as cut 3's orderByFieldNames does). All four members keep their name, docblock and signature in the class as stubs that delegate. addressPackages and the StoredOverlayEntry interface gain export in place; the overlay-cache reads (metaOverlayCacheTtlMs, readMetaOverlayCache, readOverlayCacheGeneration, writeMetaOverlayCache, MetaOverlayCacheKey) leave protocol.ts' import with their last readers. Nothing re-exports the new module, so the published surface does not move. Claude-Session: https://claude.ai/code/session_016SP1spj613tyGCgxoD1An6 Co-authored-by: Claude --- .../src/protocol-stored-overlay-reads.ts | 218 ++++++++++++++++++ packages/metadata-protocol/src/protocol.ts | 159 ++----------- 2 files changed, 241 insertions(+), 136 deletions(-) create mode 100644 packages/metadata-protocol/src/protocol-stored-overlay-reads.ts diff --git a/packages/metadata-protocol/src/protocol-stored-overlay-reads.ts b/packages/metadata-protocol/src/protocol-stored-overlay-reads.ts new file mode 100644 index 00000000000..d8b09da3a2b --- /dev/null +++ b/packages/metadata-protocol/src/protocol-stored-overlay-reads.ts @@ -0,0 +1,218 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * [#25467 cut 16] The stored-overlay row reads of + * `ObjectStackProtocolImplementation`: a type's active `sys_metadata` rows + * through the cross-request overlay cache (`readActiveOverlayRows`), each row + * parsed as the list read parses it (`storedOverlayEntries`), the names that + * have a stored row of their own (`namesWithOwnStoredRow`), and one item's + * active rows (`readItemActiveRows`), moved out of `protocol.ts` whole. + * + * A MOVE, not a redesign. Each function below is the body of the + * `ObjectStackProtocolImplementation` method of the same name as it stood, + * re-indented one level and otherwise unedited: the same reads, the same + * cache key, epoch and TTL readings in the same order, and the same parse. + * + * ## How the moved bodies reach the class + * + * The shape #23050 set for `rest-server.ts` (its `data-action-routes.ts` + * header carries the full reasoning), as `protocol-batch-data.ts` applies it. + * The class keeps every method — the published `dist/*.d.ts` lists each one's + * name, docblock and declared signature — and each body is one literal of + * exactly the {@link StoredOverlayReadHost} members that method reads, plus a + * `.call`. The engine is handed over as a getter, so the body reads the + * protocol's engine at call time (the overlay cache keys its bucket on it); a + * method is handed over as a closure that calls it on the protocol at call + * time, so a test double or spy set on the instance is still the one the body + * reaches. `namesWithOwnStoredRow` reads no member at all, so its method calls + * it directly, as `orderByFieldNames` does (cut 3). Every other reader of + * these four methods, `protocol-save-door-refusals.ts` among them, keeps + * calling the class method. + * + * What the bodies read of `protocol.ts`'s own module-level declarations they + * import from it; what they import from elsewhere, they import from where + * `protocol.ts` imported it. Nothing re-exports this module, so the published + * surface does not move. + */ + +import { PLURAL_TO_SINGULAR, SINGULAR_TO_PLURAL } from '@objectstack/spec/shared'; +import type { StoredOverlayRow } from './item-lock.js'; +import { + metaOverlayCacheTtlMs, + readMetaOverlayCache, + readOverlayCacheGeneration, + writeMetaOverlayCache, + type MetaOverlayCacheKey, +} from './meta-overlay-cache.js'; +import type { ObjectStackProtocolImplementation, StoredOverlayEntry } from './protocol.js'; +import { addressPackages } from './protocol.js'; + +/** + * The `ObjectStackProtocolImplementation` members the functions below read — + * every one of them, and no other; each function's `this` is the `Pick` of the + * members IT reads. Each is typed AS the class member (an indexed access on + * the class), so a change to a member's signature on the class is a change + * here too, and naming a member the class does not have is a compile error. + */ +export interface StoredOverlayReadHost { + readonly engine: ObjectStackProtocolImplementation['engine']; + findEnvironmentRows: ObjectStackProtocolImplementation['findEnvironmentRows']; + convertStoredItem: ObjectStackProtocolImplementation['convertStoredItem']; +} + +/** The body of `ObjectStackProtocolImplementation#readActiveOverlayRows`; the method's docblock there is its contract. */ +export async function readActiveOverlayRows( + this: Pick, + request: { type: string; packageId?: string }, +): Promise { + const { packageId } = request; + const queryActive = async (): Promise => { + const whereClause: Record = { + type: request.type, + state: 'active', + }; + if (packageId) whereClause.package_id = packageId; + // [#21911] The explicit system opt-in — see findServedOverlayRow. + // [ADR-0131 D6/D7] The environment's rows only, on every find — + // see {@link findEnvironmentRows}. + let rs = await this.findEnvironmentRows('sys_metadata', { where: whereClause, context: { isSystem: true } }); + if ((!rs || rs.length === 0)) { + const alt = PLURAL_TO_SINGULAR[request.type] ?? SINGULAR_TO_PLURAL[request.type]; + if (alt) { + const altWhere: Record = { type: alt, state: 'active' }; + if (packageId) altWhere.package_id = packageId; + rs = await this.findEnvironmentRows('sys_metadata', { where: altWhere, context: { isSystem: true } }); + } + } + return rs ?? []; + }; + // ── Leg D of #11633 (#11967): the cross-request overlay cache ── + // + // ⭐ The cache sits HERE, and its position IS the resolution of the + // SchemaRegistry-hydration trap #11633 §4 names. What is cached is + // the ROW SET — the value the `queryActive` call produces — + // never the merged answer below it. Everything downstream of this + // point still runs on every call, hit or miss: the overlay parse, + // the package-aware merge, `hydrateOverlayIntoRegistry`, the + // MetadataService merge, the disabled-package filter, the nav + // contributions, the decorations. A hit changes where the rows came + // from and nothing about what is done with them, so the read-side + // registry hydration cannot be skipped by one. + // + // ⛔ Do NOT move this below the merge. That is precisely the naive + // shape the trap describes, and it would additionally serve three + // mutable sources — the SchemaRegistry, the MetadataService and the + // artifact table — whose changes nothing in this key can observe. + // See `meta-overlay-cache.ts` for the measured four-source table. + // + // ⭐ The epoch reading is taken BEFORE the read, and it is this + // pre-read value that is stored with the rows. A write landing + // WHILE this read is in flight therefore moves the epoch past what + // the entry records, so the entry is already dead when it is + // written — the safe direction. Reading it afterwards would stamp + // pre-write rows with a post-write epoch and make that staleness + // permanent: the clear-then-repopulate-from-a-stale-read failure + // #11633 §7 pin 2 names. + const overlayCacheKey: MetaOverlayCacheKey = { + type: request.type, + packageId, + }; + const overlayCacheTtlMs = metaOverlayCacheTtlMs(); + const overlayCacheEpoch = overlayCacheTtlMs > 0 ? readOverlayCacheGeneration(this.engine) : undefined; + const overlayCacheNow = Date.now(); + const cachedRecords = readMetaOverlayCache( + this.engine, overlayCacheKey, overlayCacheEpoch, overlayCacheTtlMs, overlayCacheNow, + ); + + let records: any[]; + if (cachedRecords !== undefined) { + records = cachedRecords as any[]; + } else { + records = await queryActive(); + // ⭐ An EMPTY row set is cached too, and that is the main point + // rather than an edge case: the empty result is what triggers the + // alt-type retry above, so "no overlay rows for this type" is the + // answer whose caching removes BOTH reads. #11633 §1 measured that + // an app whose objects are all code-authored pays the doubled read + // on every request; this is the line that stops it. + writeMetaOverlayCache( + this.engine, overlayCacheKey, overlayCacheEpoch, records, overlayCacheTtlMs, overlayCacheNow, + ); + } + return records; +} + +/** The body of `ObjectStackProtocolImplementation#storedOverlayEntries`; the method's docblock there is its contract. */ +export function storedOverlayEntries( + this: Pick, + request: { type: string }, + records: any[], +): StoredOverlayEntry[] { + // Parse each overlay body once — replaying the stored-row + // conversion chain (#3903) so every consumer of this list sees + // the canonical protocol shape — and surface its persisted + // software-package binding so the sidebar package filter and + // provenance classification see overlay rows the way they see + // registry items. + return records.map((record) => { + const data = this.convertStoredItem( + String(record.type ?? request.type), + typeof record.metadata === 'string' + ? JSON.parse(record.metadata) + : record.metadata, + ) as any; + const recPkg = (record as { package_id?: string | null }).package_id ?? undefined; + if (recPkg && data && typeof data === 'object' && (data as any)._packageId === undefined) { + (data as any)._packageId = recPkg; + } + // [ADR-0131 D6/D7] Every row is the environment's: the family + // declares no organization column, and the reads that feed this + // ({@link readActiveOverlayRows}) leave a legacy organization's + // rows out off the physical column. (The form doors' legacy layer + // that once needed the row's own scope here is deleted.) + return { + name: String(record.name), data, packageId: recPkg, organizationId: null, + type: String(record.type ?? request.type), + }; + }); +} + +/** The body of `ObjectStackProtocolImplementation#namesWithOwnStoredRow`; the method's docblock there is its contract. */ +export function namesWithOwnStoredRow(records: readonly any[], packageId?: string): ReadonlySet { + const packages = addressPackages(packageId); + const names = new Set(); + for (const record of records) { + if (typeof record?.name !== 'string') continue; + if (packages.some((pkg) => pkg === undefined || (record?.package_id ?? null) === pkg)) names.add(record.name); + } + return names; +} + +/** The body of `ObjectStackProtocolImplementation#readItemActiveRows`; the method's docblock there is its contract. */ +export async function readItemActiveRows( + this: Pick, + type: string, name: string, cached = false, +): Promise { + const other = PLURAL_TO_SINGULAR[type] ?? SINGULAR_TO_PLURAL[type]; + const read = async (): Promise => { + // [#21911] The explicit system opt-in — see findServedOverlayRow. + const rows: StoredOverlayRow[] | null | undefined = await this.findEnvironmentRows('sys_metadata', { + where: { type: other === undefined ? type : { $in: [type, other] }, name, state: 'active' }, + context: { isSystem: true }, + }); + return rows ?? []; + }; + if (!cached) return read(); + // ⭐ The epoch is read BEFORE the read and stored with its rows, as the + // list read's is (see `readActiveOverlayRows`): a write landing while + // this read is in flight leaves the entry dead on arrival. + const key: MetaOverlayCacheKey = { type, name }; + const ttlMs = metaOverlayCacheTtlMs(); + const epoch = ttlMs > 0 ? readOverlayCacheGeneration(this.engine) : undefined; + const now = Date.now(); + const hit = readMetaOverlayCache(this.engine, key, epoch, ttlMs, now); + if (hit !== undefined) return hit as StoredOverlayRow[]; + const rows = await read(); + writeMetaOverlayCache(this.engine, key, epoch, rows, ttlMs, now); + return rows; +} diff --git a/packages/metadata-protocol/src/protocol.ts b/packages/metadata-protocol/src/protocol.ts index 5daf1011ba2..e600ddae370 100644 --- a/packages/metadata-protocol/src/protocol.ts +++ b/packages/metadata-protocol/src/protocol.ts @@ -71,11 +71,6 @@ import { } from './item-lock.js'; import { bumpWriteEpoch, - metaOverlayCacheTtlMs, - readMetaOverlayCache, - readOverlayCacheGeneration, - writeMetaOverlayCache, - type MetaOverlayCacheKey, } from './meta-overlay-cache.js'; import { ConflictError, @@ -400,6 +395,8 @@ import { originGatedRemovalRefusal, refusePackagedBaseOverride } from './protoco import { applySeedBodies, collectBatchPendingDeclarations } from './protocol-publish-batch.js'; // [#25467 cut 15] The artifact registry's lookups: an item's artifact, the packages that ship a type, and the lock's artifact layer. import { artifactLockLayerAt, lookupArtifactItem, shippedArtifactsOf, shippingPackagesOf } from './protocol-artifact-lookup.js'; +// [#25467 cut 16] The stored-overlay row reads: a type's active rows, their parsed entries, the names with a row of their own, and one item's rows. +import { namesWithOwnStoredRow, readActiveOverlayRows, readItemActiveRows, storedOverlayEntries } from './protocol-stored-overlay-reads.js'; /** * Canonical Zod schema per metadata type lives in @@ -2831,7 +2828,7 @@ function rowMatchesOverlayWhere(row: object, where: Readonly { +export function addressPackages(packageId: string | undefined): ReadonlyArray { return [...new Set(servedOverlayRowCandidates({ packageId }) .map((candidate) => candidate.packageId))]; } @@ -6230,7 +6227,7 @@ function isNonCanonicalStoredType(type: string): boolean { * row). [#21804] `type` is the spelling the row is stored under (its own * `type` column), which the served-row resolution reads. */ -interface StoredOverlayEntry { +export interface StoredOverlayEntry { name: string; data: any; packageId: string | undefined; @@ -10262,81 +10259,13 @@ export class ObjectStackProtocolImplementation implements private async readActiveOverlayRows( request: { type: string; packageId?: string }, ): Promise { - const { packageId } = request; - const queryActive = async (): Promise => { - const whereClause: Record = { - type: request.type, - state: 'active', - }; - if (packageId) whereClause.package_id = packageId; - // [#21911] The explicit system opt-in — see findServedOverlayRow. - // [ADR-0131 D6/D7] The environment's rows only, on every find — - // see {@link findEnvironmentRows}. - let rs = await this.findEnvironmentRows('sys_metadata', { where: whereClause, context: { isSystem: true } }); - if ((!rs || rs.length === 0)) { - const alt = PLURAL_TO_SINGULAR[request.type] ?? SINGULAR_TO_PLURAL[request.type]; - if (alt) { - const altWhere: Record = { type: alt, state: 'active' }; - if (packageId) altWhere.package_id = packageId; - rs = await this.findEnvironmentRows('sys_metadata', { where: altWhere, context: { isSystem: true } }); - } - } - return rs ?? []; - }; - // ── Leg D of #11633 (#11967): the cross-request overlay cache ── - // - // ⭐ The cache sits HERE, and its position IS the resolution of the - // SchemaRegistry-hydration trap #11633 §4 names. What is cached is - // the ROW SET — the value the `queryActive` call produces — - // never the merged answer below it. Everything downstream of this - // point still runs on every call, hit or miss: the overlay parse, - // the package-aware merge, `hydrateOverlayIntoRegistry`, the - // MetadataService merge, the disabled-package filter, the nav - // contributions, the decorations. A hit changes where the rows came - // from and nothing about what is done with them, so the read-side - // registry hydration cannot be skipped by one. - // - // ⛔ Do NOT move this below the merge. That is precisely the naive - // shape the trap describes, and it would additionally serve three - // mutable sources — the SchemaRegistry, the MetadataService and the - // artifact table — whose changes nothing in this key can observe. - // See `meta-overlay-cache.ts` for the measured four-source table. - // - // ⭐ The epoch reading is taken BEFORE the read, and it is this - // pre-read value that is stored with the rows. A write landing - // WHILE this read is in flight therefore moves the epoch past what - // the entry records, so the entry is already dead when it is - // written — the safe direction. Reading it afterwards would stamp - // pre-write rows with a post-write epoch and make that staleness - // permanent: the clear-then-repopulate-from-a-stale-read failure - // #11633 §7 pin 2 names. - const overlayCacheKey: MetaOverlayCacheKey = { - type: request.type, - packageId, + // [#25467 cut 16] The body moved to `protocol-stored-overlay-reads.ts` (see `StoredOverlayReadHost`). + const readEngine = () => this.engine; + const host: ThisParameterType = { + get engine() { return readEngine(); }, + findEnvironmentRows: (...a) => this.findEnvironmentRows(...a), }; - const overlayCacheTtlMs = metaOverlayCacheTtlMs(); - const overlayCacheEpoch = overlayCacheTtlMs > 0 ? readOverlayCacheGeneration(this.engine) : undefined; - const overlayCacheNow = Date.now(); - const cachedRecords = readMetaOverlayCache( - this.engine, overlayCacheKey, overlayCacheEpoch, overlayCacheTtlMs, overlayCacheNow, - ); - - let records: any[]; - if (cachedRecords !== undefined) { - records = cachedRecords as any[]; - } else { - records = await queryActive(); - // ⭐ An EMPTY row set is cached too, and that is the main point - // rather than an edge case: the empty result is what triggers the - // alt-type retry above, so "no overlay rows for this type" is the - // answer whose caching removes BOTH reads. #11633 §1 measured that - // an app whose objects are all code-authored pays the doubled read - // on every request; this is the line that stops it. - writeMetaOverlayCache( - this.engine, overlayCacheKey, overlayCacheEpoch, records, overlayCacheTtlMs, overlayCacheNow, - ); - } - return records; + return readActiveOverlayRows.call(host, request); } /** @@ -10350,33 +10279,11 @@ export class ObjectStackProtocolImplementation implements request: { type: string }, records: any[], ): StoredOverlayEntry[] { - // Parse each overlay body once — replaying the stored-row - // conversion chain (#3903) so every consumer of this list sees - // the canonical protocol shape — and surface its persisted - // software-package binding so the sidebar package filter and - // provenance classification see overlay rows the way they see - // registry items. - return records.map((record) => { - const data = this.convertStoredItem( - String(record.type ?? request.type), - typeof record.metadata === 'string' - ? JSON.parse(record.metadata) - : record.metadata, - ) as any; - const recPkg = (record as { package_id?: string | null }).package_id ?? undefined; - if (recPkg && data && typeof data === 'object' && (data as any)._packageId === undefined) { - (data as any)._packageId = recPkg; - } - // [ADR-0131 D6/D7] Every row is the environment's: the family - // declares no organization column, and the reads that feed this - // ({@link readActiveOverlayRows}) leave a legacy organization's - // rows out off the physical column. (The form doors' legacy layer - // that once needed the row's own scope here is deleted.) - return { - name: String(record.name), data, packageId: recPkg, organizationId: null, - type: String(record.type ?? request.type), - }; - }); + // [#25467 cut 16] The body moved to `protocol-stored-overlay-reads.ts` (see `StoredOverlayReadHost`). + const host: ThisParameterType = { + convertStoredItem: (...a) => this.convertStoredItem(...a), + }; + return storedOverlayEntries.call(host, request, records); } /** @@ -10429,13 +10336,8 @@ export class ObjectStackProtocolImplementation implements * other package served the expansion. */ private namesWithOwnStoredRow(records: readonly any[], packageId?: string): ReadonlySet { - const packages = addressPackages(packageId); - const names = new Set(); - for (const record of records) { - if (typeof record?.name !== 'string') continue; - if (packages.some((pkg) => pkg === undefined || (record?.package_id ?? null) === pkg)) names.add(record.name); - } - return names; + // [#25467 cut 16] The body moved to `protocol-stored-overlay-reads.ts`. + return namesWithOwnStoredRow(records, packageId); } /** @@ -10660,28 +10562,13 @@ export class ObjectStackProtocolImplementation implements * read is per call and never kept, exactly as before. */ private async readItemActiveRows(type: string, name: string, cached = false): Promise { - const other = PLURAL_TO_SINGULAR[type] ?? SINGULAR_TO_PLURAL[type]; - const read = async (): Promise => { - // [#21911] The explicit system opt-in — see findServedOverlayRow. - const rows: StoredOverlayRow[] | null | undefined = await this.findEnvironmentRows('sys_metadata', { - where: { type: other === undefined ? type : { $in: [type, other] }, name, state: 'active' }, - context: { isSystem: true }, - }); - return rows ?? []; + // [#25467 cut 16] The body moved to `protocol-stored-overlay-reads.ts` (see `StoredOverlayReadHost`). + const readEngine = () => this.engine; + const host: ThisParameterType = { + get engine() { return readEngine(); }, + findEnvironmentRows: (...a) => this.findEnvironmentRows(...a), }; - if (!cached) return read(); - // ⭐ The epoch is read BEFORE the read and stored with its rows, as the - // list read's is (see `readActiveOverlayRows`): a write landing while - // this read is in flight leaves the entry dead on arrival. - const key: MetaOverlayCacheKey = { type, name }; - const ttlMs = metaOverlayCacheTtlMs(); - const epoch = ttlMs > 0 ? readOverlayCacheGeneration(this.engine) : undefined; - const now = Date.now(); - const hit = readMetaOverlayCache(this.engine, key, epoch, ttlMs, now); - if (hit !== undefined) return hit as StoredOverlayRow[]; - const rows = await read(); - writeMetaOverlayCache(this.engine, key, epoch, rows, ttlMs, now); - return rows; + return readItemActiveRows.call(host, type, name, cached); } /**