From 78934701f3ba0d509f3428083a150daad1e30249 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 1 Nov 2026 12:46:29 +0000 Subject: [PATCH 1/7] feat(metadata-protocol): move package-less overlay rows into the base's .custom as declared overrides listPackagelessOverlays / migratePackagelessOverlays classify every package-less row of the five overlay types against the code packages that ship its name, and move a movable one into .custom: the pair ensured through the ensure door, the body saved through the save door with overrides { package, basedOn }, read back as the served override, and only then the unbound row deleted by its own address. A pair declaration with no basedOn gains it. Until the package-less write closes, a package-less save landing on a migrated row keeps that row's declaration, with or without overrides in the body. Co-Authored-By: Claude Claude-Session: https://claude.ai/code/session_01FEkQ5pKMJFQjFx8NCVZb62 --- .../protocol-packageless-overlay-migration.ts | 557 ++++++++++++++++++ packages/metadata-protocol/src/protocol.ts | 111 +++- ...ocol-packageless-overlay-migration.test.ts | 386 ++++++++++++ 3 files changed, 1053 insertions(+), 1 deletion(-) create mode 100644 packages/metadata-protocol/src/protocol-packageless-overlay-migration.ts create mode 100644 packages/objectql/src/protocol-packageless-overlay-migration.test.ts diff --git a/packages/metadata-protocol/src/protocol-packageless-overlay-migration.ts b/packages/metadata-protocol/src/protocol-packageless-overlay-migration.ts new file mode 100644 index 00000000000..de74e1b0bfd --- /dev/null +++ b/packages/metadata-protocol/src/protocol-packageless-overlay-migration.ts @@ -0,0 +1,557 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * # Package-less overlay rows become declared overrides (ADR-0146 D11, N-16a) + * + * Before ADR-0146, a Setup "customize" of a view, dashboard, report, + * translation or email template a code package ships was stored as a + * `sys_metadata` row bound to no package (`package_id` NULL, ADR-0005's + * environment layer, open for the five `allowEnvironmentOverlay` types). Such + * a row never exports and never reaches another environment. ADR-0146 D11 + * moves each one into the base's paired `.custom` package as a DECLARED + * override — `overrides: { package: , basedOn: }` + * (D2, as amended by #28486) — on the administrator's command: the + * `os migrate packageless-overlays` ceremony (`@objectstack/cli`), which calls + * {@link listPackagelessOverlays} and {@link migratePackagelessOverlays}. + * ⛔ Not a boot step. + * + * ## The population, and what is left alone + * + * Every environment row (a legacy organization's rows are left out, + * {@link ObjectStackProtocolImplementation.findEnvironmentRows}) of one of the + * five overlay types, either spelling, `active` or `draft`, bound to no package. + * Each is classified against the code packages that ship its name — the + * protocol's own reading (`shippedArtifactsOf`, the one the by-name reads and + * the write gate take), ⛔ never a second one: + * + * - `environment` — no package ships the name: environment metadata, not an + * overlay (ADR-0146 D11; ADR-0070 D5 keeps its own treatment). Not listed. + * - `multi_shipped` — several packages ship it: listed for the administrator, + * never guessed. + * - `no_pair` — the one package that ships it has an id no pair can be + * formed from (`customizationPackageId` refuses it): listed, not moved. + * + * "Ships" counts read-only code packages only, as the write gate does: a + * writable package's item is edited in place (ADR-0070), and the pair itself + * is writable. + * - `foreign_declaration` — the body already carries an `overrides` key that + * does not name the base: listed, never rewritten. + * - `shadowed` — the pair already holds a row of that name in that state (a + * Studio customization made first, N-17): listed with the two ways out, + * never merged or deleted. + * - `movable` — everything else. `--apply` moves it. + * + * Permission sets, positions and capabilities are not overlay types, so they + * are never in the population (ADR-0146 D6; their own ceremony is + * `os migrate security-catalog-overlays`). + * + * ## The move, one row at a time + * + * 1. The pair is ensured through the ensure door, + * {@link ObjectStackProtocolImplementation.ensureCustomizationPackage} — + * the only creation path (D3). One call per base. + * 2. The row's body gains `overrides: { package, basedOn }`, `basedOn` the + * installed base's `manifest.version` now, and is saved into the pair + * through {@link ObjectStackProtocolImplementation.saveMetaItem} — the + * override write gate (N-09) judges it like any other save — in the row's + * own state (`draft` stays a draft). + * 3. The pair's row is read back and must be the served override of the base + * ({@link isServedOverrideRowOf}, N-10's one predicate). + * 4. Only then is the unbound row deleted, by its own address — `package_id` + * NULL, through the repository beneath the delete door (the door naming no + * package now addresses the pair's row: `packagelessAddress`, sole binding). + * The delete writes its history tombstone in the same transaction. + * + * A failure at any step leaves the unbound row where it is and is that row's + * `failed` outcome; the next row is still tried. A second run finds nothing to + * move (idempotent). + * + * ## …and the stored declarations missing `basedOn` + * + * A row of a pair whose body declares `overrides: { package: }` with no + * `basedOn` is counted as a declaration by the read ({@link isServedOverrideRowOf} + * does not judge `basedOn`) and refused on its next save by the write gate + * (`OverrideDeclarationSchema` requires it). The same ceremony completes it: + * `basedOn` = the installed base's version now, saved through the same door. + */ + +import { + customizationBasePackageId, + customizationPackageId, +} from '@objectstack/spec/kernel'; +import { PLURAL_TO_SINGULAR } from '@objectstack/spec/shared'; +import { storedRowDocument } from './item-lock.js'; +import { OVERLAY_MEMBER_COLUMN, overlayMemberValue } from './migrations/overlay-index.js'; +import type { ObjectStackProtocolImplementation } from './protocol.js'; +import { shippingPackageIds } from './protocol-environment-rows.js'; +import { isServedOverrideRowOf } from './served-override.js'; +import { SysMetadataRepository } from './sys-metadata-repository.js'; + +/** The class members this module reads, typed as the class declares them. */ +export interface PackagelessOverlayMigrationHost { + readonly engine: ObjectStackProtocolImplementation['engine']; + /** The five `allowEnvironmentOverlay` types, both spellings (the class's static set). */ + readonly overlayTypes: ReadonlySet; + readonly findEnvironmentRows: ObjectStackProtocolImplementation['findEnvironmentRows']; + readonly rethrowUnlessMetadataStoreUnprovisioned: ObjectStackProtocolImplementation['rethrowUnlessMetadataStoreUnprovisioned']; + readonly shippedArtifactsOf: ObjectStackProtocolImplementation['shippedArtifactsOf']; + readonly isWritablePackage: ObjectStackProtocolImplementation['isWritablePackage']; + readonly ensureCustomizationPackage: ObjectStackProtocolImplementation['ensureCustomizationPackage']; + readonly saveMetaItem: ObjectStackProtocolImplementation['saveMetaItem']; +} + +/** How the ceremony classified one package-less row (module header, "The population"). */ +export type PackagelessOverlayDisposition = + | 'movable' + | 'multi_shipped' + | 'no_pair' + | 'foreign_declaration' + | 'shadowed'; + +/** One listed package-less row. `environment` rows are counted, not listed. */ +export interface PackagelessOverlayRow { + /** The `sys_metadata` row id. */ + readonly id: string; + /** The type as stored (either spelling). */ + readonly storedType: string; + /** The canonical singular type. */ + readonly type: string; + readonly name: string; + readonly state: 'active' | 'draft'; + /** The bundle member, for a type whose identity is a pair; absent otherwise. */ + readonly member?: string; + readonly disposition: PackagelessOverlayDisposition; + /** The code packages that ship the name, sorted. */ + readonly shippers: readonly string[]; + /** The base the row overlays — the one shipper — when there is exactly one. */ + readonly base?: string; + /** The base's paired package, when the base has one. */ + readonly pair?: string; + /** `shadowed` only: whether the pair's row declares the override of the base. */ + readonly shadowDeclares?: boolean; +} + +/** One pair row whose declaration lacks `basedOn` (module header, last section). */ +export interface IncompleteOverrideDeclarationRow { + readonly id: string; + readonly storedType: string; + readonly type: string; + readonly name: string; + readonly state: 'active' | 'draft'; + readonly member?: string; + readonly base: string; + readonly pair: string; +} + +/** The census {@link listPackagelessOverlays} answers. */ +export interface PackagelessOverlayCensus { + /** Every listed package-less row, ordered by type, name, member, state. */ + readonly rows: readonly PackagelessOverlayRow[]; + /** How many package-less rows of a name no package ships were left as environment metadata. */ + readonly environmentRows: number; + /** Every pair row declaring an override with no `basedOn`. */ + readonly incompleteDeclarations: readonly IncompleteOverrideDeclarationRow[]; +} + +/** What {@link migratePackagelessOverlays} did with one movable row or one incomplete declaration. */ +export type PackagelessOverlayOutcome = + | { readonly id: string; readonly outcome: 'moved' | 'declaration_completed'; readonly packageId: string; readonly basedOn: string } + | { readonly id: string; readonly outcome: 'failed'; readonly error: string }; + +/** One base's line of the report. */ +export interface PackagelessOverlayBaseReport { + readonly base: string; + readonly pair: string; + /** Whether this run created the pair (the ensure door's `created`). */ + readonly pairCreated: boolean; + readonly moved: number; + readonly declarationsCompleted: number; + readonly failed: number; + readonly shadowed: number; +} + +/** The answer of {@link migratePackagelessOverlays}: the census it acted on, and what it did. */ +export interface PackagelessOverlayMigration extends PackagelessOverlayCensus { + readonly outcomes: readonly PackagelessOverlayOutcome[]; + readonly bases: readonly PackagelessOverlayBaseReport[]; +} + +const SYSTEM = { isSystem: true } as const; + +function memberOf(row: any): string | undefined { + const member = row?.[OVERLAY_MEMBER_COLUMN]; + return typeof member === 'string' && member !== '' ? member : undefined; +} + +function stateOf(row: any): 'active' | 'draft' | undefined { + return row?.state === 'active' || row?.state === 'draft' ? row.state : undefined; +} + +function isUnbound(row: any): boolean { + return row?.package_id === null || row?.package_id === undefined || row?.package_id === ''; +} + +function bodyOf(row: any): Record | undefined { + try { + const body = storedRowDocument(row); + return body !== null && typeof body === 'object' && !Array.isArray(body) + ? (body as Record) + : undefined; + } catch { + return undefined; + } +} + +function messageOf(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} + +/** One item's identity across spellings and packages: `type|name|member|state`. */ +const identityKey = (type: string, name: string, member: string | undefined, state: string): string => + `${type}\u0000${name}\u0000${member ?? ''}\u0000${state}`; + +/** The rows of the five overlay types, either spelling, the environment's only. */ +async function overlayRows(this: Pick): Promise { + let rows: any[] = []; + try { + rows = await this.findEnvironmentRows('sys_metadata', { + where: { type: { $in: [...this.overlayTypes].sort() } }, + context: SYSTEM, + }); + } catch (error) { + // [#5532] An unprovisioned store holds no row; any other failure is a + // read that did not happen, never "nothing to migrate". + this.rethrowUnlessMetadataStoreUnprovisioned(error, 'sys_metadata'); + } + return (rows ?? []).filter((row) => typeof row?.name === 'string' && row.name !== '' && stateOf(row) !== undefined); +} + +/** The base version a declaration records: the installed base's `manifest.version`, or `undefined`. */ +function baseVersionOf(host: Pick, base: string): string | undefined { + const pkg = (host.engine?.registry as { getPackage?: (id: string) => any } | undefined)?.getPackage?.(base); + const version = pkg?.manifest?.version; + return typeof version === 'string' && version !== '' ? version : undefined; +} + +/** + * The census (module header, "The population"): every package-less overlay + * row classified, and every pair row whose declaration lacks `basedOn`. + * Writes nothing. + * + * @throws the `sys_metadata` read failure, classified as every overlay read + * classifies it; an unprovisioned table answers an empty census. + */ +export async function listPackagelessOverlays( + this: Pick, +): Promise { + const rows = await overlayRows.call(this); + // What every pair holds, per item identity and state. + const pairRows = new Map(); + for (const row of rows) { + if (isUnbound(row) || customizationBasePackageId(row.package_id) === null) continue; + const type = PLURAL_TO_SINGULAR[row.type] ?? row.type; + pairRows.set(`${row.package_id}\u0000${identityKey(type, row.name, memberOf(row), row.state)}`, row); + } + + const listed: PackagelessOverlayRow[] = []; + let environmentRows = 0; + for (const row of rows) { + if (!isUnbound(row)) continue; + const type = PLURAL_TO_SINGULAR[row.type] ?? row.type; + const name = String(row.name); + const state = stateOf(row)!; + const member = memberOf(row); + // The read-only code packages that ship the name: a writable package's + // item is edited in place (ADR-0070) and has no pair (ADR-0146 D3), and + // the pair itself is writable — the write gate's own narrowing. + const shippers = [...shippingPackageIds(this.shippedArtifactsOf(type, name))] + .filter((id) => !this.isWritablePackage(id)) + .sort(); + if (shippers.length === 0) { + environmentRows += 1; + continue; + } + const common = { + id: String(row.id), + storedType: String(row.type), + type, + name, + state, + ...(member !== undefined ? { member } : {}), + shippers, + }; + if (shippers.length > 1) { + listed.push({ ...common, disposition: 'multi_shipped' }); + continue; + } + const base = shippers[0]!; + const pairId = customizationPackageId(base); + if (!pairId.ok) { + listed.push({ ...common, base, disposition: 'no_pair' }); + continue; + } + const pair = pairId.value; + const declared = (bodyOf(row) as { overrides?: unknown } | undefined)?.overrides; + if (declared !== undefined && (declared === null || typeof declared !== 'object' + || (declared as { package?: unknown }).package !== base)) { + listed.push({ ...common, base, pair, disposition: 'foreign_declaration' }); + continue; + } + const shadow = pairRows.get(`${pair}\u0000${identityKey(type, name, member, state)}`); + if (shadow !== undefined) { + listed.push({ ...common, base, pair, disposition: 'shadowed', shadowDeclares: isServedOverrideRowOf(shadow, base) }); + continue; + } + listed.push({ ...common, base, pair, disposition: 'movable' }); + } + + const incompleteDeclarations: IncompleteOverrideDeclarationRow[] = []; + for (const row of pairRows.values()) { + const base = customizationBasePackageId(row.package_id); + if (base === null || !isServedOverrideRowOf(row, base)) continue; + const basedOn = (bodyOf(row) as { overrides?: { basedOn?: unknown } } | undefined)?.overrides?.basedOn; + if (typeof basedOn === 'string' && basedOn !== '') continue; + const member = memberOf(row); + incompleteDeclarations.push({ + id: String(row.id), + storedType: String(row.type), + type: PLURAL_TO_SINGULAR[row.type] ?? row.type, + name: String(row.name), + state: stateOf(row)!, + ...(member !== undefined ? { member } : {}), + base, + pair: String(row.package_id), + }); + } + + const order = (a: { type: string; name: string; member?: string; state: string; id: string }, b: typeof a) => + a.type.localeCompare(b.type) || a.name.localeCompare(b.name) + || (a.member ?? '').localeCompare(b.member ?? '') || a.state.localeCompare(b.state) || a.id.localeCompare(b.id); + return { + rows: listed.sort(order), + environmentRows, + incompleteDeclarations: incompleteDeclarations.sort(order), + }; +} + +/** The pair's row of an item, as stored now (read back after a write). */ +async function storedPairRow( + host: Pick, + item: { type: string; name: string; state: string; member?: string }, + pair: string, +): Promise { + const rows = await host.findEnvironmentRows('sys_metadata', { + where: { type: item.type, name: item.name, state: item.state, package_id: pair }, + context: SYSTEM, + }); + return (rows ?? []).find((row) => memberOf(row) === item.member); +} + +/** + * Move every `movable` row of the census into its base's pair, and complete + * every declaration missing `basedOn` (module header, "The move"). One outcome + * per acted-on row; never throws for a row. + * + * @param request.actor recorded on every write (the ceremony's own name). + */ +export async function migratePackagelessOverlays( + this: PackagelessOverlayMigrationHost, + request: { actor: string }, +): Promise { + const census = await listPackagelessOverlays.call(this); + const outcomes: PackagelessOverlayOutcome[] = []; + const bases = new Map(); + const lineOf = (base: string, pair: string) => { + let line = bases.get(base); + if (line === undefined) { + line = { pair, pairCreated: false, moved: 0, declarationsCompleted: 0, failed: 0, shadowed: 0 }; + bases.set(base, line); + } + return line; + }; + for (const row of census.rows) { + if (row.disposition === 'shadowed' && row.base && row.pair) lineOf(row.base, row.pair).shadowed += 1; + } + + // Step 1, once per base: the ensure door. A refusal fails that base's rows. + const ensured = new Map(); + const ensure = async (base: string, pair: string) => { + let answer = ensured.get(base); + if (answer === undefined) { + try { + const result = await this.ensureCustomizationPackage({ basePackageId: base }); + if (result.created) lineOf(base, pair).pairCreated = true; + answer = { ok: true }; + } catch (error) { + answer = { ok: false, error: `The customization package '${pair}' could not be ensured: ${messageOf(error)}` }; + } + ensured.set(base, answer); + } + return answer; + }; + const repository = new SysMetadataRepository({ engine: this.engine as never, orgLabel: 'env' }); + const rowsById = new Map(); + for (const row of await overlayRows.call(this)) rowsById.set(String(row.id), row); + + for (const row of census.rows) { + if (row.disposition !== 'movable' || row.base === undefined || row.pair === undefined) continue; + const line = lineOf(row.base, row.pair); + const fail = (error: string) => { + line.failed += 1; + outcomes.push({ id: row.id, outcome: 'failed', error }); + }; + const ensuredPair = await ensure(row.base, row.pair); + if (!ensuredPair.ok) { fail(ensuredPair.error); continue; } + const basedOn = baseVersionOf(this, row.base); + if (basedOn === undefined) { fail(`The installed package '${row.base}' declares no version, so no basedOn can be recorded.`); continue; } + const stored = rowsById.get(row.id); + const body = stored === undefined ? undefined : bodyOf(stored); + if (body === undefined) { fail('The stored body could not be read as a document.'); continue; } + try { + // Step 2: the declared body, saved into the pair through the save door. + await this.saveMetaItem({ + type: row.type, + name: row.name, + item: { ...body, overrides: { package: row.base, basedOn } }, + packageId: row.pair, + mode: row.state === 'draft' ? 'draft' : 'publish', + actor: request.actor, + } as never); + // Step 3: read back — the pair's row must be the served override. + const written = await storedPairRow(this, row, row.pair); + if (!isServedOverrideRowOf(written, row.base)) { + fail(`The row saved into '${row.pair}' does not read back as the declared override of '${row.base}'; the unbound row is kept.`); + continue; + } + // Step 4: only now, the unbound row, by its own address. + await repository.delete( + { type: row.storedType, name: row.name, org: 'env' } as never, + { + parentVersion: (typeof stored.checksum === 'string' ? stored.checksum : null) as string, + actor: request.actor, + source: request.actor, + message: `Moved into '${row.pair}' as a declared override of '${row.base}' (ADR-0146 D11)`, + intent: 'override-artifact', + state: row.state, + packageId: null, + ...(row.member !== undefined ? { member: row.member } : {}), + } as never, + ); + line.moved += 1; + outcomes.push({ id: row.id, outcome: 'moved', packageId: row.pair, basedOn }); + } catch (error) { + fail(messageOf(error)); + } + } + + for (const decl of census.incompleteDeclarations) { + const line = lineOf(decl.base, decl.pair); + const fail = (error: string) => { + line.failed += 1; + outcomes.push({ id: decl.id, outcome: 'failed', error }); + }; + const basedOn = baseVersionOf(this, decl.base); + if (basedOn === undefined) { fail(`The installed package '${decl.base}' declares no version, so no basedOn can be recorded.`); continue; } + const stored = rowsById.get(decl.id); + const body = stored === undefined ? undefined : bodyOf(stored); + if (body === undefined) { fail('The stored body could not be read as a document.'); continue; } + try { + await this.saveMetaItem({ + type: decl.type, + name: decl.name, + item: { ...body, overrides: { ...(body.overrides as Record), basedOn } }, + packageId: decl.pair, + mode: decl.state === 'draft' ? 'draft' : 'publish', + actor: request.actor, + } as never); + const written = await storedPairRow(this, decl, decl.pair); + const recorded = (bodyOf(written) as { overrides?: { basedOn?: unknown } } | undefined)?.overrides?.basedOn; + if (!isServedOverrideRowOf(written, decl.base) || recorded !== basedOn) { + fail(`The row of '${decl.pair}' does not read back with basedOn '${basedOn}'.`); + continue; + } + line.declarationsCompleted += 1; + outcomes.push({ id: decl.id, outcome: 'declaration_completed', packageId: decl.pair, basedOn }); + } catch (error) { + fail(messageOf(error)); + } + } + + return { + ...census, + outcomes, + bases: [...bases.entries()] + .map(([base, line]) => ({ base, ...line })) + .sort((a, b) => a.base.localeCompare(b.base)), + }; +} + +// ───────────────────────────────────────────────────────────────────────── +// Until #28576 closes the package-less write: a package-less save landing on +// a migrated row keeps that row's declaration (N-16a, Wanted 4) +// ───────────────────────────────────────────────────────────────────────── + +/** The class members {@link packagelessPairLanding} reads. */ +export interface PackagelessPairLandingHost { + readonly overlayTypes: ReadonlySet; + readonly packagelessAddress: ObjectStackProtocolImplementation['packagelessAddress']; + readonly packagelessDraftAddress: ObjectStackProtocolImplementation['packagelessDraftAddress']; + readonly findEnvironmentRows: ObjectStackProtocolImplementation['findEnvironmentRows']; +} + +/** + * The declared override a package-less save of an overlay type lands on: when + * the save's own address — {@link ObjectStackProtocolImplementation.packagelessAddress} + * for a publish-mode save, {@link ObjectStackProtocolImplementation.packagelessDraftAddress}'s + * `into` for a draft — is a base's paired `.custom`, and that pair's row + * of the item (the draft's own row first for a draft save, then the active + * row) is the served override of the base ({@link isServedOverrideRowOf}), + * answers the pair, the base and the row's `overrides` block. Otherwise + * `undefined`: nothing changes for the save. + * + * Why the save asks it. After `os migrate packageless-overlays` moves Setup's + * customization into the pair, Setup still saves and resets it naming no + * package, and the package-less address lands that save on the pair's row + * (sole binding). Setup's form sends the item as it read it — its `overrides` + * key included, or not, depending on the editor — and a pair row saved without + * its declaration is refused by the override write gate as an undeclared same + * name (ADR-0146 D2, N-09). So the save keeps the row's declaration when the + * body carries none, and judges a body that carries one as a write into the + * pair, which is where it lands. #28576 closes the package-less write; this + * goes with it. + */ +export async function packagelessPairLanding( + this: PackagelessPairLandingHost, + request: { type: string; name: string; mode?: string }, + member: string | undefined, +): Promise<{ pair: string; base: string; declaration: Record } | undefined> { + const type = PLURAL_TO_SINGULAR[request.type] ?? request.type; + if (!this.overlayTypes.has(type)) return undefined; + const draft = request.mode === 'draft'; + let into: string | null; + if (draft) { + const address = await this.packagelessDraftAddress(type, request.name, member); + if (address.kind === 'ambiguous') return undefined; + into = address.into; + } else { + const address = await this.packagelessAddress(type, request.name, member, 'active'); + into = address.candidates.length === 1 ? address.packageId : null; + } + const base = typeof into === 'string' ? customizationBasePackageId(into) : null; + if (base === null || into === null) return undefined; + const rows = await this.findEnvironmentRows('sys_metadata', { + where: { type, name: request.name, package_id: into }, + context: SYSTEM, + }); + // The member as the row stores it: `null` (absent here) for the canonical one. + const storedMember = overlayMemberValue(type, member) ?? undefined; + const at = (state: 'active' | 'draft') => + (rows ?? []).find((row) => row?.state === state && memberOf(row) === storedMember); + for (const row of draft ? [at('draft'), at('active')] : [at('active')]) { + if (row === undefined || !isServedOverrideRowOf(row, base)) continue; + const declaration = (bodyOf(row) as { overrides?: unknown } | undefined)?.overrides; + if (declaration === null || typeof declaration !== 'object' || Array.isArray(declaration)) continue; + return { pair: into, base, declaration: { ...(declaration as Record) } }; + } + return undefined; +} diff --git a/packages/metadata-protocol/src/protocol.ts b/packages/metadata-protocol/src/protocol.ts index b2b42b025ec..29f2529bfc5 100644 --- a/packages/metadata-protocol/src/protocol.ts +++ b/packages/metadata-protocol/src/protocol.ts @@ -185,6 +185,15 @@ import { deriveNamespaceFromPackageId } from '@objectstack/spec/kernel'; import { customizationPackageId } from '@objectstack/spec/kernel'; import { isServedOverrideRowOf } from './served-override.js'; import { listServedOverrides, type ServedOverrideEntry } from './protocol-served-overrides.js'; +// [#28413, ADR-0146 D11] The package-less overlay migration (`os migrate +// packageless-overlays`), and the save's landing on a migrated row. +import { + listPackagelessOverlays, + migratePackagelessOverlays, + packagelessPairLanding, + type PackagelessOverlayCensus, + type PackagelessOverlayMigration, +} from './protocol-packageless-overlay-migration.js'; // [#19417] The package-id declaration and its remedy text, imported to be // SURFACED rather than restated — see the gate at the top of `installPackage`. import { ManifestSchema, manifestIdRefusal } from '@objectstack/spec/kernel'; @@ -9770,6 +9779,79 @@ export class ObjectStackProtocolImplementation implements return listServedOverrides.call(host, request); } + /** + * [#28413, ADR-0146 D11] The census `os migrate packageless-overlays` + * previews: every environment `sys_metadata` row of one of the five + * `allowEnvironmentOverlay` types bound to no package, classified against + * the code packages that ship its name — `movable` (exactly one ships it, + * and its pair holds no row of the name in that state), `shadowed` (the + * pair already does), `multi_shipped`, `foreign_declaration` or `no_pair` + * — plus the count of rows of a name no package ships (environment + * metadata, left alone), and every pair row declaring an override with no + * `basedOn`. Writes nothing; the body and its rules are + * `protocol-packageless-overlay-migration.ts`'s. + * + * @throws the `sys_metadata` read failure, classified by + * {@link rethrowUnlessMetadataStoreUnprovisioned}; an unprovisioned + * table answers an empty census. + */ + async listPackagelessOverlays(): Promise { + const host: ThisParameterType = { + overlayTypes: ObjectStackProtocolImplementation.OVERLAY_ALLOWED_TYPES, + findEnvironmentRows: (...a) => this.findEnvironmentRows(...a), + rethrowUnlessMetadataStoreUnprovisioned: (...a) => this.rethrowUnlessMetadataStoreUnprovisioned(...a), + shippedArtifactsOf: (...a) => this.shippedArtifactsOf(...a), + isWritablePackage: (...a) => this.isWritablePackage(...a), + }; + return listPackagelessOverlays.call(host); + } + + /** + * [#28413, ADR-0146 D11] `os migrate packageless-overlays --apply`: move + * each `movable` row of {@link listPackagelessOverlays}'s census into its + * base's paired `.custom` as a declared override — the pair ensured + * through {@link ensureCustomizationPackage}, the body given + * `overrides: { package: , basedOn: }` and + * saved through {@link saveMetaItem} in the row's own state, read back as + * the served override (`isServedOverrideRowOf`), and only then the unbound + * row deleted by its own address — and complete every pair declaration + * missing `basedOn` the same way. One outcome per row, never thrown for a + * row; a second run moves nothing. ⛔ Not a boot step. + */ + async migratePackagelessOverlays(request: { actor: string }): Promise { + const readEngine = () => this.engine; + const host: ThisParameterType = { + get engine() { return readEngine(); }, + overlayTypes: ObjectStackProtocolImplementation.OVERLAY_ALLOWED_TYPES, + findEnvironmentRows: (...a) => this.findEnvironmentRows(...a), + rethrowUnlessMetadataStoreUnprovisioned: (...a) => this.rethrowUnlessMetadataStoreUnprovisioned(...a), + shippedArtifactsOf: (...a) => this.shippedArtifactsOf(...a), + isWritablePackage: (...a) => this.isWritablePackage(...a), + ensureCustomizationPackage: (...a) => this.ensureCustomizationPackage(...a), + saveMetaItem: (...a) => this.saveMetaItem(...a), + }; + return migratePackagelessOverlays.call(host, request); + } + + /** + * [#28413] The declared override a package-less save of an overlay type + * lands on, or `undefined` — see `packagelessPairLanding` in + * `protocol-packageless-overlay-migration.ts`. Asked by + * {@link saveMetaItem} before the override write gate. + */ + private packagelessPairLanding( + request: { type: string; name: string; mode?: string }, + member: string | undefined, + ): Promise<{ pair: string; base: string; declaration: Record } | undefined> { + const host: ThisParameterType = { + overlayTypes: ObjectStackProtocolImplementation.OVERLAY_ALLOWED_TYPES, + packagelessAddress: (...a) => this.packagelessAddress(...a), + packagelessDraftAddress: (...a) => this.packagelessDraftAddress(...a), + findEnvironmentRows: (...a) => this.findEnvironmentRows(...a), + }; + return packagelessPairLanding.call(host, request, member); + } + /** * The one body behind {@link getMetaItems}, * {@link getMetaItemsForExecution} and {@link getMetaItemsPerPackage}. @@ -20965,7 +21047,34 @@ export class ObjectStackProtocolImplementation implements // door already answered for that write, so no existing refusal's code // or status changes (a view container's name collision keeps its // 400, a lock its ITEM_LOCKED). - const overrideVerdict = this.overrideWriteVerdict(request); + // + // [#28413] Until #28576 closes the package-less write: a package-less + // authoring save whose address is a pair row declaring the override of + // its base (a row `os migrate packageless-overlays` moved there) keeps + // that row's `overrides` block when the body carries none, and is + // judged as the write into the pair it is — so Setup's save of a + // migrated item, which sends the item as it read it, with or without + // the declaration, is not refused as undeclared (or as a package-less + // declaration). See `packagelessPairLanding`. + let overrideSubject: OverrideWriteSubject = request; + if ( + request.packageId == null + && request.source === undefined + && request.writeFace !== 'package-duplicate' + && request.item !== null && typeof request.item === 'object' && !Array.isArray(request.item) + ) { + const landingType = PLURAL_TO_SINGULAR[request.type] ?? request.type; + const landing = await this.packagelessPairLanding( + request, itemDiscriminator(landingType, request.item), + ); + if (landing !== undefined) { + if (!('overrides' in (request.item as Record))) { + request = { ...request, item: { ...(request.item as Record), overrides: landing.declaration } }; + } + overrideSubject = { ...request, packageId: landing.pair }; + } + } + const overrideVerdict = this.overrideWriteVerdict(overrideSubject); if (overrideVerdict.verdict === 'refuse' && overrideVerdict.rule !== 'undeclared_same_name') { throw overrideWriteRefusalError(overrideVerdict); } diff --git a/packages/objectql/src/protocol-packageless-overlay-migration.test.ts b/packages/objectql/src/protocol-packageless-overlay-migration.test.ts new file mode 100644 index 00000000000..95e1b06c7fd --- /dev/null +++ b/packages/objectql/src/protocol-packageless-overlay-migration.test.ts @@ -0,0 +1,386 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * [#28413, ADR-0146 D11, N-16a] A package-less overlay row of a name a code + * package X ships moves into X's paired `.custom` as a declared override — + * `listPackagelessOverlays` / `migratePackagelessOverlays`, the protocol half + * of `os migrate packageless-overlays` — and, until #28576 closes the + * package-less write, a package-less save landing on the moved row keeps its + * declaration. + * + * Pinned through the real protocol over the real `ObjectQL` engine and an + * in-memory driver (the harness of `protocol-custom-override-effective-read.test.ts`): + * + * - THE MOVE: the row lands in the pair, created through the ensure door, + * declaring `{ package: X, basedOn: X's version }`, in its own state; the + * unbound row is gone; the five reads serve the same content as before; a + * re-run moves 0; + * - LEFT ALONE: a name no package ships, a name two packages ship, and a row + * the pair already overrides (`shadowed`), each untouched; + * - a stored pair declaration with no `basedOn` gains it (#28644's reading); + * - THE SAVE (Wanted 4): after the move, a package-less draft save and publish + * land on the pair's row with or without `overrides` in the body, and the + * item stays the declared override; a package-less DELETE removes the pair's + * row and X's document is served again; + * - ADOPT-ORPHANS (Wanted 5): adopting into a writable package skips the + * shipped-name row and rebinds an environment one. + * + * Test-path resolution: the protocol is `@objectstack/metadata-protocol`'s + * `dist/`, so a change there is read here after that package is rebuilt. + */ +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import type { ServiceObject } from '@objectstack/spec/data'; +import { + ObjectStackProtocolImplementation, + isServedOverrideRowOf, + resetEnvWritableMetadataTypes, +} from '@objectstack/metadata-protocol'; +import { SysMetadataAuditObject, SysMetadataCommitObject, SysMetadataHistoryObject } from '@objectstack/metadata-core'; +import { ObjectQL } from './engine.js'; + +const BASE = 'com.example.showcase'; +const PAIR = 'com.example.showcase.custom'; +const OTHER = 'com.example.other'; +const WORKSPACE = 'com.default.workspace'; +const VIEW = 'showcase_task.default'; +const SHARED_VIEW = 'shared_task.default'; +const ENV_VIEW = 'env_only.default'; +const BASE_COLUMNS = ['title', 'project', 'assignee', 'status', 'priority', 'due_date']; +const UNBOUND_COLUMNS = ['title', 'priority']; +const EDITED_COLUMNS = ['title', 'status']; +const ACTOR = 'os migrate packageless-overlays'; + +const sysMetadataObject: ServiceObject = { + name: 'sys_metadata', + label: 'System Metadata', + fields: { + id: { name: 'id', label: 'ID', type: 'text' as const }, + type: { name: 'type', label: 'Type', type: 'text' as const, required: true }, + name: { name: 'name', label: 'Name', type: 'text' as const, required: true }, + organization_id: { name: 'organization_id', label: 'Org', type: 'text' as const }, + package_id: { name: 'package_id', label: 'Package', type: 'text' as const }, + bundle_member: { name: 'bundle_member', label: 'Bundle Member', type: 'text' as const, maxLength: 100 }, + metadata: { name: 'metadata', label: 'Body', type: 'textarea' as const }, + draft_changes: { name: 'draft_changes', label: 'Draft Changes', type: 'textarea' as const }, + checksum: { name: 'checksum', label: 'Checksum', type: 'text' as const, maxLength: 71 }, + state: { name: 'state', label: 'State', type: 'text' as const }, + version: { name: 'version', label: 'Version', type: 'number' as const }, + created_at: { name: 'created_at', label: 'Created', type: 'datetime' as const }, + updated_at: { name: 'updated_at', label: 'Updated', type: 'datetime' as const }, + }, +}; + +function makeMemoryDriver() { + const stores = new Map>>(); + const storeFor = (obj: string) => { + let s = stores.get(obj); + if (!s) { s = new Map(); stores.set(obj, s); } + return s; + }; + let nextId = 0; + const matchesWhere = (row: Record, where: any): boolean => { + if (!where || typeof where !== 'object') return true; + for (const [k, v] of Object.entries(where)) { + if (k === '$and' && Array.isArray(v)) { + if (!v.every((w: any) => matchesWhere(row, w))) return false; + continue; + } + if (k === '$or' && Array.isArray(v)) { + if (!v.some((w: any) => matchesWhere(row, w))) return false; + continue; + } + if (k.startsWith('$')) continue; + const rowVal = row[k]; + if (v && typeof v === 'object' && Array.isArray((v as any).$in)) { + if (!(v as any).$in.includes(rowVal === undefined ? null : rowVal)) return false; + continue; + } + const expected = (v && typeof v === 'object' && '$eq' in (v as any)) ? (v as any).$eq : v; + const a = rowVal === undefined ? null : rowVal; + const b = expected === undefined ? null : expected; + if (a !== b) return false; + } + return true; + }; + const driver: any = { + name: 'memory', version: '0.0.0', supports: {} as any, + async connect() {}, async disconnect() {}, async checkHealth() { return true; }, + async execute() { return null; }, + async find(object: string, ast: any) { + const rows = Array.from(storeFor(object).values()).filter((r) => matchesWhere(r, ast?.where)); + return typeof ast?.limit === 'number' ? rows.slice(0, ast.limit) : rows; + }, + async findOne(object: string, ast: any) { + for (const r of storeFor(object).values()) if (matchesWhere(r, ast?.where)) return r; + return null; + }, + async create(object: string, data: Record) { + nextId += 1; + const id = (data.id as string) ?? `r_${nextId}`; + const row = { ...data, id }; + storeFor(object).set(id, row); + return row; + }, + async update(object: string, id: string, data: Record) { + const s = storeFor(object); + const cur = s.get(id); + if (!cur) throw new Error(`not found: ${object}/${id}`); + const updated = { ...cur, ...data, id }; + s.set(id, updated); + return updated; + }, + async upsert(object: string, data: Record) { + const id = data.id as string | undefined; + if (id && storeFor(object).has(id)) return this.update(object, id, data); + return this.create(object, data); + }, + async delete(object: string, id: string) { return storeFor(object).delete(id); }, + async count(object: string, ast: any) { return (await this.find(object, ast)).length; }, + async bulkCreate(object: string, rows: Record[]) { + return Promise.all(rows.map((r) => this.create(object, r))); + }, + async bulkUpdate() { return []; }, async bulkDelete() {}, + async beginTransaction() { return { commit: async () => {}, rollback: async () => {} }; }, + async commit() {}, async rollback() {}, + }; + return { driver, stores }; +} +const listBody = (name: string, object: string, columns: readonly string[]) => ({ + name, + object, + viewKind: 'list', + label: 'All', + config: { + type: 'grid', + data: { provider: 'object', object }, + columns: columns.map((field) => ({ field })), + }, +}); + +const objectOf = (name: string, fields: readonly string[]) => ({ + name, + label: name, + sharingModel: 'private', + fields: Object.fromEntries(fields.map((f) => [f, { name: f, label: f, type: 'text' }])), +}); +const listView = (object: string, columns: readonly string[]) => ({ + object, + list: { label: 'All', type: 'grid', data: { provider: 'object', object }, columns: columns.map((field) => ({ field })) }, +}); + +async function boot() { + const engine = new ObjectQL(); + const { driver, stores } = makeMemoryDriver(); + engine.registerDriver(driver, true); + await engine.init(); + engine.registry.registerObject(sysMetadataObject, 'test-package'); + for (const o of [SysMetadataHistoryObject, SysMetadataAuditObject, SysMetadataCommitObject]) { + engine.registry.registerObject(o as any, 'test-package'); + } + // X ships `showcase_task.default` and `shared_task.default`; another code + // package ships `shared_task.default` too (a name two packages ship). + engine.registerApp({ + id: BASE, name: 'showcase', version: '1.2.0', namespace: 'showcase', + objects: [objectOf('showcase_task', BASE_COLUMNS), objectOf('shared_task', BASE_COLUMNS)], + views: [listView('showcase_task', BASE_COLUMNS), listView('shared_task', BASE_COLUMNS)], + }); + engine.registerApp({ + id: OTHER, name: 'other', version: '2.0.0', namespace: 'other', + views: [listView('shared_task', BASE_COLUMNS)], + }); + const protocol = new ObjectStackProtocolImplementation(engine, () => new Map()); + const rows = (name: string) => Array.from(stores.get('sys_metadata')?.values() ?? []) + .filter((r) => r.type === 'view' && r.name === name) + .map((r) => [r.package_id ?? null, r.state] as const); + const rowOf = (name: string, packageId: string | null, state = 'active') => + Array.from(stores.get('sys_metadata')?.values() ?? []) + .find((r) => r.type === 'view' && r.name === name && (r.package_id ?? null) === packageId && r.state === state); + return { engine, protocol, rows, rowOf, stores }; +} + +/** A row laid down through the engine as an older release stored it. */ +const storeRow = ( + engine: ObjectQL, + name: string, + object: string, + columns: readonly string[], + packageId: string | null = null, + state = 'active', + extra: Record = {}, +) => engine.insert('sys_metadata', { + type: 'view', name, organization_id: null, package_id: packageId, + metadata: JSON.stringify({ ...listBody(name, object, columns), ...extra }), state, version: 1, +}, { context: { isSystem: true } }); + +const columnsOf = (item: any): string[] => (item?.config?.columns ?? item?.columns ?? []).map((c: any) => c?.field ?? c); +const read = async (protocol: ObjectStackProtocolImplementation, packageId?: string) => + ((await protocol.getMetaItem({ type: 'view', name: VIEW, ...(packageId ? { packageId } : {}) })) as any).item; +const layered = async (protocol: ObjectStackProtocolImplementation, packageId?: string): Promise => + (protocol as any).getMetaItemLayered({ type: 'view', name: VIEW, ...(packageId ? { packageId } : {}) }); +const listed = async (protocol: ObjectStackProtocolImplementation, packageId?: string) => { + const res: any = await protocol.getMetaItems({ type: 'view', ...(packageId ? { packageId } : {}) }); + return (res.items as any[]).filter((i) => i?.name === VIEW).map((i) => columnsOf(i)); +}; +/** The five reads of `VIEW` (N-10): no package, naming X, layered effective, X's list, env-wide list. */ +const fiveReads = async (protocol: ObjectStackProtocolImplementation) => [ + columnsOf(await read(protocol)), + columnsOf(await read(protocol, BASE)), + columnsOf((await layered(protocol)).effective), + ...(await listed(protocol, BASE)), + ...(await listed(protocol)), +]; + +const census = (protocol: ObjectStackProtocolImplementation): Promise => (protocol as any).listPackagelessOverlays(); +const migrate = (protocol: ObjectStackProtocolImplementation): Promise => + (protocol as any).migratePackagelessOverlays({ actor: ACTOR }); +const dispositions = (c: any) => (c.rows as any[]).map((r) => `${r.name}@${r.state} ${r.disposition}`); + +const savedWritable = process.env.OS_METADATA_WRITABLE; +beforeEach(() => { + delete process.env.OS_METADATA_WRITABLE; + ObjectStackProtocolImplementation.resetEnvWritableCache(); + resetEnvWritableMetadataTypes(); +}); +afterEach(() => { + if (savedWritable === undefined) delete process.env.OS_METADATA_WRITABLE; + else process.env.OS_METADATA_WRITABLE = savedWritable; + ObjectStackProtocolImplementation.resetEnvWritableCache(); + resetEnvWritableMetadataTypes(); +}); + +describe('[#28413] the census and the move', () => { + it('THE MOVE: the row lands in the pair as the declared override, the unbound row goes, the five reads serve the same content, a re-run moves 0', async () => { + const { engine, protocol, rows, rowOf } = await boot(); + await storeRow(engine, VIEW, 'showcase_task', UNBOUND_COLUMNS); + await storeRow(engine, VIEW, 'showcase_task', EDITED_COLUMNS, null, 'draft'); + const before = await fiveReads(protocol); + expect(before).toEqual([UNBOUND_COLUMNS, UNBOUND_COLUMNS, UNBOUND_COLUMNS, UNBOUND_COLUMNS, UNBOUND_COLUMNS]); + expect(dispositions(await census(protocol))).toEqual([`${VIEW}@active movable`, `${VIEW}@draft movable`]); + expect((engine.registry as any).getPackage(PAIR)).toBeUndefined(); + + const result = await migrate(protocol); + expect(result.outcomes.map((o: any) => o.outcome)).toEqual(['moved', 'moved']); + expect(result.bases).toEqual([{ base: BASE, pair: PAIR, pairCreated: true, moved: 2, declarationsCompleted: 0, failed: 0, shadowed: 0 }]); + // The pair came through the ensure door: installed, with the pair namespace. + expect((engine.registry as any).getPackage(PAIR)?.manifest?.namespace).toBe('showcase_custom'); + expect(rows(VIEW).slice().sort()).toEqual([[PAIR, 'active'], [PAIR, 'draft']]); + for (const state of ['active', 'draft']) { + const moved = rowOf(VIEW, PAIR, state)!; + expect(isServedOverrideRowOf(moved as any, BASE), state).toBe(true); + expect(JSON.parse(String(moved.metadata)).overrides).toEqual({ package: BASE, basedOn: '1.2.0' }); + } + expect(columnsOf(JSON.parse(String(rowOf(VIEW, PAIR, 'draft')!.metadata)))).toEqual(EDITED_COLUMNS); + + expect(await fiveReads(protocol)).toEqual(before); + + const again = await migrate(protocol); + expect(again.outcomes).toEqual([]); + expect(again.rows).toEqual([]); + expect(rows(VIEW).slice().sort()).toEqual([[PAIR, 'active'], [PAIR, 'draft']]); + }); + + it('LEFT ALONE: a name no package ships, a name two packages ship, and a row the pair already overrides', async () => { + const { engine, protocol, rows } = await boot(); + await (protocol as any).ensureCustomizationPackage({ basePackageId: BASE }); + // A Studio customization made first (N-17): the pair's declared row. + await storeRow(engine, VIEW, 'showcase_task', EDITED_COLUMNS, PAIR, 'active', { overrides: { package: BASE, basedOn: '1.0.0' } }); + await storeRow(engine, VIEW, 'showcase_task', UNBOUND_COLUMNS); + await storeRow(engine, SHARED_VIEW, 'shared_task', UNBOUND_COLUMNS); + await storeRow(engine, ENV_VIEW, 'showcase_task', UNBOUND_COLUMNS); + + const listedCensus = await census(protocol); + expect(dispositions(listedCensus)).toEqual([`${SHARED_VIEW}@active multi_shipped`, `${VIEW}@active shadowed`]); + expect(listedCensus.rows.find((r: any) => r.disposition === 'multi_shipped').shippers).toEqual([OTHER, BASE]); + expect(listedCensus.rows.find((r: any) => r.disposition === 'shadowed').shadowDeclares).toBe(true); + expect(listedCensus.environmentRows).toBe(1); + + const result = await migrate(protocol); + expect(result.outcomes).toEqual([]); + expect(result.bases).toEqual([{ base: BASE, pair: PAIR, pairCreated: false, moved: 0, declarationsCompleted: 0, failed: 0, shadowed: 1 }]); + expect(rows(VIEW).slice().sort((a, b) => String(a[0]).localeCompare(String(b[0])))).toEqual([[PAIR, 'active'], [null, 'active']]); + expect(rows(SHARED_VIEW)).toEqual([[null, 'active']]); + expect(rows(ENV_VIEW)).toEqual([[null, 'active']]); + // The pair's declared row is still the one served. + expect(columnsOf(await read(protocol))).toEqual(EDITED_COLUMNS); + }); + + it('a stored pair declaration with no basedOn gains the base\'s version', async () => { + const { engine, protocol, rowOf } = await boot(); + await (protocol as any).ensureCustomizationPackage({ basePackageId: BASE }); + await storeRow(engine, VIEW, 'showcase_task', EDITED_COLUMNS, PAIR, 'active', { overrides: { package: BASE } }); + const listedCensus = await census(protocol); + expect(listedCensus.incompleteDeclarations.map((d: any) => [d.name, d.pair, d.base])).toEqual([[VIEW, PAIR, BASE]]); + + const result = await migrate(protocol); + expect(result.outcomes.map((o: any) => [o.outcome, o.basedOn])).toEqual([['declaration_completed', '1.2.0']]); + expect(JSON.parse(String(rowOf(VIEW, PAIR)!.metadata)).overrides).toEqual({ package: BASE, basedOn: '1.2.0' }); + expect(columnsOf(await read(protocol))).toEqual(EDITED_COLUMNS); + expect((await census(protocol)).incompleteDeclarations).toEqual([]); + }); +}); + +/** + * The state the move leaves, laid down without the ceremony (so these cases + * read the save door alone): the pair ensured through its door, and the + * item's row bound to it, declaring the override. + */ +async function movedState(engine: ObjectQL, protocol: ObjectStackProtocolImplementation): Promise { + await (protocol as any).ensureCustomizationPackage({ basePackageId: BASE }); + await storeRow(engine, VIEW, 'showcase_task', UNBOUND_COLUMNS, PAIR, 'active', { overrides: { package: BASE, basedOn: '1.2.0' } }); +} + +describe('[#28413] until #28576: a package-less save of a moved item keeps its declaration', () => { + for (const withOverrides of [false, true]) { + it(`a package-less draft save and publish ${withOverrides ? 'carrying' : 'without'} \`overrides\` land on the pair's row, still the declared override`, async () => { + const { engine, protocol, rows, rowOf } = await boot(); + await movedState(engine, protocol); + // Setup's form save: the item as the editor read it, naming no package. + const served = await read(protocol); + const body: Record = { ...listBody(VIEW, 'showcase_task', EDITED_COLUMNS) }; + if (withOverrides) body.overrides = served.overrides; + await protocol.saveMetaItem({ type: 'view', name: VIEW, item: body, mode: 'draft' } as any); + await protocol.publishMetaItem({ type: 'view', name: VIEW } as any); + + expect(rows(VIEW)).toEqual([[PAIR, 'active']]); + expect(isServedOverrideRowOf(rowOf(VIEW, PAIR) as any, BASE)).toBe(true); + expect(JSON.parse(String(rowOf(VIEW, PAIR)!.metadata)).overrides).toEqual({ package: BASE, basedOn: '1.2.0' }); + expect(await fiveReads(protocol)).toEqual([EDITED_COLUMNS, EDITED_COLUMNS, EDITED_COLUMNS, EDITED_COLUMNS, EDITED_COLUMNS]); + + // …and a publish-mode save naming no package, the other spelling of the same write. + await protocol.saveMetaItem({ type: 'view', name: VIEW, item: { ...listBody(VIEW, 'showcase_task', UNBOUND_COLUMNS) } } as any); + expect(rows(VIEW)).toEqual([[PAIR, 'active']]); + expect(isServedOverrideRowOf(rowOf(VIEW, PAIR) as any, BASE)).toBe(true); + expect(columnsOf(await read(protocol))).toEqual(UNBOUND_COLUMNS); + }); + } + + it('a package-less DELETE removes the pair\'s row and X\'s document is served again', async () => { + const { engine, protocol, rows } = await boot(); + await movedState(engine, protocol); + await protocol.deleteMetaItem({ type: 'view', name: VIEW } as any); + expect(rows(VIEW)).toEqual([]); + expect(await fiveReads(protocol)).toEqual([BASE_COLUMNS, BASE_COLUMNS, BASE_COLUMNS, BASE_COLUMNS, BASE_COLUMNS]); + }); + + it('CONTROL: a package-less save carrying `overrides` over an unmoved package-less row is still refused (the pair is not its address)', async () => { + const { engine, protocol } = await boot(); + await storeRow(engine, VIEW, 'showcase_task', UNBOUND_COLUMNS); + await expect(protocol.saveMetaItem({ + type: 'view', name: VIEW, mode: 'draft', + item: { ...listBody(VIEW, 'showcase_task', EDITED_COLUMNS), overrides: { package: BASE, basedOn: '1.2.0' } }, + } as any)).rejects.toMatchObject({ code: 'NOT_OVERRIDABLE', status: 403 }); + }); +}); + +describe('[#28413] adopt-orphans leaves a shipped name to the ceremony', () => { + it('adopting into a writable package skips the shipped-name row and rebinds an environment one', async () => { + const { engine, protocol, rows } = await boot(); + await (protocol as any).installPackage({ manifest: { id: WORKSPACE, name: 'workspace', version: '0.1.0', type: 'app' } }); + await storeRow(engine, VIEW, 'showcase_task', UNBOUND_COLUMNS); + await storeRow(engine, ENV_VIEW, 'showcase_task', UNBOUND_COLUMNS); + const result = await (protocol as any).reassignOrphanedMetadata({ targetPackageId: WORKSPACE }); + expect(result.reassigned).toEqual([{ type: 'view', name: ENV_VIEW }]); + expect(rows(VIEW)).toEqual([[null, 'active']]); + expect(rows(ENV_VIEW)).toEqual([[WORKSPACE, 'active']]); + }); +}); From a6529800df9d9525851b082ccc39c25681e218f8 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 1 Nov 2026 12:55:45 +0000 Subject: [PATCH 2/7] feat(cli): os migrate packageless-overlays, the operator step that moves Setup overlays into .custom Preview by default; --apply (with --yes or the prompt) moves through the protocol's migratePackagelessOverlays, with a live package registry so the ensured pair persists to sys_packages. --json, --force, --database-url, --preset and --dev as the family has them. Joins the migrate family's confirmation, one-shot boot and JSON purity censuses. Co-Authored-By: Claude Claude-Session: https://claude.ai/code/session_01FEkQ5pKMJFQjFx8NCVZb62 --- .changeset/28413-packageless-overlays.md | 19 + .../packageless-overlays.integration.test.ts | 313 +++++++++++++ .../commands/migrate/packageless-overlays.ts | 436 ++++++++++++++++++ ...igrate.one-shot-family.integration.test.ts | 11 + .../cli/test/json-stdout-purity.e2e.test.ts | 4 + .../migrate-confirmation-exit.pin.test.ts | 1 + ...ocol-packageless-overlay-migration.test.ts | 12 + 7 files changed, 796 insertions(+) create mode 100644 .changeset/28413-packageless-overlays.md create mode 100644 packages/cli/src/commands/migrate/packageless-overlays.integration.test.ts create mode 100644 packages/cli/src/commands/migrate/packageless-overlays.ts diff --git a/.changeset/28413-packageless-overlays.md b/.changeset/28413-packageless-overlays.md new file mode 100644 index 00000000000..c4c46386c8f --- /dev/null +++ b/.changeset/28413-packageless-overlays.md @@ -0,0 +1,19 @@ +--- +'@objectstack/metadata-protocol': minor +'@objectstack/cli': minor +--- + +feat(cli,metadata-protocol): `os migrate packageless-overlays` moves package-less Setup customizations of a package's view, dashboard, report, translation or email template into its paired `.custom` package as declared overrides + +Clause-②: yes (widening) +Metadata-protocol: added + +**The one operator step of this release.** After upgrading, run `os migrate packageless-overlays` once (preview), then `os migrate packageless-overlays --apply`, on each deployment that has Setup customizations. Until it runs, those customizations are still served, but they stay rows of no package: they are not exported and do not reach another environment. + +- **What moves.** Every `sys_metadata` row bound to no package, of the five overlay types (`view`, `dashboard`, `report`, `translation`, `email_template`, active or draft), over a name exactly one installed read-only package ships. It moves into that package's paired `.custom` — created through the ensure door when absent and persisted to `sys_packages` — with `overrides: { package: , basedOn: }` in its body, saved through the metadata save door in its own state. The new row is read back as the served override, and only then is the package-less row deleted (with its history tombstone). Every read of the name serves the same content as before. A second run moves nothing. +- **What is left alone, and listed.** A name several packages ship; a row whose body already declares an override of another package; and a row the pair already overrides (a customization made in Studio first — the report names the two ways out: delete the package-less row, or "Restore package original" in Studio and re-run). A name no package ships is environment metadata and is not touched. Permission sets and positions are not overlay types; `os migrate security-catalog-overlays` is theirs. +- **Declarations missing `basedOn`.** A row of a pair that declares `overrides: { package }` with no `basedOn` is read as an override but refused on its next save. The same command records `basedOn` (the base's installed version) on it. +- **Setup keeps working on a moved item.** Until the package-less write closes, a save that names no package and lands on a moved row (Setup's form save and reset) keeps that row's `overrides` block when the body carries none, and is judged as the write into the pair when the body carries one — so it is no longer refused `403 NOT_OVERRIDABLE`. A package-less delete removes the pair's row, and the package's own document is served again. +- **Flags.** `--apply` (with `--yes` or the prompt), `--json`, `--force` past a busy SQLite file, `--database-url`, and `serve`'s `--preset` / `--dev`. The command is the operator's; the HTTP door that creates a pair keeps its walled refusal. + +`ObjectStackProtocolImplementation` gains `listPackagelessOverlays()` and `migratePackagelessOverlays({ actor })`. diff --git a/packages/cli/src/commands/migrate/packageless-overlays.integration.test.ts b/packages/cli/src/commands/migrate/packageless-overlays.integration.test.ts new file mode 100644 index 00000000000..93ae3b5748b --- /dev/null +++ b/packages/cli/src/commands/migrate/packageless-overlays.integration.test.ts @@ -0,0 +1,313 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * `os migrate packageless-overlays` against a real SQLite database: the + * ceremony that moves package-less overlay rows of a shipped name into the + * base's paired `.custom` as declared overrides (ADR-0146 D11, N-16a). + * + * ## The fixture + * + * A compiled artifact (`dist/objectstack.json`, the shape a deployed app boots + * from) whose package ships an object and its list view, over a database an + * older release left behind with, bound to no package: + * + * - MOVED: an active row and a draft row of the package's view; + * - CONTROL, never moved: a view row of a name no package ships (environment + * metadata), and a draft `permission` row over a name the package ships + * (the security catalog's, ADR-0146 D6 — not an overlay type). + * + * ## What each case measures + * + * The command runs in-process through oclif, as the binary runs it. The + * preview writes nothing; `--apply` creates the pair through its door + * (`sys_packages` holds it), moves both rows with the declaration, leaves the + * controls, and a FRESH boot of the same database then serves the moved + * content under the base's name on the package-less read, the read naming the + * base, the layered read and the base's list; a second `--apply` moves 0. + */ + +import { describe, it, expect, beforeAll, afterAll, beforeEach, afterEach, vi } from 'vitest'; +import { copyFileSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { dirname, join, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { SqlDriver } from '@objectstack/driver-sql'; +import { PackageServicePlugin } from '@objectstack/service-package'; +import { isExitSignal } from '../../utils/format.js'; +import { bootSchemaStack } from '../../utils/schema-migrate.js'; +import MigratePackagelessOverlays from './packageless-overlays.js'; + +// [#10126] Pay the first transform of these dist-resolved workspace deps at +// MODULE LOAD: the command reaches them through dynamic `import()`s inside +// `run()`, which vitest clocks (`scripts/check-test-source-alias.mjs`). +import '@objectstack/runtime'; +import '@objectstack/metadata-protocol'; +import '@objectstack/platform-objects/plugin'; + +const HERE = dirname(fileURLToPath(import.meta.url)); +const CLI_ROOT = resolve(HERE, '..', '..', '..'); +const CASE_TIMEOUT_MS = 180_000; + +const PKG = 'com.example.m28413'; +const PAIR = `${PKG}.custom`; +const VIEW = 'm28413_task.default'; +const ENV_VIEW = 'm28413_env.default'; +const SHIPPED_COLUMNS = ['title', 'status', 'owner']; +const STORED_COLUMNS = ['title', 'owner']; +const DRAFT_COLUMNS = ['title']; +const ARTIFACT = { + manifest: { id: PKG, name: 'm28413', version: '1.4.0', type: 'app', namespace: 'm28413' }, + objects: [{ + name: 'm28413_task', + label: 'Task', + sharingModel: 'private', + fields: Object.fromEntries(SHIPPED_COLUMNS.map((f) => [f, { name: f, label: f, type: 'text' }])), + }], + views: [{ + object: 'm28413_task', + list: { label: 'All', type: 'grid', data: { provider: 'object', object: 'm28413_task' }, columns: SHIPPED_COLUMNS.map((field) => ({ field })) }, + }], + permissions: [{ name: 'm28413_rep', label: 'Rep', objects: {} }], +}; + +const viewBody = (name: string, columns: readonly string[]) => ({ + name, + object: 'm28413_task', + viewKind: 'list', + label: 'Saved in Setup', + config: { type: 'grid', data: { provider: 'object', object: 'm28413_task' }, columns: columns.map((field) => ({ field })) }, +}); + +interface SeedRow { type: string; name: string; state: string; body: Record } +const SEED: SeedRow[] = [ + { type: 'view', name: VIEW, state: 'active', body: viewBody(VIEW, STORED_COLUMNS) }, + { type: 'view', name: VIEW, state: 'draft', body: viewBody(VIEW, DRAFT_COLUMNS) }, + // Controls. + { type: 'view', name: ENV_VIEW, state: 'active', body: viewBody(ENV_VIEW, STORED_COLUMNS) }, + // A DRAFT: an active one over a package's name refuses the cold boot, which + // is \`os migrate security-catalog-overlays\`' to clear, not this ceremony's. + { type: 'permission', name: 'm28413_rep', state: 'draft', body: { name: 'm28413_rep', label: 'Saved in the environment', objects: {} } }, +]; + +let dir: string; +let templateDb: string; +const savedCwd = process.cwd(); +const ENV_KEYS = ['OS_ARTIFACT_PATH', 'OS_DATABASE_URL', 'DATABASE_URL', 'OS_DATABASE_DRIVER', 'OS_HOME', 'NODE_ENV', + 'OS_AUTH_SECRET', 'AUTH_SECRET', 'BETTER_AUTH_SECRET', 'OS_ENVIRONMENT_ID'] as const; +const savedEnv: Record = {}; + +let caseSeq = 0; +function caseDb(): string { + caseSeq += 1; + const file = join(dir, 'cases', `${caseSeq}.db`); + mkdirSync(dirname(file), { recursive: true }); + copyFileSync(templateDb, file); + return file; +} + +/** Every `sys_metadata` row, `type/name@binding/state`, read on a connection of our own. */ +async function storedRows(dbFile: string): Promise { + const driver = new SqlDriver({ client: 'better-sqlite3', connection: { filename: dbFile }, useNullAsDefault: true }); + try { + const rows = await (driver as any).knex('sys_metadata').select('type', 'name', 'package_id', 'state'); + return rows.map((r: any) => `${r.type}/${r.name}@${r.package_id ?? '-'}/${r.state}`).sort(); + } finally { + await driver.disconnect(); + } +} + +/** One stored body, parsed. */ +async function storedBody(dbFile: string, where: Record): Promise { + const driver = new SqlDriver({ client: 'better-sqlite3', connection: { filename: dbFile }, useNullAsDefault: true }); + try { + const row = await (driver as any).knex('sys_metadata').where(where).first(); + return row ? JSON.parse(row.metadata) : undefined; + } finally { + await driver.disconnect(); + } +} + +/** The `sys_packages` ids stored. */ +async function storedPackages(dbFile: string): Promise { + const driver = new SqlDriver({ client: 'better-sqlite3', connection: { filename: dbFile }, useNullAsDefault: true }); + try { + const rows = await (driver as any).knex('sys_packages').select('id'); + return rows.map((r: any) => String(r.id)).sort(); + } finally { + await driver.disconnect(); + } +} + +interface RunResult { payload: any; exitCode: number } + +/** Run the command through oclif and capture its one JSON document. */ +async function runJson(argv: string[]): Promise { + const savedExit = process.exitCode; + const swallow = ((_chunk: unknown, ...rest: unknown[]) => { + const cb = rest.find((a) => typeof a === 'function') as (() => void) | undefined; + if (cb) cb(); + return true; + }) as typeof process.stdout.write; + const stdout = vi.spyOn(process.stdout, 'write').mockImplementation(swallow); + vi.spyOn(process.stderr, 'write').mockImplementation(swallow); + vi.spyOn(console, 'log').mockImplementation(() => {}); + vi.spyOn(console, 'warn').mockImplementation(() => {}); + vi.spyOn(console, 'error').mockImplementation(() => {}); + let thrownExit: number | undefined; + try { + try { + await MigratePackagelessOverlays.run(argv, { root: CLI_ROOT }); + } catch (error) { + if (!isExitSignal(error)) throw error; + thrownExit = (error as { oclif?: { exit?: number } }).oclif?.exit; + } + const out = stdout.mock.calls.map((c) => String(c[0])).join('').trim(); + let payload: any; + try { payload = JSON.parse(out); } catch { payload = { unparsed: out.slice(-400) }; } + return { payload, exitCode: thrownExit ?? (process.exitCode as number | undefined) ?? 0 }; + } finally { + process.exitCode = savedExit; + vi.restoreAllMocks(); + } +} + +const columnsOf = (item: any): string[] => (item?.config?.columns ?? item?.columns ?? []).map((c: any) => c?.field ?? c); + +/** What a fresh boot of `dbFile` serves for `VIEW`: the package-less read, the read naming the base, the layered effective, the base's list. */ +async function servedColumns(dbFile: string): Promise { + const stack = await bootSchemaStack({ + jsonOutput: false, databaseUrl: `file:${dbFile}`, projectRoot: dir, composeHostStack: true, + // The package registry `os serve` composes, so the pair installs from `sys_packages` as it does there. + extraPlugins: [new PackageServicePlugin()], + }); + try { + const protocol = stack.kernel.getService('protocol') as any; + const read = async (packageId?: string) => + (await protocol.getMetaItem({ type: 'view', name: VIEW, ...(packageId ? { packageId } : {}) }))?.item; + const layered = await protocol.getMetaItemLayered({ type: 'view', name: VIEW }); + const list = await protocol.getMetaItems({ type: 'view', packageId: PKG }); + return [ + columnsOf(await read()), + columnsOf(await read(PKG)), + columnsOf(layered?.effective), + ...((list?.items ?? []) as any[]).filter((i) => i?.name === VIEW).map((i) => columnsOf(i)), + ]; + } finally { + await stack.shutdown(); + } +} + +beforeAll(async () => { + for (const key of ENV_KEYS) savedEnv[key] = process.env[key]; + for (const key of ENV_KEYS) delete process.env[key]; + process.env.NODE_ENV = 'production'; + dir = mkdtempSync(join(tmpdir(), 'os-28413-')); + mkdirSync(join(dir, 'dist'), { recursive: true }); + process.env.OS_HOME = join(dir, 'home'); + process.env.OS_ARTIFACT_PATH = join(dir, 'dist', 'objectstack.json'); + writeFileSync(process.env.OS_ARTIFACT_PATH, JSON.stringify(ARTIFACT)); + + // The database an older release left: tables provisioned, then the + // package-less rows written at the driver the way that release stored them. + templateDb = join(dir, 'template.db'); + const seed = await bootSchemaStack({ + jsonOutput: false, databaseUrl: `file:${templateDb}`, projectRoot: dir, hydrateMetadata: false, + extraPlugins: [new PackageServicePlugin()], + }); + await seed.shutdown(); + const driver = new SqlDriver({ client: 'better-sqlite3', connection: { filename: templateDb }, useNullAsDefault: true }); + try { + const k = (driver as any).knex; + const now = new Date().toISOString(); + let n = 0; + for (const row of SEED) { + n += 1; + await k('sys_metadata').insert({ + id: `seed_${n}`, type: row.type, name: row.name, package_id: null, state: row.state, + version: 1, checksum: null, created_at: now, updated_at: now, metadata: JSON.stringify(row.body), + }); + } + } finally { + await driver.disconnect(); + } +}, CASE_TIMEOUT_MS); + +afterAll(() => { + process.chdir(savedCwd); + for (const [key, value] of Object.entries(savedEnv)) { + if (value === undefined) delete process.env[key]; + else process.env[key] = value; + } + try { rmSync(dir, { recursive: true, force: true }); } catch { /* ignore */ } +}); + +// The command takes `process.cwd()` as the project root, as a real invocation does. +beforeEach(() => { process.chdir(dir); }); +afterEach(() => { process.chdir(savedCwd); }); + +describe('os migrate packageless-overlays', () => { + it('the preview lists the shipped name\'s rows to move, counts the environment row, and writes nothing', async () => { + const db = caseDb(); + const before = await storedRows(db); + const { payload, exitCode } = await runJson(['--database-url', `file:${db}`, '--json']); + expect(payload.error, JSON.stringify(payload).slice(0, 600)).toBeUndefined(); + expect(exitCode).toBe(0); + expect(payload.apply).toBe(false); + expect(payload.rows.map((r: any) => `${r.type}/${r.name}/${r.state} ${r.disposition} → ${r.pair}`)).toEqual([ + `view/${VIEW}/active movable → ${PAIR}`, + `view/${VIEW}/draft movable → ${PAIR}`, + ]); + expect(payload.environmentRows).toBe(1); + expect(await storedRows(db)).toEqual(before); + expect(await storedPackages(db)).not.toContain(PAIR); + }, CASE_TIMEOUT_MS); + + it('--apply moves both rows into the pair with the declaration, keeps the controls, serves the same content, and a re-run moves 0', async () => { + const db = caseDb(); + const served = await servedColumns(db); + expect(served).toEqual([STORED_COLUMNS, STORED_COLUMNS, STORED_COLUMNS, STORED_COLUMNS]); + + const { payload, exitCode } = await runJson(['--apply', '--yes', '--database-url', `file:${db}`, '--json']); + expect(payload.error, JSON.stringify(payload).slice(0, 600)).toBeUndefined(); + expect(exitCode).toBe(0); + expect(payload.moved).toBe(2); + expect(payload.failed).toBe(0); + expect(payload.bases).toEqual([{ base: PKG, pair: PAIR, pairCreated: true, moved: 2, declarationsCompleted: 0, failed: 0, shadowed: 0 }]); + expect(payload.rows.map((r: any) => `${r.state} ${r.outcome} ${r.packageId} ${r.basedOn}`)).toEqual([ + `active moved ${PAIR} 1.4.0`, + `draft moved ${PAIR} 1.4.0`, + ]); + + expect(await storedRows(db)).toEqual([ + `permission/m28413_rep@-/draft`, + `view/${ENV_VIEW}@-/active`, + `view/${VIEW}@${PAIR}/active`, + `view/${VIEW}@${PAIR}/draft`, + ]); + for (const state of ['active', 'draft']) { + const body = await storedBody(db, { type: 'view', name: VIEW, package_id: PAIR, state }); + expect(body.overrides, state).toEqual({ package: PKG, basedOn: '1.4.0' }); + } + expect(columnsOf(await storedBody(db, { type: 'view', name: VIEW, package_id: PAIR, state: 'draft' }))).toEqual(DRAFT_COLUMNS); + // The pair came through its door, so it outlives this process. + expect(await storedPackages(db)).toContain(PAIR); + + // A fresh boot serves the moved content under the base's name. + expect(await servedColumns(db)).toEqual(served); + + const again = await runJson(['--apply', '--yes', '--database-url', `file:${db}`, '--json']); + expect(again.payload.error, JSON.stringify(again.payload).slice(0, 600)).toBeUndefined(); + expect(again.exitCode).toBe(0); + expect(again.payload.moved).toBe(0); + expect(again.payload.rows).toEqual([]); + }, CASE_TIMEOUT_MS * 3); + + it('--apply without --yes in --json refuses and moves nothing', async () => { + const db = caseDb(); + const before = await storedRows(db); + const { payload, exitCode } = await runJson(['--apply', '--database-url', `file:${db}`, '--json']); + expect(payload.error, JSON.stringify(payload).slice(0, 600)).toBe('confirmation_required'); + expect(exitCode).toBe(1); + expect(await storedRows(db)).toEqual(before); + }, CASE_TIMEOUT_MS); +}); diff --git a/packages/cli/src/commands/migrate/packageless-overlays.ts b/packages/cli/src/commands/migrate/packageless-overlays.ts new file mode 100644 index 00000000000..fd89c89f6bf --- /dev/null +++ b/packages/cli/src/commands/migrate/packageless-overlays.ts @@ -0,0 +1,436 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +import { Command, Flags } from '@oclif/core'; +import { createInterface } from 'node:readline'; +import chalk from 'chalk'; +import { STACK_TIER_PRESETS } from '@objectstack/core'; +import { + printHeader, + printSuccess, + printWarning, + printError, + printInfo, + printStep, + createTimer, + emitJson, + errorCodeFields, + isExitSignal, +} from '../../utils/format.js'; +import { PackageServicePlugin } from '@objectstack/service-package'; +import { bootSchemaStack } from '../../utils/schema-migrate.js'; +import { describeUnloadableHostConfig } from '../../utils/schema-migration-plugins.js'; +import { absentTableReads } from '../../utils/absent-table-reads.js'; +import { OCCUPANCY_HINT, probeMigrationTarget } from '../../utils/migrate-occupancy-gate.js'; +import { describeOccupancy, JOURNAL_MODE_SWITCH_NOTE } from '../../utils/sqlite-occupancy.js'; + +/** The command's own name, the `actor` its writes record. */ +export const PACKAGELESS_OVERLAYS_ACTOR = 'os migrate packageless-overlays'; + +async function confirm(question: string): Promise { + if (!process.stdin.isTTY) return false; // non-interactive → require --yes + const rl = createInterface({ input: process.stdin, output: process.stdout }); + try { + const answer: string = await new Promise((res) => rl.question(question, res)); + return /^y(es)?$/i.test(answer.trim()); + } finally { + rl.close(); + } +} + +/** The census row shape `listPackagelessOverlays` answers (`@objectstack/metadata-protocol`). */ +interface CensusRow { + id: string; + storedType: string; + type: string; + name: string; + state: 'active' | 'draft'; + member?: string; + disposition: 'movable' | 'multi_shipped' | 'no_pair' | 'foreign_declaration' | 'shadowed'; + shippers: string[]; + base?: string; + pair?: string; + shadowDeclares?: boolean; +} +interface IncompleteDeclaration { + id: string; + type: string; + name: string; + state: 'active' | 'draft'; + member?: string; + base: string; + pair: string; +} +interface Census { + rows: CensusRow[]; + environmentRows: number; + incompleteDeclarations: IncompleteDeclaration[]; +} +type Outcome = + | { id: string; outcome: 'moved' | 'declaration_completed'; packageId: string; basedOn: string } + | { id: string; outcome: 'failed'; error: string }; +interface Migration extends Census { + outcomes: Outcome[]; + bases: Array<{ base: string; pair: string; pairCreated: boolean; moved: number; declarationsCompleted: number; failed: number; shadowed: number }>; +} +interface MigrationProtocol { + listPackagelessOverlays?: () => Promise; + migratePackagelessOverlays?: (request: { actor: string }) => Promise; +} + +/** One item as the report names it. */ +function itemOf(row: { type: string; name: string; state: string; member?: string }): string { + return `${row.type} "${row.name}"${row.member ? ` (${row.member})` : ''}${row.state === 'draft' ? ' [draft]' : ''}`; +} + +/** Why a listed row is not moved, and what the administrator can do. */ +function reasonOf(row: CensusRow): string { + switch (row.disposition) { + case 'multi_shipped': + return `several packages ship this name (${row.shippers.join(', ')}), so it overlays none of them for certain. ` + + 'Left as it is: move it by hand into the pair of the package it customizes, or delete it.'; + case 'no_pair': + return `'${row.base}' has no paired .custom package id, so there is nowhere to move it. Left as it is.`; + case 'foreign_declaration': + return `its body already carries an \`overrides\` key that does not name '${row.base}'. Left as it is: ` + + 'correct or remove the key, then re-run.'; + case 'shadowed': + return `'${row.pair}' already ${row.shadowDeclares ? 'overrides this item (a customization made in Studio)' : 'holds a row of this name'}, ` + + 'and that row is the one served — this row is never merged into it. Two ways out: keep the Studio ' + + `customization and delete this package-less row (id ${row.id}), or use "Restore package original" in ` + + `Studio to delete '${row.pair}''s row and re-run this command to move this one.`; + default: + return ''; + } +} + +function jsonRow(row: CensusRow, outcome?: Outcome): Record { + return { + id: row.id, + type: row.type, + storedType: row.storedType, + name: row.name, + state: row.state, + ...(row.member !== undefined ? { member: row.member } : {}), + disposition: row.disposition, + shippers: row.shippers, + ...(row.base !== undefined ? { base: row.base } : {}), + ...(row.pair !== undefined ? { pair: row.pair } : {}), + ...(row.shadowDeclares !== undefined ? { shadowDeclares: row.shadowDeclares } : {}), + outcome: outcome?.outcome ?? (row.disposition === 'movable' ? 'listed' : 'left'), + ...(outcome && outcome.outcome !== 'failed' ? { packageId: outcome.packageId, basedOn: outcome.basedOn } : {}), + ...(outcome?.outcome === 'failed' ? { error: outcome.error } : {}), + }; +} + +/** + * `os migrate packageless-overlays` — list, and with `--apply` move, every + * package-less overlay row of a name exactly one installed code package ships + * into that package's paired `.custom`, as a declared override + * (ADR-0146 D11, the migration half of N-16). + * + * ## Why a ceremony + * + * A Setup "customize" of a view, dashboard, report, translation or email + * template a package ships was stored bound to no package: it is served, but it + * is never exported and never reaches another environment. ADR-0146 moves every + * customization of an installed package into its paired `.custom` package, + * declared in the item's body (`overrides: { package, basedOn }`). Moving the + * existing rows is the administrator's call, so it is this command and not a + * boot step; the release note names it as the one operator step. + * + * ## What it does + * + * The protocol's census and move (`listPackagelessOverlays` / + * `migratePackagelessOverlays`, `@objectstack/metadata-protocol`, whose module + * header holds the rules): the pair is ensured through its one creation door, + * each body is saved into the pair through the save door with the declaration + * (`basedOn` = the installed base's version now), read back as the served + * override, and only then is the package-less row deleted. A name no package + * ships is environment metadata and is not touched; a name several packages + * ship, and a row the pair already overrides (a Studio customization made + * first), are listed and left alone. A pair row declaring an override with no + * `basedOn` gains it. A second run moves nothing. + * + * The deployment is composed as `os serve` composes it — the host config's + * plugins and the application or the compiled artifact — so the packages that + * ship a name are the ones the server serves. `--preset` and `--dev` are + * `serve`'s two flags that move that composition, with its meaning. + * + * ## Who runs it + * + * The operator, on the deployment's database. The HTTP door that creates a pair + * (`POST /packages/:id/ensure-customization`) keeps its walled refusal: under a + * wall a tenant administrator is refused there (403), and this command adds no + * door of its own. + * + * ## The family's conventions + * + * Preview by default; `--apply` is the only writing mode, confirmed with + * `[y/N]` or `--yes`; `--force` past a busy SQLite file; `--json` for one + * document; `--database-url` / `$OS_DATABASE_URL` for the target. The preview + * boots read-only (deferred DDL, no seed, no file brought into existence), and + * `--apply` boots plain. + * + * Exit status: 0 when the preview ran, or when everything `--apply` acted on + * succeeded; 1 when a row could not be moved or completed, or when the + * composition could not be read. + */ +export default class MigratePackagelessOverlays extends Command { + static override description = + 'List, and with --apply move, the package-less view, dashboard, report, translation and email-template rows ' + + 'over a name one installed package ships into that package\'s paired .custom package, as declared overrides ' + + '(ADR-0146 D11). Run once after upgrading.'; + + static override examples = [ + '$ os migrate packageless-overlays', + '$ os migrate packageless-overlays --json', + '$ os migrate packageless-overlays --apply', + '$ os migrate packageless-overlays --apply --yes --json', + ]; + + static override flags = { + 'database-url': Flags.string({ + description: 'Database to examine (defaults to $OS_DATABASE_URL / the project DB)', + env: 'OS_DATABASE_URL', + }), + apply: Flags.boolean({ + description: 'Move the listed rows (default is a read-only preview)', + default: false, + }), + yes: Flags.boolean({ + char: 'y', + description: 'Skip the --apply confirmation prompt', + default: false, + }), + force: Flags.boolean({ + description: 'Apply even when another process is using the database (SQLite occupancy check)', + default: false, + }), + json: Flags.boolean({ description: 'Output the machine-readable result as JSON.' }), + preset: Flags.string({ + description: 'Compose as `os serve --preset` does: the tier preset the stack falls back to when it declares ' + + 'no tiers. Pass what the deployment boots with.', + options: Object.keys(STACK_TIER_PRESETS), + }), + dev: Flags.boolean({ + description: 'Compose as `os serve --dev` does: the config\'s devPlugins and the development .env files. ' + + 'Pass it when the deployment boots with --dev (`os dev`).', + }), + }; + + async run(): Promise { + const { flags } = await this.parse(MigratePackagelessOverlays); + const timer = createTimer(); + const apply = flags.apply; + if (!flags.json) printHeader('Migrate · packageless-overlays'); + + // Occupancy gate — probed BEFORE boot and before the prompt, as every + // writing member of the family does. + const occupancy = await probeMigrationTarget(flags['database-url'], { dev: flags.dev === true }); + if (occupancy.status === 'busy' && apply && !flags.force) { + if (flags.json) { + await emitJson({ + error: 'database_busy', + database: occupancy.filename, + signal: occupancy.signal, + detail: occupancy.detail, + hint: OCCUPANCY_HINT, + }, 0, { compact: true }); + this.exit(1); + return; + } + printError(describeOccupancy(occupancy)); + printWarning(OCCUPANCY_HINT); + this.exit(1); + return; + } + if (occupancy.status === 'busy' && !flags.json) { + printWarning(apply + ? `--force: ${describeOccupancy(occupancy)} Moving anyway — restart that process afterwards so it serves the moved rows.` + : `${describeOccupancy(occupancy)} The preview below writes no row or table. ${JOURNAL_MODE_SWITCH_NOTE}`); + } + if (occupancy.status === 'unknown' && !flags.json) { + printWarning(`Could not check whether the database is in use — ${occupancy.detail}`); + } + + if (apply && !flags.yes) { + if (flags.json || !process.stdin.isTTY) { + if (flags.json) { + await emitJson({ error: 'confirmation_required', hint: 'pass --yes' }, 0, { compact: true }); + this.exit(1); + return; + } + printWarning( + 'Apply mode moves the listed sys_metadata rows into each base\'s paired .custom package (created when ' + + 'absent) and deletes the package-less rows. Re-run with --yes to confirm, or run without --apply to preview.', + ); + this.exit(1); + return; + } + const ok = await confirm( + chalk.bold('\nMove the package-less overlay rows into each base\'s .custom package as declared overrides? [y/N] '), + ); + if (!ok) { + printInfo('Aborted — nothing moved.'); + return; + } + } + + if (!flags.json) { + printStep(apply ? 'Composing the deployment (APPLY mode)…' : 'Composing the deployment (preview only)…'); + } + + let stack; + try { + stack = await bootSchemaStack({ + jsonOutput: flags.json, + ...(flags['database-url'] ? { databaseUrl: flags['database-url'] } : {}), + composeHostStack: true, + composeAuthGatedSecurity: true, + // The package registry, live, under `--apply` only: the pair the + // ensure door creates is persisted to `sys_packages` (the server + // installs it from there at its next boot), and a pair an earlier run + // created is installed again here. The preview writes nothing, so it + // composes none (its `start()` provisions the table). + ...(apply ? { extraPlugins: [new PackageServicePlugin()] } : {}), + serveFlags: { dev: flags.dev === true, ...(flags.preset ? { preset: flags.preset } : {}) }, + ...(apply ? {} : { deferSchemaDdl: true, readOnlyProbe: true }), + }); + } catch (error: any) { + if (flags.json) { await emitJson({ error: error.message, ...errorCodeFields(error) }, 0, { compact: true }); this.exit(1); return; } + printError(error.message || String(error)); + this.exit(1); + return; + } + + // Collected rather than thrown: `this.exit()` raises an oclif ExitError. + let exitCode = 0; + try { + exitCode = await this.report(stack, { apply, json: flags.json === true }, timer); + } catch (error: any) { + if (isExitSignal(error)) throw error; + exitCode = 1; + if (flags.json) await emitJson({ error: error.message, ...errorCodeFields(error) }, 0, { compact: true }); + else printError(error.message || String(error)); + } finally { + await stack.shutdown(); + } + if (exitCode !== 0) this.exit(exitCode); + } + + /** Census, move under `--apply`, and report on the booted stack; answers the exit status. */ + private async report( + stack: Awaited>, + opts: { apply: boolean; json: boolean }, + timer: { elapsed: () => number; display: () => string }, + ): Promise { + const { apply, json } = opts; + // A host config that exists and could not be read leaves the shipping + // packages incomplete: a row would be classified against the wrong set. + const unloadable = describeUnloadableHostConfig(stack.composition); + if (unloadable !== null) { + if (json) await emitJson({ error: 'host_config_unloadable', message: unloadable }, 0, { compact: true }); + else printError(unloadable); + return 1; + } + const protocol = stack.kernel.getService('protocol') as MigrationProtocol | undefined; + if (typeof protocol?.listPackagelessOverlays !== 'function' || typeof protocol.migratePackagelessOverlays !== 'function') { + throw new Error( + 'No metadata protocol on this stack, so sys_metadata cannot be read. ' + + 'Run this from the project root whose stack registers the ObjectQL engine.', + ); + } + + // The preview's read-only boot measured whether `sys_metadata` exists; a + // table that does not exist stores no row. + const reads = absentTableReads(stack); + const empty: Migration = { rows: [], environmentRows: 0, incompleteDeclarations: [], outcomes: [], bases: [] }; + const result: Migration = reads.absent('sys_metadata') + ? empty + : apply + ? await protocol.migratePackagelessOverlays({ actor: PACKAGELESS_OVERLAYS_ACTOR }) + : { ...(await protocol.listPackagelessOverlays()), outcomes: [], bases: [] }; + const outcomeOf = new Map(result.outcomes.map((o) => [o.id, o])); + const movable = result.rows.filter((r) => r.disposition === 'movable'); + const left = result.rows.filter((r) => r.disposition !== 'movable'); + const failed = result.outcomes.filter((o) => o.outcome === 'failed').length; + const exitCode = apply && failed > 0 ? 1 : 0; + + if (json) { + reads.notice(true); + await emitJson({ + database: stack.dbLabel, + apply, + hostConfig: stack.composition.hostConfigPath, + movable: movable.length, + moved: result.outcomes.filter((o) => o.outcome === 'moved').length, + declarationsCompleted: result.outcomes.filter((o) => o.outcome === 'declaration_completed').length, + failed, + left: left.length, + environmentRows: result.environmentRows, + bases: result.bases, + rows: result.rows.map((row) => jsonRow(row, outcomeOf.get(row.id))), + incompleteDeclarations: result.incompleteDeclarations.map((d) => ({ + ...d, + outcome: outcomeOf.get(d.id)?.outcome ?? 'listed', + ...(outcomeOf.get(d.id)?.outcome === 'failed' ? { error: (outcomeOf.get(d.id) as { error: string }).error } : {}), + })), + duration: timer.elapsed(), + }); + return exitCode; + } + + printInfo(`Database: ${chalk.white(stack.dbLabel)}`); + if (stack.composition.hostConfigPath) printInfo(`Host config: ${chalk.white(stack.composition.hostConfigPath)}`); + reads.notice(false); + if (result.environmentRows > 0) { + printInfo(`${result.environmentRows} package-less row(s) of a name no package ships are environment metadata and are not touched.`); + } + console.log(''); + if (movable.length === 0 && left.length === 0 && result.incompleteDeclarations.length === 0) { + printSuccess(`No package-less overlay row is stored over a name a package ships — nothing to move ${chalk.dim(`(${timer.display()})`)}`); + return exitCode; + } + if (left.length > 0) { + console.log(chalk.bold(`${left.length} row(s) left as they are:`)); + for (const row of left) console.log(` • ${itemOf(row)} — ${reasonOf(row)}`); + console.log(''); + } + if (!apply) { + if (movable.length > 0) { + console.log(chalk.bold(`${movable.length} row(s) to move:`)); + for (const row of movable) console.log(` • ${itemOf(row)} → ${row.pair} (overrides ${row.base})`); + } + if (result.incompleteDeclarations.length > 0) { + console.log(chalk.bold(`${result.incompleteDeclarations.length} declared override(s) missing basedOn:`)); + for (const d of result.incompleteDeclarations) console.log(` • ${itemOf(d)} in ${d.pair} (overrides ${d.base})`); + } + console.log(''); + if (movable.length > 0 || result.incompleteDeclarations.length > 0) { + printWarning('Re-run with --apply to move them (each base\'s .custom package is created when absent).'); + } + return exitCode; + } + for (const row of [...movable, ...result.incompleteDeclarations]) { + const outcome = outcomeOf.get(row.id); + if (outcome === undefined) continue; + if (outcome.outcome === 'failed') { + console.log(` ${chalk.red('✗')} ${itemOf(row)} — ${outcome.error}`); + } else { + const verb = outcome.outcome === 'moved' ? 'moved into' : 'basedOn recorded in'; + console.log(` ${chalk.green('✓')} ${itemOf(row)} ${verb} ${outcome.packageId} ${chalk.dim(`(basedOn ${outcome.basedOn})`)}`); + } + } + console.log(''); + for (const line of result.bases) { + printInfo(`${line.base} → ${line.pair}${line.pairCreated ? ' (created)' : ''}: ${line.moved} moved, ` + + `${line.declarationsCompleted} basedOn recorded, ${line.shadowed} shadowed, ${line.failed} failed`); + } + if (failed > 0) { + printError(`${failed} row(s) could not be moved; each package-less row that failed is kept as it was.`); + } else { + printSuccess(`Done ${chalk.dim(`(${timer.display()})`)}. Restart the server so it serves the moved rows.`); + } + return exitCode; + } +} diff --git a/packages/cli/src/utils/schema-migrate.one-shot-family.integration.test.ts b/packages/cli/src/utils/schema-migrate.one-shot-family.integration.test.ts index 6c2f37531ae..dd6eff19856 100644 --- a/packages/cli/src/utils/schema-migrate.one-shot-family.integration.test.ts +++ b/packages/cli/src/utils/schema-migrate.one-shot-family.integration.test.ts @@ -71,6 +71,7 @@ import MigrateFilesToReferences from '../commands/migrate/files-to-references.js import MigrateMeta from '../commands/migrate/meta.js'; import MigrateMultiValueColumns from '../commands/migrate/multi-value-columns.js'; import MigrateOrganizationOwnership from '../commands/migrate/organization-ownership.js'; +import MigratePackagelessOverlays from '../commands/migrate/packageless-overlays.js'; import MigratePlan from '../commands/migrate/plan.js'; import MigrateRecordCurrencyCodes from '../commands/migrate/record-currency-codes.js'; import MigrateRecordedBy from '../commands/migrate/recorded-by.js'; @@ -261,6 +262,16 @@ const CALLERS: Record = { ], }], }, + 'commands/migrate/packageless-overlays.ts': { + run: invoke(MigratePackagelessOverlays), + // The fixture stores no package-less overlay row, so the run lists + // nothing and `--apply` moves nothing — the boot is what these cases are about. + noWrite: [{ label: 'migrate packageless-overlays', argv: ['--database-url', '@DB@', '--json'] }], + write: [{ + label: 'migrate packageless-overlays --apply', + argv: ['--apply', '--yes', '--database-url', '@DB@', '--json'], + }], + }, 'commands/migrate/plan.ts': { run: invoke(MigratePlan), noWrite: [{ label: 'migrate plan', argv: ['--database-url', '@DB@', '--json'] }], diff --git a/packages/cli/test/json-stdout-purity.e2e.test.ts b/packages/cli/test/json-stdout-purity.e2e.test.ts index a22344bae82..abd2fc1efc7 100644 --- a/packages/cli/test/json-stdout-purity.e2e.test.ts +++ b/packages/cli/test/json-stdout-purity.e2e.test.ts @@ -134,6 +134,10 @@ const FAMILY: Record = { // stack is shut down. A refused plan writes no plan file, so the fixture // directory gains nothing. 'migrate organization-ownership': [], + // A preview is its DEFAULT and the only form driven here: without `--apply` + // it boots read-only and moves nothing. No table exists in the fixture, so + // the face driven here is its empty listing. + 'migrate packageless-overlays': [], 'migrate plan': [], // [ADR-0144] `--object`, `--field` and `--code` are all required, so without // them the run stops at flag parsing and never boots; the pair named here is diff --git a/packages/cli/test/migrate-confirmation-exit.pin.test.ts b/packages/cli/test/migrate-confirmation-exit.pin.test.ts index 0c77f597aa9..e364a5f62b2 100644 --- a/packages/cli/test/migrate-confirmation-exit.pin.test.ts +++ b/packages/cli/test/migrate-confirmation-exit.pin.test.ts @@ -251,6 +251,7 @@ const ROSTER: Record = { meta: { argv: ['--stored', '--apply'], confirm: '--yes', writes: BOOT }, 'multi-value-columns': { argv: ['--apply'], confirm: '--yes', writes: BOOT }, 'organization-ownership': { argv: ['--apply', '--plan-file', PLAN_FILE], confirm: '--backup-taken', writes: BOOT }, + 'packageless-overlays': { argv: ['--apply'], confirm: '--yes', writes: BOOT }, 'recorded-by': { argv: ['--apply'], confirm: '--yes', diff --git a/packages/objectql/src/protocol-packageless-overlay-migration.test.ts b/packages/objectql/src/protocol-packageless-overlay-migration.test.ts index 95e1b06c7fd..3d662af06c5 100644 --- a/packages/objectql/src/protocol-packageless-overlay-migration.test.ts +++ b/packages/objectql/src/protocol-packageless-overlay-migration.test.ts @@ -304,6 +304,18 @@ describe('[#28413] the census and the move', () => { expect(columnsOf(await read(protocol))).toEqual(EDITED_COLUMNS); }); + it('READ BACK FIRST: a save that leaves no declared row in the pair keeps the unbound row, and the row is reported failed', async () => { + const { engine, protocol, rows } = await boot(); + await storeRow(engine, VIEW, 'showcase_task', UNBOUND_COLUMNS); + // A save door that answers and writes nothing. + (protocol as any).saveMetaItem = async () => ({ success: true }); + const result = await migrate(protocol); + expect(result.outcomes.map((o: any) => o.outcome)).toEqual(['failed']); + expect(result.outcomes[0].error).toContain('does not read back as the declared override'); + expect(rows(VIEW)).toEqual([[null, 'active']]); + expect(result.bases[0]).toMatchObject({ moved: 0, failed: 1 }); + }); + it('a stored pair declaration with no basedOn gains the base\'s version', async () => { const { engine, protocol, rowOf } = await boot(); await (protocol as any).ensureCustomizationPackage({ basePackageId: BASE }); From c14148beb039df466544355a48f86610cb3e2131 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 1 Nov 2026 13:12:08 +0000 Subject: [PATCH 3/7] fix(metadata-protocol): read the pair landing only where a package-less save would be refused The lock gate stays the first store reader of a package-less save (#5706); the landing is looked up at the declared_outside_pair refusal and at the late undeclared_same_name re-judgement only. Co-Authored-By: Claude --- packages/metadata-protocol/src/protocol.ts | 75 ++++++++++++++++------ 1 file changed, 54 insertions(+), 21 deletions(-) diff --git a/packages/metadata-protocol/src/protocol.ts b/packages/metadata-protocol/src/protocol.ts index 29f2529bfc5..93749bf30ff 100644 --- a/packages/metadata-protocol/src/protocol.ts +++ b/packages/metadata-protocol/src/protocol.ts @@ -9833,6 +9833,19 @@ export class ObjectStackProtocolImplementation implements return migratePackagelessOverlays.call(host, request); } + /** + * [#28413] Whether a save is a package-less AUTHORING save — the one + * {@link packagelessPairLanding} may find a pair row for: it names no + * package, is no runtime-source write or package duplicate, and carries a + * body object. + */ + private isPackagelessAuthoringSave(request: SaveMetaItemRequest): boolean { + return request.packageId == null + && request.source === undefined + && request.writeFace !== 'package-duplicate' + && request.item !== null && typeof request.item === 'object' && !Array.isArray(request.item); + } + /** * [#28413] The declared override a package-less save of an overlay type * lands on, or `undefined` — see `packagelessPairLanding` in @@ -21050,35 +21063,30 @@ export class ObjectStackProtocolImplementation implements // // [#28413] Until #28576 closes the package-less write: a package-less // authoring save whose address is a pair row declaring the override of - // its base (a row `os migrate packageless-overlays` moved there) keeps - // that row's `overrides` block when the body carries none, and is - // judged as the write into the pair it is — so Setup's save of a - // migrated item, which sends the item as it read it, with or without - // the declaration, is not refused as undeclared (or as a package-less - // declaration). See `packagelessPairLanding`. - let overrideSubject: OverrideWriteSubject = request; + // its base (a row `os migrate packageless-overlays` moved there) is the + // write into the pair it is — so Setup's save of a migrated item, which + // sends the item as it read it, is not refused. Carrying the + // declaration, it is judged here as the write into that pair; carrying + // none, the late re-judgement below keeps the row's `overrides` block + // (see `packagelessPairLanding`). The landing is read ONLY where the + // save would otherwise be refused, so no admitted save, and no refusal + // ranked before these, reads one row more (#5706's lock gate stays the + // first reader). + let overrideVerdict = this.overrideWriteVerdict(request); if ( - request.packageId == null - && request.source === undefined - && request.writeFace !== 'package-duplicate' - && request.item !== null && typeof request.item === 'object' && !Array.isArray(request.item) + overrideVerdict.verdict === 'refuse' + && overrideVerdict.rule === 'declared_outside_pair' + && this.isPackagelessAuthoringSave(request) ) { - const landingType = PLURAL_TO_SINGULAR[request.type] ?? request.type; const landing = await this.packagelessPairLanding( - request, itemDiscriminator(landingType, request.item), + request, itemDiscriminator(PLURAL_TO_SINGULAR[request.type] ?? request.type, request.item), ); - if (landing !== undefined) { - if (!('overrides' in (request.item as Record))) { - request = { ...request, item: { ...(request.item as Record), overrides: landing.declaration } }; - } - overrideSubject = { ...request, packageId: landing.pair }; - } + if (landing !== undefined) overrideVerdict = this.overrideWriteVerdict({ ...request, packageId: landing.pair }); } - const overrideVerdict = this.overrideWriteVerdict(overrideSubject); if (overrideVerdict.verdict === 'refuse' && overrideVerdict.rule !== 'undeclared_same_name') { throw overrideWriteRefusalError(overrideVerdict); } - const declaredOverride = overrideVerdict.verdict === 'admit'; + let declaredOverride = overrideVerdict.verdict === 'admit'; if (request.source === undefined && request.writeFace !== 'package-duplicate') { const authored = await this.tenantAuthoredWriteRefusal({ type: request.type, @@ -22267,6 +22275,31 @@ export class ObjectStackProtocolImplementation implements lateOverrideVerdict = this.overrideWriteVerdict({ ...request, packageId: inheritedPackage }); } } + // [#28413] …and a package-less save carrying no declaration whose + // address is a pair row declaring the override (the early gate's + // comment above) keeps that row's `overrides` block: the body as + // persisted is the declared override it replaces, judged again. + if ( + lateOverrideVerdict.verdict === 'refuse' + && lateOverrideVerdict.rule === 'undeclared_same_name' + && namedNoPackage + // (its address may have named the pair on the request since: `namedNoPackage` is what it named) + && this.isPackagelessAuthoringSave({ ...request, packageId: null }) + && !('overrides' in (request.item as Record)) + ) { + const landing = await this.packagelessPairLanding( + request, itemDiscriminator(PLURAL_TO_SINGULAR[request.type] ?? request.type, request.item), + ); + if (landing !== undefined) { + const declared = { ...request, item: { ...(request.item as Record), overrides: landing.declaration } }; + const verdict = this.overrideWriteVerdict({ ...declared, packageId: landing.pair }); + if (verdict.verdict === 'admit') { + request = declared; + lateOverrideVerdict = verdict; + declaredOverride = true; + } + } + } if (lateOverrideVerdict.verdict === 'refuse') throw overrideWriteRefusalError(lateOverrideVerdict); const singularTypeForRepo = PLURAL_TO_SINGULAR[request.type] ?? request.type; const artifactBacked = this.isArtifactBacked(singularTypeForRepo, request.name); From 77207ad78caef2487adf4ee4ae3a4a6446c64c36 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 1 Nov 2026 13:12:55 +0000 Subject: [PATCH 4/7] chore(scripts): declare the packageless-overlay migration's per-row outcome report as its failure delivery Co-Authored-By: Claude --- scripts/check-durability-degradation-log-level.mjs | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/scripts/check-durability-degradation-log-level.mjs b/scripts/check-durability-degradation-log-level.mjs index a13cfebf4f3..6e1adca3f55 100644 --- a/scripts/check-durability-degradation-log-level.mjs +++ b/scripts/check-durability-degradation-log-level.mjs @@ -585,6 +585,18 @@ const FAILURE_PROPAGATION_SITES = new Map([ + 'baseline in #4754).', }, ], + [ + 'packages/metadata-protocol/src/protocol-packageless-overlay-migration.ts::migratePackagelessOverlays', + { + callees: [['fail', 'effect']], + why: + "`fail()` is this function's per-row outcome recorder: it increments the base's `failed` count " + + "and appends `{ id, outcome: 'failed', error }` to the RETURNED report, which " + + '`os migrate packageless-overlays` prints row by row and turns into exit 1 under `--apply`. ' + + 'The unbound row is kept on every failed path (its delete follows the read-back), so the ' + + "report names the row whose move did not land and why — the operation's contract (#28413).", + }, + ], [ 'packages/cli/src/utils/schema-migration-plugins.ts::measureComposedCoverage', { From 8e95731650cdf0b9d769dc50a01a9b6a09446f57 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 1 Nov 2026 13:14:20 +0000 Subject: [PATCH 5/7] docs(changeset): declare Skills-docs for the packageless-overlays ceremony Co-Authored-By: Claude --- .changeset/28413-packageless-overlays.md | 1 + 1 file changed, 1 insertion(+) diff --git a/.changeset/28413-packageless-overlays.md b/.changeset/28413-packageless-overlays.md index c4c46386c8f..830320f5e33 100644 --- a/.changeset/28413-packageless-overlays.md +++ b/.changeset/28413-packageless-overlays.md @@ -7,6 +7,7 @@ feat(cli,metadata-protocol): `os migrate packageless-overlays` moves package-les Clause-②: yes (widening) Metadata-protocol: added +Skills-docs: none — the one taught line this touches, `skills/objectstack-platform/SKILL.md`'s "a package-unbound Setup row still wins over the base's until D11", stays true: the ceremony moves such rows when an operator runs it, and the package-less write that makes new ones stays open until #28576, which is the change that retires that line. No file under `skills/**` teaches the `os migrate` operator ceremonies (a `git grep` of `skills/` for `packageless` and `security-catalog-overlays` names nothing). **The one operator step of this release.** After upgrading, run `os migrate packageless-overlays` once (preview), then `os migrate packageless-overlays --apply`, on each deployment that has Setup customizations. Until it runs, those customizations are still served, but they stay rows of no package: they are not exported and do not reach another environment. From 34c1aa5d997079f8636392d81d925a80a65672c6 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 1 Nov 2026 13:25:51 +0000 Subject: [PATCH 6/7] test(cli): resolve @objectstack/service-package to source in the cli suite Co-Authored-By: Claude --- packages/cli/vitest.config.ts | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/packages/cli/vitest.config.ts b/packages/cli/vitest.config.ts index 397ae9d490a..7d5ad3dacb7 100644 --- a/packages/cli/vitest.config.ts +++ b/packages/cli/vitest.config.ts @@ -962,6 +962,18 @@ export default defineConfig({ find: /^@objectstack\/plugin-sharing$/, replacement: path.resolve(__dirname, '../plugins/plugin-sharing/src/index.ts'), }, + // `commands/migrate/packageless-overlays.ts` mounts a live + // `@objectstack/service-package` `PackageServicePlugin` under `--apply`, so + // the `.custom` pair the ceremony creates persists to `sys_packages`; + // `packageless-overlays.integration.test.ts` boots it the same way. + // Resolved to source for the reason its neighbours give. Anchored + // (service-package's bare entry is the one imported). + // `check:test-source-alias` names this import and is the gate that fails + // without the entry. + { + find: /^@objectstack\/service-package$/, + replacement: path.resolve(__dirname, '../services/service-package/src/index.ts'), + }, ], }, test: { From 5865a0ceceed9e86ad8fc9ec8803c30ea5d84934 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 1 Nov 2026 14:24:46 +0000 Subject: [PATCH 7/7] fix(metadata-protocol): a pair row in either state shadows a package-less row; basedOn completion over every overridable type Co-Authored-By: Claude --- .changeset/28413-packageless-overlays.md | 4 +- .../commands/migrate/packageless-overlays.ts | 41 ++------------ packages/metadata-protocol/src/index.ts | 1 + .../protocol-packageless-overlay-migration.ts | 55 ++++++++++++++++--- ...ocol-packageless-overlay-migration.test.ts | 36 ++++++++++-- 5 files changed, 88 insertions(+), 49 deletions(-) diff --git a/.changeset/28413-packageless-overlays.md b/.changeset/28413-packageless-overlays.md index 830320f5e33..89c6f728989 100644 --- a/.changeset/28413-packageless-overlays.md +++ b/.changeset/28413-packageless-overlays.md @@ -12,8 +12,8 @@ Skills-docs: none — the one taught line this touches, `skills/objectstack-plat **The one operator step of this release.** After upgrading, run `os migrate packageless-overlays` once (preview), then `os migrate packageless-overlays --apply`, on each deployment that has Setup customizations. Until it runs, those customizations are still served, but they stay rows of no package: they are not exported and do not reach another environment. - **What moves.** Every `sys_metadata` row bound to no package, of the five overlay types (`view`, `dashboard`, `report`, `translation`, `email_template`, active or draft), over a name exactly one installed read-only package ships. It moves into that package's paired `.custom` — created through the ensure door when absent and persisted to `sys_packages` — with `overrides: { package: , basedOn: }` in its body, saved through the metadata save door in its own state. The new row is read back as the served override, and only then is the package-less row deleted (with its history tombstone). Every read of the name serves the same content as before. A second run moves nothing. -- **What is left alone, and listed.** A name several packages ship; a row whose body already declares an override of another package; and a row the pair already overrides (a customization made in Studio first — the report names the two ways out: delete the package-less row, or "Restore package original" in Studio and re-run). A name no package ships is environment metadata and is not touched. Permission sets and positions are not overlay types; `os migrate security-catalog-overlays` is theirs. -- **Declarations missing `basedOn`.** A row of a pair that declares `overrides: { package }` with no `basedOn` is read as an override but refused on its next save. The same command records `basedOn` (the base's installed version) on it. +- **What is left alone, and listed.** A name several packages ship; a row whose body already declares an override of another package; and a row the pair already overrides, active or draft (a customization made in Studio first — the report names the two ways out: delete the package-less row, or "Restore package original" in Studio and re-run). A name no package ships is environment metadata and is not touched. Permission sets and positions are not overlay types; `os migrate security-catalog-overlays` is theirs. +- **Declarations missing `basedOn`.** A row of a pair, of any type a declared override applies to, that declares `overrides: { package }` with no `basedOn` is read as an override but refused on its next save. The same command records `basedOn` (the base's installed version) on it. - **Setup keeps working on a moved item.** Until the package-less write closes, a save that names no package and lands on a moved row (Setup's form save and reset) keeps that row's `overrides` block when the body carries none, and is judged as the write into the pair when the body carries one — so it is no longer refused `403 NOT_OVERRIDABLE`. A package-less delete removes the pair's row, and the package's own document is served again. - **Flags.** `--apply` (with `--yes` or the prompt), `--json`, `--force` past a busy SQLite file, `--database-url`, and `serve`'s `--preset` / `--dev`. The command is the operator's; the HTTP door that creates a pair keeps its walled refusal. diff --git a/packages/cli/src/commands/migrate/packageless-overlays.ts b/packages/cli/src/commands/migrate/packageless-overlays.ts index fd89c89f6bf..485dea35dcb 100644 --- a/packages/cli/src/commands/migrate/packageless-overlays.ts +++ b/packages/cli/src/commands/migrate/packageless-overlays.ts @@ -16,6 +16,7 @@ import { errorCodeFields, isExitSignal, } from '../../utils/format.js'; +import type { PackagelessOverlayCensus, PackagelessOverlayMigration } from '@objectstack/metadata-protocol'; import { PackageServicePlugin } from '@objectstack/service-package'; import { bootSchemaStack } from '../../utils/schema-migrate.js'; import { describeUnloadableHostConfig } from '../../utils/schema-migration-plugins.js'; @@ -37,41 +38,11 @@ async function confirm(question: string): Promise { } } -/** The census row shape `listPackagelessOverlays` answers (`@objectstack/metadata-protocol`). */ -interface CensusRow { - id: string; - storedType: string; - type: string; - name: string; - state: 'active' | 'draft'; - member?: string; - disposition: 'movable' | 'multi_shipped' | 'no_pair' | 'foreign_declaration' | 'shadowed'; - shippers: string[]; - base?: string; - pair?: string; - shadowDeclares?: boolean; -} -interface IncompleteDeclaration { - id: string; - type: string; - name: string; - state: 'active' | 'draft'; - member?: string; - base: string; - pair: string; -} -interface Census { - rows: CensusRow[]; - environmentRows: number; - incompleteDeclarations: IncompleteDeclaration[]; -} -type Outcome = - | { id: string; outcome: 'moved' | 'declaration_completed'; packageId: string; basedOn: string } - | { id: string; outcome: 'failed'; error: string }; -interface Migration extends Census { - outcomes: Outcome[]; - bases: Array<{ base: string; pair: string; pairCreated: boolean; moved: number; declarationsCompleted: number; failed: number; shadowed: number }>; -} +/** The census and the move's report, as `@objectstack/metadata-protocol` declares them. */ +type Census = PackagelessOverlayCensus; +type Migration = PackagelessOverlayMigration; +type CensusRow = Census['rows'][number]; +type Outcome = Migration['outcomes'][number]; interface MigrationProtocol { listPackagelessOverlays?: () => Promise; migratePackagelessOverlays?: (request: { actor: string }) => Promise; diff --git a/packages/metadata-protocol/src/index.ts b/packages/metadata-protocol/src/index.ts index 7b7680e54a5..01d862e5ebd 100644 --- a/packages/metadata-protocol/src/index.ts +++ b/packages/metadata-protocol/src/index.ts @@ -238,6 +238,7 @@ export type { PackageWritabilityEngine } from './package-writability.js'; export { isServedOverrideRowOf } from './served-override.js'; export type { ServedOverrideRow } from './served-override.js'; export type { ServedOverrideEntry } from './protocol-served-overrides.js'; +export type { PackagelessOverlayCensus, PackagelessOverlayMigration } from './protocol-packageless-overlay-migration.js'; // [ADR-0131 D6] The `unmanaged` install mode's walled-posture refusal — the // function `importUnmanagedPackage` raises, exported so the install door diff --git a/packages/metadata-protocol/src/protocol-packageless-overlay-migration.ts b/packages/metadata-protocol/src/protocol-packageless-overlay-migration.ts index de74e1b0bfd..083d33d4f6f 100644 --- a/packages/metadata-protocol/src/protocol-packageless-overlay-migration.ts +++ b/packages/metadata-protocol/src/protocol-packageless-overlay-migration.ts @@ -36,9 +36,10 @@ * is writable. * - `foreign_declaration` — the body already carries an `overrides` key that * does not name the base: listed, never rewritten. - * - `shadowed` — the pair already holds a row of that name in that state (a - * Studio customization made first, N-17): listed with the two ways out, - * never merged or deleted. + * - `shadowed` — the pair already holds a row of that item, in EITHER state + * (a Studio customization made first, N-17): listed with the two ways out, + * never merged or deleted. Per item, not per state — an unbound draft moved + * in beside the pair's active override would replace it on its publish. * - `movable` — everything else. `--apply` moves it. * * Permission sets, positions and capabilities are not overlay types, so they @@ -73,13 +74,19 @@ * does not judge `basedOn`) and refused on its next save by the write gate * (`OverrideDeclarationSchema` requires it). The same ceremony completes it: * `basedOn` = the installed base's version now, saved through the same door. + * Its population is every type a declaration applies to + * (`isOverridableMetadataType`), as the read counts one — not only the five + * overlay types the move acts on. A type whose declared write the gate does + * not yet open (`flow`, `action`: ADR-0146 D9) is refused by that save, and the + * row is that refusal's `failed` outcome, its sentence the reason. */ import { + OVERRIDABLE_METADATA_TYPES, customizationBasePackageId, customizationPackageId, } from '@objectstack/spec/kernel'; -import { PLURAL_TO_SINGULAR } from '@objectstack/spec/shared'; +import { PLURAL_TO_SINGULAR, SINGULAR_TO_PLURAL } from '@objectstack/spec/shared'; import { storedRowDocument } from './item-lock.js'; import { OVERLAY_MEMBER_COLUMN, overlayMemberValue } from './migrations/overlay-index.js'; import type { ObjectStackProtocolImplementation } from './protocol.js'; @@ -210,12 +217,30 @@ function messageOf(error: unknown): string { const identityKey = (type: string, name: string, member: string | undefined, state: string): string => `${type}\u0000${name}\u0000${member ?? ''}\u0000${state}`; +/** + * Every type a declared override applies to (`isOverridableMetadataType`), + * both spellings: the population of the `basedOn` completion, which reads a + * declaration wherever `isServedOverrideRowOf` counts one — not only on the + * five overlay types the move acts on. + */ +const OVERRIDABLE_TYPE_SPELLINGS: readonly string[] = Object.freeze( + [...new Set(OVERRIDABLE_METADATA_TYPES.flatMap((type) => [type, SINGULAR_TO_PLURAL[type] ?? type]))].sort(), +); + /** The rows of the five overlay types, either spelling, the environment's only. */ -async function overlayRows(this: Pick): Promise { +function overlayRows(this: Pick): Promise { + return environmentRowsOfTypes.call(this, [...this.overlayTypes].sort()); +} + +/** The environment's `sys_metadata` rows of the given stored type spellings. */ +async function environmentRowsOfTypes( + this: Pick, + types: readonly string[], +): Promise { let rows: any[] = []; try { rows = await this.findEnvironmentRows('sys_metadata', { - where: { type: { $in: [...this.overlayTypes].sort() } }, + where: { type: { $in: [...types] } }, context: SYSTEM, }); } catch (error) { @@ -252,6 +277,15 @@ export async function listPackagelessOverlays( const type = PLURAL_TO_SINGULAR[row.type] ?? row.type; pairRows.set(`${row.package_id}\u0000${identityKey(type, row.name, memberOf(row), row.state)}`, row); } + // The pair's row of an item that shadows an unbound row in `state`: the + // pair's override is per ITEM, not per state, so a row of the pair in + // EITHER state shadows it — the one in the same state when there is one. + // (An unbound draft beside the pair's active override, moved in as the + // pair's draft, would replace that override on its publish: the merge + // the census exists to refuse.) + const shadowOf = (pair: string, type: string, name: string, member: string | undefined, state: 'active' | 'draft') => + pairRows.get(`${pair}\u0000${identityKey(type, name, member, state)}`) + ?? pairRows.get(`${pair}\u0000${identityKey(type, name, member, state === 'active' ? 'draft' : 'active')}`); const listed: PackagelessOverlayRow[] = []; let environmentRows = 0; @@ -297,7 +331,7 @@ export async function listPackagelessOverlays( listed.push({ ...common, base, pair, disposition: 'foreign_declaration' }); continue; } - const shadow = pairRows.get(`${pair}\u0000${identityKey(type, name, member, state)}`); + const shadow = shadowOf(pair, type, name, member, state); if (shadow !== undefined) { listed.push({ ...common, base, pair, disposition: 'shadowed', shadowDeclares: isServedOverrideRowOf(shadow, base) }); continue; @@ -305,8 +339,10 @@ export async function listPackagelessOverlays( listed.push({ ...common, base, pair, disposition: 'movable' }); } + // Every type a declaration applies to, not only the five the move acts on. const incompleteDeclarations: IncompleteOverrideDeclarationRow[] = []; - for (const row of pairRows.values()) { + for (const row of await environmentRowsOfTypes.call(this, OVERRIDABLE_TYPE_SPELLINGS)) { + if (isUnbound(row)) continue; const base = customizationBasePackageId(row.package_id); if (base === null || !isServedOverrideRowOf(row, base)) continue; const basedOn = (bodyOf(row) as { overrides?: { basedOn?: unknown } } | undefined)?.overrides?.basedOn; @@ -392,6 +428,9 @@ export async function migratePackagelessOverlays( const repository = new SysMetadataRepository({ engine: this.engine as never, orgLabel: 'env' }); const rowsById = new Map(); for (const row of await overlayRows.call(this)) rowsById.set(String(row.id), row); + if (census.incompleteDeclarations.length > 0) { + for (const row of await environmentRowsOfTypes.call(this, OVERRIDABLE_TYPE_SPELLINGS)) rowsById.set(String(row.id), row); + } for (const row of census.rows) { if (row.disposition !== 'movable' || row.base === undefined || row.pair === undefined) continue; diff --git a/packages/objectql/src/protocol-packageless-overlay-migration.test.ts b/packages/objectql/src/protocol-packageless-overlay-migration.test.ts index 3d662af06c5..a38d38a0c43 100644 --- a/packages/objectql/src/protocol-packageless-overlay-migration.test.ts +++ b/packages/objectql/src/protocol-packageless-overlay-migration.test.ts @@ -145,6 +145,8 @@ function makeMemoryDriver() { }; return { driver, stores }; } +const PAGE = 'showcase_home'; +const pageBody = (name: string, label: string) => ({ name, label, type: 'app', regions: [] }); const listBody = (name: string, object: string, columns: readonly string[]) => ({ name, object, @@ -285,19 +287,26 @@ describe('[#28413] the census and the move', () => { // A Studio customization made first (N-17): the pair's declared row. await storeRow(engine, VIEW, 'showcase_task', EDITED_COLUMNS, PAIR, 'active', { overrides: { package: BASE, basedOn: '1.0.0' } }); await storeRow(engine, VIEW, 'showcase_task', UNBOUND_COLUMNS); + // …and a Setup DRAFT of the same name: the pair's override is per + // name, so a draft beside its ACTIVE override is shadowed too — moved + // in as the pair's draft, its publish would replace the Studio + // customization (the merge Wanted 2 forbids). + await storeRow(engine, VIEW, 'showcase_task', UNBOUND_COLUMNS, null, 'draft'); await storeRow(engine, SHARED_VIEW, 'shared_task', UNBOUND_COLUMNS); await storeRow(engine, ENV_VIEW, 'showcase_task', UNBOUND_COLUMNS); const listedCensus = await census(protocol); - expect(dispositions(listedCensus)).toEqual([`${SHARED_VIEW}@active multi_shipped`, `${VIEW}@active shadowed`]); + expect(dispositions(listedCensus)).toEqual([ + `${SHARED_VIEW}@active multi_shipped`, `${VIEW}@active shadowed`, `${VIEW}@draft shadowed`, + ]); expect(listedCensus.rows.find((r: any) => r.disposition === 'multi_shipped').shippers).toEqual([OTHER, BASE]); - expect(listedCensus.rows.find((r: any) => r.disposition === 'shadowed').shadowDeclares).toBe(true); + expect(listedCensus.rows.filter((r: any) => r.disposition === 'shadowed').map((r: any) => r.shadowDeclares)).toEqual([true, true]); expect(listedCensus.environmentRows).toBe(1); const result = await migrate(protocol); expect(result.outcomes).toEqual([]); - expect(result.bases).toEqual([{ base: BASE, pair: PAIR, pairCreated: false, moved: 0, declarationsCompleted: 0, failed: 0, shadowed: 1 }]); - expect(rows(VIEW).slice().sort((a, b) => String(a[0]).localeCompare(String(b[0])))).toEqual([[PAIR, 'active'], [null, 'active']]); + expect(result.bases).toEqual([{ base: BASE, pair: PAIR, pairCreated: false, moved: 0, declarationsCompleted: 0, failed: 0, shadowed: 2 }]); + expect(rows(VIEW).slice().sort((a, b) => `${a[0]}${a[1]}`.localeCompare(`${b[0]}${b[1]}`))).toEqual([[PAIR, 'active'], [null, 'active'], [null, 'draft']]); expect(rows(SHARED_VIEW)).toEqual([[null, 'active']]); expect(rows(ENV_VIEW)).toEqual([[null, 'active']]); // The pair's declared row is still the one served. @@ -329,6 +338,25 @@ describe('[#28413] the census and the move', () => { expect(columnsOf(await read(protocol))).toEqual(EDITED_COLUMNS); expect((await census(protocol)).incompleteDeclarations).toEqual([]); }); + + it('…on every overridable type, not only the five overlay types (a page)', async () => { + const { engine, protocol, stores } = await boot(); + engine.registerApp({ id: BASE, name: 'showcase', version: '1.2.0', namespace: 'showcase', pages: [pageBody(PAGE, 'Home')] } as any); + await (protocol as any).ensureCustomizationPackage({ basePackageId: BASE }); + await engine.insert('sys_metadata', { + type: 'page', name: PAGE, organization_id: null, package_id: PAIR, + metadata: JSON.stringify({ ...pageBody(PAGE, 'Edited home'), overrides: { package: BASE } }), state: 'active', version: 1, + }, { context: { isSystem: true } }); + const pageRow = () => Array.from(stores.get('sys_metadata')?.values() ?? []) + .find((r) => r.type === 'page' && r.name === PAGE && r.package_id === PAIR && r.state === 'active'); + + expect((await census(protocol)).incompleteDeclarations.map((d: any) => [d.type, d.name, d.pair])).toEqual([['page', PAGE, PAIR]]); + const result = await migrate(protocol); + expect(result.outcomes.map((o: any) => [o.outcome, o.basedOn, o.error])).toEqual([['declaration_completed', '1.2.0', undefined]]); + const body = JSON.parse(String(pageRow()!.metadata)); + expect(body.overrides).toEqual({ package: BASE, basedOn: '1.2.0' }); + expect(body.label).toBe('Edited home'); + }); }); /**