From 635ab80279c01128f7925dde67234357a7a7eaf3 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 26 Oct 2026 12:25:02 +0000 Subject: [PATCH 01/13] wip(spec): seed file reference and bundled seed file bytes Co-Authored-By: Claude Claude-Session: https://claude.ai/code/session_012eJQ8dtut5bZsnhFE3XGHF --- packages/spec/src/data/index.ts | 3 + packages/spec/src/data/seed-file.zod.ts | 143 ++++++++++++++++++++++++ packages/spec/src/data/seed.zod.ts | 107 +++++++++++++++++- 3 files changed, 252 insertions(+), 1 deletion(-) create mode 100644 packages/spec/src/data/seed-file.zod.ts diff --git a/packages/spec/src/data/index.ts b/packages/spec/src/data/index.ts index ddbfa841ef6..bd06fe5295a 100644 --- a/packages/spec/src/data/index.ts +++ b/packages/spec/src/data/index.ts @@ -235,6 +235,9 @@ export * from './driver-nosql.zod'; // policy, shared by the seed dataset column (`seed.zod`) and the import // mapping entry (`api/export.zod`), so neither door spells it twice. export * from './column-apply-on.zod'; +// [#25779] A file the package ships, written into a file field by a seed: +// `seedFile(path)`, and the bytes `os build` bundles under `Seed.files`. +export * from './seed-file.zod'; export * from './seed.zod'; // Form Layouts diff --git a/packages/spec/src/data/seed-file.zod.ts b/packages/spec/src/data/seed-file.zod.ts new file mode 100644 index 00000000000..468dd2f7ee2 --- /dev/null +++ b/packages/spec/src/data/seed-file.zod.ts @@ -0,0 +1,143 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +import { z } from 'zod'; + +import { lazySchema } from '../shared/lazy-schema'; +import { strictObject } from '../shared/strict-object'; + +/** + * # Seed files — a file a package ships, written into a file field by a seed + * + * A `file` / `image` / `avatar` / `video` / `audio` field stores an opaque + * `sys_file` id (ADR-0104 D3 wave 2), and only an upload mints one. A seed has + * no upload, so a seed record names the file by the PATH the package ships it + * at instead, and the platform carries the bytes from there (#25779, ruled on + * hotclm#19: "the platform's seed loader should carry bundled fixture files; + * the app keeps `file` required"): + * + * 1. **Authoring.** A record writes `seedFile('fixtures/nda-v1.pdf')` — the + * value {@link SeedFileReferenceSchema} — into a file-class field. The path + * is relative to the package root: the directory holding + * `objectstack.config.ts`, the root a declarative connector's file ref + * resolves against too. + * 2. **Bundling.** `os build` / `os compile` reads every referenced file and + * writes its bytes into the dataset's `files` map + * ({@link SeedFileContentSchema}), keyed by the same path. The compiled + * artifact is what every boot reads — `os dev`, `os serve`, a published + * package, an artifact URL — so the bytes travel with the seed wherever it + * goes. A path that names no file fails the build. + * 3. **Loading.** The seed loader stores the bytes as a `sys_file`, writes its + * id into the field, and has the written record own it — (object, record, + * field), the ownership an upload gets. A replay that finds the row still + * holding a file with the same bytes keeps it, so a boot does not mint a + * new file for an unchanged fixture. A reference whose bytes the dataset + * does not carry is refused, by row, naming the path. + * + * 300 rows may name the same path: each row gets a file of its own, because a + * file has one owning field (ADR-0104 D3). + */ + +/** Why a seed file path was refused — the one sentence both refusals print. */ +const SEED_FILE_PATH_RULE = + 'a seed file path is relative to the package root, uses `/` separators, and has no empty, `.` or `..` ' + + 'segment (e.g. `fixtures/nda-v1.pdf`)'; + +/** + * Is `path` a package-relative seed file path? Relative (no leading `/`, no + * drive letter), `/`-separated (no `\`), and confined to the package: no + * empty, `.` or `..` segment, so the path cannot climb out of the root it is + * resolved against and two spellings of one file cannot key two entries. + */ +export function isSeedFilePath(path: unknown): path is string { + if (typeof path !== 'string' || path.length === 0) return false; + if (path.startsWith('/') || path.includes('\\') || /^[A-Za-z]:/.test(path)) return false; + return path.split('/').every((segment) => segment !== '' && segment !== '.' && segment !== '..'); +} + +/** A package-relative seed file path ({@link isSeedFilePath}). */ +export const SeedFilePathSchema = lazySchema(() => z.string() + .refine(isSeedFilePath, { message: SEED_FILE_PATH_RULE }) + .describe('Path of a file the package ships, relative to the package root (e.g. `fixtures/nda-v1.pdf`)')); + +/** A MIME type: `type/subtype`, the shape a `sys_file.mime_type` carries. */ +const MIME_TYPE = /^[a-z0-9][a-z0-9!#$&^_.+-]*\/[a-z0-9][a-z0-9!#$&^_.+-]*$/i; + +/** + * The value a seed record writes into a file-class field to have the platform + * store a file the package ships: `{ $file: 'fixtures/nda-v1.pdf' }`, built by + * {@link seedFile}. A `multiple: true` file field takes an array of them. + */ +export const SeedFileReferenceSchema = lazySchema(() => strictObject({ + surface: 'a seed file reference', + history: 'Declared closed from its first day: an unrecognized key is refused, never dropped.', + aliases: { path: '$file', file: '$file', mimetype: 'contentType', mime: 'contentType', filename: 'name' }, +}, { + $file: SeedFilePathSchema, + name: z.string().min(1).optional() + .describe('File name the stored file carries (what a download is saved as); defaults to the last segment of the path'), + contentType: z.string().regex(MIME_TYPE, 'expected a MIME type such as `application/pdf`').optional() + .describe('MIME type of the file; defaults to the one `os build` infers from the path\'s extension'), +}).describe('A file the package ships, by package-relative path, written into a file field by a seed')); + +/** Author-time shape of {@link SeedFileReferenceSchema}. */ +export type SeedFileReference = z.input; + +/** + * The bytes of one seed file, as `os build` bundles them into a dataset's + * `files` map: base64 `data`, with the `size` and SHA-256 the loader checks + * them against before it stores anything. + */ +export const SeedFileContentSchema = lazySchema(() => strictObject({ + surface: 'a bundled seed file', + history: 'Declared closed from its first day: an unrecognized key is refused, never dropped.', +}, { + contentType: z.string().regex(MIME_TYPE, 'expected a MIME type such as `application/pdf`') + .describe('MIME type the stored file carries'), + size: z.number().int().nonnegative().describe('Byte length of the file'), + sha256: z.string().regex(/^[a-f0-9]{64}$/, 'expected a lowercase hex SHA-256 digest') + .describe('SHA-256 of the bytes, lowercase hex — the loader refuses bytes that do not match it'), + data: z.string().describe('The bytes, base64-encoded'), +}).describe('One bundled seed file: its bytes and what they must hash to')); + +/** Author-time shape of {@link SeedFileContentSchema}. */ +export type SeedFileContent = z.input; + +/** + * Write a file the package ships into a file field from a seed record. + * + * ```ts + * defineSeed(ContractVersion, { + * externalId: 'code', + * records: [ + * { code: 'NDA-001-V1', file: seedFile('fixtures/nda-v1.pdf') }, + * ], + * }); + * ``` + * + * `path` is relative to the package root (the directory holding + * `objectstack.config.ts`). `os build` bundles the bytes; the seed loader + * stores them and the written record owns the file. + */ +export function seedFile( + path: string, + options: { name?: string; contentType?: string } = {}, +): SeedFileReference { + return { + $file: path, + ...(options.name !== undefined ? { name: options.name } : {}), + ...(options.contentType !== undefined ? { contentType: options.contentType } : {}), + }; +} + +/** + * Does `value` claim to be a seed file reference — a plain object carrying an + * own `$file` key? The CLAIM, not its validity: a malformed reference is still + * one, so the place that finds it can refuse it with the schema's own issues + * instead of writing an object into a file field. + */ +export function isSeedFileReference(value: unknown): value is SeedFileReference { + return value !== null + && typeof value === 'object' + && !Array.isArray(value) + && Object.prototype.hasOwnProperty.call(value, '$file'); +} diff --git a/packages/spec/src/data/seed.zod.ts b/packages/spec/src/data/seed.zod.ts index 15b68653583..404d450969f 100644 --- a/packages/spec/src/data/seed.zod.ts +++ b/packages/spec/src/data/seed.zod.ts @@ -13,6 +13,13 @@ import { LocaleSchema } from '../system/translation.zod'; import { findClosestMatches, formatSuggestion } from '../shared/suggestions.zod'; import { resolveInjectedSystemColumns, type InjectedSystemColumnName } from './injected-system-columns'; import { ColumnApplyOnSchema } from './column-apply-on.zod'; +import { FILE_REFERENCE_TYPES } from './field-value.zod'; +import { + SeedFileContentSchema, + SeedFilePathSchema, + SeedFileReferenceSchema, + isSeedFileReference, +} from './seed-file.zod'; export const SeedMode = z.enum([ 'insert', // Try to insert, fail on duplicate 'update', // Only update found records, ignore new @@ -65,6 +72,31 @@ export type SeedColumn = z.input; /** Post-parse shape of {@link SeedColumn} — defaults applied, transforms run (ADR-0122). */ export type SeedColumnParsed = z.infer; +/** + * One value of a seed record. The target object's field decides what it may + * be, so this schema takes anything — except a value that CLAIMS to be a seed + * file reference (an object carrying `$file`, {@link isSeedFileReference}), + * alone or as an element of a `multiple: true` file field's array, which must + * be a well-formed one (#25779): a misspelt key or a path that climbs out of + * the package is refused here, by name, instead of reaching a file field as an + * object the field cannot store. + */ +const SeedRecordValueSchema = z.unknown().superRefine((value, ctx) => { + const elements = Array.isArray(value) ? value : [value]; + elements.forEach((element, index) => { + if (!isSeedFileReference(element)) return; + const parsed = SeedFileReferenceSchema.safeParse(element); + if (parsed.success) return; + for (const issue of parsed.error.issues) { + ctx.addIssue({ + code: 'custom', + message: issue.message, + path: Array.isArray(value) ? [index, ...issue.path] : [...issue.path], + }); + } + }); +}); + /** * Seed Schema (Seed Data / Fixtures) * @@ -216,8 +248,37 @@ export const SeedSchema = lazySchema(() => strictObject({ /** * The Payload * Array of raw JSON objects matching the Object Schema. + * + * A file-class field (`file`, `image`, `avatar`, `video`, `audio`) takes a + * file the package ships as `seedFile('fixtures/nda-v1.pdf')` — see + * {@link SeedSchema.shape.files}. */ - records: z.array(z.record(z.string(), z.unknown())).describe('Data records'), + records: z.array(z.record(z.string(), SeedRecordValueSchema)).describe('Data records'), + + /** + * Bundled Files + * + * The bytes of every file the records name with `seedFile(path)` (a + * `{ $file: path }` value), keyed by that path (#25779). + * + * `os build` / `os compile` writes this map: it reads each referenced path + * relative to the package root (the directory holding + * `objectstack.config.ts`) and bundles the bytes into the compiled + * artifact, so the seed carries its files to every boot that reads the + * artifact — `os dev`, `os serve`, a published package. An author writes the + * references, not this map. + * + * The loader checks each entry's `size` and `sha256` against its `data`, + * stores the bytes as a `sys_file`, writes the new id into the field and + * has the written record own the file. A reference with no entry here, or + * an entry whose bytes do not match, refuses its row by name. + */ + files: z.record(SeedFilePathSchema, SeedFileContentSchema).optional() + .describe( + 'Bytes of the package files the records name with `seedFile(path)`, keyed by path. Written by ' + + '`os build`, which reads each path relative to the package root; the loader stores each one as a ' + + 'file the written record owns.', + ), // ADR-0010 — runtime protection envelope (internal — set by the loader). // `seed` is a registered metadata type, so `MetadataPlugin`'s artifact loader @@ -353,6 +414,44 @@ function assertSeedKeysDeclared( ); } +/** + * The call-time check that a seed file reference ({@link isSeedFileReference}) + * is written only into a file-class field (#25779): a `seedFile(...)` value on + * a `text` field would reach the engine as an object it cannot store. Every + * finding is refused together, one line per field and record. A field the + * object does not declare is {@link assertSeedKeysDeclared}'s finding, and a + * caller passing no field map is not judged. + */ +function assertSeedFilesOnFileFields( + objectDef: { name: string; fields: unknown }, + records: ReadonlyArray>, +): void { + const { fields } = objectDef; + if (!fields || typeof fields !== 'object' || Array.isArray(fields)) return; + const lines: string[] = []; + records.forEach((record, index) => { + for (const [key, value] of Object.entries(record)) { + const elements = Array.isArray(value) ? value : [value]; + if (!elements.some(isSeedFileReference)) continue; + const def = (fields as Record)[key]; + if (!def || typeof def !== 'object') continue; + const type = (def as { type?: unknown }).type; + if (typeof type === 'string' && FILE_REFERENCE_TYPES.has(type)) continue; + lines.push( + ` • records[${index}]: \`${key}\` is a \`${String(type)}\` field, and \`seedFile(...)\` writes only a file field.`, + ); + } + }); + if (lines.length === 0) return; + throw new Error( + `defineSeed('${objectDef.name}'): seedFile(...) written into a field that does not store a file.\n` + + 'A seed file reference stores a file the package ships, so it goes only into a field of type ' + + `${[...FILE_REFERENCE_TYPES].map((t) => `\`${t}\``).join(', ')}.\n\n` + + `${lines.join('\n')}\n\n` + + 'Write the value the field stores instead, or move the reference to the file field it was meant for.', + ); +} + /** * Type-safe factory for creating seed definitions. Every record key is checked * against the object definition passed in, twice: @@ -375,6 +474,10 @@ function assertSeedKeysDeclared( * The keys of {@link SeedSchema.shape.columns} are held to the same set, both * times ({@link SeedColumns}). * + * A `seedFile(path)` value is refused when it runs on a field that is not a + * file field (`file`, `image`, `avatar`, `video`, `audio`) — see + * {@link SeedSchema.shape.files} for how the bytes reach the loader. + * * @example * ```ts * export const leadSeed = defineSeed(Lead, { @@ -386,6 +489,7 @@ function assertSeedKeysDeclared( * { first_name: 'Bob', created_at: cel`daysAgo(3)` }, // ✅ an injected system column * { first_name: 'Bob', account: 'Acme Corp' }, // ✅ reference by natural key * { first_name: 'Bob', account: { externalId: 'Acme Corp' } }, // ❌ object not allowed + * { first_name: 'Eve', resume: seedFile('fixtures/eve-resume.pdf') }, // ✅ a file the package ships * ], * }); * ``` @@ -401,5 +505,6 @@ export function defineSeed< ): Seed { const seed = SeedSchema.parse({ ...config, object: objectDef.name }); assertSeedKeysDeclared(objectDef, seed.records, seed.columns); + assertSeedFilesOnFileFields(objectDef, seed.records); return seed; } From 483df50ec7b878b046989c3f776925ddfc9225db Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 26 Oct 2026 12:38:29 +0000 Subject: [PATCH 02/13] wip: field-files storage verb, seed loader seed files, compile-time bundling Co-Authored-By: Claude Claude-Session: https://claude.ai/code/session_012eJQ8dtut5bZsnhFE3XGHF --- packages/cli/src/commands/compile.ts | 21 + packages/cli/src/utils/bundle-seed-files.ts | 193 +++++++++ packages/metadata-protocol/src/index.ts | 2 +- .../src/protocol-publish-batch.ts | 7 +- packages/metadata-protocol/src/protocol.ts | 7 +- packages/metadata-protocol/src/seed-files.ts | 387 ++++++++++++++++++ packages/metadata-protocol/src/seed-loader.ts | 124 +++++- packages/runtime/src/app-plugin.ts | 43 +- packages/runtime/src/domains/packages.ts | 8 +- .../service-storage/src/field-file-service.ts | 311 ++++++++++++++ .../src/file-reference-lifecycle.ts | 4 +- .../src/storage-service-plugin.ts | 25 +- .../spec/src/contracts/storage-service.ts | 118 ++++++ 13 files changed, 1235 insertions(+), 15 deletions(-) create mode 100644 packages/cli/src/utils/bundle-seed-files.ts create mode 100644 packages/metadata-protocol/src/seed-files.ts create mode 100644 packages/services/service-storage/src/field-file-service.ts diff --git a/packages/cli/src/commands/compile.ts b/packages/cli/src/commands/compile.ts index db06378abf0..71fae50d8ca 100644 --- a/packages/cli/src/commands/compile.ts +++ b/packages/cli/src/commands/compile.ts @@ -15,6 +15,7 @@ import { type ConversionNotice, } from '@objectstack/spec'; import { loadConfig, namedExportRejectionHints } from '../utils/config.js'; +import { bundleSeedFiles } from '../utils/bundle-seed-files.js'; import { refuseUnbuiltStack } from '../utils/stack-provenance-refusal.js'; import { isExtractionRefusal, lowerCallables } from '../utils/lower-callables.js'; import { authoringRuleUnionStack } from '../utils/stack-collections.js'; @@ -1035,6 +1036,26 @@ export default class Compile extends Command { finalBundle.packages = placed.packages; } + // 4a. [#25779] Bundle the files seed records name with `seedFile(path)` + // into their datasets' `files` maps, read relative to the directory + // holding the config. The artifact is what every boot loads, so the + // bytes travel with the seed; a path that names no file fails the + // build here rather than a row at boot. + const seedFiles = bundleSeedFiles(finalBundle, path.dirname(absolutePath)); + if (seedFiles.issues.length > 0) { + if (flags.json) { + await emitJson({ success: false, error: 'seed file bundling failed', issues: seedFiles.issues, warnings: warningsSoFar(), conversions: conversionNotices }, 0, { compact: true }); + this.exit(1); + } + console.log(''); + printError(`Seed file bundling failed (${seedFiles.issues.length} issue${seedFiles.issues.length > 1 ? 's' : ''})`); + for (const issue of seedFiles.issues) console.log(chalk.red(` ✗ ${issue.location}: ${issue.message}`)); + this.exit(1); + } + if (seedFiles.files > 0 && !flags.json) { + printStep(`Bundling seed files... ${seedFiles.files} file${seedFiles.files === 1 ? '' : 's'} (${(seedFiles.bytes / 1024).toFixed(1)} KB)`); + } + // 4b. Bundle handler functions into `/objectstack-runtime.{hash}.mjs` // and stamp the relative path into the JSON so the runtime can // dynamic-import it at boot. `runtimeModule` is part of the diff --git a/packages/cli/src/utils/bundle-seed-files.ts b/packages/cli/src/utils/bundle-seed-files.ts new file mode 100644 index 00000000000..417c5db7d34 --- /dev/null +++ b/packages/cli/src/utils/bundle-seed-files.ts @@ -0,0 +1,193 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +import { createHash } from 'node:crypto'; +import fs from 'node:fs'; +import path from 'node:path'; +import { isSeedFileReference, type SeedFileContent, type SeedFileReference } from '@objectstack/spec/data'; + +/** + * [#25779] `os build` / `os compile`'s half of a seed file: read each file a + * seed record names with `seedFile(path)` and write its bytes into the + * dataset's `files` map, so the compiled artifact carries them to every boot + * that loads it — `os dev`, `os serve`, a published package, an artifact URL. + * + * The path is relative to the package root, the directory holding + * `objectstack.config.ts` (the root a declarative connector's file ref resolves + * against too), and confined to it. A path that names no file there, or a file + * whose type cannot be told from its extension and that no reference names a + * `contentType` for, fails the build: the seed would otherwise load without the + * file its field requires. + */ + +/** A refusal, located in the artifact like a schema issue is. */ +export interface SeedFileBundleIssue { + /** Where the reference sits, e.g. `data[2].records[0].file`. */ + location: string; + message: string; +} + +export interface SeedFileBundleResult { + /** Files bundled, counted once per dataset that carries them. */ + files: number; + /** Their bytes, before base64. */ + bytes: number; + issues: SeedFileBundleIssue[]; +} + +/** + * Media types by extension: the seed file's `contentType` when no reference + * names one. Deliberately the common document, image and media types only; any + * other file states its type with `seedFile(path, { contentType })`. + */ +const SEED_FILE_TYPES: Readonly> = { + '.pdf': 'application/pdf', + '.txt': 'text/plain', + '.md': 'text/markdown', + '.csv': 'text/csv', + '.json': 'application/json', + '.xml': 'application/xml', + '.html': 'text/html', + '.rtf': 'application/rtf', + '.doc': 'application/msword', + '.docx': 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', + '.xls': 'application/vnd.ms-excel', + '.xlsx': 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', + '.ppt': 'application/vnd.ms-powerpoint', + '.pptx': 'application/vnd.openxmlformats-officedocument.presentationml.presentation', + '.odt': 'application/vnd.oasis.opendocument.text', + '.zip': 'application/zip', + '.png': 'image/png', + '.jpg': 'image/jpeg', + '.jpeg': 'image/jpeg', + '.gif': 'image/gif', + '.webp': 'image/webp', + '.svg': 'image/svg+xml', + '.mp3': 'audio/mpeg', + '.wav': 'audio/wav', + '.ogg': 'audio/ogg', + '.m4a': 'audio/mp4', + '.mp4': 'video/mp4', + '.webm': 'video/webm', + '.mov': 'video/quicktime', +}; + +/** One dataset of the artifact, as far as this pass reads it. */ +interface DatasetLike { + records?: unknown; + files?: Record; +} + +/** Every seed dataset in the artifact — the top level's and each package body's — with where it sits. */ +function datasetsOf(bundle: Record): Array<{ dataset: DatasetLike; location: string }> { + const found: Array<{ dataset: DatasetLike; location: string }> = []; + const seen = new Set(); + const take = (list: unknown, location: string): void => { + if (!Array.isArray(list)) return; + list.forEach((dataset, i) => { + if (!dataset || typeof dataset !== 'object' || seen.has(dataset)) return; + seen.add(dataset); + found.push({ dataset: dataset as DatasetLike, location: `${location}[${i}]` }); + }); + }; + take(bundle.data, 'data'); + const manifest = bundle.manifest as { data?: unknown } | undefined; + if (manifest && typeof manifest === 'object') take(manifest.data, 'manifest.data'); + if (Array.isArray(bundle.packages)) { + bundle.packages.forEach((pkg, i) => { + const body = (pkg as { manifest?: { data?: unknown } } | null)?.manifest; + if (body && typeof body === 'object') take(body.data, `packages[${i}].manifest.data`); + }); + } + return found; +} + +/** The seed file references of one dataset, by path, with where each one sits. */ +function referencesOf( + dataset: DatasetLike, + location: string, +): Map> { + const byPath = new Map>(); + if (!Array.isArray(dataset.records)) return byPath; + dataset.records.forEach((record, r) => { + if (!record || typeof record !== 'object') return; + for (const [field, value] of Object.entries(record as Record)) { + const elements = Array.isArray(value) ? value : [value]; + elements.forEach((element, e) => { + if (!isSeedFileReference(element) || typeof element.$file !== 'string') return; + const at = `${location}.records[${r}].${field}${Array.isArray(value) ? `[${e}]` : ''}`; + const list = byPath.get(element.$file) ?? []; + list.push({ reference: element, location: at }); + byPath.set(element.$file, list); + }); + } + }); + return byPath; +} + +/** + * Write the bytes of every seed file the artifact's datasets name into their + * `files` maps, reading each path under `packageRoot`. Changes `bundle` in + * place; answers what it bundled and what it refused. + */ +export function bundleSeedFiles(bundle: Record, packageRoot: string): SeedFileBundleResult { + const result: SeedFileBundleResult = { files: 0, bytes: 0, issues: [] }; + const root = path.resolve(packageRoot); + const read = new Map(); + for (const { dataset, location } of datasetsOf(bundle)) { + const byPath = referencesOf(dataset, location); + if (byPath.size === 0) continue; + const files: Record = { ...(dataset.files ?? {}) }; + for (const [seedPath, references] of byPath) { + const first = references[0].location; + const absolute = path.resolve(root, seedPath); + const relative = path.relative(root, absolute); + if (relative === '' || relative.startsWith('..') || path.isAbsolute(relative)) { + result.issues.push({ + location: first, + message: `seedFile('${seedPath}') climbs out of the package root ${root}; a seed file path stays inside it.`, + }); + continue; + } + if (!read.has(absolute)) { + let bytes: Buffer | null = null; + try { + if (fs.statSync(absolute).isFile()) bytes = fs.readFileSync(absolute); + } catch { + bytes = null; + } + read.set(absolute, bytes); + } + const bytes = read.get(absolute) ?? null; + if (bytes === null) { + result.issues.push({ + location: first, + message: + `seedFile('${seedPath}') names no file: ${absolute} does not exist or is not a file. A seed file path is ` + + 'relative to the directory holding objectstack.config.ts.', + }); + continue; + } + const declared = references.map((r) => r.reference.contentType).find((t) => typeof t === 'string'); + const contentType = SEED_FILE_TYPES[path.extname(seedPath).toLowerCase()] ?? declared; + if (!contentType) { + result.issues.push({ + location: first, + message: + `seedFile('${seedPath}'): the file's type cannot be told from its extension. Name it on the reference: ` + + `seedFile('${seedPath}', { contentType: 'application/octet-stream' }) — or the type it really is.`, + }); + continue; + } + files[seedPath] = { + contentType, + size: bytes.byteLength, + sha256: createHash('sha256').update(bytes).digest('hex'), + data: bytes.toString('base64'), + }; + result.files += 1; + result.bytes += bytes.byteLength; + } + dataset.files = files; + } + return result; +} diff --git a/packages/metadata-protocol/src/index.ts b/packages/metadata-protocol/src/index.ts index fe8d552afb4..4d99ca5445d 100644 --- a/packages/metadata-protocol/src/index.ts +++ b/packages/metadata-protocol/src/index.ts @@ -244,7 +244,7 @@ export { } from './migrations/recorded-by-sentinel.js'; export type { SentinelHistoryRow } from './migrations/recorded-by-sentinel.js'; -export { SeedLoaderService } from './seed-loader.js'; +export { SeedLoaderService, type SeedLoaderServiceOptions } from './seed-loader.js'; export { runBuildProbes } from './build-probes.js'; export type { RuntimeBuildIssue, diff --git a/packages/metadata-protocol/src/protocol-publish-batch.ts b/packages/metadata-protocol/src/protocol-publish-batch.ts index 4ddc22d3565..75bf877afdb 100644 --- a/packages/metadata-protocol/src/protocol-publish-batch.ts +++ b/packages/metadata-protocol/src/protocol-publish-batch.ts @@ -36,6 +36,7 @@ * surface does not move. */ +import type { IFieldFileService } from '@objectstack/spec/contracts'; import type { ObjectStackProtocolImplementation } from './protocol.js'; import { canonicalMetaType, @@ -62,7 +63,10 @@ export interface PublishBatchHost { /** The body of `ObjectStackProtocolImplementation#applySeedBodies`; the method's docblock there is its contract. */ export async function applySeedBodies( - this: Pick, + this: Pick & { + /** [#25779] The `field-files` service a seed file is stored through, read from the kernel's registry. */ + getFieldFiles?: () => IFieldFileService | null | undefined; + }, bodies: unknown[], ): Promise<{ success: boolean; inserted: number; updated: number; error?: string; errors?: unknown[]; @@ -90,6 +94,7 @@ export async function applySeedBodies( this.engine as any, metadataAdapter as any, console as any, + { getFieldFiles: () => this.getFieldFiles?.() }, ); // [#8333 · P9] `safeParse`, not `parse` — see {@link // seedRequestValidationError}. A malformed seed body used to reach diff --git a/packages/metadata-protocol/src/protocol.ts b/packages/metadata-protocol/src/protocol.ts index 727252769c7..0842f371c30 100644 --- a/packages/metadata-protocol/src/protocol.ts +++ b/packages/metadata-protocol/src/protocol.ts @@ -169,8 +169,8 @@ import { PLURAL_TO_SINGULAR, SINGULAR_TO_PLURAL, canonicalMetaUrlType, metaUrlSp // [#13331] The cluster fan-out transport type only — the protocol never // depends on `@objectstack/service-cluster`; a bridge plugin there hands the // live transport in through `attachMetadataMutationPubSub`. -import type { IPubSub } from '@objectstack/spec/contracts'; -import { RLS_MEMBERSHIP_RESOLVER_SERVICE } from '@objectstack/spec/contracts'; +import type { IFieldFileService, IPubSub } from '@objectstack/spec/contracts'; +import { FIELD_FILE_SERVICE, RLS_MEMBERSHIP_RESOLVER_SERVICE } from '@objectstack/spec/contracts'; import { ALL_CONVERSIONS, applyConversionsToFlow, applyConversionsToStoredItem, type ConversionNotice, type ConversionTodoNotice } from '@objectstack/spec'; import { type FormView, type I18nLabel, isAggregatedViewContainer, expandViewContainer } from '@objectstack/spec/ui'; // [commit ece4dad31] Emitted-specifier pin. This module's inferred public declarations @@ -23339,6 +23339,9 @@ export class ObjectStackProtocolImplementation implements const host: ThisParameterType = { get engine() { return readEngine(); }, getMetaItem: (...a) => this.getMetaItem(...a), + // [#25779] The `field-files` service, so a published seed's files + // are stored as the boot seed's are. + getFieldFiles: () => this.getServicesRegistry?.().get(FIELD_FILE_SERVICE) as IFieldFileService | undefined, }; return applySeedBodies.call(host, bodies); } diff --git a/packages/metadata-protocol/src/seed-files.ts b/packages/metadata-protocol/src/seed-files.ts new file mode 100644 index 00000000000..b151539ae15 --- /dev/null +++ b/packages/metadata-protocol/src/seed-files.ts @@ -0,0 +1,387 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +import { createHash } from 'node:crypto'; +import type { FieldFileDescription, IFieldFileService } from '@objectstack/spec/contracts'; +import type { Seed, SeedFileContent, SeedFileReference } from '@objectstack/spec/data'; +import { FILE_REFERENCE_TYPES, isFileIdToken, isSeedFileReference } from '@objectstack/spec/data'; +// The platform's one answer to "did this read fail because the table has not +// been provisioned yet?" — a fresh install whose `sys_file` table the first +// upload provisions holds no file, so none is found. +import { isMissingTableError } from '@objectstack/metadata/errors'; + +/** + * [#25779] The seed loader's half of a seed file — `seedFile(path)` written + * into a file-class field (`@objectstack/spec/data`, `seed-file.zod.ts`). + * + * A file field holds an opaque `sys_file` id, and a seed record names the file + * by the path the package ships it at instead. `os build` bundled the bytes + * into the dataset's `files` map; this module turns each reference into a + * file the written record owns: + * + * 1. **Before the write** ({@link SeedDatasetFiles.prepareRow}): every + * reference is checked against the bytes the dataset carries (present, + * the declared size, the declared SHA-256). A row the existing data already + * holds keeps the file it holds when that file was stored from the same + * path with the same bytes, name and type — a boot does not mint a new file + * for an unchanged fixture. Otherwise the bytes are stored through the + * `field-files` service as a `pending` file, and the reference is replaced + * by the file's id. + * 2. **After the write** ({@link SeedDatasetFiles.settleRow}): the row's slot + * — (object, record, field) — is made the owner of the files, which commits + * them. A row that never lands leaves its files `pending`, for the + * pending-upload retention to reclaim. + * + * And one refusal for a value that is not a reference (shape 3): an id-shaped + * string in a file field that names no file in the row's organization is + * refused, by row, before anything is written. It used to be written as given, + * where the read path shows no file and the reference hooks own nothing. + * + * Every refusal is the row's: it names the field, the value and the fix, and + * the loader drops the row with it, counted and logged like any other row it + * cannot write. + */ + +/** One file-class field of the seeded object. */ +interface FileField { + readonly multiple: boolean; +} + +/** The files a written row must own, per field. */ +export type SeedFileClaims = ReadonlyArray<{ field: string; fileIds: string[] }>; + +/** What {@link SeedDatasetFiles.prepareRow} decided for one row. */ +export type SeedFileRowOutcome = + | { ok: true; claims: SeedFileClaims } + | { ok: false; field: string; attemptedValue: unknown; message: string }; + +/** The provenance a seed file's `sys_file` row keeps (`StoreFieldFileInput.metadata`). */ +const PROVENANCE_PATH = 'seedFile'; +const PROVENANCE_SHA256 = 'sha256'; + +/** How many file ids one `describeFiles` read names. */ +const DESCRIBE_CHUNK = 500; + +/** The file-class fields of an object definition, by name. */ +function fileFieldsOf(definition: unknown): Map { + const found = new Map(); + const fields = (definition as { fields?: Record } | null) + ?.fields; + if (!fields || typeof fields !== 'object') return found; + for (const [name, field] of Object.entries(fields)) { + if (typeof field?.type === 'string' && (FILE_REFERENCE_TYPES as readonly string[]).includes(field.type)) { + found.set(name, { multiple: field.multiple === true }); + } + } + return found; +} + +/** The values a field holds, one per element (a `multiple` field holds an array). */ +function elementsOf(value: unknown): unknown[] { + return Array.isArray(value) ? value : [value]; +} + +/** A stored `multiple` value may come back as JSON text; the ids it holds either way. */ +function heldIdsOf(value: unknown): string[] { + let parsed = value; + if (typeof value === 'string' && value.startsWith('[')) { + try { + parsed = JSON.parse(value); + } catch { + parsed = value; + } + } + return elementsOf(parsed).filter(isFileIdToken); +} + +/** The last segment of a seed file path: the file name a download is saved as by default. */ +function baseNameOf(path: string): string { + return path.slice(path.lastIndexOf('/') + 1); +} + +/** The bytes of one bundled seed file, checked against what `os build` recorded. */ +type CheckedContent = { ok: true; bytes: Buffer; content: SeedFileContent } | { ok: false; reason: string }; + +/** + * The files of one dataset: built once per dataset by {@link open}, asked once + * per row by the loader. + */ +export class SeedDatasetFiles { + private readonly checked = new Map(); + /** `describeFiles` answers, by organization (`''` for none), then by id. */ + private readonly described = new Map>(); + + private constructor( + private readonly objectName: string, + private readonly fields: ReadonlyMap, + private readonly files: Readonly>, + private readonly service: IFieldFileService | null, + ) {} + + /** + * The dataset's file handling, or `null` when the seeded object declares no + * file-class field — nothing below runs for such an object. + * + * `prefetch` names the ids the load will ask about up front (the ids the + * matched rows hold and the id-shaped values the records author), so a + * dataset is described in a few reads rather than one per row. + */ + static async open(args: { + dataset: Seed; + objectDefinition: unknown; + service: IFieldFileService | null; + existingRecords: ReadonlyMap> | undefined; + organizationId: string | null; + dryRun: boolean; + }): Promise { + const fields = fileFieldsOf(args.objectDefinition); + if (fields.size === 0) return null; + const handler = new SeedDatasetFiles( + args.dataset.object, + fields, + (args.dataset as { files?: Record }).files ?? {}, + args.service, + ); + if (!args.dryRun && args.service) { + const ids = new Set(); + for (const record of args.dataset.records as Array>) { + for (const field of fields.keys()) { + for (const element of elementsOf(record[field])) { + if (isFileIdToken(element)) ids.add(element); + } + } + } + for (const row of args.existingRecords?.values() ?? []) { + for (const field of fields.keys()) for (const id of heldIdsOf(row[field])) ids.add(id); + } + await handler.describe([...ids], args.organizationId); + } + return handler; + } + + /** + * Turn the row's seed file references into file ids, and refuse the row + * when one of its file values cannot be honoured. `record` is changed in + * place: each reference becomes the id of the file the row will hold. + * + * `existing` is the row the load matched, read with its STORED file values; + * `willWrite` is whether the row is written at all (an `ignore`-mode match, + * or an `update`-mode miss, writes nothing, so nothing is stored for it). A + * dry run checks the references against the bundled bytes and touches + * nothing else. + */ + async prepareRow(args: { + record: Record; + existing: Record | undefined; + organizationId: string | null; + willWrite: boolean; + dryRun: boolean; + }): Promise { + const claims: Array<{ field: string; fileIds: string[] }> = []; + for (const [field, def] of this.fields) { + const value = args.record[field]; + if (value === undefined || value === null) continue; + const elements = elementsOf(value); + const references = elements.filter(isSeedFileReference); + + if (references.length === 0) { + // Shape 3: an id-shaped value must name a file the row's organization + // holds. Anything else (an external URL, an inline blob) is kept as + // written, as the reference hooks keep it. + if (args.dryRun || !this.service) continue; + const tokens = elements.filter(isFileIdToken); + if (tokens.length === 0) continue; + const known = await this.describe(tokens, args.organizationId); + const missing = tokens.filter((id) => !known.get(id)); + if (missing.length > 0) { + return this.refusal( + field, + value, + `${this.objectName}.${field} holds ${missing.map((id) => `'${id}'`).join(', ')}, which names no stored ` + + `file${args.organizationId ? ` in organization ${args.organizationId}` : ''}. A file field holds the id ` + + 'of a stored file; a seed names the file a package ships with seedFile(path) instead ' + + "(e.g. seedFile('fixtures/contract.pdf')), and the loader stores it.", + ); + } + continue; + } + + // A field holding seed files holds nothing else: the loader settles the + // field's ownership to exactly the files it stored. + if (references.length !== elements.length) { + return this.refusal( + field, + value, + `${this.objectName}.${field} mixes seedFile(...) values with other values. A field that takes seed files ` + + 'holds only seed files; write every file it holds as seedFile(path).', + ); + } + if (references.length > 1 && !def.multiple) { + return this.refusal( + field, + value, + `${this.objectName}.${field} holds one file, and the record gives it ${references.length} seedFile(...) ` + + 'values. Give it one, or declare the field `multiple: true`.', + ); + } + + const contents: Array<{ reference: SeedFileReference; bytes: Buffer; content: SeedFileContent }> = []; + for (const reference of references) { + const checked = this.check(reference.$file); + if (!checked.ok) return this.refusal(field, reference, checked.reason); + contents.push({ reference, bytes: checked.bytes, content: checked.content }); + } + if (args.dryRun || !args.willWrite) continue; + if (!this.service) { + return this.refusal( + field, + value, + `${this.objectName}.${field} names seedFile('${references[0].$file}'), and this load has no file service to ` + + 'store it with: seed files are stored where @objectstack/service-storage runs (the field-files service). ' + + 'Load this seed through a server that composes the storage service.', + ); + } + + // A file the matched row already holds, stored from the same path with + // the same bytes, name and type, is kept; each held file serves once. + const held = args.existing ? heldIdsOf(args.existing[field]) : []; + const heldFiles = held.length > 0 ? await this.describe(held, args.organizationId) : new Map(); + const existingId = args.existing?.id == null ? undefined : String(args.existing.id); + const unused = new Set(held); + const fileIds: string[] = []; + for (const { reference, bytes, content } of contents) { + const name = reference.name ?? baseNameOf(reference.$file); + const mimeType = reference.contentType ?? content.contentType; + const kept = [...unused].find((id) => { + const file = heldFiles.get(id); + return file != null + && (file.owner === null + || (file.owner.object === this.objectName && file.owner.field === field && file.owner.recordId === existingId)) + && file.metadata[PROVENANCE_PATH] === reference.$file + && file.metadata[PROVENANCE_SHA256] === content.sha256 + && file.name === name + && file.mimeType === mimeType; + }); + if (kept !== undefined) { + unused.delete(kept); + fileIds.push(kept); + continue; + } + try { + const stored = await this.service.storeFile({ + organizationId: args.organizationId, + filename: name, + mimeType, + data: bytes, + metadata: { [PROVENANCE_PATH]: reference.$file, [PROVENANCE_SHA256]: content.sha256 }, + }); + fileIds.push(stored.fileId); + } catch (err) { + return this.refusal( + field, + reference, + `${this.objectName}.${field}: seedFile('${reference.$file}') could not be stored — ` + + `${(err as Error)?.message ?? String(err)}`, + ); + } + } + args.record[field] = def.multiple ? fileIds : fileIds[0]; + claims.push({ field, fileIds }); + } + return { ok: true, claims }; + } + + /** + * After the row landed as `recordId`: make each field's slot the owner of + * its files, committing them. Answers why the files are not the row's when a + * claim fails, else `null`. A file the slot already owns, committed, costs + * no write. + */ + async settleRow(recordId: string, claims: SeedFileClaims, organizationId: string | null): Promise { + if (!this.service || claims.length === 0) return null; + for (const { field, fileIds } of claims) { + const known = await this.describe(fileIds, organizationId); + const settled = fileIds.every((id) => { + const file = known.get(id); + return file?.status === 'committed' + && file.owner?.object === this.objectName + && file.owner.recordId === recordId + && file.owner.field === field; + }); + if (settled) continue; + try { + await this.service.claimFiles({ object: this.objectName, recordId, field }, fileIds, organizationId); + this.forget(fileIds, organizationId); + } catch (err) { + return `${this.objectName}/${recordId}.${field} was written, but its seed file(s) ${fileIds.join(', ')} could ` + + `not be made the record's own: ${(err as Error)?.message ?? String(err)}. The files stay pending and are ` + + 'reclaimed by the pending-upload retention, which leaves the field naming no file; re-run the seed to store ' + + 'them again.'; + } + } + return null; + } + + private refusal(field: string, attemptedValue: unknown, message: string): SeedFileRowOutcome { + return { ok: false, field, attemptedValue, message }; + } + + /** The bundled bytes of `path`, decoded once and checked against their recorded size and SHA-256. */ + private check(path: string): CheckedContent { + const cached = this.checked.get(path); + if (cached) return cached; + let result: CheckedContent; + const content = Object.prototype.hasOwnProperty.call(this.files, path) ? this.files[path] : undefined; + if (!content) { + result = { + ok: false, + reason: + `seedFile('${path}') names a file this ${this.objectName} dataset does not carry: its \`files\` map has no ` + + `entry for '${path}'. \`os build\` bundles each referenced file from the package root; build the package ` + + 'again (and check that the file exists there), or, for a dataset written by hand, add the bytes to `files`.', + }; + } else { + const bytes = Buffer.from(content.data, 'base64'); + const sha256 = createHash('sha256').update(bytes).digest('hex'); + result = bytes.byteLength !== content.size || sha256 !== content.sha256 + ? { + ok: false, + reason: + `seedFile('${path}'): the bytes this ${this.objectName} dataset carries do not match what was recorded ` + + `for them (${bytes.byteLength} bytes, SHA-256 ${sha256}; recorded ${content.size} bytes, SHA-256 ` + + `${content.sha256}). The bundle was altered after \`os build\`; build the package again.`, + } + : { ok: true, bytes, content }; + } + this.checked.set(path, result); + return result; + } + + /** The files among `ids` that exist in `organizationId`, read once per id and organization. */ + private async describe( + ids: readonly string[], + organizationId: string | null, + ): Promise> { + const scope = organizationId ?? ''; + let cache = this.described.get(scope); + if (!cache) this.described.set(scope, (cache = new Map())); + const unknown = [...new Set(ids)].filter((id) => !cache!.has(id)); + for (let i = 0; i < unknown.length && this.service; i += DESCRIBE_CHUNK) { + const chunk = unknown.slice(i, i + DESCRIBE_CHUNK); + let found: Map; + try { + found = await this.service.describeFiles(chunk, organizationId); + } catch (err) { + if (!isMissingTableError(err, 'sys_file')) throw err; + found = new Map(); + } + for (const id of chunk) cache.set(id, found.get(id) ?? null); + } + return cache; + } + + /** Drop what a claim just changed, so a later row reads it fresh. */ + private forget(ids: readonly string[], organizationId: string | null): void { + const cache = this.described.get(organizationId ?? ''); + for (const id of ids) cache?.delete(id); + } +} diff --git a/packages/metadata-protocol/src/seed-loader.ts b/packages/metadata-protocol/src/seed-loader.ts index c6d8d47e664..b7295f76eb6 100644 --- a/packages/metadata-protocol/src/seed-loader.ts +++ b/packages/metadata-protocol/src/seed-loader.ts @@ -1,6 +1,6 @@ // Copyright (c) 2025 ObjectStack. Licensed under the Apache-2.0 license. -import type { IDataEngine, IMetadataService, ISeedLoaderService } from '@objectstack/spec/contracts'; +import type { IDataEngine, IFieldFileService, IMetadataService, ISeedLoaderService } from '@objectstack/spec/contracts'; import type { SeedLoaderRequestParsed, SeedLoaderResultParsed, @@ -24,6 +24,9 @@ import { // (and which `managedBy` buckets exempt them), and the mask itself (ADR-0100). isMaskedOnReadFieldType, SECRET_MASK, + // [#25779] A replay reads a matched row's file fields in their STORED form + // (the bare `sys_file` id), the value a seed file is compared and kept by. + RAW_FILE_VALUES_CONTEXT_KEY, } from '@objectstack/spec/data'; import { SEED_WRITE_EXECUTION_CONTEXT } from '@objectstack/spec/kernel'; import { resolveSeedRecord } from '@objectstack/formula'; @@ -41,6 +44,9 @@ import { validationFailureDetails } from '@objectstack/types'; // `SysMetadataRepository` (#4867) and `cascadeDeleteRelations` (#8895) ask, so // a driver quirk is taught to the platform once instead of per seam. import { isMissingTableError } from '@objectstack/metadata/errors'; +// [#25779] `seedFile(path)` values: stored, kept across replays, and owned by +// the row that holds them; an id-shaped file value that names no file refused. +import { SeedDatasetFiles, type SeedFileClaims } from './seed-files.js'; interface Logger { info(message: string, meta?: Record): void; @@ -552,6 +558,17 @@ interface ReplayComparands { readonly masked: ReadonlySet; } +/** What a {@link SeedLoaderService} is built with beyond its engine, metadata and logger. */ +export interface SeedLoaderServiceOptions { + /** + * [#25779] The `field-files` service (`FIELD_FILE_SERVICE`, + * `@objectstack/spec/contracts`) seed files are stored through, resolved + * when a dataset loads. A caller with a kernel passes + * `() => ctx.getService(FIELD_FILE_SERVICE)`. + */ + getFieldFiles?: () => IFieldFileService | null | undefined; +} + export class SeedLoaderService implements ISeedLoaderService { private engine: IDataEngine; private metadata: IMetadataService; @@ -686,10 +703,20 @@ export class SeedLoaderService implements ISeedLoaderService { */ private uncomparedCredentials = new Map; rows: number }>(); - constructor(engine: IDataEngine, metadata: IMetadataService, logger: Logger) { + /** + * [#25779] The `field-files` service (`FIELD_FILE_SERVICE`) a seed file is + * stored through, read per dataset. Absent — a loader built without one, or + * a composition without `@objectstack/service-storage` — a row naming a seed + * file is refused, and an id-shaped file value is written unchecked, as the + * platform keeps one where it manages no files. + */ + private readonly getFieldFiles: () => IFieldFileService | null | undefined; + + constructor(engine: IDataEngine, metadata: IMetadataService, logger: Logger, options: SeedLoaderServiceOptions = {}) { this.engine = engine; this.metadata = metadata; this.logger = logger; + this.getFieldFiles = options.getFieldFiles ?? (() => undefined); } // ========================================================================== @@ -1321,6 +1348,7 @@ export class SeedLoaderService implements ISeedLoaderService { if (externalIdValue && internalId) { insertedRecords.get(objectName)!.set(externalIdValue, internalId); } + await settleSeedFiles(recordIndex, internalId); // [#25779] } else { errored++; const error = this.buildWriteError(objectName, record, externalId, recordIndex, res.error); @@ -1360,6 +1388,45 @@ export class SeedLoaderService implements ISeedLoaderService { temporal: temporalFieldKinds(objectDefinition), masked: maskedOnReadFields(objectDefinition), }; + // [#25779] The dataset's file fields: seed files stored and kept, and an + // id-shaped file value that names no file refused (`seed-files.ts`). + // `null` for an object with no file-class field — the common case. + const datasetFiles = await SeedDatasetFiles.open({ + dataset, + objectDefinition, + service: this.getFieldFiles() ?? null, + existingRecords, + organizationId: config.organizationId ?? this.fallbackOrgId ?? null, + dryRun: config.dryRun === true, + }); + /** Record index → the files that row's slots must own once it lands. */ + const fileClaimsByRecordIndex = new Map(); + /** + * [#25779] Make row `recordIndex`, landed as `recordId`, the owner of the + * seed files it holds. Every write site below calls it once the row is + * there — an insert, an update, a skipped replay — and a skipped replay + * whose files are already its own costs a cached read. + */ + const settleSeedFiles = async (recordIndex: number, recordId: string | undefined): Promise => { + const pending = fileClaimsByRecordIndex.get(recordIndex); + if (!datasetFiles || !pending || !recordId) return; + const failure = await datasetFiles.settleRow(recordId, pending.claims, pending.organizationId); + if (failure === null) return; + const error: ReferenceResolutionError = { + sourceObject: objectName, + field: pending.claims.map((c) => c.field).join(', '), + targetObject: 'sys_file', + targetField: 'id', + attemptedValue: pending.claims.flatMap((c) => c.fileIds), + recordIndex, + message: failure, + }; + errors.push(error); + allErrors.push(error); + // `error`: counted in `allErrors`, and the row looks seeded while the + // file its field names is on its way to being reclaimed. + this.logger.error(`[SeedLoader] ${failure}`, undefined, { object: objectName, recordIndex }); + }; // [ADR-0131 D9] With no organization pinned and none derivable on an install // that registers the organization object, a row of an organization-owned // object has no owner: it is REFUSED below, never written NULL. @@ -1867,6 +1934,52 @@ export class SeedLoaderService implements ISeedLoaderService { for (const column of createOnlyColumns) delete record[column]; } + // [#25779] Seed files, once the row's references and organization are + // final: each `seedFile(path)` becomes the id of a file stored for (or + // kept by) this row, and a file value that cannot be honoured refuses + // the row before anything of it is written. + if (datasetFiles) { + const matched = existingRecords?.get(this.externalIdKey(record, externalId)); + const organizationOfRow = + typeof record['organization_id'] === 'string' && record['organization_id'] !== '' + ? (record['organization_id'] as string) + : null; + const outcome = await datasetFiles.prepareRow({ + record, + existing: matched, + organizationId: organizationOfRow, + willWrite: mode === 'update' ? matched !== undefined : mode === 'ignore' ? matched === undefined : true, + dryRun: config.dryRun === true, + }); + if (!outcome.ok) { + errored++; + const error: ReferenceResolutionError = { + sourceObject: objectName, + field: outcome.field, + targetObject: 'sys_file', + targetField: 'id', + attemptedValue: outcome.attemptedValue, + recordIndex: i, + message: outcome.message, + }; + errors.push(error); + allErrors.push(error); + // `error` (#4729 / #4632): counted, and the record is dropped. A dry + // run reports it like any other refusal it finds. + if (!config.dryRun) { + this.logger.error( + `[SeedLoader] ${outcome.message} ${objectName} record #${i} was NOT seeded.`, + undefined, + { object: objectName, field: outcome.field, recordIndex: i }, + ); + } + continue; + } + if (outcome.claims.length > 0) { + fileClaimsByRecordIndex.set(i, { claims: outcome.claims, organizationId: organizationOfRow }); + } + } + // [commit 1cba33f16] EARLY SIGNAL — emitted here, before this row reaches the // engine, for the deferrals it took on columns the write contract // requires on INSERT. @@ -1934,6 +2047,7 @@ export class SeedLoaderService implements ISeedLoaderService { if (externalIdValue && internalId) { insertedRecords.get(objectName)!.set(externalIdValue, String(internalId)); } + await settleSeedFiles(i, internalId ? String(internalId) : undefined); // [#25779] } catch (err: any) { errored++; // Same cascade guard as the batched update path: the row may @@ -1969,6 +2083,7 @@ export class SeedLoaderService implements ISeedLoaderService { if (decision.id && externalIdValue) { insertedRecords.get(objectName)!.set(externalIdValue, decision.id); } + await settleSeedFiles(i, decision.id); // [#25779] } else if (decision.action === 'update') { // Register the externalId → id mapping BEFORE attempting the // write: the row exists and its id is known regardless of whether @@ -1984,6 +2099,7 @@ export class SeedLoaderService implements ISeedLoaderService { try { await this.writeRecoveringSummary(objectName, () => withTransientRetry(() => this.engine.update(objectName, { ...record, id: decision.id }, opts))); updated++; + await settleSeedFiles(i, decision.id); // [#25779] } catch (err: any) { errored++; const error = this.buildWriteError(objectName, record, externalId, i, err); @@ -3470,8 +3586,10 @@ export class SeedLoaderService implements ISeedLoaderService { try { // Full rows (not just id + externalId): the write decision compares the // incoming seed record against the existing row to skip no-op replays. + // [#25779] File fields in their STORED form: the ids a replay keeps a + // seed file by, and compares an authored id against. const findArgs: Record = { - context: { isSystem: true }, + context: { isSystem: true, [RAW_FILE_VALUES_CONTEXT_KEY]: true }, }; // Per-tenant replay: restrict to the target tenant's own rows // so upsert key matching never returns another tenant's record diff --git a/packages/runtime/src/app-plugin.ts b/packages/runtime/src/app-plugin.ts index a868c37df88..b0edb872bf0 100644 --- a/packages/runtime/src/app-plugin.ts +++ b/packages/runtime/src/app-plugin.ts @@ -28,7 +28,8 @@ import { contributeCodeDatasourceNames } from './code-datasource-names.js'; import { mergeSeedDatasets, readSeedDatasets, registerSeedReplayerOnce } from './seed-datasets.js'; import { declareSeedSource, recordOrganizationReplay, type SeedLandingReport } from './seed-settlement.js'; import { loadDisabledPackageIds } from './package-state-store.js'; -import type { IMetadataService, IObjectQLEngine, II18nService } from '@objectstack/spec/contracts'; +import type { IFieldFileService, IMetadataService, IObjectQLEngine, II18nService } from '@objectstack/spec/contracts'; +import { FIELD_FILE_SERVICE } from '@objectstack/spec/contracts'; import { normalizeFlowFunctionEntry, type NormalizedFlowFunction } from '@objectstack/spec/automation'; import { readServiceSelfInfo } from '@objectstack/spec/api'; import { SEED_WRITE_EXECUTION_CONTEXT } from '@objectstack/spec/kernel'; @@ -54,6 +55,30 @@ import { countServerTiming } from '@objectstack/observability'; */ const SEED_WRITE_OPTIONS = { context: SEED_WRITE_EXECUTION_CONTEXT } as const; +/** + * [#25779] What every seed loader this plugin builds is given beyond its + * engine: the `field-files` service a seed file (`seedFile(path)`) is stored + * through, resolved when a dataset loads. A kernel without + * `@objectstack/service-storage` answers none, and the loader refuses a row + * naming a seed file, by name, rather than write an object into a file field. + */ +function seedLoaderOptionsFor(ctx: PluginContext): { getFieldFiles: () => IFieldFileService | undefined } { + return { + getFieldFiles: () => { + try { + return ctx.getService(FIELD_FILE_SERVICE); + } catch { + return undefined; + } + }, + }; +} + +/** [#25779] Does any of these datasets carry the bytes of a seed file? */ +function datasetsCarrySeedFiles(datasets: ReadonlyArray<{ files?: unknown }>): boolean { + return datasets.some((d) => d.files !== null && typeof d.files === 'object' && Object.keys(d.files).length > 0); +} + /** * Optional per-project context attached when AppPlugin is instantiated by the * project kernel factory. Required for the `app:registered` / `app:unregistered` @@ -1837,7 +1862,7 @@ export class AppPlugin implements Plugin { if (!Array.isArray(datasetsNow) || datasetsNow.length === 0) { return { inserted: 0, updated: 0, skipped: 0, errors: [] as any[] }; } - const seedLoader = new SeedLoaderService(ql, md, loggerRef); + const seedLoader = new SeedLoaderService(ql, md, loggerRef, seedLoaderOptionsFor(ctx)); const { SeedLoaderRequestSchema } = await import('@objectstack/spec/data'); const request = SeedLoaderRequestSchema.parse({ seeds: datasetsNow, @@ -1936,7 +1961,7 @@ export class AppPlugin implements Plugin { try { const metadata = ctx.getService('metadata') as IMetadataService | undefined; if (metadata) { - const seedLoader = new SeedLoaderService(ql, metadata, ctx.logger); + const seedLoader = new SeedLoaderService(ql, metadata, ctx.logger, seedLoaderOptionsFor(ctx)); const { SeedLoaderRequestSchema } = await import('@objectstack/spec/data'); // [#16595] `Seed.locale`'s producer on the DEFAULT boot // path — see {@link resolveSeedLocale}. @@ -2116,9 +2141,19 @@ export class AppPlugin implements Plugin { // seed source is already declared, so every consumer of the // settlement tally waits for `app:seeded` exactly as it does for a // seed that overran its budget. + // + // [#25779] The same anchor when a dataset carries seed files: the + // storage plugin settles the deployment's file store from its + // `storage` settings at `kernel:ready`, so a file stored now could + // land in the constructor default that a saved setting is about + // to replace — and the reference hooks it is owned under are + // installed then too. if (this.demoStaffingArmed(ctx)) { ctx.logger.info('[Seeder] inline seed deferred to kernel:bootstrapped — the demo accounts this stack declares (demoStaff) are created at kernel:ready'); ctx.hook('kernel:bootstrapped', runInlineSeed); + } else if (datasetsCarrySeedFiles(normalizedDatasets)) { + ctx.logger.info('[Seeder] inline seed deferred to kernel:bootstrapped — its datasets carry seed files, which are stored once the storage service has read its settings at kernel:ready'); + ctx.hook('kernel:bootstrapped', runInlineSeed); } else { await runInlineSeed(); } @@ -2348,7 +2383,7 @@ export class AppPlugin implements Plugin { ctx.logger.warn('[Seeder] hot-reload seed skipped — metadata service unavailable'); return; } - const seedLoader = new SeedLoaderService(ql, metadata, ctx.logger); + const seedLoader = new SeedLoaderService(ql, metadata, ctx.logger, seedLoaderOptionsFor(ctx)); const { SeedLoaderRequestSchema } = await import('@objectstack/spec/data'); // [#16595] `Seed.locale`'s producer on the dev hot-reload // path. Resolved from the BOOT bundle, not from `payload`: a diff --git a/packages/runtime/src/domains/packages.ts b/packages/runtime/src/domains/packages.ts index 6311687ba31..336fc8ec659 100644 --- a/packages/runtime/src/domains/packages.ts +++ b/packages/runtime/src/domains/packages.ts @@ -11,6 +11,7 @@ import { CoreServiceName } from '@objectstack/spec/system'; import { PLURAL_TO_SINGULAR } from '@objectstack/spec/shared'; +import { FIELD_FILE_SERVICE, type IFieldFileService } from '@objectstack/spec/contracts'; // [#5320] A's mechanical half of the fork ruling: on export, view artifacts // are PARTITIONED — containers travel in `views:`, expanded items a travelling // container re-derives exactly are folded away (the import side's own @@ -2872,7 +2873,12 @@ _context: HttpProtocolContext, const { SeedLoaderService } = await import('../seed-loader.js'); const { SeedLoaderRequestSchema } = await import('@objectstack/spec/data'); - const loader = new SeedLoaderService(ql, metadata, deps.logger ?? console); + // [#25779] The service a seed file is stored through; absent, a row naming + // one is refused by the loader, by name. + const fieldFiles = await Promise.resolve() + .then(() => deps.resolveService(_context, FIELD_FILE_SERVICE)) + .catch(() => undefined) as IFieldFileService | undefined; + const loader = new SeedLoaderService(ql, metadata, deps.logger ?? console, { getFieldFiles: () => fieldFiles }); // [#8443] `safeParse`, not `parse` — the same producer-side declaration // #8333's P9 made next door, for the same reason. This catch's caller now // withholds anything undeclared, and a raw `ZodError` declares nothing, so diff --git a/packages/services/service-storage/src/field-file-service.ts b/packages/services/service-storage/src/field-file-service.ts new file mode 100644 index 00000000000..71dc1a6ed7e --- /dev/null +++ b/packages/services/service-storage/src/field-file-service.ts @@ -0,0 +1,311 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +import { randomUUID } from 'node:crypto'; +import type { RegisteredErrorCode, StandardErrorCode } from '@objectstack/spec/api'; +import type { + FieldFileDescription, + FieldFileSlot, + IDataEngine, + IFieldFileService, + IStorageService, + StoreFieldFileInput, + StoredFieldFile, +} from '@objectstack/spec/contracts'; +import { StorageMetadataStore, type StorageWriteContext } from './metadata-store.js'; +// The field-reference lifecycle's own release, so a slot this verb settles +// lets go of its old file exactly as a record write that replaces the value +// does — the tombstone policy included. +import { + ownedBySlot, + releaseOwnership, + type FileReferenceEngine, + type FileReferenceLogger, +} from './file-reference-lifecycle.js'; +// The upload doors' own size verdict and refusal, asked here rather than +// restated. +import { + resolveStorageLimits, + uploadSizeRefusal, + UPLOAD_TOO_LARGE_CODE, + UPLOAD_TOO_LARGE_STATUS, + type StorageLimitsSnapshot, +} from './storage-limits.js'; +// The upload doors' own storage-key layout. +import { buildKey } from './storage-routes.js'; + +/** + * [#25779] The server-side field-file verb — `IFieldFileService`, registered + * by `StorageServicePlugin` under `FIELD_FILE_SERVICE` + * (`@objectstack/spec/contracts`): store bytes as a `sys_file` no record owns + * yet, then hand it to the (object, record, field) slot whose value holds its + * id. + * + * Its producer is the seed loader, writing the files a package ships + * (`seedFile(path)`) into the file fields of the rows it seeds. A seed has no + * upload, so it acts as the SYSTEM, and each check is the one an upload meets: + * + * - **Size.** `uploadSizeRefusal` against the resolved `max_upload_mb`, the + * verdict the REST upload doors and the attach verb ask, before anything is + * stored. + * - **The `sys_file` row.** `StorageMetadataStore`, the upload doors' writer, + * with the organization threaded through `StorageWriteContext`, so the + * driver stamps the row exactly as it stamps an upload. + * - **The storage key.** `buildKey`, under the `tenant` scope: the file is + * the organization's, not a user's upload. + * + * ## Pending until a record owns it + * + * The write order is the presigned upload door's: the row first, `pending`, + * then the bytes. The row turns `committed` only when {@link claimFiles} hands + * it to the slot that holds it, so a file whose record never lands — a row the + * engine refused, a process that died between the two — is reclaimed, row and + * bytes, by the pending-upload retention. + * + * ## The claim is the lifecycle's claim, made explicit + * + * When the record write that holds the id runs under the field-reference hooks + * (`installFileReferenceHooks`, armed at `kernel:ready`), the hooks claim the + * file themselves; {@link claimFiles} then finds it already owned by the slot + * and only commits it. A write that runs before the hooks are installed — a + * seed in a plugin's `start()` — gets its ownership here instead. Either way a + * file has one owner: a file another slot owns is refused, never transferred + * (the hooks' own rule), and the slot's previous file is released through the + * lifecycle's {@link releaseOwnership}. + */ + +/** What {@link FieldFileService} is composed over — each read at call time. */ +export interface FieldFileServiceDeps { + /** The data engine `sys_file` lives in; `null` when none is wired. */ + getEngine(): (IDataEngine & FileReferenceEngine) | null; + /** The `storage` service the bytes go to. */ + getStorage(): IStorageService | null; + /** The `storage` namespace's Limits group as last read — the upload doors' own source. */ + getLimitsSnapshot(): StorageLimitsSnapshot | undefined; + logger: FileReferenceLogger; +} + +/** The storage scope every field file this verb stores is filed under. */ +const FIELD_FILE_SCOPE = 'tenant'; + +const NOT_READY_MESSAGE = + 'The field-files service cannot store a file: no data engine is wired into this kernel, so there is no ' + + 'sys_file table to record it in.'; + +/** An ADR-0112-enveloped refusal: `code` and `status` on the thrown error. */ +function refuse(code: StandardErrorCode | RegisteredErrorCode, status: number, message: string): never { + const err = new Error(message) as Error & { code: string; status: number }; + err.code = code; + err.status = status; + throw err; +} + +function nonEmptyString(value: unknown): value is string { + return typeof value === 'string' && value.length > 0; +} + +/** The system context for a `sys_file` read or write in `organizationId`. */ +function systemContextIn(organizationId: string | null): Record { + return organizationId === null ? { isSystem: true } : { isSystem: true, tenantId: organizationId }; +} + +function validateStoreInput(input: StoreFieldFileInput): StoreFieldFileInput { + const fail = (message: string): never => refuse('VALIDATION_ERROR', 400, `storeFile: ${message}`); + if (!input || typeof input !== 'object') fail('the input must be an object'); + for (const field of ['filename', 'mimeType'] as const) { + if (!nonEmptyString(input[field])) fail(`'${field}' must be a non-empty string`); + } + if (input.organizationId !== null && !nonEmptyString(input.organizationId)) { + fail( + "'organizationId' must be the organization the file is stored in, or null for none — it is required, " + + 'so storing in no organization is stated rather than left out', + ); + } + if (!Buffer.isBuffer(input.data)) fail("'data' must be a Buffer holding the file's bytes"); + if (input.metadata !== undefined) { + const entries = input.metadata && typeof input.metadata === 'object' ? Object.values(input.metadata) : null; + if (!entries || entries.some((v) => typeof v !== 'string')) { + fail("'metadata' must map names to string values"); + } + } + return input; +} + +function validateSlot(slot: FieldFileSlot, fileIds: readonly string[]): void { + const fail = (message: string): never => refuse('VALIDATION_ERROR', 400, `claimFiles: ${message}`); + if (!slot || typeof slot !== 'object') fail('the slot must be an object'); + for (const field of ['object', 'recordId', 'field'] as const) { + if (!nonEmptyString(slot[field])) fail(`the slot's '${field}' must be a non-empty string`); + } + if (!Array.isArray(fileIds) || fileIds.some((id) => !nonEmptyString(id))) { + fail("'fileIds' must be an array of file ids"); + } +} + +/** `sys_file.metadata` as written by {@link FieldFileService.storeFile}: a JSON object of strings. */ +function parseMetadata(raw: unknown): Record { + if (typeof raw !== 'string' || raw.length === 0) return {}; + try { + const parsed: unknown = JSON.parse(raw); + if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) return {}; + return Object.fromEntries( + Object.entries(parsed as Record).filter( + (entry): entry is [string, string] => typeof entry[1] === 'string', + ), + ); + } catch { + return {}; + } +} + +function describeRow(row: Record): FieldFileDescription { + const status = row.status === 'committed' || row.status === 'deleted' ? row.status : 'pending'; + const owner = + row.ref_object != null && row.ref_id != null && row.ref_field != null + ? { object: String(row.ref_object), recordId: String(row.ref_id), field: String(row.ref_field) } + : null; + return { + fileId: String(row.id), + name: typeof row.name === 'string' ? row.name : '', + mimeType: typeof row.mime_type === 'string' ? row.mime_type : null, + status, + owner, + metadata: parseMetadata(row.metadata), + }; +} + +function isSlot(owner: FieldFileSlot | null, slot: FieldFileSlot): boolean { + return owner !== null && owner.object === slot.object && owner.recordId === slot.recordId && owner.field === slot.field; +} + +export class FieldFileService implements IFieldFileService { + constructor(private readonly deps: FieldFileServiceDeps) {} + + async storeFile(input: StoreFieldFileInput): Promise { + const request = validateStoreInput(input); + const engine = this.deps.getEngine(); + const storage = this.deps.getStorage(); + if (!engine || !storage) refuse('SERVICE_UNAVAILABLE', 503, NOT_READY_MESSAGE); + + // ① Size — before a byte is stored or a row is written. + const size = request.data.byteLength; + const tooLarge = uploadSizeRefusal( + size, + resolveStorageLimits(this.deps.getLimitsSnapshot()).maxUploadBytes, + (n) => `The file ${request.filename} is ${n} bytes`, + ); + if (tooLarge !== null) refuse(UPLOAD_TOO_LARGE_CODE, UPLOAD_TOO_LARGE_STATUS, tooLarge); + + // ② The `sys_file` row, `pending` — the pointer the bytes are reclaimed by. + const writeContext: StorageWriteContext = { organizationId: request.organizationId }; + const store = new StorageMetadataStore(engine); + const fileId = randomUUID(); + const key = buildKey(FIELD_FILE_SCOPE, fileId, request.filename); + await store.createFile( + { + id: fileId, + key, + name: request.filename, + mime_type: request.mimeType, + size, + scope: FIELD_FILE_SCOPE, + acl: 'private', + status: 'pending', + ...(request.metadata ? { metadata: JSON.stringify(request.metadata) } : {}), + }, + writeContext, + ); + + // ③ The bytes. A failed upload removes the row it would have filled; if + // even that fails, the pending row is the retention's to reclaim. + try { + await storage.upload(key, request.data, { contentType: request.mimeType }); + } catch (err) { + try { + await store.deleteFile(fileId, writeContext); + } catch (deleteErr) { + this.deps.logger.warn( + `[storage] field-files: could not delete the pending sys_file row ${fileId} of a failed upload ` + + `(${(deleteErr as Error)?.message ?? deleteErr}). The pending-upload retention removes it.`, + ); + } + throw err; + } + return { fileId, size }; + } + + async describeFiles( + fileIds: readonly string[], + organizationId: string | null, + ): Promise> { + const engine = this.deps.getEngine(); + if (!engine) refuse('SERVICE_UNAVAILABLE', 503, NOT_READY_MESSAGE); + const ids = [...new Set(fileIds.filter(nonEmptyString))]; + const found = new Map(); + if (ids.length === 0) return found; + const rows = + (await engine.find('sys_file', { + where: { id: { $in: ids } }, + limit: ids.length, + context: systemContextIn(organizationId), + })) ?? []; + for (const row of rows) { + if (row?.id == null) continue; + found.set(String(row.id), describeRow(row)); + } + return found; + } + + async claimFiles(slot: FieldFileSlot, fileIds: readonly string[], organizationId: string | null): Promise { + validateSlot(slot, fileIds); + const engine = this.deps.getEngine(); + if (!engine) refuse('SERVICE_UNAVAILABLE', 503, NOT_READY_MESSAGE); + const ids = [...new Set(fileIds)]; + const files = await this.describeFiles(ids, organizationId); + for (const id of ids) { + const file = files.get(id); + if (!file) { + refuse( + 'RECORD_NOT_FOUND', + 404, + `claimFiles: no file ${id} exists in ${organizationId === null ? 'this deployment' : `organization ${organizationId}`}, ` + + `so ${slot.object}/${slot.recordId}.${slot.field} cannot own it.`, + ); + } + if (file.owner !== null && !isSlot(file.owner, slot)) { + refuse( + 'RESOURCE_CONFLICT', + 409, + `claimFiles: file ${id} is owned by ${file.owner.object}/${file.owner.recordId}.${file.owner.field}; a field ` + + `file has one owner and is never transferred, so ${slot.object}/${slot.recordId}.${slot.field} cannot take it.`, + ); + } + } + + // The slot's other files are released first — the slot now holds `ids` and + // nothing else — through the lifecycle's own release. + const held = await ownedBySlot(engine, slot.object, slot.recordId, slot.field); + await releaseOwnership( + engine, + this.deps.logger, + held.filter((row) => !ids.includes(String(row.id))), + ); + + const store = new StorageMetadataStore(engine); + const writeContext: StorageWriteContext = { organizationId }; + for (const id of ids) { + const file = files.get(id)!; + if (file.status === 'committed' && isSlot(file.owner, slot)) continue; + await store.updateFile( + id, + { + ref_object: slot.object, + ref_id: slot.recordId, + ref_field: slot.field, + status: 'committed', + deleted_at: null, + }, + writeContext, + ); + } + } +} diff --git a/packages/services/service-storage/src/file-reference-lifecycle.ts b/packages/services/service-storage/src/file-reference-lifecycle.ts index a20601e3c08..7e119648369 100644 --- a/packages/services/service-storage/src/file-reference-lifecycle.ts +++ b/packages/services/service-storage/src/file-reference-lifecycle.ts @@ -849,7 +849,7 @@ async function collectionOpen(engine: FileReferenceEngine): Promise { * own never-completed reap policy, and re-tombstoning a `deleted` row would * reset its grace clock. */ -async function releaseOwnership( +export async function releaseOwnership( engine: FileReferenceEngine, logger: FileReferenceLogger, rows: Array>, @@ -884,7 +884,7 @@ async function releaseOwnership( } /** Files currently owned by a specific slot. */ -async function ownedBySlot( +export async function ownedBySlot( engine: FileReferenceEngine, object: string, recordId: string, diff --git a/packages/services/service-storage/src/storage-service-plugin.ts b/packages/services/service-storage/src/storage-service-plugin.ts index 329a8af6636..86e1359b0bb 100644 --- a/packages/services/service-storage/src/storage-service-plugin.ts +++ b/packages/services/service-storage/src/storage-service-plugin.ts @@ -66,6 +66,7 @@ import { installFileReferenceHooks, type FileReferenceReadAuthorizer } from './f import { installAttachmentAccessHooks, installAttachmentReadVisibility } from './attachment-access-hooks.js'; import { contributeAttachmentDeleteFloorAlternate } from './attachment-delete-floor-alternate.js'; import { FileAttachmentService, type FileAttachmentEngine } from './file-attachment-service.js'; +import { FieldFileService } from './field-file-service.js'; import { SystemFile, SystemUploadSession } from './objects/index.js'; // ADR-0052 §3 ownership: `sys_attachment` (a file↔record link) belongs with the // storage domain, not the audit/compliance ledger. Definition stays in @@ -196,8 +197,12 @@ export class StorageServicePlugin implements Plugin { * [#23306] `file-attachments` (`FILE_ATTACHMENT_SERVICE`) is the server-side * attach verb (`file-attachment-service.ts`): a slot of its own, because the * `storage` slot's contract is the adapters' blob store. + * + * [#25779] `field-files` (`FIELD_FILE_SERVICE`) is the server-side verb for a + * file FIELD's file (`field-file-service.ts`), which the seed loader stores + * the files a package ships through. */ - providesServices = ['storage', 'file-storage', 'file-attachments']; + providesServices = ['storage', 'file-storage', 'file-attachments', 'field-files']; /** * `com.objectstack.engine.objectql` — init() registers sys_file / * sys_upload_session / sys_attachment through the `manifest` service @@ -395,6 +400,24 @@ export class StorageServicePlugin implements Plugin { }, }), ); + // [#25779] The server-side field-file verb, under `FIELD_FILE_SERVICE`. + // Its engine is read per call, so it serves from the moment the data + // engine is registered — a seed loaded in a plugin's `start()` included. + ctx.registerService( + 'field-files', + new FieldFileService({ + getEngine: () => { + try { + return ctx.getService('objectql') ?? null; + } catch { + return null; + } + }, + getStorage: () => storage, + getLimitsSnapshot: () => storage.getLimitsSnapshot(), + logger: ctx.logger, + }), + ); ctx.logger.info( `StorageServicePlugin: registered ${adapter} storage adapter (swappable, metrics=${this.metrics.constructor?.name ?? 'unknown'})`, ); diff --git a/packages/spec/src/contracts/storage-service.ts b/packages/spec/src/contracts/storage-service.ts index 59091e64009..660bbdb7259 100644 --- a/packages/spec/src/contracts/storage-service.ts +++ b/packages/spec/src/contracts/storage-service.ts @@ -526,3 +526,121 @@ export interface IFileAttachmentService { /** Store `input.data` and attach it to the parent record, as one file and one attachment. */ attachFile(input: AttachFileInput): Promise; } + +// ========================================== +// Server-side field files: store bytes a record's file field will own (#25779) +// ========================================== + +/** + * Service-registry slot for {@link IFieldFileService}. + * + * `@objectstack/service-storage` registers it, beside + * {@link FILE_ATTACHMENT_SERVICE}: that package owns `sys_file` and the + * field-reference ownership it keeps (ADR-0104 D3 wave 2). The seed loader + * reaches it to store a file a seed record writes into a file field + * (`seedFile(path)`, `@objectstack/spec/data`). + */ +export const FIELD_FILE_SERVICE = 'field-files' as const; + +/** What {@link IFieldFileService.storeFile} stores. */ +export interface StoreFieldFileInput { + /** + * The organization the file is stored in, or `null` for none. Required, so + * that storing in no organization is stated, never a field left out. The + * record that will own the file is written in the same organization. + */ + organizationId: string | null; + /** The file name a download is saved as. Its extension is kept on the storage key. */ + filename: string; + /** The file's media type. */ + mimeType: string; + /** + * The bytes. Their length is judged against the deployment's upload size + * limit (the `max_upload_mb` storage setting) before anything is stored. + */ + data: Buffer; + /** + * Provenance the stored file keeps (on `sys_file.metadata`), read back by + * {@link IFieldFileService.describeFiles}: the seed loader records the seed + * file path and the SHA-256 of the bytes, so a later load can tell an + * unchanged file from a changed one. + */ + metadata?: Record; +} + +/** What {@link IFieldFileService.storeFile} stored. */ +export interface StoredFieldFile { + /** The new `sys_file` row's id: the value a file field holds. */ + fileId: string; + /** The number of bytes stored. */ + size: number; +} + +/** The (object, record, field) slot that owns a field file (ADR-0104 D3 wave 2). */ +export interface FieldFileSlot { + /** The owning record's object, by its short name. */ + object: string; + /** The owning record's id. */ + recordId: string; + /** The file-class field on that record whose value is the file's id. */ + field: string; +} + +/** One stored file, as {@link IFieldFileService.describeFiles} answers it. */ +export interface FieldFileDescription { + /** The `sys_file` row's id. */ + fileId: string; + /** The file name a download is saved as. */ + name: string; + /** The file's media type, or `null` when none was recorded. */ + mimeType: string | null; + /** `pending` until a record owns it; `deleted` for a tombstone in its grace window. */ + status: 'pending' | 'committed' | 'deleted'; + /** The slot that owns the file, or `null` while none does. */ + owner: FieldFileSlot | null; + /** The provenance {@link StoreFieldFileInput.metadata} recorded; `{}` when none. */ + metadata: Record; +} + +/** + * The server-side door for a FILE FIELD's file: store bytes as a `sys_file` + * no record owns yet, then hand that file to the record whose field holds its + * id — the ownership an upload gets when a record write references it. + * + * For a producer with no upload in front of it — the seed loader, which writes + * the files a package ships (`seedFile(path)`) — so it acts as the SYSTEM: + * there is no caller whose read of the file could be asked. What it does run is + * what an upload runs: the upload size limit, the organization the file is + * stored in, and the storage key layout of the upload doors. + * + * Refusals carry `code` and `status` (ADR-0112): + * - `VALIDATION_ERROR` / 400: a malformed input; + * - `PAYLOAD_TOO_LARGE` / 413: the bytes are over the upload size limit, + * refused before anything is stored; + * - `RESOURCE_CONFLICT` / 409: `claimFiles` named a file another slot owns — + * a field file has one owner, and it is never transferred; + * - `RECORD_NOT_FOUND` / 404: `claimFiles` named a file that does not exist + * in the organization; + * - `SERVICE_UNAVAILABLE` / 503: no data engine is wired into this kernel. + * + * A stored file stays `pending` until `claimFiles` commits it, so a file whose + * record never lands is reclaimed — row and bytes — by the platform's + * pending-upload retention. + */ +export interface IFieldFileService { + /** Store `input.data` as a `pending` `sys_file` no record owns yet. */ + storeFile(input: StoreFieldFileInput): Promise; + /** + * The files among `fileIds` that exist in `organizationId` (every + * organization's, for `null`), by id. An id missing from the answer names + * no file there. + */ + describeFiles(fileIds: readonly string[], organizationId: string | null): Promise>; + /** + * Make `slot` the owner of exactly `fileIds`: each one is committed and + * owned by the slot, and every other file the slot owned is released, as a + * record write that replaces the field's value releases it. Idempotent: a + * file the slot already owns, committed, is left as it is. + */ + claimFiles(slot: FieldFileSlot, fileIds: readonly string[], organizationId: string | null): Promise; +} From 95123a81d3624ec5e257e7d238d5e8015ce018f2 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 26 Oct 2026 12:51:07 +0000 Subject: [PATCH 03/13] wip: seed file path regex, ledger, liveness, tests, changeset, docs Co-Authored-By: Claude Claude-Session: https://claude.ai/code/session_012eJQ8dtut5bZsnhFE3XGHF --- .changeset/25779-seed-files.md | 39 ++ content/docs/data-modeling/seed-data.mdx | 43 ++ .../cli/src/utils/bundle-seed-files.test.ts | 102 +++++ packages/metadata-protocol/src/seed-files.ts | 2 +- .../src/seed-loader-seed-files.test.ts | 419 ++++++++++++++++++ .../src/field-file-service.test.ts | 279 ++++++++++++ packages/spec/authorable-surface/data.json | 8 + .../spec/dropped-refinements.baseline.json | 34 ++ packages/spec/json-schema.manifest/data.json | 3 + packages/spec/liveness/seed.json | 37 ++ packages/spec/liveness/state-counts/seed.md | 2 +- .../src/contracts/storage-service.test.ts | 47 ++ packages/spec/src/data/seed-file.zod.test.ts | 170 +++++++ packages/spec/src/data/seed-file.zod.ts | 22 +- 14 files changed, 1196 insertions(+), 11 deletions(-) create mode 100644 .changeset/25779-seed-files.md create mode 100644 packages/cli/src/utils/bundle-seed-files.test.ts create mode 100644 packages/metadata-protocol/src/seed-loader-seed-files.test.ts create mode 100644 packages/services/service-storage/src/field-file-service.test.ts create mode 100644 packages/spec/src/data/seed-file.zod.test.ts diff --git a/.changeset/25779-seed-files.md b/.changeset/25779-seed-files.md new file mode 100644 index 00000000000..80e62ce211b --- /dev/null +++ b/.changeset/25779-seed-files.md @@ -0,0 +1,39 @@ +--- +'@objectstack/spec': minor +'@objectstack/metadata-protocol': minor +'@objectstack/service-storage': minor +'@objectstack/runtime': minor +'@objectstack/cli': minor +--- + +feat(spec,metadata-protocol,service-storage,runtime,cli)!: a seed record can carry a file the package ships, `seedFile(path)`, and a file field's id that names no stored file is refused + +Clause-②: yes (narrowing) +Metadata-protocol: narrowed + + + +A `file`, `image`, `avatar`, `video` or `audio` field stores the id of a stored file, and only an upload made one, so a seed could not fill a required file field at all. A seed record now names the file by the path the package ships it at: + +```ts +import { defineSeed, seedFile } from '@objectstack/spec/data'; + +defineSeed(ContractVersion, { + externalId: 'code', + records: [ + { code: 'NDA-001-V1', file: seedFile('fixtures/nda-v1.pdf') }, + { code: 'NDA-002-V1', file: seedFile('fixtures/nda-v2', { name: 'NDA v2.pdf', contentType: 'application/pdf' }) }, + ], +}); +``` + +- **The path** is relative to the directory holding `objectstack.config.ts`, uses `/`, and stays inside it: an absolute path, a drive letter, a `\`, or an empty, `.` or `..` segment is refused at parse. A `multiple: true` file field takes an array of `seedFile(...)` values. `defineSeed` refuses `seedFile(...)` in a field that does not store a file. +- **`os build` / `os compile`** reads each referenced file and writes its bytes into the dataset's new `files` map (`{ contentType, size, sha256, data }`, base64), so the compiled artifact carries them to every boot that loads it: `os dev`, `os serve`, a published package, an artifact URL. A path that names no file, or a file whose type is neither known by its extension nor named with `contentType`, fails the build. +- **The seed loader** stores the bytes through the new `field-files` service (`FIELD_FILE_SERVICE`, `IFieldFileService`, registered by `@objectstack/service-storage`), writes the file's id into the field, and makes the written record the file's owner, the (object, record, field) ownership an upload gets. Each row gets a file of its own. A replay that finds the row holding a file stored from the same path with the same bytes, name and type keeps it, so a restart does not store a new file for an unchanged fixture. A file stays `pending` until its row lands, so a file whose row is refused is reclaimed by the pending-upload retention. +- **Refused, by row, before the row is written**: a reference whose bytes the dataset does not carry, bytes that do not match their recorded size and SHA-256, a load with no `field-files` service, a field mixing `seedFile(...)` with other values, and several `seedFile(...)` values in a one-file field. Each is counted, reported in the load's errors and logged at `error`. +- **The inline boot seed** of an app whose datasets carry seed files runs at `kernel:bootstrapped` (as one with `demoStaff` already does), once the storage service has read its settings, so the files land in the store the deployment serves from. +- `SeedLoaderService` takes an optional fourth argument, `{ getFieldFiles }`; every platform caller passes the kernel's `field-files` service. + +**BREAKING** (a seed-load narrowing). A seed record whose file field holds an id-shaped string (letters, digits, `_` and `-`, at most 64 characters) that names no stored file in the row's organization is now refused, by row, with an error naming the field and the value. Before, it was written as given, and the field then read as holding no file. An external URL or an inline value is not an id and is written as before; so is any value on a deployment without `@objectstack/service-storage`. + +**What to do.** If a seed load now reports `… holds '', which names no stored file`, replace the value with the file the field should hold, `seedFile('')`, or remove it. diff --git a/content/docs/data-modeling/seed-data.mdx b/content/docs/data-modeling/seed-data.mdx index 902950dcd8f..d5b83be0282 100644 --- a/content/docs/data-modeling/seed-data.mdx +++ b/content/docs/data-modeling/seed-data.mdx @@ -440,6 +440,46 @@ duplicates on every replay boot ([#3434](https://github.com/objectstack-ai/objec --- +## File Fields — `seedFile(path)` + +A `file`, `image`, `avatar`, `video` or `audio` field stores the id of a stored +file, and a seed has no upload to make one. Name the file by the path your +package ships it at instead, and the platform stores it for the record: + +```typescript +import { defineSeed, seedFile } from '@objectstack/spec/data'; + +export const contractVersionsSeed = defineSeed(ContractVersion, { + externalId: 'code', + records: [ + { code: 'NDA-001-V1', file: seedFile('fixtures/nda-v1.pdf') }, + // `name` is what a download is saved as; `contentType` for an extension + // the build does not know. + { code: 'MSA-001-V1', file: seedFile('fixtures/msa-v1', { name: 'MSA v1.pdf', contentType: 'application/pdf' }) }, + // A `multiple: true` file field takes an array. + { code: 'SOW-001-V1', file: seedFile('fixtures/sow.pdf'), scans: [seedFile('fixtures/p1.png'), seedFile('fixtures/p2.png')] }, + ], +}); +``` + +- **The path** is relative to the directory holding `objectstack.config.ts` and + stays inside it — `/` separators, no leading `/`, no `.` or `..` segment. +- **`os build`** reads each referenced file and bundles its bytes into the + compiled artifact, so the seed carries them wherever the artifact goes + (`os dev`, `os serve`, a published package). A path that names no file fails + the build. +- **The seed load** stores each file and writes its id into the field; the + record owns the file exactly as if it had been uploaded into that field. Every + row gets a file of its own, even when two rows name the same path. A replay + keeps a file whose bytes did not change, so restarting does not store it + again; changing the file and rebuilding replaces it. +- **Refused, by row, loudly**: a reference whose bytes the build did not bundle + (a dataset written by hand without its `files`), bytes that do not match what + the build recorded, and a file field holding an id-shaped string that names no + stored file — write `seedFile(...)` there instead. + +--- + ## Dynamic Values (CEL) Any field value may be a **CEL expression** evaluated at install time against a @@ -639,6 +679,9 @@ function defineSeed< records: Array>; // declared fields + injectable system columns } ): Seed + +// A file the package ships, for a file-class field (see "File Fields"). +function seedFile(path: string, options?: { name?: string; contentType?: string }): { $file: string; name?: string; contentType?: string } ``` `SeedRecord` keys are the object's declared fields (a `lookup` / `master_detail` diff --git a/packages/cli/src/utils/bundle-seed-files.test.ts b/packages/cli/src/utils/bundle-seed-files.test.ts new file mode 100644 index 00000000000..815e62ed906 --- /dev/null +++ b/packages/cli/src/utils/bundle-seed-files.test.ts @@ -0,0 +1,102 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +import { createHash } from 'node:crypto'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { afterEach, describe, expect, it } from 'vitest'; +import { SeedSchema, seedFile } from '@objectstack/spec/data'; +import { bundleSeedFiles } from './bundle-seed-files.js'; + +/** + * [#25779] `os build` reads each file a seed record names with + * `seedFile(path)`, relative to the directory holding the config, and writes + * its bytes into the dataset's `files` map — the shape the seed loader checks + * them against. A path that names no file, climbs out of the root, or has a + * type nobody can tell fails the build. + */ + +const roots: string[] = []; +afterEach(() => { + for (const root of roots.splice(0)) fs.rmSync(root, { recursive: true, force: true }); +}); + +function packageRoot(files: Record): string { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'os-25779-')); + roots.push(root); + for (const [rel, content] of Object.entries(files)) { + fs.mkdirSync(path.dirname(path.join(root, rel)), { recursive: true }); + fs.writeFileSync(path.join(root, rel), content); + } + return root; +} + +const sha = (text: string) => createHash('sha256').update(Buffer.from(text)).digest('hex'); + +describe('bundleSeedFiles', () => { + it('writes each referenced file into its dataset, once per path, in the shape SeedSchema accepts', () => { + const root = packageRoot({ 'fixtures/nda.pdf': '%PDF nda', 'scans/p1.png': 'png' }); + const dataset = { + object: 'clm_version', + records: [ + { code: 'V1', file: seedFile('fixtures/nda.pdf'), scans: [seedFile('scans/p1.png')] }, + { code: 'V2', file: seedFile('fixtures/nda.pdf') }, + ], + }; + const bundle: Record = { data: [dataset] }; + + const result = bundleSeedFiles(bundle, root); + expect(result.issues).toEqual([]); + expect(result.files).toBe(2); + expect((dataset as any).files).toEqual({ + 'fixtures/nda.pdf': { + contentType: 'application/pdf', + size: 8, + sha256: sha('%PDF nda'), + data: Buffer.from('%PDF nda').toString('base64'), + }, + 'scans/p1.png': { contentType: 'image/png', size: 3, sha256: sha('png'), data: Buffer.from('png').toString('base64') }, + }); + expect(SeedSchema.safeParse(dataset).success).toBe(true); + }); + + it('reaches the datasets of every package body too', () => { + const root = packageRoot({ 'a.txt': 'hello' }); + const inner = { object: 'x', records: [{ f: seedFile('a.txt') }] }; + bundleSeedFiles({ packages: [{ manifest: { data: [inner] } }] }, root); + expect((inner as any).files['a.txt'].contentType).toBe('text/plain'); + }); + + it('leaves a dataset that names no seed file exactly as it was', () => { + const dataset = { object: 'x', records: [{ name: 'a' }] }; + const result = bundleSeedFiles({ data: [dataset] }, packageRoot({})); + expect(result).toEqual({ files: 0, bytes: 0, issues: [] }); + expect(dataset).toEqual({ object: 'x', records: [{ name: 'a' }] }); + }); + + it('refuses a path that names no file, saying where the reference sits', () => { + const result = bundleSeedFiles( + { data: [{ object: 'x', records: [{ code: 'a' }, { file: seedFile('fixtures/missing.pdf') }] }] }, + packageRoot({}), + ); + expect(result.issues).toHaveLength(1); + expect(result.issues[0].location).toBe('data[0].records[1].file'); + expect(result.issues[0].message).toContain("seedFile('fixtures/missing.pdf') names no file"); + }); + + it('refuses a type it cannot tell, unless the reference names one', () => { + const root = packageRoot({ 'blob.bin': 'xx' }); + const refused = bundleSeedFiles({ data: [{ object: 'x', records: [{ f: seedFile('blob.bin') }] }] }, root); + expect(refused.issues[0].message).toContain("the file's type cannot be told from its extension"); + + const named = { object: 'x', records: [{ f: seedFile('blob.bin', { contentType: 'application/octet-stream' }) }] }; + expect(bundleSeedFiles({ data: [named] }, root).issues).toEqual([]); + expect((named as any).files['blob.bin'].contentType).toBe('application/octet-stream'); + }); + + it('never reads outside the package root', () => { + const root = packageRoot({}); + const result = bundleSeedFiles({ data: [{ object: 'x', records: [{ f: { $file: '../outside.pdf' } }] }] }, root); + expect(result.issues[0].message).toContain('climbs out of the package root'); + }); +}); diff --git a/packages/metadata-protocol/src/seed-files.ts b/packages/metadata-protocol/src/seed-files.ts index b151539ae15..56b5cbbded3 100644 --- a/packages/metadata-protocol/src/seed-files.ts +++ b/packages/metadata-protocol/src/seed-files.ts @@ -68,7 +68,7 @@ function fileFieldsOf(definition: unknown): Map { ?.fields; if (!fields || typeof fields !== 'object') return found; for (const [name, field] of Object.entries(fields)) { - if (typeof field?.type === 'string' && (FILE_REFERENCE_TYPES as readonly string[]).includes(field.type)) { + if (typeof field?.type === 'string' && FILE_REFERENCE_TYPES.has(field.type)) { found.set(name, { multiple: field.multiple === true }); } } diff --git a/packages/metadata-protocol/src/seed-loader-seed-files.test.ts b/packages/metadata-protocol/src/seed-loader-seed-files.test.ts new file mode 100644 index 00000000000..47aff50312d --- /dev/null +++ b/packages/metadata-protocol/src/seed-loader-seed-files.test.ts @@ -0,0 +1,419 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +import { createHash } from 'node:crypto'; +import { describe, it, expect, vi } from 'vitest'; +import { SeedLoaderService } from './seed-loader.js'; +import type { + FieldFileDescription, + FieldFileSlot, + IDataEngine, + IFieldFileService, + IMetadataService, + StoreFieldFileInput, +} from '@objectstack/spec/contracts'; +import { RAW_FILE_VALUES_CONTEXT_KEY, seedFile } from '@objectstack/spec/data'; + +/** + * [#25779] A seed record writes `seedFile(path)` into a file field; `os build` + * bundled the bytes into the dataset's `files` map. The loader stores them + * through the `field-files` service, writes the file's id into the field, makes + * the written row the file's owner, and keeps the file on a replay that finds + * the same bytes. It refuses — by row, before anything of the row is written — + * a reference whose bytes the dataset does not carry or that do not match what + * was recorded, a load with no file service, and (shape 3) an id-shaped value + * that names no stored file. + */ + +function createLogger() { + return { info: vi.fn(), warn: vi.fn(), error: vi.fn(), debug: vi.fn() }; +} + +/** An engine whose `find` honours `where` and records the read context, like a real one. */ +function createEngine(): { engine: IDataEngine; store: Record; findContexts: unknown[] } { + const store: Record = {}; + const findContexts: unknown[] = []; + let next = 0; + const engine = { + find: vi.fn(async (object: string, query?: any) => { + findContexts.push(query?.context); + let rows = store[object] ?? []; + if (query?.where) rows = rows.filter((r) => Object.entries(query.where).every(([k, v]) => r[k] === v)); + return rows.map((r) => ({ ...r })); + }), + findOne: vi.fn(async (object: string, query?: any) => { + const rows = (store[object] ?? []).filter((r) => + Object.entries(query?.where ?? {}).every(([k, v]) => r[k] === v), + ); + return rows[0] ? { ...rows[0] } : null; + }), + insert: vi.fn(async (object: string, data: any) => { + store[object] ??= []; + const one = (d: any) => { + const row = { id: `row-${++next}`, ...d }; + store[object].push(row); + return { ...row }; + }; + return Array.isArray(data) ? data.map(one) : one(data); + }), + update: vi.fn(async (object: string, data: any) => { + const rows = store[object] ?? []; + const i = rows.findIndex((r) => r.id === data.id); + if (i >= 0) rows[i] = { ...rows[i], ...data }; + return data; + }), + delete: vi.fn(async () => ({ deleted: 1 })), + count: vi.fn(async (object: string) => (store[object] ?? []).length), + aggregate: vi.fn(async () => []), + } as unknown as IDataEngine; + return { engine, store, findContexts }; +} + +/** An in-memory `field-files` service that records what it was asked to do. */ +function createFieldFiles(organizationOf: (id: string) => string | null = () => null) { + const files = new Map(); + let next = 0; + const stored: StoreFieldFileInput[] = []; + const claims: Array<{ slot: FieldFileSlot; fileIds: string[]; organizationId: string | null }> = []; + const service: IFieldFileService & { failClaims?: boolean } = { + storeFile: vi.fn(async (input: StoreFieldFileInput) => { + stored.push(input); + const fileId = `file-${++next}`; + files.set(fileId, { + fileId, + name: input.filename, + mimeType: input.mimeType, + status: 'pending', + owner: null, + metadata: { ...(input.metadata ?? {}) }, + organizationId: input.organizationId, + data: input.data, + }); + return { fileId, size: input.data.byteLength }; + }), + describeFiles: vi.fn(async (ids: readonly string[], organizationId: string | null) => { + const found = new Map(); + for (const id of ids) { + const file = files.get(id); + if (!file) continue; + if (organizationId !== null && (file.organizationId ?? organizationOf(id)) !== organizationId) continue; + const { organizationId: _o, data: _d, ...description } = file; + found.set(id, structuredClone(description)); + } + return found; + }), + claimFiles: vi.fn(async (slot: FieldFileSlot, fileIds: readonly string[], organizationId: string | null) => { + if (service.failClaims) throw new Error('the storage service is down'); + claims.push({ slot, fileIds: [...fileIds], organizationId }); + for (const file of files.values()) { + const owner = file.owner; + if (owner && owner.object === slot.object && owner.recordId === slot.recordId && owner.field === slot.field + && !fileIds.includes(file.fileId)) file.owner = null; + } + for (const id of fileIds) { + const file = files.get(id)!; + file.owner = { ...slot }; + file.status = 'committed'; + } + }), + }; + return { service, files, stored, claims }; +} + +function createMetadata(): IMetadataService { + const objects: Record = { + clm_version: { + name: 'clm_version', + fields: { + code: { type: 'text' }, + file: { type: 'file', required: true }, + scans: { type: 'image', multiple: true }, + logo: { type: 'image' }, + parent: { type: 'lookup', reference: 'clm_version' }, + }, + }, + }; + return { + getObject: vi.fn(async (name: string) => objects[name]), + listObjects: vi.fn(async () => Object.values(objects)), + register: vi.fn(async () => {}), + get: vi.fn(async (_t: string, name: string) => objects[name]), + list: vi.fn(async () => []), + unregister: vi.fn(async () => {}), + exists: vi.fn(async () => false), + listNames: vi.fn(async () => []), + } as unknown as IMetadataService; +} + +const CONFIG = { dryRun: false, haltOnError: false, multiPass: true, defaultMode: 'upsert', batchSize: 1000, transaction: false } as any; + +/** The bundled entry `os build` writes for `bytes`. */ +function bundled(bytes: string, contentType = 'application/pdf') { + const buffer = Buffer.from(bytes); + return { + contentType, + size: buffer.byteLength, + sha256: createHash('sha256').update(buffer).digest('hex'), + data: buffer.toString('base64'), + }; +} + +function dataset(records: any[], files: Record = { 'fixtures/v1.pdf': bundled('%PDF v1') }, extra: any = {}) { + return { object: 'clm_version', externalId: 'code', mode: 'upsert', env: ['prod', 'dev', 'test'], records, files, ...extra }; +} + +function loader(engine: IDataEngine, service: IFieldFileService | undefined, logger = createLogger()) { + return new SeedLoaderService(engine, createMetadata(), logger, { getFieldFiles: () => service }); +} + +describe('seed files: stored, owned by the written row, kept across replays (#25779)', () => { + it('stores the bundled bytes, writes the file id into the field, and makes the row its owner', async () => { + const { engine, store } = createEngine(); + const { service, files, stored, claims } = createFieldFiles(); + const result = await loader(engine, service).load({ + seeds: [dataset([{ code: 'V1', file: seedFile('fixtures/v1.pdf') }])], + config: CONFIG, + }); + + expect(result.success).toBe(true); + expect(result.summary.totalInserted).toBe(1); + const row = store.clm_version[0]; + expect(row.file).toBe('file-1'); + expect(stored).toHaveLength(1); + expect(stored[0]).toMatchObject({ + organizationId: null, + filename: 'v1.pdf', + mimeType: 'application/pdf', + metadata: { seedFile: 'fixtures/v1.pdf', sha256: bundled('%PDF v1').sha256 }, + }); + expect(stored[0].data.toString()).toBe('%PDF v1'); + expect(claims).toEqual([{ slot: { object: 'clm_version', recordId: row.id, field: 'file' }, fileIds: ['file-1'], organizationId: null }]); + expect(files.get('file-1')).toMatchObject({ status: 'committed', owner: { object: 'clm_version', recordId: row.id, field: 'file' } }); + }); + + it('takes the reference\'s name and type over the bundled defaults, and the organization of the row', async () => { + const { engine } = createEngine(); + const { service, stored, claims } = createFieldFiles(); + await loader(engine, service).load({ + seeds: [dataset([{ code: 'V1', file: seedFile('fixtures/v1.pdf', { name: 'NDA.pdf', contentType: 'application/x-pdf' }) }])], + config: { ...CONFIG, organizationId: 'org_a' }, + }); + expect(stored[0]).toMatchObject({ organizationId: 'org_a', filename: 'NDA.pdf', mimeType: 'application/x-pdf' }); + expect(claims[0].organizationId).toBe('org_a'); + }); + + it('keeps the file on a replay of unchanged bytes: no new file, no write, no claim', async () => { + const { engine, store, findContexts } = createEngine(); + const { service, stored, claims } = createFieldFiles(); + const seeds = [dataset([{ code: 'V1', file: seedFile('fixtures/v1.pdf') }])]; + await loader(engine, service).load({ seeds, config: CONFIG }); + (engine.update as any).mockClear(); + + const replay = await loader(engine, service).load({ seeds, config: CONFIG }); + expect(replay.success).toBe(true); + expect(replay.summary.totalSkipped).toBe(1); + expect(stored).toHaveLength(1); + expect(claims).toHaveLength(1); + expect(engine.update).not.toHaveBeenCalled(); + expect(store.clm_version[0].file).toBe('file-1'); + // The matched row is read with its STORED file values. + expect(findContexts.some((c: any) => c?.[RAW_FILE_VALUES_CONTEXT_KEY] === true)).toBe(true); + }); + + it('stores a new file when the bytes changed, writes it, and the slot owns it alone', async () => { + const { engine, store } = createEngine(); + const { service, files, stored } = createFieldFiles(); + await loader(engine, service).load({ seeds: [dataset([{ code: 'V1', file: seedFile('fixtures/v1.pdf') }])], config: CONFIG }); + + const changed = await loader(engine, service).load({ + seeds: [dataset([{ code: 'V1', file: seedFile('fixtures/v1.pdf') }], { 'fixtures/v1.pdf': bundled('%PDF v1, revised') })], + config: CONFIG, + }); + expect(changed.summary.totalUpdated).toBe(1); + expect(stored).toHaveLength(2); + expect(store.clm_version[0].file).toBe('file-2'); + expect(files.get('file-2')?.owner?.recordId).toBe(store.clm_version[0].id); + expect(files.get('file-1')?.owner).toBeNull(); + }); + + it('stores one file per reference in a multiple field, in order', async () => { + const { engine, store } = createEngine(); + const { service, claims } = createFieldFiles(); + const result = await loader(engine, service).load({ + seeds: [dataset( + [{ code: 'V1', file: seedFile('fixtures/v1.pdf'), scans: [seedFile('scans/p1.png'), seedFile('scans/p2.png')] }], + { + 'fixtures/v1.pdf': bundled('%PDF v1'), + 'scans/p1.png': bundled('png one', 'image/png'), + 'scans/p2.png': bundled('png two', 'image/png'), + }, + )], + config: CONFIG, + }); + expect(result.success).toBe(true); + expect(store.clm_version[0].scans).toEqual(['file-2', 'file-3']); + expect(claims.find((c) => c.slot.field === 'scans')?.fileIds).toEqual(['file-2', 'file-3']); + }); + + it('gives every row its own file, even when rows name the same path', async () => { + const { engine, store } = createEngine(); + const { service, stored } = createFieldFiles(); + await loader(engine, service).load({ + seeds: [dataset([ + { code: 'V1', file: seedFile('fixtures/v1.pdf') }, + { code: 'V2', file: seedFile('fixtures/v1.pdf') }, + ])], + config: CONFIG, + }); + expect(stored).toHaveLength(2); + expect(store.clm_version.map((r) => r.file)).toEqual(['file-1', 'file-2']); + }); + + it('settles the files of a self-referencing dataset, written row by row', async () => { + const { engine, store } = createEngine(); + const { service, claims } = createFieldFiles(); + const result = await loader(engine, service).load({ + seeds: [dataset([ + { code: 'V1', file: seedFile('fixtures/v1.pdf') }, + { code: 'V2', file: seedFile('fixtures/v1.pdf'), parent: 'V1' }, + ])], + config: CONFIG, + }); + expect(result.success).toBe(true); + expect(claims.map((c) => c.slot.recordId)).toEqual(store.clm_version.map((r) => r.id)); + }); +}); + +describe('seed files: refusals, by row, before the row is written (#25779)', () => { + it('refuses a reference whose bytes the dataset does not carry, naming the path', async () => { + const { engine, store } = createEngine(); + const { service, stored } = createFieldFiles(); + const logger = createLogger(); + const result = await loader(engine, service, logger).load({ + seeds: [dataset([{ code: 'V1', file: seedFile('fixtures/missing.pdf') }, { code: 'V2', file: seedFile('fixtures/v1.pdf') }])], + config: CONFIG, + }); + expect(result.success).toBe(false); + expect(result.summary.totalErrored).toBe(1); + expect(result.errors[0]).toMatchObject({ field: 'file', recordIndex: 0, targetObject: 'sys_file' }); + expect(result.errors[0].message).toContain("seedFile('fixtures/missing.pdf') names a file this clm_version dataset does not carry"); + expect(store.clm_version.map((r) => r.code)).toEqual(['V2']); + expect(stored).toHaveLength(1); + expect(logger.error).toHaveBeenCalledWith(expect.stringContaining('record #0 was NOT seeded'), undefined, expect.anything()); + }); + + it('refuses bytes that do not match their recorded digest', async () => { + const { engine, store } = createEngine(); + const { service } = createFieldFiles(); + const tampered = { ...bundled('%PDF v1'), data: Buffer.from('%PDF evil').toString('base64') }; + const result = await loader(engine, service).load({ + seeds: [dataset([{ code: 'V1', file: seedFile('fixtures/v1.pdf') }], { 'fixtures/v1.pdf': tampered })], + config: CONFIG, + }); + expect(result.errors[0].message).toContain('do not match what was recorded'); + expect(store.clm_version ?? []).toEqual([]); + }); + + it('refuses a seed file when the load has no file service', async () => { + const { engine, store } = createEngine(); + const result = await loader(engine, undefined).load({ + seeds: [dataset([{ code: 'V1', file: seedFile('fixtures/v1.pdf') }])], + config: CONFIG, + }); + expect(result.errors[0].message).toContain('this load has no file service to store it with'); + expect(store.clm_version ?? []).toEqual([]); + }); + + it('refuses a field mixing seed files with other values, and several seed files in a one-file field', async () => { + const { engine } = createEngine(); + const { service, stored } = createFieldFiles(); + const result = await loader(engine, service).load({ + seeds: [dataset( + [ + { code: 'V1', file: seedFile('fixtures/v1.pdf'), scans: [seedFile('fixtures/v1.pdf'), 'https://example.com/a.png'] }, + { code: 'V2', file: [seedFile('fixtures/v1.pdf'), seedFile('fixtures/v1.pdf')] }, + ], + )], + config: CONFIG, + }); + expect(result.errors.map((e) => e.message)).toEqual([ + expect.stringContaining('mixes seedFile(...) values with other values'), + expect.stringContaining('holds one file, and the record gives it 2 seedFile(...) values'), + ]); + expect(stored).toEqual([]); + }); + + it('refuses an id-shaped value that names no stored file (shape 3), and keeps one that does', async () => { + const { engine, store } = createEngine(); + const { service } = createFieldFiles(); + await loader(engine, service).load({ seeds: [dataset([{ code: 'V1', file: seedFile('fixtures/v1.pdf') }])], config: CONFIG }); + + const result = await loader(engine, service).load({ + seeds: [dataset([ + { code: 'V2', file: 'file-1' }, + { code: 'V3', file: 'sf_contract_template' }, + { code: 'V4', file: 'file-1', logo: 'https://cdn.example.com/logo.png' }, + ], {})], + config: CONFIG, + }); + expect(result.errors).toHaveLength(1); + expect(result.errors[0]).toMatchObject({ recordIndex: 1, field: 'file', attemptedValue: 'sf_contract_template' }); + expect(result.errors[0].message).toContain("holds 'sf_contract_template', which names no stored file"); + expect(store.clm_version.map((r) => r.code)).toEqual(['V1', 'V2', 'V4']); + // An external URL is not an id: written as given. + expect(store.clm_version.find((r) => r.code === 'V4')?.logo).toBe('https://cdn.example.com/logo.png'); + }); + + it('answers shape 3 in the row\'s organization: a file of another organization does not resolve', async () => { + const { engine } = createEngine(); + const { service } = createFieldFiles(); + await service.storeFile({ organizationId: 'org_b', filename: 'x.pdf', mimeType: 'application/pdf', data: Buffer.from('x') }); + const result = await loader(engine, service).load({ + seeds: [dataset([{ code: 'V1', file: 'file-1', organization_id: 'org_a' }], {})], + config: CONFIG, + }); + expect(result.errors[0]?.message).toContain('which names no stored file in organization org_a'); + }); + + it('a dry run checks the references against the bundled bytes and stores nothing', async () => { + const { engine } = createEngine(); + const { service, stored } = createFieldFiles(); + const result = await loader(engine, service).load({ + seeds: [dataset([{ code: 'V1', file: seedFile('fixtures/v1.pdf') }, { code: 'V2', file: seedFile('nope.pdf') }])], + config: { ...CONFIG, dryRun: true }, + }); + expect(result.errors).toHaveLength(1); + expect(result.errors[0].message).toContain("seedFile('nope.pdf')"); + expect(stored).toEqual([]); + expect(service.describeFiles).not.toHaveBeenCalled(); + }); + + it('reports a row whose files could not be made its own, loudly, without unwriting the row', async () => { + const { engine, store } = createEngine(); + const { service } = createFieldFiles(); + service.failClaims = true; + const logger = createLogger(); + const result = await loader(engine, service, logger).load({ + seeds: [dataset([{ code: 'V1', file: seedFile('fixtures/v1.pdf') }])], + config: CONFIG, + }); + expect(result.success).toBe(false); + expect(result.summary.totalInserted).toBe(1); + expect(store.clm_version).toHaveLength(1); + expect(result.errors[0].message).toContain('could not be made the record\'s own: the storage service is down'); + expect(logger.error).toHaveBeenCalled(); + }); + + it('leaves an object without file fields untouched: the service is never asked', async () => { + const { engine } = createEngine(); + const { service } = createFieldFiles(); + const metadata = createMetadata(); + (metadata.getObject as any).mockImplementation(async (name: string) => + name === 'plain' ? { name: 'plain', fields: { code: { type: 'text' } } } : undefined); + const result = await new SeedLoaderService(engine, metadata, createLogger(), { getFieldFiles: () => service }).load({ + seeds: [{ object: 'plain', externalId: 'code', mode: 'upsert', env: ['prod', 'dev', 'test'], records: [{ code: 'a' }] }] as any, + config: CONFIG, + }); + expect(result.success).toBe(true); + expect(service.describeFiles).not.toHaveBeenCalled(); + }); +}); diff --git a/packages/services/service-storage/src/field-file-service.test.ts b/packages/services/service-storage/src/field-file-service.test.ts new file mode 100644 index 00000000000..ed48a09e6a2 --- /dev/null +++ b/packages/services/service-storage/src/field-file-service.test.ts @@ -0,0 +1,279 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * [#25779] The server-side field-file verb — `FIELD_FILE_SERVICE`, registered + * by `StorageServicePlugin` — through the plugin's own wiring: a real + * `ObjectQL` over sqlite `:memory:`, the plugin's local adapter in a temp + * directory, and a settings namespace carrying `max_upload_mb`. + * + * - `storeFile` writes one `pending` `sys_file` in the organization it names, + * under the `tenant` scope, with its provenance, and the bytes behind it — + * and serves before `kernel:ready`, where a plugin's `start()` seeds; + * - over the size limit it is refused with the upload doors' envelope and + * leaves nothing; + * - `describeFiles` answers the files that exist in the organization, only; + * - `claimFiles` commits the files and makes the slot their owner, releases + * the slot's previous file, never takes a file another slot owns, and is + * idempotent — and agrees with the reference hooks when they claimed first. + */ + +import { describe, it, expect, afterEach, vi } from 'vitest'; +import { promises as fs } from 'node:fs'; +import { join } from 'node:path'; +import { tmpdir } from 'node:os'; +import { ObjectQL } from '@objectstack/objectql'; +import { SqlDriver } from '@objectstack/driver-sql'; +import { FIELD_FILE_SERVICE } from '@objectstack/spec/contracts'; +import type { IFieldFileService, IStorageService, StoreFieldFileInput } from '@objectstack/spec/contracts'; +import { SystemFile } from './objects/system-file.object.js'; +import { StorageServicePlugin } from './storage-service-plugin.js'; +import { FieldFileService } from './field-file-service.js'; + +const SYS = { context: { isSystem: true } } as const; +/** `max_upload_mb` as saved: 0.001 MB ⇒ floor(0.001 × 1 MiB) = 1048 bytes. */ +const MAX_UPLOAD_MB = 0.001; + +function quietLogger() { + const logger: any = { info: vi.fn(), warn: vi.fn(), debug: vi.fn(), error: vi.fn(), trace: vi.fn(), fatal: vi.fn() }; + logger.child = () => logger; + return logger; +} + +function makeCtx() { + const services = new Map(); + const phases = new Map unknown>>(); + const ctx: any = { + logger: quietLogger(), + registerService: (name: string, svc: unknown) => { + services.set(name, svc); + }, + getService: (name: string) => { + if (!services.has(name)) throw new Error(`service '${name}' not registered`); + return services.get(name); + }, + hook: (event: string, fn: () => unknown) => { + phases.set(event, [...(phases.get(event) ?? []), fn]); + }, + }; + const run = async (event: string) => { + for (const fn of phases.get(event) ?? []) await fn(); + }; + return { ctx, run }; +} + +function storageSettings() { + return { + createClient: () => ({}), + getNamespace: async () => ({ values: { max_upload_mb: { value: MAX_UPLOAD_MB, source: 'global' } } }), + subscribe: () => undefined, + registerAction: () => undefined, + }; +} + +const engines: ObjectQL[] = []; +afterEach(async () => { + for (const ql of engines.splice(0)) { + try { + await ql.destroy(); + } catch { + /* already torn down */ + } + } +}); + +/** A real engine with `sys_file` and `ff_doc`, whose `file` field holds one file and `scans` several. */ +async function engine(): Promise { + const ql = new ObjectQL({ logger: quietLogger() } as any); + ql.registerDriver( + new SqlDriver({ client: 'better-sqlite3', connection: { filename: ':memory:' }, useNullAsDefault: true }) as any, + true, + ); + await ql.init(); + engines.push(ql); + ql.registry.registerObject(SystemFile as any, 'test-25779'); + ql.registry.registerObject( + { + name: 'ff_doc', + fields: { name: { type: 'text' }, file: { type: 'file' }, scans: { type: 'file', multiple: true } }, + } as any, + 'test-25779', + ); + await ql.syncSchemas(); + return ql; +} + +/** Boot the plugin over a fresh engine; `ready: false` stops after `start()`, as a seed in `start()` sees it. */ +async function boot(opts: { ready?: boolean } = {}) { + const ql = await engine(); + const rootDir = await fs.mkdtemp(join(tmpdir(), 'oss-25779-')); + const plugin = new StorageServicePlugin({ adapter: 'local', local: { rootDir }, registerRoutes: false }); + const { ctx, run } = makeCtx(); + ctx.registerService('objectql', ql); + ctx.registerService('settings', storageSettings()); + await plugin.init(ctx); + await plugin.start(ctx); + if (opts.ready !== false) { + await run('kernel:ready'); + await run('kernel:bootstrapped'); + } + const files = ctx.getService(FIELD_FILE_SERVICE) as IFieldFileService; + const storage = ctx.getService('storage') as IStorageService; + return { ql, files, storage }; +} + +const input = (over: Partial = {}): StoreFieldFileInput => ({ + organizationId: 'org_a', + filename: 'nda-v1.pdf', + mimeType: 'application/pdf', + data: Buffer.from('%PDF-1.7 a seeded contract'), + metadata: { seedFile: 'fixtures/nda-v1.pdf', sha256: 'f'.repeat(64) }, + ...over, +}); + +const fileRows = (ql: ObjectQL) => ql.find('sys_file', { ...SYS }) as Promise>>; +const slot = (recordId: string, field = 'file') => ({ object: 'ff_doc', recordId, field }); + +async function refusalOf(call: Promise): Promise<{ code?: string; status?: number; message?: string }> { + try { + await call; + } catch (err) { + return err as { code?: string; status?: number }; + } + throw new Error('expected the call to be refused'); +} + +describe('FIELD_FILE_SERVICE: the server-side field-file verb (#25779)', () => { + it('is registered under its contract slot and declared in providesServices', async () => { + expect(new StorageServicePlugin().providesServices).toContain(FIELD_FILE_SERVICE); + const { files } = await boot(); + expect(files).toBeInstanceOf(FieldFileService); + }); + + it('stores a pending, unowned file in the named organization, with its provenance and bytes — before kernel:ready too', async () => { + const { ql, files, storage } = await boot({ ready: false }); + const data = Buffer.from('%PDF-1.7 seeded'); + const out = await files.storeFile(input({ data })); + expect(out.size).toBe(data.byteLength); + + const rows = await fileRows(ql); + expect(rows).toHaveLength(1); + expect(rows[0]).toMatchObject({ + id: out.fileId, + key: `tenant/${out.fileId}.pdf`, + name: 'nda-v1.pdf', + mime_type: 'application/pdf', + size: data.byteLength, + scope: 'tenant', + acl: 'private', + status: 'pending', + organization_id: 'org_a', + }); + expect(rows[0].ref_id ?? null).toBeNull(); + expect(JSON.parse(rows[0].metadata)).toEqual({ seedFile: 'fixtures/nda-v1.pdf', sha256: 'f'.repeat(64) }); + expect((await storage.download(rows[0].key)).equals(data)).toBe(true); + }); + + it('refuses bytes over the upload size limit with the upload doors\' envelope, leaving nothing', async () => { + const { ql, files } = await boot(); + const refusal = await refusalOf(files.storeFile(input({ data: Buffer.alloc(2000) }))); + expect(refusal).toMatchObject({ code: 'PAYLOAD_TOO_LARGE', status: 413 }); + expect(await fileRows(ql)).toEqual([]); + // The same call under the limit is stored. + await files.storeFile(input({ data: Buffer.alloc(1000) })); + expect(await fileRows(ql)).toHaveLength(1); + }); + + it('refuses a malformed input before anything is written', async () => { + const { ql, files } = await boot(); + expect(await refusalOf(files.storeFile(input({ filename: '' })))).toMatchObject({ code: 'VALIDATION_ERROR', status: 400 }); + expect(await refusalOf(files.storeFile({ ...input(), organizationId: undefined as any }))).toMatchObject({ status: 400 }); + expect(await refusalOf(files.storeFile(input({ data: 'bytes' as any })))).toMatchObject({ status: 400 }); + expect(await fileRows(ql)).toEqual([]); + }); + + it('describes the files that exist in the organization, and no other', async () => { + const { files } = await boot(); + const a = await files.storeFile(input({ organizationId: 'org_a' })); + const b = await files.storeFile(input({ organizationId: 'org_b' })); + const inA = await files.describeFiles([a.fileId, b.fileId, 'missing'], 'org_a'); + expect([...inA.keys()]).toEqual([a.fileId]); + expect(inA.get(a.fileId)).toEqual({ + fileId: a.fileId, + name: 'nda-v1.pdf', + mimeType: 'application/pdf', + status: 'pending', + owner: null, + metadata: { seedFile: 'fixtures/nda-v1.pdf', sha256: 'f'.repeat(64) }, + }); + }); + + it('claims: commits the files, makes the slot their owner, and releases the slot\'s previous file', async () => { + const { ql, files } = await boot({ ready: false }); + await ql.insert('ff_doc', { id: 'doc_1', name: 'one', organization_id: 'org_a' }, SYS); + const first = await files.storeFile(input()); + await files.claimFiles(slot('doc_1'), [first.fileId], 'org_a'); + let described = await files.describeFiles([first.fileId], 'org_a'); + expect(described.get(first.fileId)).toMatchObject({ status: 'committed', owner: slot('doc_1') }); + + // Idempotent: claiming again changes nothing. + await files.claimFiles(slot('doc_1'), [first.fileId], 'org_a'); + described = await files.describeFiles([first.fileId], 'org_a'); + expect(described.get(first.fileId)).toMatchObject({ status: 'committed', owner: slot('doc_1') }); + + // A new file for the same slot releases the first. + const second = await files.storeFile(input({ filename: 'nda-v2.pdf' })); + await files.claimFiles(slot('doc_1'), [second.fileId], 'org_a'); + described = await files.describeFiles([first.fileId, second.fileId], 'org_a'); + expect(described.get(first.fileId)?.owner).toBeNull(); + expect(described.get(second.fileId)).toMatchObject({ status: 'committed', owner: slot('doc_1') }); + }); + + it('never takes a file another slot owns, and refuses a file that does not exist', async () => { + const { files } = await boot(); + const owned = await files.storeFile(input()); + await files.claimFiles(slot('doc_1'), [owned.fileId], 'org_a'); + expect(await refusalOf(files.claimFiles(slot('doc_2'), [owned.fileId], 'org_a'))).toMatchObject({ + code: 'RESOURCE_CONFLICT', + status: 409, + }); + expect((await files.describeFiles([owned.fileId], 'org_a')).get(owned.fileId)?.owner).toEqual(slot('doc_1')); + expect(await refusalOf(files.claimFiles(slot('doc_2'), ['no-such-file'], 'org_a'))).toMatchObject({ + code: 'RECORD_NOT_FOUND', + status: 404, + }); + // A file of another organization is not one this organization can claim. + const elsewhere = await files.storeFile(input({ organizationId: 'org_b' })); + expect(await refusalOf(files.claimFiles(slot('doc_2'), [elsewhere.fileId], 'org_a'))).toMatchObject({ status: 404 }); + }); + + it('agrees with the reference hooks: a record write claims the stored file, and claimFiles then only commits it', async () => { + const { ql, files } = await boot(); + const stored = await files.storeFile(input()); + const row = (await ql.insert( + 'ff_doc', + { name: 'hooked', file: stored.fileId, organization_id: 'org_a' }, + { context: { isSystem: true, tenantId: 'org_a' } }, + )) as { id: string }; + let described = await files.describeFiles([stored.fileId], 'org_a'); + expect(described.get(stored.fileId)).toMatchObject({ status: 'pending', owner: slot(row.id) }); + + await files.claimFiles(slot(row.id), [stored.fileId], 'org_a'); + described = await files.describeFiles([stored.fileId], 'org_a'); + expect(described.get(stored.fileId)).toMatchObject({ status: 'committed', owner: slot(row.id) }); + }); + + it('settles a multiple field to exactly the files it is given', async () => { + const { files } = await boot({ ready: false }); + const [p1, p2, p3] = [ + await files.storeFile(input({ filename: 'p1.png', mimeType: 'image/png' })), + await files.storeFile(input({ filename: 'p2.png', mimeType: 'image/png' })), + await files.storeFile(input({ filename: 'p3.png', mimeType: 'image/png' })), + ]; + await files.claimFiles(slot('doc_1', 'scans'), [p1.fileId, p2.fileId], 'org_a'); + await files.claimFiles(slot('doc_1', 'scans'), [p2.fileId, p3.fileId], 'org_a'); + const described = await files.describeFiles([p1.fileId, p2.fileId, p3.fileId], 'org_a'); + expect(described.get(p1.fileId)?.owner).toBeNull(); + expect(described.get(p2.fileId)?.owner).toEqual(slot('doc_1', 'scans')); + expect(described.get(p3.fileId)?.owner).toEqual(slot('doc_1', 'scans')); + }); +}); diff --git a/packages/spec/authorable-surface/data.json b/packages/spec/authorable-surface/data.json index 5fc0f149aed..5434cc89c77 100644 --- a/packages/spec/authorable-surface/data.json +++ b/packages/spec/authorable-surface/data.json @@ -904,11 +904,19 @@ "data/Seed:columns", "data/Seed:env", "data/Seed:externalId", + "data/Seed:files", "data/Seed:locale", "data/Seed:mode", "data/Seed:object", "data/Seed:records", "data/SeedColumn:applyOn", + "data/SeedFileContent:contentType", + "data/SeedFileContent:data", + "data/SeedFileContent:sha256", + "data/SeedFileContent:size", + "data/SeedFileReference:$file", + "data/SeedFileReference:contentType", + "data/SeedFileReference:name", "data/SeedIdentity:org", "data/SeedIdentity:user", "data/SeedLoadResult:errored", diff --git a/packages/spec/dropped-refinements.baseline.json b/packages/spec/dropped-refinements.baseline.json index 23014d0a90e..e72b84d47f4 100644 --- a/packages/spec/dropped-refinements.baseline.json +++ b/packages/spec/dropped-refinements.baseline.json @@ -80,6 +80,7 @@ "manifest.dashboards.element.header.actions.element", "manifest.dashboards.element.widgets.element", "manifest.dashboards.element.widgets.element.filter", + "manifest.data.element.records.element.valueType", "manifest.datasets.element", "manifest.datasets.element.filter", "manifest.datasets.element.include.element", @@ -159,12 +160,14 @@ }, "api/DisablePackageResponse": { "sites": [ + "package.manifest.data.element.records.element.valueType", "package.manifest.navigationContributions.element.items.element.lazy", "package.manifest.navigationContributions.element.items.element.lazy.options[0]" ] }, "api/EnablePackageResponse": { "sites": [ + "package.manifest.data.element.records.element.valueType", "package.manifest.navigationContributions.element.items.element.lazy", "package.manifest.navigationContributions.element.items.element.lazy.options[0]" ] @@ -191,6 +194,7 @@ }, "api/GetInstalledPackageResponse": { "sites": [ + "data.options[0].manifest.data.element.records.element.valueType", "data.options[0].manifest.navigationContributions.element.items.element.lazy", "data.options[0].manifest.navigationContributions.element.items.element.lazy.options[0]", "data.options[1].manifest.actions.element.in", @@ -261,6 +265,7 @@ }, "api/GetPackageResponse": { "sites": [ + "package.manifest.data.element.records.element.valueType", "package.manifest.navigationContributions.element.items.element.lazy", "package.manifest.navigationContributions.element.items.element.lazy.options[0]" ] @@ -300,18 +305,21 @@ }, "api/InstallPackageRequest": { "sites": [ + "manifest.data.element.records.element.valueType", "manifest.navigationContributions.element.items.element.lazy", "manifest.navigationContributions.element.items.element.lazy.options[0]" ] }, "api/InstallPackageResponse": { "sites": [ + "package.manifest.data.element.records.element.valueType", "package.manifest.navigationContributions.element.items.element.lazy", "package.manifest.navigationContributions.element.items.element.lazy.options[0]" ] }, "api/InstalledPackageAtEitherStage": { "sites": [ + "options[0].manifest.data.element.records.element.valueType", "options[0].manifest.navigationContributions.element.items.element.lazy", "options[0].manifest.navigationContributions.element.items.element.lazy.options[0]", "options[1].manifest.actions.element.in", @@ -379,6 +387,7 @@ }, "api/ListInstalledPackagesResponse": { "sites": [ + "data.packages.element.options[0].manifest.data.element.records.element.valueType", "data.packages.element.options[0].manifest.navigationContributions.element.items.element.lazy", "data.packages.element.options[0].manifest.navigationContributions.element.items.element.lazy.options[0]", "data.packages.element.options[1].manifest.actions.element.in", @@ -441,6 +450,7 @@ }, "api/ListPackagesResponse": { "sites": [ + "packages.element.manifest.data.element.records.element.valueType", "packages.element.manifest.navigationContributions.element.items.element.lazy", "packages.element.manifest.navigationContributions.element.items.element.lazy.options[0]" ] @@ -480,36 +490,42 @@ }, "api/PackageInstallBody": { "sites": [ + "options[1].data.element.records.element.valueType", "options[1].navigationContributions.element.items.element.lazy", "options[1].navigationContributions.element.items.element.lazy.options[0]" ] }, "api/PackageInstallManifest": { "sites": [ + "data.element.records.element.valueType", "navigationContributions.element.items.element.lazy", "navigationContributions.element.items.element.lazy.options[0]" ] }, "api/PackageInstallRequest": { "sites": [ + "manifest.data.element.records.element.valueType", "manifest.navigationContributions.element.items.element.lazy", "manifest.navigationContributions.element.items.element.lazy.options[0]" ] }, "api/PackageInstallResponse": { "sites": [ + "data.package.manifest.data.element.records.element.valueType", "data.package.manifest.navigationContributions.element.items.element.lazy", "data.package.manifest.navigationContributions.element.items.element.lazy.options[0]" ] }, "api/PackageUpgradeRequest": { "sites": [ + "manifest.data.element.records.element.valueType", "manifest.navigationContributions.element.items.element.lazy", "manifest.navigationContributions.element.items.element.lazy.options[0]" ] }, "api/ResolveDependenciesRequest": { "sites": [ + "manifest.data.element.records.element.valueType", "manifest.navigationContributions.element.items.element.lazy", "manifest.navigationContributions.element.items.element.lazy.options[0]" ] @@ -1006,6 +1022,16 @@ "" ] }, + "data/Seed": { + "sites": [ + "records.element.valueType" + ] + }, + "data/SeedLoaderRequest": { + "sites": [ + "seeds.element.records.element.valueType" + ] + }, "data/SetOperator": { "sites": [ "$in", @@ -1065,48 +1091,56 @@ }, "kernel/DisablePackageResponse": { "sites": [ + "package.manifest.data.element.records.element.valueType", "package.manifest.navigationContributions.element.items.element.lazy", "package.manifest.navigationContributions.element.items.element.lazy.options[0]" ] }, "kernel/EnablePackageResponse": { "sites": [ + "package.manifest.data.element.records.element.valueType", "package.manifest.navigationContributions.element.items.element.lazy", "package.manifest.navigationContributions.element.items.element.lazy.options[0]" ] }, "kernel/GetPackageResponse": { "sites": [ + "package.manifest.data.element.records.element.valueType", "package.manifest.navigationContributions.element.items.element.lazy", "package.manifest.navigationContributions.element.items.element.lazy.options[0]" ] }, "kernel/InstallPackageRequest": { "sites": [ + "manifest.data.element.records.element.valueType", "manifest.navigationContributions.element.items.element.lazy", "manifest.navigationContributions.element.items.element.lazy.options[0]" ] }, "kernel/InstallPackageResponse": { "sites": [ + "package.manifest.data.element.records.element.valueType", "package.manifest.navigationContributions.element.items.element.lazy", "package.manifest.navigationContributions.element.items.element.lazy.options[0]" ] }, "kernel/InstalledPackage": { "sites": [ + "manifest.data.element.records.element.valueType", "manifest.navigationContributions.element.items.element.lazy", "manifest.navigationContributions.element.items.element.lazy.options[0]" ] }, "kernel/ListPackagesResponse": { "sites": [ + "packages.element.manifest.data.element.records.element.valueType", "packages.element.manifest.navigationContributions.element.items.element.lazy", "packages.element.manifest.navigationContributions.element.items.element.lazy.options[0]" ] }, "kernel/Manifest": { "sites": [ + "data.element.records.element.valueType", "navigationContributions.element.items.element.lazy", "navigationContributions.element.items.element.lazy.options[0]" ] diff --git a/packages/spec/json-schema.manifest/data.json b/packages/spec/json-schema.manifest/data.json index 6391af09b01..75a86da7b6c 100644 --- a/packages/spec/json-schema.manifest/data.json +++ b/packages/spec/json-schema.manifest/data.json @@ -157,6 +157,9 @@ "data/ScriptValidation", "data/Seed", "data/SeedColumn", + "data/SeedFileContent", + "data/SeedFilePath", + "data/SeedFileReference", "data/SeedIdentity", "data/SeedLoadResult", "data/SeedLoaderConfig", diff --git a/packages/spec/liveness/seed.json b/packages/spec/liveness/seed.json index 705164500c5..4e3d251d586 100644 --- a/packages/spec/liveness/seed.json +++ b/packages/spec/liveness/seed.json @@ -57,6 +57,43 @@ "verifiedAt": "2026-08-28", "evidence": "packages/metadata-protocol/src/seed-loader.ts#loadDataset (`for (let i = 0; i < dataset.records.length; i++)` — the payload rows, written one at a time so a failure can name the row that caused it)", "note": "the payload rows. WALK BOUNDARY: each record is a z.record — the keys an author actually writes are the TARGET OBJECT's field names, governed by that object's own field definitions (and the defineSeed factory's compile-time key check), not by this ledger. Recorded here rather than left implicit, per the datasource `config` precedent. 2026-08-28: RE-ANCHORED (commit 8f10a79f7) and REPOINTED — `:434` had rotted onto the `referenceVia` pointer-pair scan inside `load`, ~384 lines above the row loop. Re-closed by hand against 8cb96ec41." + }, + "files": { + "status": "live", + "verifiedAt": "2026-10-26", + "evidenceScope": "in-repo", + "evidence": "packages/metadata-protocol/src/seed-files.ts#SeedDatasetFiles (`check`: `Object.prototype.hasOwnProperty.call(this.files, path)` — the bundled bytes a `seedFile(path)` reference is stored from, decoded once per dataset and checked against their recorded size and SHA-256); packages/cli/src/utils/bundle-seed-files.ts#bundleSeedFiles (the producer: `os build` writes each referenced file's bytes here, read relative to the package root)", + "note": "Seeded 2026-10-26 (#25779, ruled on hotclm#19 letter D). The bytes of the package files the records name with `seedFile(path)`, keyed by that path; written by `os build`, never by an author. The keys are package-relative paths (the `records` walk boundary's rule applies to the records, not to this map), so the ledger drills only the bundled file's own keys.", + "children": { + "contentType": { + "status": "live", + "verifiedAt": "2026-10-26", + "evidenceScope": "in-repo", + "evidence": "packages/metadata-protocol/src/seed-files.ts#SeedDatasetFiles (`prepareRow`: `reference.contentType ?? content.contentType` — the stored file's media type)", + "note": "The MIME type the stored `sys_file` carries unless the reference names one." + }, + "size": { + "status": "live", + "verifiedAt": "2026-10-26", + "evidenceScope": "in-repo", + "evidence": "packages/metadata-protocol/src/seed-files.ts#SeedDatasetFiles (`check`: `bytes.byteLength !== content.size` — bytes of another length are refused, by row)", + "note": "The byte length the decoded bytes must have." + }, + "sha256": { + "status": "live", + "verifiedAt": "2026-10-26", + "evidenceScope": "in-repo", + "evidence": "packages/metadata-protocol/src/seed-files.ts#SeedDatasetFiles (`check`: `sha256 !== content.sha256` — bytes that hash otherwise are refused, by row); packages/metadata-protocol/src/seed-files.ts#SeedDatasetFiles (`prepareRow`: recorded on the stored file, so a replay keeps an unchanged file)", + "note": "The digest the bytes are checked against, and the provenance a replay keeps a file by." + }, + "data": { + "status": "live", + "verifiedAt": "2026-10-26", + "evidenceScope": "in-repo", + "evidence": "packages/metadata-protocol/src/seed-files.ts#SeedDatasetFiles (`check`: `Buffer.from(content.data, 'base64')` — the bytes the `field-files` service stores)", + "note": "The bytes, base64." + } + } } } } diff --git a/packages/spec/liveness/state-counts/seed.md b/packages/spec/liveness/state-counts/seed.md index 4b437cea0f8..76912db6bb8 100644 --- a/packages/spec/liveness/state-counts/seed.md +++ b/packages/spec/liveness/state-counts/seed.md @@ -12,4 +12,4 @@ committed anywhere: `check:liveness` sums the shards when it reads them. | Type | live | exp | elsewhere | dead | planned | classified | |---|---|---|---|---|---|---| -| `seed` | 14 | 0 | 0 | 0 | 0 | 14 | +| `seed` | 18 | 0 | 0 | 0 | 0 | 18 | diff --git a/packages/spec/src/contracts/storage-service.test.ts b/packages/spec/src/contracts/storage-service.test.ts index fb1be6e4883..0eb7e5818d9 100644 --- a/packages/spec/src/contracts/storage-service.test.ts +++ b/packages/spec/src/contracts/storage-service.test.ts @@ -1,6 +1,8 @@ import { describe, it, expect } from 'vitest'; import type { AttachFileInput, + FieldFileSlot, + IFieldFileService, IFileAttachmentService, IStorageService, StorageFileInfo, @@ -8,6 +10,7 @@ import type { } from './storage-service'; import { DEFAULT_STORAGE_LIST_LIMIT, + FIELD_FILE_SERVICE, FILE_ATTACHMENT_SERVICE, decodeStorageListCursor, encodeStorageListCursor, @@ -399,3 +402,47 @@ describe('File Attachment Service Contract (#23306)', () => { expect(seen[0].organizationId).toBeNull(); }); }); + +describe('Field File Service Contract (#25779)', () => { + it('names its own registry slot, beside the attach verb', () => { + expect(FIELD_FILE_SERVICE).toBe('field-files'); + }); + + it('stores a file no record owns, describes it, and hands it to one slot', async () => { + const files = new Map(); + const service: IFieldFileService = { + storeFile: async (input) => { + files.set('f1', { owner: null, status: 'pending' }); + return { fileId: 'f1', size: input.data.byteLength }; + }, + describeFiles: async (ids) => new Map( + ids.filter((id) => files.has(id)).map((id) => [id, { + fileId: id, + name: 'nda.pdf', + mimeType: 'application/pdf', + status: files.get(id)!.status, + owner: files.get(id)!.owner, + metadata: {}, + }]), + ), + claimFiles: async (slot, ids) => { + for (const id of ids) files.set(id, { owner: slot, status: 'committed' }); + }, + }; + const stored = await service.storeFile({ + organizationId: null, + filename: 'nda.pdf', + mimeType: 'application/pdf', + data: Buffer.from('%PDF'), + metadata: { seedFile: 'fixtures/nda.pdf' }, + }); + expect(stored).toEqual({ fileId: 'f1', size: 4 }); + await service.claimFiles({ object: 'clm_contract_version', recordId: 'r1', field: 'file' }, ['f1'], null); + const described = await service.describeFiles(['f1', 'missing'], null); + expect([...described.keys()]).toEqual(['f1']); + expect(described.get('f1')).toMatchObject({ + status: 'committed', + owner: { object: 'clm_contract_version', recordId: 'r1', field: 'file' }, + }); + }); +}); diff --git a/packages/spec/src/data/seed-file.zod.test.ts b/packages/spec/src/data/seed-file.zod.test.ts new file mode 100644 index 00000000000..a3a5d9a6964 --- /dev/null +++ b/packages/spec/src/data/seed-file.zod.test.ts @@ -0,0 +1,170 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * [#25779] A seed record names a file the package ships with `seedFile(path)`; + * `os build` bundles the bytes into the dataset's `files` map. These pin the + * spec half: the path rule, the reference's closed shape inside a record, the + * bundled bytes' shape, and `defineSeed`'s refusal of a reference in a field + * that does not store a file. + */ + +import { describe, expect, it } from 'vitest'; +import { Field } from './field.zod'; +import { ObjectSchema } from './object.zod'; +import { defineSeed, SeedSchema } from './seed.zod'; +import { + SeedFileContentSchema, + SeedFilePathSchema, + SeedFileReferenceSchema, + isSeedFilePath, + isSeedFileReference, + seedFile, +} from './seed-file.zod'; + +const SHA = 'a'.repeat(64); + +describe('seed file paths', () => { + it.each([ + 'fixtures/nda-v1.pdf', + 'a', + '.hidden/logo.png', + 'dir/file name.pdf', + '.../odd-but-legal', + ])('accepts the package-relative path %j', (path) => { + expect(isSeedFilePath(path)).toBe(true); + expect(SeedFilePathSchema.safeParse(path).success).toBe(true); + }); + + it.each([ + ['', 'empty'], + ['/etc/passwd', 'absolute'], + ['C:/files/a.pdf', 'drive letter'], + ['fixtures\\a.pdf', 'backslash'], + ['../secret.pdf', 'climbs out'], + ['fixtures/../../secret.pdf', 'climbs out midway'], + ['fixtures/./a.pdf', '`.` segment'], + ['fixtures//a.pdf', 'empty segment'], + ['fixtures/', 'trailing slash'], + ])('refuses %j (%s)', (path) => { + expect(isSeedFilePath(path)).toBe(false); + const parsed = SeedFilePathSchema.safeParse(path); + expect(parsed.success).toBe(false); + expect(parsed.error?.issues[0]?.message).toContain('relative to the package root'); + }); +}); + +describe('seedFile()', () => { + it('builds the reference, carrying only the options given', () => { + expect(seedFile('fixtures/nda.pdf')).toEqual({ $file: 'fixtures/nda.pdf' }); + expect(seedFile('fixtures/nda', { name: 'NDA.pdf', contentType: 'application/pdf' })).toEqual({ + $file: 'fixtures/nda', + name: 'NDA.pdf', + contentType: 'application/pdf', + }); + }); + + it('is what isSeedFileReference recognises — an own `$file` key, valid or not', () => { + expect(isSeedFileReference(seedFile('a.pdf'))).toBe(true); + expect(isSeedFileReference({ $file: '../bad' })).toBe(true); + expect(isSeedFileReference({ path: 'a.pdf' })).toBe(false); + expect(isSeedFileReference('a.pdf')).toBe(false); + expect(isSeedFileReference([seedFile('a.pdf')])).toBe(false); + expect(isSeedFileReference(null)).toBe(false); + }); + + it('refuses an unknown key on the reference, pointing at the key it meant', () => { + const parsed = SeedFileReferenceSchema.safeParse({ $file: 'a.pdf', mime: 'application/pdf' }); + expect(parsed.success).toBe(false); + expect(JSON.stringify(parsed.error?.issues)).toContain('contentType'); + }); +}); + +describe('SeedSchema with seed files', () => { + it('accepts references in records, single and multiple, and the bundled bytes', () => { + const parsed = SeedSchema.safeParse({ + object: 'clm_contract_version', + externalId: 'code', + records: [ + { code: 'V1', file: seedFile('fixtures/v1.pdf') }, + { code: 'V2', scans: [seedFile('fixtures/p1.png'), seedFile('fixtures/p2.png')] }, + ], + files: { + 'fixtures/v1.pdf': { contentType: 'application/pdf', size: 3, sha256: SHA, data: 'YWJj' }, + }, + }); + expect(parsed.success).toBe(true); + // The reference reaches the loader as authored. + expect(parsed.data?.records[0].file).toEqual({ $file: 'fixtures/v1.pdf' }); + }); + + it('refuses a malformed reference where it sits in the record', () => { + const parsed = SeedSchema.safeParse({ + object: 'clm_contract_version', + records: [{ code: 'V1', scans: [seedFile('ok.png'), { $file: '../escape.png' }] }], + }); + expect(parsed.success).toBe(false); + expect(parsed.error?.issues[0]?.path).toEqual(['records', 0, 'scans', 1, '$file']); + }); + + it('leaves every other record value alone', () => { + const parsed = SeedSchema.safeParse({ + object: 'account', + records: [{ name: 'Acme', meta: { file: 'not a reference' }, tags: ['a', 'b'] }], + }); + expect(parsed.success).toBe(true); + }); + + it('refuses a bundled file keyed by a path that is not package-relative, or with a bad digest', () => { + expect(SeedSchema.safeParse({ + object: 'a', + records: [], + files: { '/abs.pdf': { contentType: 'application/pdf', size: 1, sha256: SHA, data: 'YQ==' } }, + }).success).toBe(false); + expect(SeedFileContentSchema.safeParse({ + contentType: 'application/pdf', + size: 1, + sha256: 'ABC', + data: 'YQ==', + }).success).toBe(false); + expect(SeedFileContentSchema.safeParse({ + contentType: 'application/pdf', + size: 1, + sha256: SHA, + data: 'YQ==', + extra: true, + }).success).toBe(false); + }); +}); + +describe('defineSeed with seed files', () => { + const Version = ObjectSchema.create({ + name: 'clm_contract_version', + fields: { + code: Field.text({ label: 'Code' }), + file: Field.file({ label: 'File', required: true }), + notes: Field.textarea({ label: 'Notes' }), + }, + }); + + it('accepts seedFile() in a file field', () => { + const seed = defineSeed(Version, { + externalId: 'code', + records: [{ code: 'V1', file: seedFile('fixtures/v1.pdf') }], + }); + expect(seed.records[0].file).toEqual({ $file: 'fixtures/v1.pdf' }); + }); + + it('refuses seedFile() in a field that does not store a file, naming the record and the field', () => { + let message = ''; + try { + defineSeed(Version, { + externalId: 'code', + records: [{ code: 'V1', file: seedFile('a.pdf'), notes: seedFile('fixtures/notes.txt') }], + }); + } catch (error) { + message = (error as Error).message; + } + expect(message).toContain("defineSeed('clm_contract_version'): seedFile(...) written into a field that does not store a file."); + expect(message).toContain('records[0]: `notes` is a `textarea` field'); + }); +}); diff --git a/packages/spec/src/data/seed-file.zod.ts b/packages/spec/src/data/seed-file.zod.ts index 468dd2f7ee2..b197c19ac3b 100644 --- a/packages/spec/src/data/seed-file.zod.ts +++ b/packages/spec/src/data/seed-file.zod.ts @@ -43,20 +43,24 @@ const SEED_FILE_PATH_RULE = + 'segment (e.g. `fixtures/nda-v1.pdf`)'; /** - * Is `path` a package-relative seed file path? Relative (no leading `/`, no - * drive letter), `/`-separated (no `\`), and confined to the package: no - * empty, `.` or `..` segment, so the path cannot climb out of the root it is - * resolved against and two spellings of one file cannot key two entries. + * A package-relative seed file path: relative (no leading `/`, no drive + * letter), `/`-separated (no `\`), and confined to the package — every segment + * non-empty and neither `.` nor `..`, so the path cannot climb out of the root + * it is resolved against and two spellings of one file cannot key two entries. + * + * One regular expression rather than a predicate, so the published JSON Schema + * states the rule as the `pattern` the runtime applies. */ +const SEED_FILE_PATH_PATTERN = /^(?![A-Za-z]:)(?!\.{1,2}(?:\/|$))[^/\\]+(?:\/(?!\.{1,2}(?:\/|$))[^/\\]+)*$/; + +/** Is `path` a package-relative seed file path ({@link SeedFilePathSchema})? */ export function isSeedFilePath(path: unknown): path is string { - if (typeof path !== 'string' || path.length === 0) return false; - if (path.startsWith('/') || path.includes('\\') || /^[A-Za-z]:/.test(path)) return false; - return path.split('/').every((segment) => segment !== '' && segment !== '.' && segment !== '..'); + return typeof path === 'string' && SEED_FILE_PATH_PATTERN.test(path); } -/** A package-relative seed file path ({@link isSeedFilePath}). */ +/** A package-relative seed file path, e.g. `fixtures/nda-v1.pdf`. */ export const SeedFilePathSchema = lazySchema(() => z.string() - .refine(isSeedFilePath, { message: SEED_FILE_PATH_RULE }) + .regex(SEED_FILE_PATH_PATTERN, SEED_FILE_PATH_RULE) .describe('Path of a file the package ships, relative to the package root (e.g. `fixtures/nda-v1.pdf`)')); /** A MIME type: `type/subtype`, the shape a `sys_file.mime_type` carries. */ From 8ce259f5479bed3704e6e9cd286477856592423b Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 26 Oct 2026 13:59:55 +0000 Subject: [PATCH 04/13] fix(cli,storage,loader): os validate runs the seed-file gate; refused rows store nothing Co-Authored-By: Claude Claude-Session: https://claude.ai/code/session_012eJQ8dtut5bZsnhFE3XGHF --- packages/cli/src/commands/compile.ts | 16 ++- packages/cli/src/commands/validate.ts | 23 ++++ .../cli/src/utils/bundle-seed-files.test.ts | 63 ++++++--- packages/cli/src/utils/bundle-seed-files.ts | 79 +++++++---- .../build-seed-files-artifact.e2e.test.ts | 130 ++++++++++++++++++ .../cli/test/normalized-call-sites.test.ts | 4 + .../test/validate-build-gate-parity.test.ts | 11 +- packages/metadata-protocol/src/seed-files.ts | 14 +- .../src/field-file-service.test.ts | 19 ++- .../service-storage/src/field-file-service.ts | 2 + 10 files changed, 299 insertions(+), 62 deletions(-) create mode 100644 packages/cli/test/build-seed-files-artifact.e2e.test.ts diff --git a/packages/cli/src/commands/compile.ts b/packages/cli/src/commands/compile.ts index 71fae50d8ca..4b915f87768 100644 --- a/packages/cli/src/commands/compile.ts +++ b/packages/cli/src/commands/compile.ts @@ -15,7 +15,7 @@ import { type ConversionNotice, } from '@objectstack/spec'; import { loadConfig, namedExportRejectionHints } from '../utils/config.js'; -import { bundleSeedFiles } from '../utils/bundle-seed-files.js'; +import { collectSeedFiles, placeSeedFiles } from '../utils/bundle-seed-files.js'; import { refuseUnbuiltStack } from '../utils/stack-provenance-refusal.js'; import { isExtractionRefusal, lowerCallables } from '../utils/lower-callables.js'; import { authoringRuleUnionStack } from '../utils/stack-collections.js'; @@ -1036,12 +1036,13 @@ export default class Compile extends Command { finalBundle.packages = placed.packages; } - // 4a. [#25779] Bundle the files seed records name with `seedFile(path)` - // into their datasets' `files` maps, read relative to the directory - // holding the config. The artifact is what every boot loads, so the - // bytes travel with the seed; a path that names no file fails the - // build here rather than a row at boot. - const seedFiles = bundleSeedFiles(finalBundle, path.dirname(absolutePath)); + // 4a. [#25779] The files seed records name with `seedFile(path)`, read + // relative to the directory holding the config — the gate + // `os validate` runs too — then written into their datasets' + // `files` maps. The artifact is what every boot loads, so the bytes + // travel with the seed; a path that names no file fails the build + // here rather than a row at boot. + const seedFiles = collectSeedFiles(absolutePath, finalBundle); if (seedFiles.issues.length > 0) { if (flags.json) { await emitJson({ success: false, error: 'seed file bundling failed', issues: seedFiles.issues, warnings: warningsSoFar(), conversions: conversionNotices }, 0, { compact: true }); @@ -1052,6 +1053,7 @@ export default class Compile extends Command { for (const issue of seedFiles.issues) console.log(chalk.red(` ✗ ${issue.location}: ${issue.message}`)); this.exit(1); } + placeSeedFiles(seedFiles); if (seedFiles.files > 0 && !flags.json) { printStep(`Bundling seed files... ${seedFiles.files} file${seedFiles.files === 1 ? '' : 's'} (${(seedFiles.bytes / 1024).toFixed(1)} KB)`); } diff --git a/packages/cli/src/commands/validate.ts b/packages/cli/src/commands/validate.ts index fbf91411e09..48d68475519 100644 --- a/packages/cli/src/commands/validate.ts +++ b/packages/cli/src/commands/validate.ts @@ -25,6 +25,7 @@ import { runAuthoringRules, splitBySeverity, authoringRulesFor } from '@objectst import { resolveJsxGateManifest, printJsxGateNotices } from '../utils/sdui-manifest.js'; import { preflightDeclaredCapabilities, renderCapabilityMessage } from '../utils/capability-preflight.js'; import { collectAndLintDocs, type DocIssue } from '../utils/collect-docs.js'; +import { collectSeedFiles } from '../utils/bundle-seed-files.js'; import { printHeader, printKV, @@ -768,6 +769,28 @@ export default class Validate extends Command { this.exit(1); } + // 3d. [#25779] Seed files — each `seedFile(path)` a seed record names + // must name a file under the directory holding the config, of a type + // the extension or the reference tells. `os build` refuses the same + // paths before it bundles the bytes into the artifact. + const seedFiles = collectSeedFiles(absolutePath, result.data as Record); + if (seedFiles.issues.length > 0) { + if (flags.json) { + await emitJson({ + valid: false, + errors: seedFiles.issues, + warnings: warningsSoFar(), + conversions: conversionNotices, + duration: timer.elapsed(), + }); + this.exit(1); + } + console.log(''); + printError(`Seed file validation failed (${seedFiles.issues.length} issue${seedFiles.issues.length > 1 ? 's' : ''})`); + for (const issue of seedFiles.issues) console.log(chalk.red(` ✗ ${issue.location}: ${issue.message}`)); + this.exit(1); + } + // 4. Collect and display stats const stats = collectMetadataStats(config); diff --git a/packages/cli/src/utils/bundle-seed-files.test.ts b/packages/cli/src/utils/bundle-seed-files.test.ts index 815e62ed906..dbe1d9f30bf 100644 --- a/packages/cli/src/utils/bundle-seed-files.test.ts +++ b/packages/cli/src/utils/bundle-seed-files.test.ts @@ -6,14 +6,15 @@ import os from 'node:os'; import path from 'node:path'; import { afterEach, describe, expect, it } from 'vitest'; import { SeedSchema, seedFile } from '@objectstack/spec/data'; -import { bundleSeedFiles } from './bundle-seed-files.js'; +import { collectSeedFiles, placeSeedFiles } from './bundle-seed-files.js'; /** - * [#25779] `os build` reads each file a seed record names with - * `seedFile(path)`, relative to the directory holding the config, and writes - * its bytes into the dataset's `files` map — the shape the seed loader checks - * them against. A path that names no file, climbs out of the root, or has a - * type nobody can tell fails the build. + * [#25779] `os validate` and `os build` read each file a seed record names + * with `seedFile(path)`, relative to the directory holding the config + * (`collectSeedFiles`); `os build` then writes its bytes into the dataset's + * `files` map (`placeSeedFiles`) — the shape the seed loader checks them + * against. A path that names no file, climbs out of the root, or has a type + * nobody can tell is refused. */ const roots: string[] = []; @@ -21,6 +22,7 @@ afterEach(() => { for (const root of roots.splice(0)) fs.rmSync(root, { recursive: true, force: true }); }); +/** A package root holding `files`; answers the path of its (unread) config. */ function packageRoot(files: Record): string { const root = fs.mkdtempSync(path.join(os.tmpdir(), 'os-25779-')); roots.push(root); @@ -28,12 +30,19 @@ function packageRoot(files: Record): string { fs.mkdirSync(path.dirname(path.join(root, rel)), { recursive: true }); fs.writeFileSync(path.join(root, rel), content); } - return root; + return path.join(root, 'objectstack.config.ts'); +} + +/** Both halves, as `os build` runs them. */ +function bundle(stack: Record, config: string) { + const collected = collectSeedFiles(config, stack); + if (collected.issues.length === 0) placeSeedFiles(collected); + return collected; } const sha = (text: string) => createHash('sha256').update(Buffer.from(text)).digest('hex'); -describe('bundleSeedFiles', () => { +describe('collectSeedFiles / placeSeedFiles', () => { it('writes each referenced file into its dataset, once per path, in the shape SeedSchema accepts', () => { const root = packageRoot({ 'fixtures/nda.pdf': '%PDF nda', 'scans/p1.png': 'png' }); const dataset = { @@ -43,11 +52,10 @@ describe('bundleSeedFiles', () => { { code: 'V2', file: seedFile('fixtures/nda.pdf') }, ], }; - const bundle: Record = { data: [dataset] }; - - const result = bundleSeedFiles(bundle, root); + const result = bundle({ data: [dataset] }, root); expect(result.issues).toEqual([]); expect(result.files).toBe(2); + expect(result.bytes).toBe(11); expect((dataset as any).files).toEqual({ 'fixtures/nda.pdf': { contentType: 'application/pdf', @@ -63,21 +71,21 @@ describe('bundleSeedFiles', () => { it('reaches the datasets of every package body too', () => { const root = packageRoot({ 'a.txt': 'hello' }); const inner = { object: 'x', records: [{ f: seedFile('a.txt') }] }; - bundleSeedFiles({ packages: [{ manifest: { data: [inner] } }] }, root); + bundle({ packages: [{ manifest: { data: [inner] } }] }, root); expect((inner as any).files['a.txt'].contentType).toBe('text/plain'); }); it('leaves a dataset that names no seed file exactly as it was', () => { const dataset = { object: 'x', records: [{ name: 'a' }] }; - const result = bundleSeedFiles({ data: [dataset] }, packageRoot({})); - expect(result).toEqual({ files: 0, bytes: 0, issues: [] }); + const result = bundle({ data: [dataset] }, packageRoot({})); + expect(result).toEqual({ datasets: new Map(), files: 0, bytes: 0, issues: [] }); expect(dataset).toEqual({ object: 'x', records: [{ name: 'a' }] }); }); it('refuses a path that names no file, saying where the reference sits', () => { - const result = bundleSeedFiles( - { data: [{ object: 'x', records: [{ code: 'a' }, { file: seedFile('fixtures/missing.pdf') }] }] }, + const result = collectSeedFiles( packageRoot({}), + { data: [{ object: 'x', records: [{ code: 'a' }, { file: seedFile('fixtures/missing.pdf') }] }] }, ); expect(result.issues).toHaveLength(1); expect(result.issues[0].location).toBe('data[0].records[1].file'); @@ -86,17 +94,34 @@ describe('bundleSeedFiles', () => { it('refuses a type it cannot tell, unless the reference names one', () => { const root = packageRoot({ 'blob.bin': 'xx' }); - const refused = bundleSeedFiles({ data: [{ object: 'x', records: [{ f: seedFile('blob.bin') }] }] }, root); + const refused = collectSeedFiles(root, { data: [{ object: 'x', records: [{ f: seedFile('blob.bin') }] }] }); expect(refused.issues[0].message).toContain("the file's type cannot be told from its extension"); const named = { object: 'x', records: [{ f: seedFile('blob.bin', { contentType: 'application/octet-stream' }) }] }; - expect(bundleSeedFiles({ data: [named] }, root).issues).toEqual([]); + expect(bundle({ data: [named] }, root).issues).toEqual([]); expect((named as any).files['blob.bin'].contentType).toBe('application/octet-stream'); }); + it('a type the reference names wins over the one its extension suggests', () => { + const root = packageRoot({ 'scan.pdf': 'x' }); + const dataset = { object: 'x', records: [{ f: seedFile('scan.pdf', { contentType: 'application/x-pdf' }) }] }; + bundle({ data: [dataset] }, root); + expect((dataset as any).files['scan.pdf'].contentType).toBe('application/x-pdf'); + }); + + it('the gate writes nothing: the stack it judged is unchanged until placeSeedFiles', () => { + const root = packageRoot({ 'a.txt': 'hello' }); + const dataset = { object: 'x', records: [{ f: seedFile('a.txt') }] }; + const collected = collectSeedFiles(root, { data: [dataset] }); + expect(collected.files).toBe(1); + expect('files' in dataset).toBe(false); + placeSeedFiles(collected); + expect((dataset as any).files['a.txt'].size).toBe(5); + }); + it('never reads outside the package root', () => { const root = packageRoot({}); - const result = bundleSeedFiles({ data: [{ object: 'x', records: [{ f: { $file: '../outside.pdf' } }] }] }, root); + const result = collectSeedFiles(root, { data: [{ object: 'x', records: [{ f: { $file: '../outside.pdf' } }] }] }); expect(result.issues[0].message).toContain('climbs out of the package root'); }); }); diff --git a/packages/cli/src/utils/bundle-seed-files.ts b/packages/cli/src/utils/bundle-seed-files.ts index 417c5db7d34..d12318b39bb 100644 --- a/packages/cli/src/utils/bundle-seed-files.ts +++ b/packages/cli/src/utils/bundle-seed-files.ts @@ -6,17 +6,23 @@ import path from 'node:path'; import { isSeedFileReference, type SeedFileContent, type SeedFileReference } from '@objectstack/spec/data'; /** - * [#25779] `os build` / `os compile`'s half of a seed file: read each file a - * seed record names with `seedFile(path)` and write its bytes into the - * dataset's `files` map, so the compiled artifact carries them to every boot - * that loads it — `os dev`, `os serve`, a published package, an artifact URL. + * [#25779] The CLI's half of a seed file — the files seed records name with + * `seedFile(path)`, read off disk and carried in the artifact, so every boot + * that loads it has the bytes: `os dev`, `os serve`, a published package, an + * artifact URL. * - * The path is relative to the package root, the directory holding - * `objectstack.config.ts` (the root a declarative connector's file ref resolves - * against too), and confined to it. A path that names no file there, or a file - * whose type cannot be told from its extension and that no reference names a - * `contentType` for, fails the build: the seed would otherwise load without the - * file its field requires. + * Two calls, split the way the package docs are (`collectAndLintDocs` / + * `placeCollectedDocs`): + * + * - {@link collectSeedFiles} — the GATE, run by `os validate` and `os build` + * alike. It reads each file, relative to the directory holding the config + * (the root a declarative connector's file ref resolves against too) and + * confined to it, and refuses a path that names no file, or a file whose + * type cannot be told from its extension and that no reference names a + * `contentType` for: the seed would otherwise load without the file its + * field requires. It writes nothing. + * - {@link placeSeedFiles} — `os build`'s ASSEMBLY: each dataset's bytes into + * its `files` map. It judges nothing. */ /** A refusal, located in the artifact like a schema issue is. */ @@ -26,8 +32,10 @@ export interface SeedFileBundleIssue { message: string; } -export interface SeedFileBundleResult { - /** Files bundled, counted once per dataset that carries them. */ +export interface CollectedSeedFiles { + /** Each dataset that names a seed file, with the `files` map it is to carry. */ + datasets: Map>; + /** Files read, counted once per dataset that carries them. */ files: number; /** Their bytes, before base64. */ bytes: number; @@ -77,8 +85,13 @@ interface DatasetLike { files?: Record; } -/** Every seed dataset in the artifact — the top level's and each package body's — with where it sits. */ -function datasetsOf(bundle: Record): Array<{ dataset: DatasetLike; location: string }> { +/** + * Every seed dataset in the stack, with where it sits: the top level's + * `data`, the manifest's, and each `packages[i].manifest` body's. Read by + * position, off the objects themselves, because {@link placeSeedFiles} writes + * into these very datasets — the ones the artifact serializes. + */ +function datasetsOf(stack: Record): Array<{ dataset: DatasetLike; location: string }> { const found: Array<{ dataset: DatasetLike; location: string }> = []; const seen = new Set(); const take = (list: unknown, location: string): void => { @@ -89,11 +102,11 @@ function datasetsOf(bundle: Record): Array<{ dataset: DatasetLi found.push({ dataset: dataset as DatasetLike, location: `${location}[${i}]` }); }); }; - take(bundle.data, 'data'); - const manifest = bundle.manifest as { data?: unknown } | undefined; + take(stack.data, 'data'); + const manifest = stack.manifest as { data?: unknown } | undefined; if (manifest && typeof manifest === 'object') take(manifest.data, 'manifest.data'); - if (Array.isArray(bundle.packages)) { - bundle.packages.forEach((pkg, i) => { + if (Array.isArray(stack.packages)) { + stack.packages.forEach((pkg, i) => { const body = (pkg as { manifest?: { data?: unknown } } | null)?.manifest; if (body && typeof body === 'object') take(body.data, `packages[${i}].manifest.data`); }); @@ -125,15 +138,16 @@ function referencesOf( } /** - * Write the bytes of every seed file the artifact's datasets name into their - * `files` maps, reading each path under `packageRoot`. Changes `bundle` in - * place; answers what it bundled and what it refused. + * Read every seed file the stack's datasets name — the top level's, the + * manifest's and each package body's, where each dataset sits — and judge it. + * `configPath` is the config's path; seed file paths resolve against its + * directory. Reads only: the stack is not changed. */ -export function bundleSeedFiles(bundle: Record, packageRoot: string): SeedFileBundleResult { - const result: SeedFileBundleResult = { files: 0, bytes: 0, issues: [] }; - const root = path.resolve(packageRoot); +export function collectSeedFiles(configPath: string, stack: Record): CollectedSeedFiles { + const result: CollectedSeedFiles = { datasets: new Map(), files: 0, bytes: 0, issues: [] }; + const root = path.dirname(path.resolve(configPath)); const read = new Map(); - for (const { dataset, location } of datasetsOf(bundle)) { + for (const { dataset, location } of datasetsOf(stack)) { const byPath = referencesOf(dataset, location); if (byPath.size === 0) continue; const files: Record = { ...(dataset.files ?? {}) }; @@ -167,8 +181,9 @@ export function bundleSeedFiles(bundle: Record, packageRoot: st }); continue; } + // A type the author names wins over the one the extension suggests. const declared = references.map((r) => r.reference.contentType).find((t) => typeof t === 'string'); - const contentType = SEED_FILE_TYPES[path.extname(seedPath).toLowerCase()] ?? declared; + const contentType = declared ?? SEED_FILE_TYPES[path.extname(seedPath).toLowerCase()]; if (!contentType) { result.issues.push({ location: first, @@ -187,7 +202,17 @@ export function bundleSeedFiles(bundle: Record, packageRoot: st result.files += 1; result.bytes += bytes.byteLength; } - dataset.files = files; + result.datasets.set(dataset, files); } return result; } + +/** + * Write what {@link collectSeedFiles} read into the datasets it read it for: + * each one's `files` map. Run after the gate passed, on the stack it read. + */ +export function placeSeedFiles(collected: CollectedSeedFiles): void { + for (const [dataset, files] of collected.datasets) { + (dataset as DatasetLike).files = files; + } +} diff --git a/packages/cli/test/build-seed-files-artifact.e2e.test.ts b/packages/cli/test/build-seed-files-artifact.e2e.test.ts new file mode 100644 index 00000000000..1ae2d104720 --- /dev/null +++ b/packages/cli/test/build-seed-files-artifact.e2e.test.ts @@ -0,0 +1,130 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * #25779 — `os build` carries the files seed records name with + * `seedFile(path)` in the artifact, and `os validate` refuses the path the + * build would refuse. + * + * Measured on the files the command actually puts on disk, and on what a + * deployment LOADS (`loadArtifactBundle`, the runtime's own loader): the + * dataset keeps the reference as authored and carries the file's bytes, size + * and SHA-256 under `files`, keyed by the path. The control for the refusal is + * the same stack with the file present, which both commands accept. + * + * Runs the CLI from source (`bin/run-dev.js` through tsx), as the other build + * suites do, so `compile.ts` and `validate.ts` are measured without a rebuild. + */ + +import { describe, it, expect, beforeAll, afterAll } from 'vitest'; +import { createHash } from 'node:crypto'; +import { execFile } from 'node:child_process'; +import { mkdirSync, mkdtempSync, rmSync, readFileSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { loadArtifactBundle } from '@objectstack/runtime'; +import { childEnv } from './helpers/serve-process.js'; +import { linkSpec } from './helpers/define-stack-fixture.js'; + +const HERE = resolve(fileURLToPath(import.meta.url), '..'); +const CLI = resolve(HERE, '../bin/run-dev.js'); +const TSX = resolve(HERE, '../../../node_modules/.bin/tsx'); + +const PDF = '%PDF-1.7 a seeded contract'; + +const config = (path: string) => `import { defineStack } from '@objectstack/spec'; +import { seedFile } from '@objectstack/spec/data'; + +export default defineStack({ + manifest: { id: 'com.example.clm', name: 'clm', version: '1.0.0', type: 'app', namespace: 'clm' }, + objects: [ + { + name: 'clm_version', + label: 'Version', + sharingModel: 'private', + fields: { + code: { type: 'text', label: 'Code' }, + file: { type: 'file', label: 'File', required: true }, + }, + }, + ], + data: [ + { object: 'clm_version', externalId: 'code', records: [{ code: 'V1', file: seedFile('${path}') }] }, + ], +}); +`; + +let dir = ''; +let missing = ''; + +beforeAll(() => { + dir = mkdtempSync(join(tmpdir(), 'os-seed-files-build-')); + mkdirSync(join(dir, 'fixtures')); + writeFileSync(join(dir, 'fixtures', 'nda.pdf'), PDF); + writeFileSync(join(dir, 'objectstack.config.ts'), config('fixtures/nda.pdf')); + linkSpec(dir); + + missing = mkdtempSync(join(tmpdir(), 'os-seed-files-missing-')); + writeFileSync(join(missing, 'objectstack.config.ts'), config('fixtures/absent.pdf')); + linkSpec(missing); +}); + +afterAll(() => { + for (const d of [dir, missing]) if (d) rmSync(d, { recursive: true, force: true }); +}); + +function run(command: 'build' | 'validate', cwd: string): Promise<{ code: number; stdout: string; stderr: string }> { + return new Promise((done) => { + execFile( + TSX, + [CLI, command, '--json'], + { cwd, maxBuffer: 16 * 1024 * 1024, env: childEnv({ NO_COLOR: '1' }) }, + (err, stdout, stderr) => + done({ + code: err ? (typeof (err as { code?: unknown }).code === 'number' ? (err as unknown as { code: number }).code : 1) : 0, + stdout: String(stdout), + stderr: String(stderr), + }), + ); + }); +} + +describe('seed files ride the artifact (#25779)', () => { + it('`os build` writes the bytes beside the reference, and a deployment that loads the artifact reads them', async () => { + const built = await run('build', dir); + expect(built.code, `os build --json failed:\n${built.stdout}\n${built.stderr}`).toBe(0); + + const artifactPath = String((JSON.parse(built.stdout) as { output?: string }).output); + const expected = { + contentType: 'application/pdf', + size: Buffer.byteLength(PDF), + sha256: createHash('sha256').update(PDF).digest('hex'), + data: Buffer.from(PDF).toString('base64'), + }; + for (const bundle of [ + JSON.parse(readFileSync(artifactPath, 'utf8')) as Record, + (await loadArtifactBundle(artifactPath)) as Record, + ]) { + const dataset = bundle.data[0]; + expect(dataset.records[0].file).toEqual({ $file: 'fixtures/nda.pdf' }); + expect(dataset.files).toEqual({ 'fixtures/nda.pdf': expected }); + } + + const validated = await run('validate', dir); + expect(validated.code, `os validate --json failed:\n${validated.stdout}\n${validated.stderr}`).toBe(0); + }, 180_000); + + it('both commands refuse a path that names no file, saying where the reference sits', async () => { + for (const command of ['build', 'validate'] as const) { + const refused = await run(command, missing); + expect(refused.code, `${command}:\n${refused.stdout}\n${refused.stderr}`).toBe(1); + const payload = JSON.parse(refused.stdout) as { issues?: unknown[]; errors?: unknown[] }; + expect(payload.issues ?? payload.errors).toEqual([ + expect.objectContaining({ + location: 'data[0].records[0].file', + message: expect.stringContaining("seedFile('fixtures/absent.pdf') names no file"), + }), + ]); + } + }, 180_000); +}); diff --git a/packages/cli/test/normalized-call-sites.test.ts b/packages/cli/test/normalized-call-sites.test.ts index f6559533902..bd6c2f36e71 100644 --- a/packages/cli/test/normalized-call-sites.test.ts +++ b/packages/cli/test/normalized-call-sites.test.ts @@ -414,6 +414,10 @@ const READS: Readonly> = { ], why: 'the doors hand it the parsed union, and `os lint` the lowered copy of its folded stack. `scaffold-validate.ts` hands it one scaffold\'s own stack, which has no `packages[]`.', }, + 'utils/bundle-seed-files.ts :: stack.data': { + reads: 'top-level', + why: 'the top level\'s own datasets; the same walk reads `manifest.data` and each `packages[i].manifest` body\'s `data` too, by position, because `placeSeedFiles` writes the bytes into those very dataset objects (#25779). Pinned on a package body in `bundle-seed-files.test.ts`.', + }, 'utils/collect-docs.ts :: stack.docs': { reads: 'top-level', why: 'the artifact\'s own top-level `docs`; the same function reads each package\'s docs off its body (`bodyDocsOf(stack.packages, …)`) and lints them against that package.', diff --git a/packages/cli/test/validate-build-gate-parity.test.ts b/packages/cli/test/validate-build-gate-parity.test.ts index 1798467dbe0..75fa29938f9 100644 --- a/packages/cli/test/validate-build-gate-parity.test.ts +++ b/packages/cli/test/validate-build-gate-parity.test.ts @@ -158,6 +158,11 @@ const SHARED_NON_REGISTRY_GATES: readonly string[] = [ // before its named-export merge), which no rule is ever handed. `os lint` is // not an author-time door under the ruling and does not run it. 'refuseUnbuiltStack', + // [#25779] Seed files: each `seedFile(path)` a seed record names must name a + // file under the config's directory, of a type its extension or the + // reference tells. Not a registry rule: it reads the filesystem. `os build` + // then writes the bytes it read into the artifact (`placeSeedFiles`, below). + 'collectSeedFiles', ]; /** @@ -284,8 +289,12 @@ const NOT_A_GATE: Readonly> = { // classification, for the same reasons — it raises nothing, and it hands // back the `docs` array and `packages` value it was given when it has // nothing to move. + // [#25779] `placeSeedFiles` writes the seed file bytes `collectSeedFiles` — + // a gate in SHARED_NON_REGISTRY_GATES above — already read and judged into + // their datasets' `files` maps. It refuses nothing and reports nothing; with + // no artifact, `os validate` has nowhere to put them. 'Artifact assembly — puts already-collected, already-linted content into the bundle `os build` writes; judges nothing': - ['placeCollectedDocs'], + ['placeCollectedDocs', 'placeSeedFiles'], 'Presentation — renders, formats or serialises a verdict something else reached; judges nothing': [ 'printHeader', diff --git a/packages/metadata-protocol/src/seed-files.ts b/packages/metadata-protocol/src/seed-files.ts index 56b5cbbded3..65dafb7f2e6 100644 --- a/packages/metadata-protocol/src/seed-files.ts +++ b/packages/metadata-protocol/src/seed-files.ts @@ -176,7 +176,13 @@ export class SeedDatasetFiles { willWrite: boolean; dryRun: boolean; }): Promise { - const claims: Array<{ field: string; fileIds: string[] }> = []; + // Every field is judged before any file is stored, so a refused row stores + // nothing. + const planned: Array<{ + field: string; + def: FileField; + contents: Array<{ reference: SeedFileReference; bytes: Buffer; content: SeedFileContent }>; + }> = []; for (const [field, def] of this.fields) { const value = args.record[field]; if (value === undefined || value === null) continue; @@ -240,7 +246,11 @@ export class SeedDatasetFiles { 'Load this seed through a server that composes the storage service.', ); } + planned.push({ field, def, contents }); + } + const claims: Array<{ field: string; fileIds: string[] }> = []; + for (const { field, def, contents } of planned) { // A file the matched row already holds, stored from the same path with // the same bytes, name and type, is kept; each held file serves once. const held = args.existing ? heldIdsOf(args.existing[field]) : []; @@ -267,7 +277,7 @@ export class SeedDatasetFiles { continue; } try { - const stored = await this.service.storeFile({ + const stored = await this.service!.storeFile({ organizationId: args.organizationId, filename: name, mimeType, diff --git a/packages/services/service-storage/src/field-file-service.test.ts b/packages/services/service-storage/src/field-file-service.test.ts index ed48a09e6a2..2a715e23f61 100644 --- a/packages/services/service-storage/src/field-file-service.test.ts +++ b/packages/services/service-storage/src/field-file-service.test.ts @@ -81,14 +81,21 @@ afterEach(async () => { } }); -/** A real engine with `sys_file` and `ff_doc`, whose `file` field holds one file and `scans` several. */ -async function engine(): Promise { +/** + * A real engine with `sys_file` and `ff_doc`, whose `file` field holds one + * file and `scans` several, under `posture`. Which organization's files a read + * reaches is the driver's tenant term — the wall the posture puts up, never a + * comparison the verb writes — so a block that pins the organization boundary + * boots `isolated`; under `single` there is one tenant and no wall. + */ +async function engine(posture: 'single' | 'isolated'): Promise { const ql = new ObjectQL({ logger: quietLogger() } as any); ql.registerDriver( new SqlDriver({ client: 'better-sqlite3', connection: { filename: ':memory:' }, useNullAsDefault: true }) as any, true, ); await ql.init(); + ql.setTenancyPostureProvider(() => posture); engines.push(ql); ql.registry.registerObject(SystemFile as any, 'test-25779'); ql.registry.registerObject( @@ -103,8 +110,8 @@ async function engine(): Promise { } /** Boot the plugin over a fresh engine; `ready: false` stops after `start()`, as a seed in `start()` sees it. */ -async function boot(opts: { ready?: boolean } = {}) { - const ql = await engine(); +async function boot(opts: { ready?: boolean; posture?: 'single' | 'isolated' } = {}) { + const ql = await engine(opts.posture ?? 'single'); const rootDir = await fs.mkdtemp(join(tmpdir(), 'oss-25779-')); const plugin = new StorageServicePlugin({ adapter: 'local', local: { rootDir }, registerRoutes: false }); const { ctx, run } = makeCtx(); @@ -192,7 +199,7 @@ describe('FIELD_FILE_SERVICE: the server-side field-file verb (#25779)', () => { }); it('describes the files that exist in the organization, and no other', async () => { - const { files } = await boot(); + const { files } = await boot({ posture: 'isolated' }); const a = await files.storeFile(input({ organizationId: 'org_a' })); const b = await files.storeFile(input({ organizationId: 'org_b' })); const inA = await files.describeFiles([a.fileId, b.fileId, 'missing'], 'org_a'); @@ -229,7 +236,7 @@ describe('FIELD_FILE_SERVICE: the server-side field-file verb (#25779)', () => { }); it('never takes a file another slot owns, and refuses a file that does not exist', async () => { - const { files } = await boot(); + const { files } = await boot({ posture: 'isolated' }); const owned = await files.storeFile(input()); await files.claimFiles(slot('doc_1'), [owned.fileId], 'org_a'); expect(await refusalOf(files.claimFiles(slot('doc_2'), [owned.fileId], 'org_a'))).toMatchObject({ diff --git a/packages/services/service-storage/src/field-file-service.ts b/packages/services/service-storage/src/field-file-service.ts index 71dc1a6ed7e..c3940884a4b 100644 --- a/packages/services/service-storage/src/field-file-service.ts +++ b/packages/services/service-storage/src/field-file-service.ts @@ -242,6 +242,8 @@ export class FieldFileService implements IFieldFileService { const ids = [...new Set(fileIds.filter(nonEmptyString))]; const found = new Map(); if (ids.length === 0) return found; + // Which organization's files the read reaches is the driver's tenant term, + // the lifecycle's own lookup channel. ⛔ No organization comparison here. const rows = (await engine.find('sys_file', { where: { id: { $in: ids } }, From b62f351a6ed175b3f65f1645f17b2843b5623884 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 26 Oct 2026 14:09:07 +0000 Subject: [PATCH 05/13] chore(spec): regenerate references, api-surface, export origins; strictness-ledger row for seed-file.zod.ts Co-Authored-By: Claude Claude-Session: https://claude.ai/code/session_012eJQ8dtut5bZsnhFE3XGHF --- content/docs/references/api/package-api.mdx | 2 + content/docs/references/data/index.mdx | 3 +- content/docs/references/data/meta.json | 3 +- content/docs/references/data/seed-file.mdx | 94 +++++++++++++++++++ content/docs/references/data/seed-loader.mdx | 1 + content/docs/references/data/seed.mdx | 12 +++ content/docs/references/index.mdx | 9 +- content/docs/references/kernel/manifest.mdx | 1 + .../data.md | 9 +- .../2026-07-unknown-key-strictness-ledger.md | 1 + packages/spec/api-surface/contracts.json | 6 ++ packages/spec/api-surface/data.json | 9 ++ packages/spec/declaration-map/data.json | 5 + packages/spec/export-origins/contracts.json | 6 ++ packages/spec/export-origins/data.json | 9 ++ packages/spec/src/data/seed-file.zod.ts | 1 + skills/objectstack-data/references/_index.md | 1 + 17 files changed, 162 insertions(+), 10 deletions(-) create mode 100644 content/docs/references/data/seed-file.mdx diff --git a/content/docs/references/api/package-api.mdx b/content/docs/references/api/package-api.mdx index 903d29313c1..a935b3487b8 100644 --- a/content/docs/references/api/package-api.mdx +++ b/content/docs/references/api/package-api.mdx @@ -250,6 +250,7 @@ Package manifest installed through POST /api/v1/packages (AUTHORING stage; scope | **env** | `Enum<'prod' \| 'dev' \| 'test'>[]` | optional (default: `["prod","dev","test"]`) | Applicable environments | | **locale** | `string[]` | optional | Applicable locales (BCP-47 tags); omitted applies to every locale. The publish and install paths do not filter by locale — they load every dataset and warn | | **records** | `Record[]` | ✅ | Data records | +| **files** | `Record` | optional | Bytes of the package files the records name with `seedFile(path)`, keyed by path. Written by `os build`, which reads each path relative to the package root; the loader stores each one as a file the written record owns. | | **_lock** | `Enum<'none' \| 'no-overlay' \| 'no-delete' \| 'full'>` | optional | Item-level lock — controls overlay & delete (ADR-0010). | | **_lockReason** | `string` | optional | Human-readable reason shown when a write is refused by _lock. | | **_lockSource** | `Enum<'artifact' \| 'package' \| 'env-forced'>` | optional | Layer that set _lock (artifact \| package \| env-forced). | @@ -358,6 +359,7 @@ Package manifest installed through POST /api/v1/packages (AUTHORING stage; scope | **env** | `Enum<'prod' \| 'dev' \| 'test'>[]` | optional (default: `["prod","dev","test"]`) | Applicable environments | | **locale** | `string[]` | optional | Applicable locales (BCP-47 tags); omitted applies to every locale. The publish and install paths do not filter by locale — they load every dataset and warn | | **records** | `Record[]` | ✅ | Data records | +| **files** | `Record` | optional | Bytes of the package files the records name with `seedFile(path)`, keyed by path. Written by `os build`, which reads each path relative to the package root; the loader stores each one as a file the written record owns. | | **_lock** | `Enum<'none' \| 'no-overlay' \| 'no-delete' \| 'full'>` | optional | Item-level lock — controls overlay & delete (ADR-0010). | | **_lockReason** | `string` | optional | Human-readable reason shown when a write is refused by _lock. | | **_lockSource** | `Enum<'artifact' \| 'package' \| 'env-forced'>` | optional | Layer that set _lock (artifact \| package \| env-forced). | diff --git a/content/docs/references/data/index.mdx b/content/docs/references/data/index.mdx index 9946692d77e..4edde16f0fb 100644 --- a/content/docs/references/data/index.mdx +++ b/content/docs/references/data/index.mdx @@ -1,7 +1,7 @@ --- title: Data Protocol — complete schema reference navTitle: Data Protocol -description: "The ObjectStack Data Protocol in 30 reference pages: every schema in @objectstack/spec with its properties, types, defaults and a TypeScript example." +description: "The ObjectStack Data Protocol in 31 reference pages: every schema in @objectstack/spec with its properties, types, defaults and a TypeScript example." --- {/* ⚠️ AUTO-GENERATED — DO NOT EDIT. Run build-docs.ts to regenerate. Hand-written docs live in the module folders under content/docs/. */} @@ -37,6 +37,7 @@ This section contains all protocol schemas for the data layer of ObjectStack. + diff --git a/content/docs/references/data/meta.json b/content/docs/references/data/meta.json index 88c37f782cc..768133a1ebf 100644 --- a/content/docs/references/data/meta.json +++ b/content/docs/references/data/meta.json @@ -35,6 +35,7 @@ "driver-sqlite", "driver-turso", "field-value", - "picklist" + "picklist", + "seed-file" ] } \ No newline at end of file diff --git a/content/docs/references/data/seed-file.mdx b/content/docs/references/data/seed-file.mdx new file mode 100644 index 00000000000..ce0356fe32f --- /dev/null +++ b/content/docs/references/data/seed-file.mdx @@ -0,0 +1,94 @@ +--- +title: Seed File schema — Data Protocol reference +navTitle: Seed File +description: "A file / image / avatar / video / audio field stores an opaque sys_file id, and only an upload mints one." +--- + +{/* ⚠️ AUTO-GENERATED — DO NOT EDIT. Run build-docs.ts to regenerate. Hand-written docs live in the module folders under content/docs/. */} + +## Seed files — a file a package ships, written into a file field by a seed + +A `file` / `image` / `avatar` / `video` / `audio` field stores an opaque +`sys_file` id (ADR-0104 D3 wave 2), and only an upload mints one. A seed has +no upload, so a seed record names the file by the PATH the package ships it +at instead, and the platform carries the bytes from there (#25779, ruled on +hotclm#19: "the platform's seed loader should carry bundled fixture files; +the app keeps `file` required"): + +1. **Authoring.** A record writes `seedFile('fixtures/nda-v1.pdf')` — the + value `SeedFileReferenceSchema` — into a file-class field. The path + is relative to the package root: the directory holding + `objectstack.config.ts`, the root a declarative connector's file ref + resolves against too. +2. **Bundling.** `os build` / `os compile` reads every referenced file and + writes its bytes into the dataset's `files` map + (`SeedFileContentSchema`), keyed by the same path. The compiled + artifact is what every boot reads — `os dev`, `os serve`, a published + package, an artifact URL — so the bytes travel with the seed wherever it + goes. A path that names no file fails the build. +3. **Loading.** The seed loader stores the bytes as a `sys_file`, writes its + id into the field, and has the written record own it — (object, record, + field), the ownership an upload gets. A replay that finds the row still + holding a file with the same bytes keeps it, so a boot does not mint a + new file for an unchanged fixture. A reference whose bytes the dataset + does not carry is refused, by row, naming the path. + +300 rows may name the same path: each row gets a file of its own, because a +file has one owning field (ADR-0104 D3). + + +**Source:** `packages/spec/src/data/seed-file.zod.ts` + + +## TypeScript Usage + +```typescript +import { SeedFileContentSchema, SeedFilePathSchema, SeedFileReferenceSchema } from '@objectstack/spec/data'; +import type { SeedFileContent, SeedFilePath, SeedFileReference } from '@objectstack/spec/data'; + +// Validate data +const result = SeedFileContentSchema.parse(data); +``` + +--- + +## SeedFileContent + +One bundled seed file: its bytes and what they must hash to + +### Properties + +| Property | Type | Required | Description | +| :--- | :--- | :--- | :--- | +| **contentType** | `string` | ✅ | MIME type the stored file carries | +| **size** | `integer` | ✅ | Byte length of the file | +| **sha256** | `string` | ✅ | SHA-256 of the bytes, lowercase hex — the loader refuses bytes that do not match it | +| **data** | `string` | ✅ | The bytes, base64-encoded | + + +--- + +## SeedFilePath + +Path of a file the package ships, relative to the package root (e.g. `fixtures/nda-v1.pdf`) + +**Type:** `string` + + +--- + +## SeedFileReference + +A file the package ships, by package-relative path, written into a file field by a seed + +### Properties + +| Property | Type | Required | Description | +| :--- | :--- | :--- | :--- | +| **$file** | `string` | ✅ | Path of a file the package ships, relative to the package root (e.g. `fixtures/nda-v1.pdf`) | +| **name** | `string` | optional | File name the stored file carries (what a download is saved as); defaults to the last segment of the path | +| **contentType** | `string` | optional | MIME type of the file; defaults to the one `os build` infers from the path's extension | + + +--- + diff --git a/content/docs/references/data/seed-loader.mdx b/content/docs/references/data/seed-loader.mdx index bc8e7d7fa6c..a217c7cba0e 100644 --- a/content/docs/references/data/seed-loader.mdx +++ b/content/docs/references/data/seed-loader.mdx @@ -234,6 +234,7 @@ Seed loader request with datasets and configuration | **env** | `Enum<'prod' \| 'dev' \| 'test'>[]` | optional (default: `["prod","dev","test"]`) | Applicable environments | | **locale** | `string[]` | optional | Applicable locales (BCP-47 tags); omitted applies to every locale. The publish and install paths do not filter by locale — they load every dataset and warn | | **records** | `Record[]` | ✅ | Data records | +| **files** | `Record` | optional | Bytes of the package files the records name with `seedFile(path)`, keyed by path. Written by `os build`, which reads each path relative to the package root; the loader stores each one as a file the written record owns. | | **_lock** | `Enum<'none' \| 'no-overlay' \| 'no-delete' \| 'full'>` | optional | Item-level lock — controls overlay & delete (ADR-0010). | | **_lockReason** | `string` | optional | Human-readable reason shown when a write is refused by _lock. | | **_lockSource** | `Enum<'artifact' \| 'package' \| 'env-forced'>` | optional | Layer that set _lock (artifact \| package \| env-forced). | diff --git a/content/docs/references/data/seed.mdx b/content/docs/references/data/seed.mdx index 444f62275de..0a92f45a5cd 100644 --- a/content/docs/references/data/seed.mdx +++ b/content/docs/references/data/seed.mdx @@ -35,6 +35,7 @@ const result = SeedSchema.parse(data); | **env** | `Enum<'prod' \| 'dev' \| 'test'>[]` | optional (default: `["prod","dev","test"]`) | Applicable environments | | **locale** | `string[]` | optional | Applicable locales (BCP-47 tags); omitted applies to every locale. The publish and install paths do not filter by locale — they load every dataset and warn | | **records** | `Record[]` | ✅ | Data records | +| **files** | `Record` | optional | Bytes of the package files the records name with `seedFile(path)`, keyed by path. Written by `os build`, which reads each path relative to the package root; the loader stores each one as a file the written record owns. | | **_lock** | `Enum<'none' \| 'no-overlay' \| 'no-delete' \| 'full'>` | optional | Item-level lock — controls overlay & delete (ADR-0010). | | **_lockReason** | `string` | optional | Human-readable reason shown when a write is refused by _lock. | | **_lockSource** | `Enum<'artifact' \| 'package' \| 'env-forced'>` | optional | Layer that set _lock (artifact \| package \| env-forced). | @@ -49,6 +50,17 @@ const result = SeedSchema.parse(data); | :--- | :--- | :--- | :--- | | **applyOn** | `Enum<'create' \| 'always'>` | optional (default: `"always"`) | When a seed load writes this column. `always` (default) writes it on every row the load creates or updates. `create` writes it only when the load creates the row: an `upsert` replay that matches an existing row leaves the row's value as it is, and an `update` dataset never writes it, so replaying the seed refreshes the `always` columns without reverting state the app wrote since the first load. A column of the dataset's `externalId` is still matched on and stays in the matched row's update. The same values and meaning as an import mapping entry's `applyOn`. | +### Nested Shape: `Seed.files[string]` + +One bundled seed file: its bytes and what they must hash to + +| Property | Type | Required | Description | +| :--- | :--- | :--- | :--- | +| **contentType** | `string` | ✅ | MIME type the stored file carries | +| **size** | `integer` | ✅ | Byte length of the file | +| **sha256** | `string` | ✅ | SHA-256 of the bytes, lowercase hex — the loader refuses bytes that do not match it | +| **data** | `string` | ✅ | The bytes, base64-encoded | + --- diff --git a/content/docs/references/index.mdx b/content/docs/references/index.mdx index 1767273fd9c..1608ee3be4e 100644 --- a/content/docs/references/index.mdx +++ b/content/docs/references/index.mdx @@ -1,7 +1,7 @@ --- title: Protocol reference — every schema by module navTitle: Protocol Reference -description: Every schema published by @objectstack/spec — 1544 schemas across 14 protocol modules +description: Every schema published by @objectstack/spec — 1547 schemas across 14 protocol modules --- {/* ⚠️ AUTO-GENERATED — DO NOT EDIT. Run build-docs.ts to regenerate. Hand-written docs live in the module folders under content/docs/. */} @@ -23,7 +23,7 @@ counts are sums of the rows they head. Regenerate with | [AI Protocol](/docs/references/ai) | 12 | 68 | Agents, tools, skills, RAG and knowledge sources, model registry, conversations. | | [API Protocol](/docs/references/api) | 32 | 438 | REST contracts, endpoints, routing, realtime, batch, discovery. | | [Automation Protocol](/docs/references/automation) | 14 | 73 | Flows and their nodes, approvals, ETL pipelines, webhooks, state machines, execution records. | -| [Data Protocol](/docs/references/data) | 30 | 180 | Objects, fields, queries, filters, datasources and drivers — the ObjectQL layer. | +| [Data Protocol](/docs/references/data) | 31 | 183 | Objects, fields, queries, filters, datasources and drivers — the ObjectQL layer. | | [Identity Protocol](/docs/references/identity) | 5 | 27 | Users and accounts, organizations, positions, SCIM provisioning. | | [Integration Protocol](/docs/references/integration) | 1 | 12 | The single connector protocol (ADR-0097) — catalog descriptors and provider-bound instances. | | [Kernel Protocol](/docs/references/kernel) | 29 | 151 | Plugin lifecycle and manifests, capabilities and security, metadata loading, service registry. | @@ -34,7 +34,7 @@ counts are sums of the rows they head. Regenerate with | [Studio Protocol](/docs/references/studio) | 3 | 35 | Studio designer metadata — the authoring surfaces for the protocols above. | | [System Protocol](/docs/references/system) | 37 | 286 | The runtime environment — logging, jobs, cache, metrics, notifications, i18n and compliance. | | [UI Protocol](/docs/references/ui) | 16 | 171 | Apps, pages, views, dashboards, reports, actions and themes — the ObjectUI layer. | -| **Total** | **199** | **1544** | 14 protocol modules | +| **Total** | **200** | **1547** | 14 protocol modules | --- @@ -131,7 +131,7 @@ Flows and their nodes, approvals, ETL pipelines, webhooks, state machines, execu ## Data Protocol -**Source:** `packages/spec/src/data/` · **Import:** `@objectstack/spec/data` · **30 pages, 180 schemas** +**Source:** `packages/spec/src/data/` · **Import:** `@objectstack/spec/data` · **31 pages, 183 schemas** Objects, fields, queries, filters, datasources and drivers — the ObjectQL layer. @@ -165,6 +165,7 @@ Objects, fields, queries, filters, datasources and drivers — the ObjectQL laye | [`picklist.zod.ts`](/docs/references/data/picklist) | `Picklist`, `PicklistExtension`, `PicklistServedField` | | [`query.zod.ts`](/docs/references/data/query) | `AggregationFunction`, `AggregationNode`, `DateGranularity`, `FieldNode`, `FullTextSearch`, `GroupByNode`, `Query`, `SortNode` | | [`seed.zod.ts`](/docs/references/data/seed) | `Seed`, `SeedColumn`, `SeedMode` | +| [`seed-file.zod.ts`](/docs/references/data/seed-file) | `SeedFileContent`, `SeedFilePath`, `SeedFileReference` | | [`seed-loader.zod.ts`](/docs/references/data/seed-loader) | `ObjectDependencyGraph`, `ObjectDependencyNode`, `ReferenceResolution`, `ReferenceResolutionError`, `SeedIdentity`, `SeedLoadResult`, `SeedLoaderConfig`, `SeedLoaderRequest`, `SeedLoaderResult` | | [`validation.zod.ts`](/docs/references/data/validation) | `ConditionalValidation`, `CrossFieldValidation`, `FormatValidation`, `JSONValidation`, `ScriptValidation`, `StateMachineValidation`, `ValidationRule` | diff --git a/content/docs/references/kernel/manifest.mdx b/content/docs/references/kernel/manifest.mdx index c0fd420abf2..04180596719 100644 --- a/content/docs/references/kernel/manifest.mdx +++ b/content/docs/references/kernel/manifest.mdx @@ -103,6 +103,7 @@ Install mode: managed (registered as code, upgradeable, sealed) | unmanaged (cop | **env** | `Enum<'prod' \| 'dev' \| 'test'>[]` | optional (default: `["prod","dev","test"]`) | Applicable environments | | **locale** | `string[]` | optional | Applicable locales (BCP-47 tags); omitted applies to every locale. The publish and install paths do not filter by locale — they load every dataset and warn | | **records** | `Record[]` | ✅ | Data records | +| **files** | `Record` | optional | Bytes of the package files the records name with `seedFile(path)`, keyed by path. Written by `os build`, which reads each path relative to the package root; the loader stores each one as a file the written record owns. | | **_lock** | `Enum<'none' \| 'no-overlay' \| 'no-delete' \| 'full'>` | optional | Item-level lock — controls overlay & delete (ADR-0010). | | **_lockReason** | `string` | optional | Human-readable reason shown when a write is refused by _lock. | | **_lockSource** | `Enum<'artifact' \| 'package' \| 'env-forced'>` | optional | Layer that set _lock (artifact \| package \| env-forced). | diff --git a/docs/audits/2026-07-unknown-key-strictness-ledger.counts/data.md b/docs/audits/2026-07-unknown-key-strictness-ledger.counts/data.md index afd86d8067f..61f4f5df382 100644 --- a/docs/audits/2026-07-unknown-key-strictness-ledger.counts/data.md +++ b/docs/audits/2026-07-unknown-key-strictness-ledger.counts/data.md @@ -21,7 +21,7 @@ The `strict` column is the one the campaign schedules against; it counts both th | Dir | Sites | strict | passthrough | catchall | strip | |---|---|---|---|---|---| -| `data/` | 159 | 82 | 2 | 0 | 75 | +| `data/` | 161 | 84 | 2 | 0 | 75 | ## `data/` — sites @@ -53,10 +53,11 @@ classify and is not listed (it becomes reportable the day it grows its first sit | `object.zod.ts` | 21 | | `picklist.zod.ts` | 3 | | `query.zod.ts` | 5 | +| `seed-file.zod.ts` | 2 | | `seed-loader.zod.ts` | 12 | | `seed.zod.ts` | 2 | | `validation.zod.ts` | 6 | -| **total** | **159** | +| **total** | **161** | ## `data/` — open @@ -64,7 +65,7 @@ Per file, how many of its sites still silently discard unknown keys. The `Class` column that decides the bucket split is hand-written in the ledger; the arithmetic over it is here. -**75 strip of 159**, in 10 file(s). +**75 strip of 161**, in 10 file(s). | File | Strip | Sites | |---|---|---| @@ -78,7 +79,7 @@ over it is here. | `hook.zod.ts` | 5 | 7 | | `query.zod.ts` | 4 | 5 | | `seed-loader.zod.ts` | 12 | 12 | -| **total** | **75** | **159** | +| **total** | **75** | **161** | | Bucket | Sites | |---|---| diff --git a/docs/audits/2026-07-unknown-key-strictness-ledger.md b/docs/audits/2026-07-unknown-key-strictness-ledger.md index 6790b5e4460..2e8dd667efd 100644 --- a/docs/audits/2026-07-unknown-key-strictness-ledger.md +++ b/docs/audits/2026-07-unknown-key-strictness-ledger.md @@ -732,6 +732,7 @@ column does not move and the `strip` column falls by the count of what left. | `external-catalog.zod.ts` | wire (p) | | | `validation.zod.ts` | authorable | **strict as of #4001 batch 3b** — a `z.lazy()` discriminated union, so the one-call conversion does not apply: each of the six variants builds its own `strictObject` from a shared `BASE_VALIDATION_SHAPE`. Closing the base alone would have rejected correctly but suggested from the SHARED keys only, so a typo of a variant's own key (`transtions` → `transitions`) would get no rename. Site count 1 → 6 because the six variants are now object sites in their own right. The ADR-0010 envelope lives in the shared shape, so all six inherit it | | `field-value.zod.ts` / `seed.zod.ts` | authorable | `seed` is strict (registered-types batch). **`field-value` strict as of #13802 (2026-09-02) — re-verdicted `open` → `authorable` by maintainer ruling 2026-09-01 (option A, director batch #26, verbatim 「同意」), overruling this row's #4001 batch D reading.** Both value contracts (`LocationValueSchema`, `AddressSchema` = `AddressValueSchema`) are `strictObject` now; `FileValueSchema` stays `z.looseObject` on purpose — the one deliberate loose site, ⛔ untouched. What batch D got wrong was not the door census but the cost model: every member of both shapes is OPTIONAL, so under `.strip` a value with a completely wrong key set still parsed green and the wrong keys vanished — the showcase seed wrote `postal_code`, the platform accepted it, dropped it, and rendered an empty ZIP box (#13388; found by objectui#6812's survey; #5143 had named the same stripping on the widget round-trip) — while any stored-value scan over the class could only ever report zero (#13802's finding: an instrument reporting a number it has no way to earn). The "legitimate extras" (`heading`/`speed`, `district`) were a hypothesis with no in-repo producer. **Corpus census at `a39b02a6`, ordered first by the ruling**: every in-repo corpus that writes address/location VALUES — `examples/**`, `packages/apps/**`, `packages/qa/**`, `packages/**/src` fixtures and tests, `content/docs/**`, `skills/**`, `.changeset/**`, `docs/**` (`.ts/.tsx/.js/.mjs/.cjs/.json/.yaml/.yml/.md/.mdx`; generated `references/`, `releases/`, CHANGELOGs excluded) — a brace-matched scan of 8459 files / 196,098 leaf object literals found 57 address- or location-shaped literals and **0 carrying a key outside the declared sets** other than batch D's own tolerance pin in `analytics-strictness-batchd.test.ts` (repinned to the closure in the same stroke; the SCIM hits are `SCIMAddressSchema`, a different contract). The spelling grep `git grep -n -E "postal_code|zip_code|zipCode|postcode|latitude|longitude|heading *:|speed *:|district *:" -- 'examples/**' 'packages/apps/**' 'packages/qa/**' 'packages/spec/**' 'content/docs/**' 'skills/**'` hits only the retired-form docs, `ListMapConfig`'s field-name keys and pins. So the repair commit the ruling ordered first was EMPTY by measurement — #13388's seed fix had already landed at #14090. **Migration note — where the refusal bites, by call site** (`git grep -n "valueSchemaFor(" -- packages`, non-test): ① **authoring, hard reject, unconditional** — a `location`/`address` field's literal `defaultValue` (`default-value-shape.ts` `checkLiteralDefaultValue` → `FieldSchema`, #7127) and an action-param value of those types (`ui/action-params.zod.ts` `validateActionParams`, D2 strict by default since 17.0); ② **record writes, per deployment** — objectql `record-validator` `validateOne` (`shapeSchemaFor(def).safeParse(value)`, insert and update) rejects with `invalid_type` / `invalid_value_shape` ONLY when `valueShapeStrictEffective` holds (the deployment attested `adr-0104-value-shapes`, or `OS_DATA_VALUE_SHAPE_STRICT_ENABLED=1`; `OS_ALLOW_LAX_VALUE_SHAPES=1` re-opens), otherwise **warn-first** — logged once per field and reported to the admitted-violation sink (#4769), unchanged; ③ **the scan** — `os migrate value-shapes` (`valueShapeViolation`, the same predicate) now COUNTS an undeclared key, so a deployment holding such values cannot attest until they are cleaned at the producer — the mechanism that keeps ② from stranding stored data; ④ **read paths: none** — no consumer calls `valueSchemaFor(def, 'expanded')` for these types outside `packages/spec` (`git grep "'expanded')" -- 'packages/**/src/**' ':!packages/spec/**'` → 0 hits), drivers return the stored JSON verbatim and renderers read it, so a stored `{ …, postal_code }` still reads back as written. ⛔ Per the ruling no read path was narrowed; the customer-database inventory is the one thing this repo cannot see — a confidence gap recorded here, not glossed: the scan is the instrument that closes it per deployment. ⛔ No consumer-side alias (AGENTS.md #0.1): the `aliases` on both shapes are did-you-mean RENAMES in the refusal message (`zipCode`/`zip`/`postcode` → `postalCode`, `latitude`/`longitude` → `lat`/`lng`), not tolerance — `postal_code` is refused, never read. objectui at the pin (`LocationField.tsx`) `safeParse`s only a widget-built `{ lat, lng, altitude?, accuracy? }` candidate, so its verdicts do not move; its `LocationField.optionalKeys.test.tsx` pins the OLD strip behaviour and flips the day objectui takes a spec carrying this — filed there, not here. D3 semantic entry `address-location-value-unknown-keys-refused` (protocol 18). Re-check: `git grep -n "strictObject(\|looseObject(\|z.object(" -- packages/spec/src/data/field-value.zod.ts` → three `strictObject(`, one `z.looseObject(`, zero `z.object(` — the third is `FileRefusedValueSchema` (#22593), the expanded read form a reader refused `sys_file` metadata is served instead of the bare id: engine-emitted, never a stored value, closed so it is told from a resolved `FileValue` by shape | +| `seed-file.zod.ts` | authorable | **strict from birth (#25779).** Both sites are `strictObject`: `SeedFileReferenceSchema` is what `seedFile(path)` writes into a seed record (`$file`, `name?`, `contentType?`, with `aliases` pointing a misspelled key at the one it meant), and `SeedFileContentSchema` is one entry of a dataset's `files` map, the bytes `os build` bundles (`contentType`, `size`, `sha256`, `data`). The reference is authored and enters through `SeedSchema`, a registered type; the content is written by the build and read back by the seed loader, and a key neither declares is refused at the parse rather than carried. | ### `automation/` — file-level triage diff --git a/packages/spec/api-surface/contracts.json b/packages/spec/api-surface/contracts.json index 7e5114fba47..77cc67cea52 100644 --- a/packages/spec/api-surface/contracts.json +++ b/packages/spec/api-surface/contracts.json @@ -98,8 +98,11 @@ "EngineTransactionInfo (interface)", "EngineTransactionOptions (interface)", "ExplainAccessRequest (interface)", + "FIELD_FILE_SERVICE (const)", "FILE_ATTACHMENT_SERVICE (const)", "FLOW_UNBOUND_CAUSES (const)", + "FieldFileDescription (interface)", + "FieldFileSlot (interface)", "FinishReason (type)", "FlowRuntimeState (interface)", "FlowUnboundCause (type)", @@ -131,6 +134,7 @@ "IEmailTransport (interface)", "IEmbedder (interface)", "IExternalDatasourceService (interface)", + "IFieldFileService (interface)", "IFileAccessDelegate (interface)", "IFileAttachmentService (interface)", "IHierarchyScopeResolver (interface)", @@ -301,6 +305,8 @@ "StorageListOptions (interface)", "StorageListPage (interface)", "StorageUploadOptions (interface)", + "StoreFieldFileInput (interface)", + "StoredFieldFile (interface)", "StrategyContext (interface)", "SubscribeOptions (interface)", "SystemModelMessage (type)", diff --git a/packages/spec/api-surface/data.json b/packages/spec/api-surface/data.json index 9449990146a..eb77a2d814b 100644 --- a/packages/spec/api-surface/data.json +++ b/packages/spec/api-surface/data.json @@ -707,6 +707,12 @@ "SeedColumn (type)", "SeedColumnParsed (type)", "SeedColumnSchema (const)", + "SeedFileContent (type)", + "SeedFileContentSchema (const)", + "SeedFilePath (type)", + "SeedFilePathSchema (const)", + "SeedFileReference (type)", + "SeedFileReferenceSchema (const)", "SeedIdentity (type)", "SeedIdentitySchema (const)", "SeedImportMode (type)", @@ -927,6 +933,8 @@ "isRecordContextToken (function)", "isRefusedTextComparand (function)", "isRuntimeDefaultToken (function)", + "isSeedFilePath (function)", + "isSeedFileReference (function)", "isTenancyDisabled (function)", "isTextFilterOperator (function)", "isTitleEligible (function)", @@ -994,6 +1002,7 @@ "resolveRecordDisplayName (function)", "resolveSearchFieldResolution (function)", "resolveSearchFields (function)", + "seedFile (function)", "sequenceWidth (function)", "stripLegacyApiMethods (function)", "suggestDefaultValueToken (function)", diff --git a/packages/spec/declaration-map/data.json b/packages/spec/declaration-map/data.json index 506e1c65832..7553f0d3de5 100644 --- a/packages/spec/declaration-map/data.json +++ b/packages/spec/declaration-map/data.json @@ -288,6 +288,11 @@ "Seed": "data/Seed", "SeedColumn": "data/SeedColumn", "SeedColumnSchema": "data/SeedColumn", + "SeedFileContent": "data/SeedFileContent", + "SeedFileContentSchema": "data/SeedFileContent", + "SeedFilePathSchema": "data/SeedFilePath", + "SeedFileReference": "data/SeedFileReference", + "SeedFileReferenceSchema": "data/SeedFileReference", "SeedIdentity": "data/SeedIdentity", "SeedIdentitySchema": "data/SeedIdentity", "SeedLoadResult": "data/SeedLoadResult", diff --git a/packages/spec/export-origins/contracts.json b/packages/spec/export-origins/contracts.json index bd493c6eda4..4ed8068441e 100644 --- a/packages/spec/export-origins/contracts.json +++ b/packages/spec/export-origins/contracts.json @@ -98,8 +98,11 @@ "EngineTransactionInfo": "src/contracts/objectql-engine.ts#EngineTransactionInfo (interface)", "EngineTransactionOptions": "src/contracts/objectql-engine.ts#EngineTransactionOptions (interface)", "ExplainAccessRequest": "src/contracts/security-service.ts#ExplainAccessRequest (interface)", + "FIELD_FILE_SERVICE": "src/contracts/storage-service.ts#FIELD_FILE_SERVICE (const)", "FILE_ATTACHMENT_SERVICE": "src/contracts/storage-service.ts#FILE_ATTACHMENT_SERVICE (const)", "FLOW_UNBOUND_CAUSES": "src/contracts/automation-service.ts#FLOW_UNBOUND_CAUSES (const)", + "FieldFileDescription": "src/contracts/storage-service.ts#FieldFileDescription (interface)", + "FieldFileSlot": "src/contracts/storage-service.ts#FieldFileSlot (interface)", "FinishReason": "node_modules/ai/dist/index.d.ts#FinishReason (type)", "FlowRuntimeState": "src/contracts/automation-service.ts#FlowRuntimeState (interface)", "FlowUnboundCause": "src/contracts/automation-service.ts#FlowUnboundCause (type)", @@ -131,6 +134,7 @@ "IEmailTransport": "src/contracts/email-service.ts#IEmailTransport (interface)", "IEmbedder": "src/contracts/embedder.ts#IEmbedder (interface)", "IExternalDatasourceService": "src/contracts/external-datasource-service.ts#IExternalDatasourceService (interface)", + "IFieldFileService": "src/contracts/storage-service.ts#IFieldFileService (interface)", "IFileAccessDelegate": "src/contracts/storage-service.ts#IFileAccessDelegate (interface)", "IFileAttachmentService": "src/contracts/storage-service.ts#IFileAttachmentService (interface)", "IHierarchyScopeResolver": "src/contracts/sharing-service.ts#IHierarchyScopeResolver (interface)", @@ -301,6 +305,8 @@ "StorageListOptions": "src/contracts/storage-service.ts#StorageListOptions (interface)", "StorageListPage": "src/contracts/storage-service.ts#StorageListPage (interface)", "StorageUploadOptions": "src/contracts/storage-service.ts#StorageUploadOptions (interface)", + "StoreFieldFileInput": "src/contracts/storage-service.ts#StoreFieldFileInput (interface)", + "StoredFieldFile": "src/contracts/storage-service.ts#StoredFieldFile (interface)", "StrategyContext": "src/contracts/analytics-service.ts#StrategyContext (interface)", "SubscribeOptions": "src/contracts/cluster-service.ts#SubscribeOptions (interface)", "SystemModelMessage": "node_modules/@ai-sdk/provider-utils/dist/index.d.ts#SystemModelMessage (type)", diff --git a/packages/spec/export-origins/data.json b/packages/spec/export-origins/data.json index 457d665180b..bc4f58f154d 100644 --- a/packages/spec/export-origins/data.json +++ b/packages/spec/export-origins/data.json @@ -695,6 +695,12 @@ "SeedColumn": "src/data/seed.zod.ts#SeedColumn (type)", "SeedColumnParsed": "src/data/seed.zod.ts#SeedColumnParsed (type)", "SeedColumnSchema": "src/data/seed.zod.ts#SeedColumnSchema (const)", + "SeedFileContent": "src/data/seed-file.zod.ts#SeedFileContent (type)", + "SeedFileContentSchema": "src/data/seed-file.zod.ts#SeedFileContentSchema (const)", + "SeedFilePath": "src/data/seed-file.zod.ts#SeedFilePath (type)", + "SeedFilePathSchema": "src/data/seed-file.zod.ts#SeedFilePathSchema (const)", + "SeedFileReference": "src/data/seed-file.zod.ts#SeedFileReference (type)", + "SeedFileReferenceSchema": "src/data/seed-file.zod.ts#SeedFileReferenceSchema (const)", "SeedIdentity": "src/data/seed-loader.zod.ts#SeedIdentity (type)", "SeedIdentitySchema": "src/data/seed-loader.zod.ts#SeedIdentitySchema (const)", "SeedImportMode": "src/data/seed.zod.ts#SeedImportMode (type)", @@ -913,6 +919,8 @@ "isRecordContextToken": "src/data/context-tokens.zod.ts#isRecordContextToken (function)", "isRefusedTextComparand": "src/data/filter-text-comparand.ts#isRefusedTextComparand (function)", "isRuntimeDefaultToken": "src/data/default-value-tokens.ts#isRuntimeDefaultToken (function)", + "isSeedFilePath": "src/data/seed-file.zod.ts#isSeedFilePath (function)", + "isSeedFileReference": "src/data/seed-file.zod.ts#isSeedFileReference (function)", "isTenancyDisabled": "src/data/object.zod.ts#isTenancyDisabled (function)", "isTextFilterOperator": "src/data/filter-text-operator-declared-type.ts#isTextFilterOperator (function)", "isTitleEligible": "src/data/display-name.ts#isTitleEligible (function)", @@ -980,6 +988,7 @@ "resolveRecordDisplayName": "src/data/display-name.ts#resolveRecordDisplayName (function)", "resolveSearchFieldResolution": "src/data/search-fields.ts#resolveSearchFieldResolution (function)", "resolveSearchFields": "src/data/search-fields.ts#resolveSearchFields (function)", + "seedFile": "src/data/seed-file.zod.ts#seedFile (function)", "sequenceWidth": "src/data/autonumber-format.ts#sequenceWidth (function)", "stripLegacyApiMethods": "src/data/object.zod.ts#stripLegacyApiMethods (function)", "suggestDefaultValueToken": "src/data/default-value-shape.ts#suggestDefaultValueToken (function)", diff --git a/packages/spec/src/data/seed-file.zod.ts b/packages/spec/src/data/seed-file.zod.ts index b197c19ac3b..a24895374de 100644 --- a/packages/spec/src/data/seed-file.zod.ts +++ b/packages/spec/src/data/seed-file.zod.ts @@ -62,6 +62,7 @@ export function isSeedFilePath(path: unknown): path is string { export const SeedFilePathSchema = lazySchema(() => z.string() .regex(SEED_FILE_PATH_PATTERN, SEED_FILE_PATH_RULE) .describe('Path of a file the package ships, relative to the package root (e.g. `fixtures/nda-v1.pdf`)')); +export type SeedFilePath = z.infer; /** A MIME type: `type/subtype`, the shape a `sys_file.mime_type` carries. */ const MIME_TYPE = /^[a-z0-9][a-z0-9!#$&^_.+-]*\/[a-z0-9][a-z0-9!#$&^_.+-]*$/i; diff --git a/skills/objectstack-data/references/_index.md b/skills/objectstack-data/references/_index.md index 911c10e155b..045ab01efe2 100644 --- a/skills/objectstack-data/references/_index.md +++ b/skills/objectstack-data/references/_index.md @@ -37,6 +37,7 @@ from `node_modules` — there is no local copy in the skill bundle. - `node_modules/@objectstack/spec/src/data/filter.zod.ts` — Unified Query DSL Specification - `node_modules/@objectstack/spec/src/data/hook-body.zod.ts` — Exports: HookBodyCapability, ExpressionBodySchema, ScriptBodySchema, HookBodySchema - `node_modules/@objectstack/spec/src/data/query.zod.ts` — QueryAST — Abstract Syntax Tree for data queries. +- `node_modules/@objectstack/spec/src/data/seed-file.zod.ts` — Seed files — a file a package ships, written into a file field by a seed - `node_modules/@objectstack/spec/src/identity/eval-user.zod.ts` — EvalUser — the one user-context contract (ADR-0068 D1). - `node_modules/@objectstack/spec/src/kernel/execution-context.zod.ts` — Exports: AuthGateCodeSchema, AuthGateScopeSchema, ExecutionContextSchema - `node_modules/@objectstack/spec/src/kernel/metadata-protection.zod.ts` — Metadata Protection Model — Phase 1 (ADR-0010) From 389fc184fecdca3a4faf632e92e0a07028998e4a Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 26 Oct 2026 14:15:51 +0000 Subject: [PATCH 06/13] fix(cli): a config boot carries the seed file bytes os build bundles Co-Authored-By: Claude Claude-Session: https://claude.ai/code/session_012eJQ8dtut5bZsnhFE3XGHF --- packages/cli/src/commands/serve.ts | 16 ++ .../serve-config-boot-seed-files.e2e.test.ts | 213 ++++++++++++++++++ 2 files changed, 229 insertions(+) create mode 100644 packages/cli/test/serve-config-boot-seed-files.e2e.test.ts diff --git a/packages/cli/src/commands/serve.ts b/packages/cli/src/commands/serve.ts index 603bdb76c10..a6b8e9c6845 100644 --- a/packages/cli/src/commands/serve.ts +++ b/packages/cli/src/commands/serve.ts @@ -2807,6 +2807,22 @@ export default class Serve extends Command { } } + // [#25779] Seed files, mirrored the same way. `os build` reads each file + // a seed record names with `seedFile(path)` into its dataset's `files` + // map (compile.ts step 4a), and the seed loader stores it from there. + // `defineStack(...)` carries the reference and not the bytes, so a config + // boot without this would refuse every such row that an artifact boot of + // the same project seeds. A path it cannot read is reported and left out; + // the loader then refuses that row, by name, and the boot goes on. + if (!useArtifactFallback) { + const { collectSeedFiles, placeSeedFiles } = await import('../utils/bundle-seed-files.js'); + const seedFiles = collectSeedFiles(absolutePath, config as Record); + placeSeedFiles(seedFiles); + for (const issue of seedFiles.issues) { + console.warn(chalk.yellow(` ⚠ ${issue.location}: ${issue.message}`)); + } + } + // Boot-mode dispatch: this open-core CLI only supports `standalone` // (and the artifact-fallback shortcut). Cloud / multi-environment // boot modes live in a separate distribution and are no longer diff --git a/packages/cli/test/serve-config-boot-seed-files.e2e.test.ts b/packages/cli/test/serve-config-boot-seed-files.e2e.test.ts new file mode 100644 index 00000000000..de04eeaae68 --- /dev/null +++ b/packages/cli/test/serve-config-boot-seed-files.e2e.test.ts @@ -0,0 +1,213 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * #25779 — a seed record fills a required file field with a file the package + * ships, `seedFile(path)`, and the booted platform serves that file. + * + * The card's acceptance, measured through the doors a user reaches: `os serve + * objectstack.config.ts --dev` on a project whose `clm_version.file` is + * required, seeded with two versions naming the same fixture. Each seeded row + * holds a file of its own, and following `GET /storage/files/:fileId` for it + * answers the fixture's bytes, so the file is stored, committed and owned (an + * uncommitted or unowned file is not served). + * + * ## Why it spawns the real command + * + * The collection and the loader are pinned in-process + * (`src/utils/bundle-seed-files.test.ts`, metadata-protocol's + * `seed-loader-seed-files.test.ts`, service-storage's + * `field-file-service.test.ts`). Only a boot reaches the seams between them: the + * config-load mirror in `serve.ts` (a config carries the reference, not the + * bytes), the `field-files` service the storage plugin registers, and the + * inline seed deferred to `kernel:bootstrapped`. + */ + +import { describe, it, expect, beforeAll, afterAll } from 'vitest'; +import { spawn, type ChildProcessWithoutNullStreams } from 'node:child_process'; +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { + CLI, + E2E_SECRET_KEY, + KERNEL_ASSIGNED_PORT, + TSX, + childEnv, + probeThroughChild, + readKernelAssignedPort, +} from './helpers/serve-process.js'; +import { linkSpec } from './helpers/define-stack-fixture.js'; + +/** The banner's tail — every row above it has printed. */ +const READY = /Press Ctrl\+C to stop/; + +const PDF = '%PDF-1.7 MARKER-25779 a seeded contract'; + +const CONFIG = `import { defineStack } from '@objectstack/spec'; +import { seedFile } from '@objectstack/spec/data'; + +export default defineStack({ + manifest: { id: 'com.example.clm', name: 'clm', version: '1.0.0', type: 'app', namespace: 'clm' }, + objects: [ + { + name: 'clm_version', + label: 'Version', + sharingModel: 'private', + fields: { + code: { type: 'text', label: 'Code' }, + file: { type: 'file', label: 'File', required: true }, + }, + }, + ], + data: [ + { + object: 'clm_version', + externalId: 'code', + records: [ + { code: 'V1', file: seedFile('fixtures/nda.pdf') }, + { code: 'V2', file: seedFile('fixtures/nda.pdf', { name: 'NDA v2.pdf' }) }, + ], + }, + ], +}, { strict: false }); +`; + +let dir = ''; +let child: ChildProcessWithoutNullStreams | null = null; +let output = ''; + +beforeAll(() => { + dir = mkdtempSync(join(tmpdir(), 'os-seed-files-serve-')); + mkdirSync(join(dir, 'fixtures')); + writeFileSync(join(dir, 'fixtures', 'nda.pdf'), PDF); + writeFileSync(join(dir, 'objectstack.config.ts'), CONFIG); + linkSpec(dir); +}); + +afterAll(async () => { + if (child && child.exitCode === null && child.signalCode === null) { + await new Promise((done) => { + const give = setTimeout(() => { + try { child!.kill('SIGKILL'); } catch { /* already gone */ } + done(); + }, 10_000); + child!.once('exit', () => { clearTimeout(give); done(); }); + try { child!.kill('SIGTERM'); } catch { clearTimeout(give); done(); } + }); + } + if (dir) rmSync(dir, { recursive: true, force: true }); +}); + +/** Boot `os serve objectstack.config.ts --dev`; `--dev` seeds the admin. Answers the port it bound. */ +function bootServe(): Promise { + return new Promise((resolveBoot, rejectBoot) => { + child = spawn(TSX, [CLI, 'serve', 'objectstack.config.ts', '-p', KERNEL_ASSIGNED_PORT, '--dev'], { + cwd: dir, + stdio: ['pipe', 'pipe', 'pipe'], + env: childEnv({ + NO_COLOR: '1', + OS_DATABASE_URL: ':memory:', + OS_LOG_LEVEL: '', + OS_DISABLE_CONSOLE: '1', + OS_SECRET_KEY: E2E_SECRET_KEY, + }), + }) as ChildProcessWithoutNullStreams; + const what = `os serve --dev in ${dir}`; + let settled = false; + const settle = (verdict: Error | number) => { + if (settled) return; + settled = true; + clearTimeout(timer); + if (verdict instanceof Error) rejectBoot(verdict); + else resolveBoot(verdict); + }; + const timer = setTimeout( + () => settle(new Error(`${what} never printed ${READY}\n--- output ---\n${output.slice(-4000)}`)), + 180_000, + ); + const onData = (d: unknown) => { + output += String(d); + if (READY.test(output)) settle(readKernelAssignedPort(output, what)); + }; + child.stdout.on('data', onData); + child.stderr.on('data', onData); + child.on('exit', (code) => + settle(new Error(`${what} exited ${String(code)} before ${READY}\n--- output ---\n${output.slice(-4000)}`)), + ); + }); +} + +/** One HTTP exchange, attributed to the child if the transport fails. ⛔ No assertion inside. */ +async function request(what: string, url: string, init: RequestInit = {}): Promise<{ status: number; headers: Headers; text: string; body: any }> { + return probeThroughChild( + { + child: child!, + transcript: () => `\n--- child output ---\n${output.slice(-4000)}`, + label: 'serve-config-boot-seed-files', + what, + }, + async () => { + const r = await fetch(url, init); + const text = await r.text(); + let body: unknown = text; + try { body = JSON.parse(text); } catch { /* keep the text */ } + return { status: r.status, headers: r.headers, text, body }; + }, + ); +} + +/** The rows a data read answers, whichever envelope carries them. */ +function rowsOf(body: any): Array> { + const candidates = [body?.data?.records, body?.data?.items, body?.records, body?.items, body?.data, body?.value]; + return (candidates.find(Array.isArray) ?? []) as Array>; +} + +/** A file field's value as read: the bare id, or the expanded form carrying it. */ +function fileIdOf(value: unknown): string | undefined { + if (typeof value === 'string') return value; + if (value && typeof value === 'object') { + const v = value as Record; + for (const key of ['id', 'fileId', 'file_id']) if (typeof v[key] === 'string') return v[key] as string; + } + return undefined; +} + +describe('a config boot seeds and serves the files seed records name (#25779)', () => { + it('each seeded version holds a stored file of its own, and the file door serves the fixture\'s bytes', async () => { + const port = await bootServe(); + const base = `http://localhost:${port}/api/v1`; + + const signIn = await request('the sign-in probe', `${base}/auth/sign-in/email`, { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ email: 'admin@objectos.ai', password: 'admin123' }), + }); + expect(signIn.status, `sign-in: ${signIn.text}`).toBe(200); + const headers = { authorization: `Bearer ${signIn.body.token}` }; + + // The inline seed of a stack carrying seed files runs at + // `kernel:bootstrapped`; read until both rows are there, bounded. + let rows: Array> = []; + let last = ''; + for (let attempt = 0; attempt < 60 && rows.length < 2; attempt++) { + const read = await request('the data read', `${base}/data/clm_version`, { headers }); + last = `${read.status} ${read.text.slice(0, 2000)}`; + rows = rowsOf(read.body); + if (rows.length < 2) await new Promise((r) => setTimeout(r, 1000)); + } + expect(rows.map((r) => r.code).sort(), `last read: ${last}\n--- output ---\n${output.slice(-4000)}`).toEqual(['V1', 'V2']); + + const ids = rows.map((r) => fileIdOf(r.file)); + expect(ids.every((id) => typeof id === 'string' && id.length > 0), JSON.stringify(rows)).toBe(true); + expect(new Set(ids).size, 'each row holds a file of its own').toBe(2); + + for (const id of ids) { + const redirect = await request(`the file door for ${id}`, `${base}/storage/files/${id}`, { headers, redirect: 'manual' }); + expect(redirect.status, `GET /storage/files/${id}: ${redirect.text}`).toBe(302); + const location = new URL(redirect.headers.get('location')!, `http://localhost:${port}`).toString(); + const bytes = await request(`the bytes of ${id}`, location, { headers }); + expect(bytes.status).toBe(200); + expect(bytes.text).toBe(PDF); + } + }, 300_000); +}); From a7fc798d1ae176ce1279525092225c14a0a8c439 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 26 Oct 2026 14:18:48 +0000 Subject: [PATCH 07/13] docs(seed): os validate and a config boot read seed files too Co-Authored-By: Claude Claude-Session: https://claude.ai/code/session_012eJQ8dtut5bZsnhFE3XGHF --- .changeset/25779-seed-files.md | 2 +- content/docs/data-modeling/seed-data.mdx | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.changeset/25779-seed-files.md b/.changeset/25779-seed-files.md index 80e62ce211b..b27ca2073d3 100644 --- a/.changeset/25779-seed-files.md +++ b/.changeset/25779-seed-files.md @@ -28,7 +28,7 @@ defineSeed(ContractVersion, { ``` - **The path** is relative to the directory holding `objectstack.config.ts`, uses `/`, and stays inside it: an absolute path, a drive letter, a `\`, or an empty, `.` or `..` segment is refused at parse. A `multiple: true` file field takes an array of `seedFile(...)` values. `defineSeed` refuses `seedFile(...)` in a field that does not store a file. -- **`os build` / `os compile`** reads each referenced file and writes its bytes into the dataset's new `files` map (`{ contentType, size, sha256, data }`, base64), so the compiled artifact carries them to every boot that loads it: `os dev`, `os serve`, a published package, an artifact URL. A path that names no file, or a file whose type is neither known by its extension nor named with `contentType`, fails the build. +- **`os build` / `os compile`** reads each referenced file and writes its bytes into the dataset's new `files` map (`{ contentType, size, sha256, data }`, base64), so the compiled artifact carries them to every boot that loads it: `os dev`, `os serve`, a published package, an artifact URL. A path that names no file, or a file whose type is neither named with `contentType` nor known by its extension, fails the build, and `os validate` refuses the same paths. `os serve` / `os dev` booting a config rather than an artifact reads the files the same way, so both boots seed the same files. - **The seed loader** stores the bytes through the new `field-files` service (`FIELD_FILE_SERVICE`, `IFieldFileService`, registered by `@objectstack/service-storage`), writes the file's id into the field, and makes the written record the file's owner, the (object, record, field) ownership an upload gets. Each row gets a file of its own. A replay that finds the row holding a file stored from the same path with the same bytes, name and type keeps it, so a restart does not store a new file for an unchanged fixture. A file stays `pending` until its row lands, so a file whose row is refused is reclaimed by the pending-upload retention. - **Refused, by row, before the row is written**: a reference whose bytes the dataset does not carry, bytes that do not match their recorded size and SHA-256, a load with no `field-files` service, a field mixing `seedFile(...)` with other values, and several `seedFile(...)` values in a one-file field. Each is counted, reported in the load's errors and logged at `error`. - **The inline boot seed** of an app whose datasets carry seed files runs at `kernel:bootstrapped` (as one with `demoStaff` already does), once the storage service has read its settings, so the files land in the store the deployment serves from. diff --git a/content/docs/data-modeling/seed-data.mdx b/content/docs/data-modeling/seed-data.mdx index d5b83be0282..3cbdac0ba62 100644 --- a/content/docs/data-modeling/seed-data.mdx +++ b/content/docs/data-modeling/seed-data.mdx @@ -467,7 +467,8 @@ export const contractVersionsSeed = defineSeed(ContractVersion, { - **`os build`** reads each referenced file and bundles its bytes into the compiled artifact, so the seed carries them wherever the artifact goes (`os dev`, `os serve`, a published package). A path that names no file fails - the build. + the build, and `os validate` refuses it too. `os serve` booting the config + itself reads the files the same way. - **The seed load** stores each file and writes its id into the field; the record owns the file exactly as if it had been uploaded into that field. Every row gets a file of its own, even when two rows name the same path. A replay From 3b4fff7af5a0ffeb49309c88a1c4640f31bc0439 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 26 Oct 2026 15:16:34 +0000 Subject: [PATCH 08/13] fix: gate findings (ADR-0122 aliases, slot typing, test double conformance, counts) Co-Authored-By: Claude Claude-Session: https://claude.ai/code/session_012eJQ8dtut5bZsnhFE3XGHF --- .../docs/getting-started/quick-reference.mdx | 2 +- .../src/seed-loader-seed-files.test.ts | 20 ++++++++++++++----- .../src/storage-service-plugin.ts | 7 +++++-- packages/spec/llms.txt | 4 ++-- packages/spec/src/data/seed-file.zod.ts | 4 +++- .../src/type-alias-convention.pin.test.ts | 17 +++++++++++++--- scripts/engine-double-contract.pinned.json | 15 ++++++++++++++ 7 files changed, 55 insertions(+), 14 deletions(-) diff --git a/content/docs/getting-started/quick-reference.mdx b/content/docs/getting-started/quick-reference.mdx index 85db14ba0af..a82cd4927fb 100644 --- a/content/docs/getting-started/quick-reference.mdx +++ b/content/docs/getting-started/quick-reference.mdx @@ -22,7 +22,7 @@ Categories that have no section here at all are named under [Categories Without a Section](#categories-without-a-section) — that curation is stated, not left implicit. -## Data Protocol (15 of 30 schemas) +## Data Protocol (15 of 31 schemas) Core business logic and data modeling schemas. diff --git a/packages/metadata-protocol/src/seed-loader-seed-files.test.ts b/packages/metadata-protocol/src/seed-loader-seed-files.test.ts index 47aff50312d..96969e4b983 100644 --- a/packages/metadata-protocol/src/seed-loader-seed-files.test.ts +++ b/packages/metadata-protocol/src/seed-loader-seed-files.test.ts @@ -12,6 +12,7 @@ import type { StoreFieldFileInput, } from '@objectstack/spec/contracts'; import { RAW_FILE_VALUES_CONTEXT_KEY, seedFile } from '@objectstack/spec/data'; +import { assertEngineDeleteDispatch, assertEngineUpdateDispatch, assertEngineFindOnePredicate } from '@objectstack/metadata-core'; /** * [#25779] A seed record writes `seedFile(path)` into a file field; `os build` @@ -33,17 +34,22 @@ function createEngine(): { engine: IDataEngine; store: Record; fi const store: Record = {}; const findContexts: unknown[] = []; let next = 0; + /** Equality on plain keys; a combinator or operator this fake does not implement is refused, never misread. */ + const matches = (row: Record, where: Record) => + Object.entries(where).every(([k, v]) => { + if (k.startsWith('$')) throw new Error(`fake driver: unsupported operator ${k}`); + return row[k] === v; + }); const engine = { find: vi.fn(async (object: string, query?: any) => { findContexts.push(query?.context); let rows = store[object] ?? []; - if (query?.where) rows = rows.filter((r) => Object.entries(query.where).every(([k, v]) => r[k] === v)); + if (query?.where) rows = rows.filter((r) => matches(r, query.where)); return rows.map((r) => ({ ...r })); }), findOne: vi.fn(async (object: string, query?: any) => { - const rows = (store[object] ?? []).filter((r) => - Object.entries(query?.where ?? {}).every(([k, v]) => r[k] === v), - ); + assertEngineFindOnePredicate(object, query); + const rows = (store[object] ?? []).filter((r) => matches(r, query?.where ?? {})); return rows[0] ? { ...rows[0] } : null; }), insert: vi.fn(async (object: string, data: any) => { @@ -56,12 +62,16 @@ function createEngine(): { engine: IDataEngine; store: Record; fi return Array.isArray(data) ? data.map(one) : one(data); }), update: vi.fn(async (object: string, data: any) => { + assertEngineUpdateDispatch(data, undefined); const rows = store[object] ?? []; const i = rows.findIndex((r) => r.id === data.id); if (i >= 0) rows[i] = { ...rows[i], ...data }; return data; }), - delete: vi.fn(async () => ({ deleted: 1 })), + delete: vi.fn(async (_object: string, options?: any) => { + assertEngineDeleteDispatch(options); + return { deleted: 1 }; + }), count: vi.fn(async (object: string) => (store[object] ?? []).length), aggregate: vi.fn(async () => []), } as unknown as IDataEngine; diff --git a/packages/services/service-storage/src/storage-service-plugin.ts b/packages/services/service-storage/src/storage-service-plugin.ts index 86e1359b0bb..bb5e337ef22 100644 --- a/packages/services/service-storage/src/storage-service-plugin.ts +++ b/packages/services/service-storage/src/storage-service-plugin.ts @@ -62,7 +62,7 @@ import type { TenancyPosture } from '@objectstack/spec/security'; import { registerStorageFamily } from './storage-routes.js'; import type { FileReadVerdict, StorageGatedSessionReader, StorageUploadSession } from './storage-routes.js'; import { installAttachmentLifecycleHooks, createSysFileReapGuard, createUploadSessionReapGuard, findFileHolder, findHeldFiles } from './attachment-lifecycle.js'; -import { installFileReferenceHooks, type FileReferenceReadAuthorizer } from './file-reference-lifecycle.js'; +import { installFileReferenceHooks, type FileReferenceEngine, type FileReferenceReadAuthorizer } from './file-reference-lifecycle.js'; import { installAttachmentAccessHooks, installAttachmentReadVisibility } from './attachment-access-hooks.js'; import { contributeAttachmentDeleteFloorAlternate } from './attachment-delete-floor-alternate.js'; import { FileAttachmentService, type FileAttachmentEngine } from './file-attachment-service.js'; @@ -408,7 +408,10 @@ export class StorageServicePlugin implements Plugin { new FieldFileService({ getEngine: () => { try { - return ctx.getService('objectql') ?? null; + // The kernel's data engine is ObjectQL, which also carries the + // hook and lookup surface the lifecycle's release reads. + const engine: (IDataEngine & FileReferenceEngine) | undefined = ctx.getService('objectql'); + return engine ?? null; } catch { return null; } diff --git a/packages/spec/llms.txt b/packages/spec/llms.txt index 75bfb5c00b7..5f37470ea65 100644 --- a/packages/spec/llms.txt +++ b/packages/spec/llms.txt @@ -77,7 +77,7 @@ const query = { --- -## 3. Schema Inventory by Domain (207 schemas) +## 3. Schema Inventory by Domain (208 schemas) Counted as `*.zod.ts` modules under `packages/spec/src//` — the sources that ship in this tarball (`files` includes `src/**/*.zod.ts`), so every number @@ -87,7 +87,7 @@ here is verifiable from the installed package. |--------|-------|-------------| | system | 37 | Auth, Cache, Demo Staff, Dev Login, Email / SMS Config, Encryption, Fate Supplement (organization ownership), HTTP Server, License, Logging, Metrics | | kernel | 30 | Plugin, Manifest, Events (6 sub-modules), Feature, Context, Package Registry | -| data | 31 | Object, Field, Picklist, Query, Filter, Driver (SQL/NoSQL/Memory/Mongo/Postgres), Cube | +| data | 32 | Object, Field, Picklist, Query, Filter, Driver (SQL/NoSQL/Memory/Mongo/Postgres), Cube, Seed | | api | 31 | Endpoint, REST Server, Discovery, OData, Batch, WebSocket, Response Envelope, Package Lifecycle, Package API (assembled stage) | | ui | 18 | View, App, Action, Dashboard, Page, Chart, Component, Animation | | automation | 14 | Flow, Approval, BPMN Interop, Control Flow, Webhook, Schedule Organization | diff --git a/packages/spec/src/data/seed-file.zod.ts b/packages/spec/src/data/seed-file.zod.ts index a24895374de..e32d6df3f82 100644 --- a/packages/spec/src/data/seed-file.zod.ts +++ b/packages/spec/src/data/seed-file.zod.ts @@ -62,7 +62,7 @@ export function isSeedFilePath(path: unknown): path is string { export const SeedFilePathSchema = lazySchema(() => z.string() .regex(SEED_FILE_PATH_PATTERN, SEED_FILE_PATH_RULE) .describe('Path of a file the package ships, relative to the package root (e.g. `fixtures/nda-v1.pdf`)')); -export type SeedFilePath = z.infer; +export type SeedFilePath = z.input; /** A MIME type: `type/subtype`, the shape a `sys_file.mime_type` carries. */ const MIME_TYPE = /^[a-z0-9][a-z0-9!#$&^_.+-]*\/[a-z0-9][a-z0-9!#$&^_.+-]*$/i; @@ -86,6 +86,7 @@ export const SeedFileReferenceSchema = lazySchema(() => strictObject({ /** Author-time shape of {@link SeedFileReferenceSchema}. */ export type SeedFileReference = z.input; +export type SeedFileReferenceParsed = z.infer; /** * The bytes of one seed file, as `os build` bundles them into a dataset's @@ -106,6 +107,7 @@ export const SeedFileContentSchema = lazySchema(() => strictObject({ /** Author-time shape of {@link SeedFileContentSchema}. */ export type SeedFileContent = z.input; +export type SeedFileContentParsed = z.infer; /** * Write a file the package ships into a file field from a seed record. diff --git a/packages/spec/src/type-alias-convention.pin.test.ts b/packages/spec/src/type-alias-convention.pin.test.ts index 40c6c20ac41..cf8886e2610 100644 --- a/packages/spec/src/type-alias-convention.pin.test.ts +++ b/packages/spec/src/type-alias-convention.pin.test.ts @@ -278,9 +278,11 @@ import type * as M190 from './kernel/execution-context.zod.js'; import type * as M191 from './automation/api-trigger-secret-ref.zod.js'; // [#25761] The one column apply-on vocabulary both write doors read -- new module, next free index. import type * as M192 from './data/column-apply-on.zod.js'; +// [#25779] A seed file's package-relative path -- new module, next free index. +import type * as M193 from './data/seed-file.zod.js'; // --------------------------------------------------------------------------- -// 789 isomorphic aliases: `z.input` === `z.infer`, so no `XParsed` is declared. +// 790 isomorphic aliases: `z.input` === `z.infer`, so no `XParsed` is declared. // // That number is machine-checked, not hand-kept. The runtime companion at the // bottom of this file recomputes the pin count from the source and asserts that @@ -907,6 +909,9 @@ export type Iso_data_query__AggregationNodeSchema = Assert, z.infer< typeof M71.DateGranularity > >>; export type Iso_data_query__GroupByNodeSchema = Assert, z.infer< typeof M71.GroupByNodeSchema > >>; +// data/seed-file.zod.ts +export type Iso_data_seedFile__SeedFilePathSchema = Assert, z.infer< typeof M193.SeedFilePathSchema > >>; + // data/seed-loader.zod.ts export type Iso_data_seedLoader__ReferenceResolutionErrorSchema = Assert, z.infer< typeof M72.ReferenceResolutionErrorSchema > >>; export type Iso_data_seedLoader__SeedIdentitySchema = Assert, z.infer< typeof M72.SeedIdentitySchema > >>; @@ -1710,7 +1715,7 @@ describe('ADR-0122 type-alias convention', () => { // this title and the section header above the pin list — are now asserted // against the recomputed count below, so neither can go stale without a red // test naming it. - it('still declares all 789 isomorphic pins', () => { + it('still declares all 790 isomorphic pins', () => { // The truth of each pin is proved by tsc, not here — an `Assert>` // that stops holding is a compile error with the alias named. What tsc // cannot notice is a pin that was DELETED: removing the assertion removes @@ -2600,7 +2605,13 @@ describe('ADR-0122 type-alias convention', () => { // (Iso_ui_dataset__DatasetMeasureDurationSchema, // Iso_ui_dataset__DatasetDurationUnit) rather than given `…Parsed` // synonyms. +2 added. - expect(pins).toHaveLength(789); + // + // 789 -> 790 is #25779: `SeedFilePathSchema` (`data/seed-file.zod.ts`), + // the package-relative path a `seedFile(...)` reference names — a plain + // string with a pattern, so it is pinned here + // (Iso_data_seedFile__SeedFilePathSchema) rather than given a `…Parsed` + // synonym. +1 added. + expect(pins).toHaveLength(790); // The count is stated in PROSE twice as well — this case's title and the // section header above the pin list — and until commit c6b05c76a nothing read either diff --git a/scripts/engine-double-contract.pinned.json b/scripts/engine-double-contract.pinned.json index 07cf3b05c6c..e462351e52f 100644 --- a/scripts/engine-double-contract.pinned.json +++ b/scripts/engine-double-contract.pinned.json @@ -2041,6 +2041,21 @@ "verb": "update", "pinned": 1 }, + { + "file": "packages/metadata-protocol/src/seed-loader-seed-files.test.ts", + "verb": "delete", + "pinned": 1 + }, + { + "file": "packages/metadata-protocol/src/seed-loader-seed-files.test.ts", + "verb": "findOne", + "pinned": 1 + }, + { + "file": "packages/metadata-protocol/src/seed-loader-seed-files.test.ts", + "verb": "update", + "pinned": 1 + }, { "file": "packages/metadata-protocol/src/seed-loader-sole-organization-read-failure.test.ts", "verb": "delete", From b8fe94f8933cb86c2c59031320072a113b68e5fd Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 26 Oct 2026 15:27:10 +0000 Subject: [PATCH 09/13] chore(spec): regenerate api-surface, export origins and declaration map for the seed file type aliases Co-Authored-By: Claude Claude-Session: https://claude.ai/code/session_012eJQ8dtut5bZsnhFE3XGHF --- packages/spec/api-surface/data.json | 2 ++ packages/spec/declaration-map/data.json | 1 + packages/spec/export-origins/data.json | 2 ++ 3 files changed, 5 insertions(+) diff --git a/packages/spec/api-surface/data.json b/packages/spec/api-surface/data.json index eb77a2d814b..f92492106a0 100644 --- a/packages/spec/api-surface/data.json +++ b/packages/spec/api-surface/data.json @@ -708,10 +708,12 @@ "SeedColumnParsed (type)", "SeedColumnSchema (const)", "SeedFileContent (type)", + "SeedFileContentParsed (type)", "SeedFileContentSchema (const)", "SeedFilePath (type)", "SeedFilePathSchema (const)", "SeedFileReference (type)", + "SeedFileReferenceParsed (type)", "SeedFileReferenceSchema (const)", "SeedIdentity (type)", "SeedIdentitySchema (const)", diff --git a/packages/spec/declaration-map/data.json b/packages/spec/declaration-map/data.json index 7553f0d3de5..30ad24cb2b0 100644 --- a/packages/spec/declaration-map/data.json +++ b/packages/spec/declaration-map/data.json @@ -290,6 +290,7 @@ "SeedColumnSchema": "data/SeedColumn", "SeedFileContent": "data/SeedFileContent", "SeedFileContentSchema": "data/SeedFileContent", + "SeedFilePath": "data/SeedFilePath", "SeedFilePathSchema": "data/SeedFilePath", "SeedFileReference": "data/SeedFileReference", "SeedFileReferenceSchema": "data/SeedFileReference", diff --git a/packages/spec/export-origins/data.json b/packages/spec/export-origins/data.json index bc4f58f154d..624fc004e45 100644 --- a/packages/spec/export-origins/data.json +++ b/packages/spec/export-origins/data.json @@ -696,10 +696,12 @@ "SeedColumnParsed": "src/data/seed.zod.ts#SeedColumnParsed (type)", "SeedColumnSchema": "src/data/seed.zod.ts#SeedColumnSchema (const)", "SeedFileContent": "src/data/seed-file.zod.ts#SeedFileContent (type)", + "SeedFileContentParsed": "src/data/seed-file.zod.ts#SeedFileContentParsed (type)", "SeedFileContentSchema": "src/data/seed-file.zod.ts#SeedFileContentSchema (const)", "SeedFilePath": "src/data/seed-file.zod.ts#SeedFilePath (type)", "SeedFilePathSchema": "src/data/seed-file.zod.ts#SeedFilePathSchema (const)", "SeedFileReference": "src/data/seed-file.zod.ts#SeedFileReference (type)", + "SeedFileReferenceParsed": "src/data/seed-file.zod.ts#SeedFileReferenceParsed (type)", "SeedFileReferenceSchema": "src/data/seed-file.zod.ts#SeedFileReferenceSchema (const)", "SeedIdentity": "src/data/seed-loader.zod.ts#SeedIdentity (type)", "SeedIdentitySchema": "src/data/seed-loader.zod.ts#SeedIdentitySchema (const)", From 2d0dd7dd1ad15f6cc68f7d5b02a7441598b0f040 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 26 Oct 2026 16:29:53 +0000 Subject: [PATCH 10/13] chore(changeset): copy the declaration lines verbatim from the claim Co-Authored-By: Claude Claude-Session: https://claude.ai/code/session_012eJQ8dtut5bZsnhFE3XGHF --- .changeset/25779-seed-files.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.changeset/25779-seed-files.md b/.changeset/25779-seed-files.md index b27ca2073d3..b55dc302599 100644 --- a/.changeset/25779-seed-files.md +++ b/.changeset/25779-seed-files.md @@ -8,8 +8,8 @@ feat(spec,metadata-protocol,service-storage,runtime,cli)!: a seed record can carry a file the package ships, `seedFile(path)`, and a file field's id that names no stored file is refused -Clause-②: yes (narrowing) -Metadata-protocol: narrowed +Clause-②: yes +Metadata-protocol: added From 6c5c49ccf765b46faad11b96021614633eff6435 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 26 Oct 2026 16:48:19 +0000 Subject: [PATCH 11/13] chore(changeset): declare the narrowing, as the amended claim reads Co-Authored-By: Claude Claude-Session: https://claude.ai/code/session_012eJQ8dtut5bZsnhFE3XGHF --- .changeset/25779-seed-files.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.changeset/25779-seed-files.md b/.changeset/25779-seed-files.md index b55dc302599..b27ca2073d3 100644 --- a/.changeset/25779-seed-files.md +++ b/.changeset/25779-seed-files.md @@ -8,8 +8,8 @@ feat(spec,metadata-protocol,service-storage,runtime,cli)!: a seed record can carry a file the package ships, `seedFile(path)`, and a file field's id that names no stored file is refused -Clause-②: yes -Metadata-protocol: added +Clause-②: yes (narrowing) +Metadata-protocol: narrowed From 976a6ee6ca4f655a3a0024c15f18f3fc17a58efd Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 26 Oct 2026 17:26:35 +0000 Subject: [PATCH 12/13] fix: the dropped-refinements ledger header counts this PR's sites; a test record names no error code The ledger gained two entries and 26 dropped-refinement sites (the seed record value's refinement) without its measured totals; they now read 247 and 866. A bundle test's record field named code read as an error-code literal to the casing guard; it is a name. Co-Authored-By: Claude Claude-Session: https://claude.ai/code/session_012eJQ8dtut5bZsnhFE3XGHF --- packages/cli/src/utils/bundle-seed-files.test.ts | 2 +- packages/spec/dropped-refinements.baseline.json | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/cli/src/utils/bundle-seed-files.test.ts b/packages/cli/src/utils/bundle-seed-files.test.ts index dbe1d9f30bf..db2fdc25cdb 100644 --- a/packages/cli/src/utils/bundle-seed-files.test.ts +++ b/packages/cli/src/utils/bundle-seed-files.test.ts @@ -85,7 +85,7 @@ describe('collectSeedFiles / placeSeedFiles', () => { it('refuses a path that names no file, saying where the reference sits', () => { const result = collectSeedFiles( packageRoot({}), - { data: [{ object: 'x', records: [{ code: 'a' }, { file: seedFile('fixtures/missing.pdf') }] }] }, + { data: [{ object: 'x', records: [{ name: 'a' }, { file: seedFile('fixtures/missing.pdf') }] }] }, ); expect(result.issues).toHaveLength(1); expect(result.issues[0].location).toBe('data[0].records[1].file'); diff --git a/packages/spec/dropped-refinements.baseline.json b/packages/spec/dropped-refinements.baseline.json index 6669008dac5..39f5070c6b8 100644 --- a/packages/spec/dropped-refinements.baseline.json +++ b/packages/spec/dropped-refinements.baseline.json @@ -2,8 +2,8 @@ "description": "Shrink-only ledger of every PUBLISHED JSON Schema that is STILL WIDER than the Zod type it was generated from, because a rule written as `.refine()` reaches the runtime and not the file (#18670). `z.toJSONSchema()` has no arm for a `custom` check: a plain record, the same record with a `.refine()`, and the same record with an ABORTING `.refine()` all project byte-identically (measured on zod 4.4.3, the version packages/spec resolves). So a document one of these files ACCEPTS can still be refused at parse time, and an author -- or an AI -- validating against packages/spec/json-schema/** finds out a release later. Each `sites` path is a position under that schema at which a refinement is dropped; the same paths are written onto the artifact itself as `x-dropped-refinements`. Item 2 closed the first patterns: a refinement DECLARED through the closed list in src/shared/refinement-projection.ts is emitted into the published file, reads `projected` rather than `dropped`, and its row LEAVES this ledger in the same PR -- which is why the ledger shrinks and never grows on a repair. Every refinement outside that closed list stays here, and adding an arm to the list is a public-contract decision, not a refactor. Hand-edited on purpose and with no `gen:` script: a generator would let a new gap be admitted by running a command instead of by a decision, which is the silence this ledger exists to end. Adding, removing or moving a site fails packages/spec/scripts/build-schemas.ts until the line moves with it, and the failure prints the corrected entry in full. ⛔ Do not delete or weaken a refinement to shorten this file -- the runtime rule is correct; it is the projection that is silent, and the remedy is to teach the closed list a NAMED pattern, never to drop the rule.", "measured": { "zod": "4.4.3", - "publishedSchemasWithDroppedRefinements": 245, - "droppedRefinementSites": 840, + "publishedSchemasWithDroppedRefinements": 247, + "droppedRefinementSites": 866, "refinementSitesThatDidProject": 369, "refinementSitesWithNoJsonFormToCompare": 0 }, From c33b95d89fa4a66dd871a0ab617f49bf870421e9 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 26 Oct 2026 17:28:17 +0000 Subject: [PATCH 13/13] fix(spec): the seed-files liveness row names placeSeedFiles; the level-1 description pin counts seed-file.zod.ts Co-Authored-By: Claude Claude-Session: https://claude.ai/code/session_012eJQ8dtut5bZsnhFE3XGHF --- packages/spec/liveness/seed.json | 2 +- packages/spec/scripts/file-description.test.ts | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/packages/spec/liveness/seed.json b/packages/spec/liveness/seed.json index 4e3d251d586..9fcb1924717 100644 --- a/packages/spec/liveness/seed.json +++ b/packages/spec/liveness/seed.json @@ -62,7 +62,7 @@ "status": "live", "verifiedAt": "2026-10-26", "evidenceScope": "in-repo", - "evidence": "packages/metadata-protocol/src/seed-files.ts#SeedDatasetFiles (`check`: `Object.prototype.hasOwnProperty.call(this.files, path)` — the bundled bytes a `seedFile(path)` reference is stored from, decoded once per dataset and checked against their recorded size and SHA-256); packages/cli/src/utils/bundle-seed-files.ts#bundleSeedFiles (the producer: `os build` writes each referenced file's bytes here, read relative to the package root)", + "evidence": "packages/metadata-protocol/src/seed-files.ts#SeedDatasetFiles (`check`: `Object.prototype.hasOwnProperty.call(this.files, path)` — the bundled bytes a `seedFile(path)` reference is stored from, decoded once per dataset and checked against their recorded size and SHA-256); packages/cli/src/utils/bundle-seed-files.ts#placeSeedFiles (the producer: `os build` writes each referenced file's bytes here, after `collectSeedFiles` reads them relative to the package root)", "note": "Seeded 2026-10-26 (#25779, ruled on hotclm#19 letter D). The bytes of the package files the records name with `seedFile(path)`, keyed by that path; written by `os build`, never by an author. The keys are package-relative paths (the `records` walk boundary's rule applies to the records, not to this map), so the ledger drills only the bundled file's own keys.", "children": { "contentType": { diff --git a/packages/spec/scripts/file-description.test.ts b/packages/spec/scripts/file-description.test.ts index 0dd904eb228..61e12aeae5d 100644 --- a/packages/spec/scripts/file-description.test.ts +++ b/packages/spec/scripts/file-description.test.ts @@ -2225,10 +2225,11 @@ describe('corpus — every rendered description is well-formed markdown', () => // level-1 opener among the four modules that moved to `marketplace/`, // still counts. 31 -> 30: `kernel/package-upgrade.zod.ts` — a level-1 // opener (`# Package Upgrade Protocol`) — was removed whole by #25608's - // ADR-0049 retirement.) - expect(shifted.length).toBe(30); + // ADR-0049 retirement. 30 -> 31: `data/seed-file.zod.ts` (#25779) is a new + // module whose header opens at level 1 (`# Seed files`).) + expect(shifted.length).toBe(31); // …and every one of them was shifted because it opened at level 1. - expect(shifted.filter(d => /^ {0,3}#(?:[ \t]|$)/m.test(withoutFences(d.untouched)))).toHaveLength(30); + expect(shifted.filter(d => /^ {0,3}#(?:[ \t]|$)/m.test(withoutFences(d.untouched)))).toHaveLength(31); }); it('never opens a page on the `@module` marker (#13796)', () => {