From 0dec20d2181258c0a623b2eb3bf153ebddd2440c Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 25 Oct 2026 13:44:32 +0000 Subject: [PATCH 01/14] wip(spec): dataset duration measure schema --- .../src/data/analytics-column-reference.ts | 14 +- packages/spec/src/ui/dataset.form.ts | 1 + packages/spec/src/ui/dataset.zod.ts | 185 +++++++++++++++++- 3 files changed, 195 insertions(+), 5 deletions(-) diff --git a/packages/spec/src/data/analytics-column-reference.ts b/packages/spec/src/data/analytics-column-reference.ts index 2bb692602dc..9666aef5a56 100644 --- a/packages/spec/src/data/analytics-column-reference.ts +++ b/packages/spec/src/data/analytics-column-reference.ts @@ -70,8 +70,11 @@ * `../ui/analytics-carrier-filter.ts` precedent). */ +/** One bare identifier — a column name, or one hop of a path. */ +const IDENTIFIER = '[A-Za-z_][A-Za-z0-9_]*'; + /** A bare identifier, then zero or more `.identifier` hops — the column path. */ -const COLUMN_PATH = '[A-Za-z_][A-Za-z0-9_]*(?:\\.[A-Za-z_][A-Za-z0-9_]*)*'; +const COLUMN_PATH = `${IDENTIFIER}(?:\\.${IDENTIFIER})*`; /** * A column of the object, a relationship path ending in one, or the row @@ -88,6 +91,15 @@ export const ANALYTICS_COLUMN_REFERENCE = new RegExp(`^(?:\\*|${COLUMN_PATH})$`) */ export const ANALYTICS_COLUMN_PATH = new RegExp(`^${COLUMN_PATH}$`); +/** + * [#25778] {@link ANALYTICS_COLUMN_PATH} without the relationship hops — one + * column of the dataset's OWN object, by its bare name: the slot of a duration + * measure's `from` / `to` (`DatasetMeasureDurationSchema`, + * `../ui/dataset.zod.ts`), which reads two columns of the same row. Built from + * the same identifier as {@link COLUMN_PATH}, never a second spelling of it. + */ +export const ANALYTICS_BASE_COLUMN = new RegExp(`^${IDENTIFIER}$`); + /** The row wildcard — what a `count` aggregates (`COUNT(*)`). */ const ROW_WILDCARD = '*'; diff --git a/packages/spec/src/ui/dataset.form.ts b/packages/spec/src/ui/dataset.form.ts index b09b2b3e878..9f729c412ee 100644 --- a/packages/spec/src/ui/dataset.form.ts +++ b/packages/spec/src/ui/dataset.form.ts @@ -91,6 +91,7 @@ export const datasetForm = defineForm({ { field: 'format', label: 'Format' }, { field: 'currency', label: 'Currency' }, { field: 'derived', label: 'Derived From' }, + { field: 'duration', label: 'Duration' }, ], }, ], diff --git a/packages/spec/src/ui/dataset.zod.ts b/packages/spec/src/ui/dataset.zod.ts index 27136740a6e..21cdd43f74b 100644 --- a/packages/spec/src/ui/dataset.zod.ts +++ b/packages/spec/src/ui/dataset.zod.ts @@ -10,6 +10,7 @@ import { SnakeCaseIdentifierSchema } from '../shared/identifiers.zod'; import { I18nLabelSchema } from './i18n.zod'; import { AggregationFunction, DateGranularity } from '../data/query.zod'; import { + ANALYTICS_BASE_COLUMN, ANALYTICS_COLUMN_PATH, ANALYTICS_COLUMN_REFERENCE, rowWildcardOutsideCount, @@ -81,8 +82,9 @@ const DATASET_HISTORY = const DATASET_NO_SQL = 'the dataset layer takes no raw SQL — it is deliberately smaller than a query (ADR-0021). ' + 'A dimension names a `field` (a base field or a `relationship[.relationship].field` path); ' - + 'a measure names an `aggregate` + `field`, and the only computed form is ' - + '`derived: { op, of: [...] }`, which combines OTHER measures in this dataset by name. ' + + 'a measure names an `aggregate` + `field`, and the two computed forms are ' + + '`derived: { op, of: [...] }`, which combines OTHER measures in this dataset by name, and ' + + '`duration: { from, to, unit }`, which aggregates the time between two datetime columns of each record. ' + 'Joins are compiled from `Dataset.include` — you never write an ON clause.'; /** @@ -128,10 +130,13 @@ const DATASET_MEASURE_FIELD_NOT_COLUMN = + `a count may also omit \`field\`. ${DATASET_FIELD_EXPRESSION_REFUSED} Name the column the measure ` + 'aggregates. A derived value is declared in the ADR-0021 form, where the platform judges every field it ' + 'reads: a conditional count or sum is a measure with its own structured `filter` ' - + '(`{ name: \'done_count\', aggregate: \'count\', filter: { status: \'done\' } }`), and a ratio, sum, ' + + '(`{ name: \'done_count\', aggregate: \'count\', filter: { status: \'done\' } }`), a ratio, sum, ' + 'difference or product of measures is `derived: { op, of: [...] }` over measures named in this dataset ' + '(`{ name: \'done_rate\', derived: { op: \'ratio\', of: [\'done_count\', \'task_count\'] }, format: ' - + '\'0.0%\' }` — a 0–1 fraction, which the `%` pattern displays as a percentage).'; + + '\'0.0%\' }` — a 0–1 fraction, which the `%` pattern displays as a percentage), and the time between two ' + + 'datetime columns of the same record is `duration: { from, to, unit }` ' + + '(`{ name: \'avg_cycle_days\', aggregate: \'avg\', duration: { from: \'submitted_at\', to: \'activated_at\', ' + + 'unit: \'days\' } }`).'; /** * Dimension — a groupable axis (e.g. "region", "close_date by quarter"). @@ -207,6 +212,104 @@ export const DatasetDimensionSchema = lazySchema(() => strictObject({ */ export const DerivedMeasureOp = z.enum(['ratio', 'sum', 'difference', 'product']); +/** + * [#25778] The unit a duration measure's number is expressed in. Fixed-length + * units only: a month or a year has no one length in seconds, so a duration in + * either would depend on which calendar months the interval crosses. + */ +export const DatasetDurationUnit = z.enum(['seconds', 'minutes', 'hours', 'days']); + +/** + * [#25778] Seconds per {@link DatasetDurationUnit} — the one conversion every + * consumer divides by (the SQL compiler, the draft-preview evaluator, a + * renderer suffixing the unit), so no two of them can disagree about a day. + */ +export const DATASET_DURATION_UNIT_SECONDS: Readonly, number>> = + Object.freeze({ seconds: 1, minutes: 60, hours: 3600, days: 86400 }); + +/** + * [#25778] The aggregates a duration measure takes. Each reads the per-record + * duration as a number: `avg` and `sum` add them up, `min` and `max` pick one. + * `count` and `count_distinct` are not here: a count of records with both + * instants set is a `count` measure with a `filter`, not a duration. + */ +export const DATASET_DURATION_AGGREGATES = ['avg', 'min', 'max', 'sum'] as const; + +const DATASET_DURATION_COLUMN_NOT_BASE = + 'a duration reads two columns of the SAME record: name a `datetime` field of the dataset\'s own object ' + + '(`submitted_at`), by its bare name. A relationship path, a SQL expression, `\'*\'` or an empty string ' + + 'is refused: the dataset layer takes no raw SQL or expressions (ADR-0021).'; + +/** + * [#25778] A duration measure's column pair: the time from `from` to `to` on + * each record, in `unit`, which the measure's `aggregate` then reduces. + * + * Both columns are `datetime` fields of the dataset's own object — the time + * between two instants of one record (cycle time, time in stage, time to + * first response). The type is judged where the declarations are in hand: at + * authoring (`measure-duration-field-type-refused`) and when the dataset is + * compiled (`400 DATASET_INVALID`). + * + * A record whose `from` or `to` is null contributes nothing: it is excluded + * from `avg`, `min`, `max` and `sum` alike, so `avg` is the mean over records + * with both instants set. A record whose `to` is before its `from` contributes + * a NEGATIVE duration, as written — the measure reports the data, it does not + * repair it; filter such records out with the measure's `filter` if they are + * errors. + * + * The result is a number of `unit`s (`1.5` with `unit: 'days'` is a day and a + * half), the same on every SQL dialect the analytics runtime serves. No + * expression language and no raw SQL: the pair is two column names and a unit + * word, so every field it reads stays judgeable (ADR-0021). + */ +export const DatasetMeasureDurationSchema = lazySchema(() => strictObject({ + surface: 'this duration-measure spec', + history: DATASET_HISTORY, + aliases: { + start: 'from', + begin: 'from', + since: 'from', + startField: 'from', + fromField: 'from', + end: 'to', + until: 'to', + endField: 'to', + toField: 'to', + units: 'unit', + interval: 'unit', + }, + guidance: { + sql: DATASET_NO_SQL, + expression: DATASET_NO_SQL, + formula: DATASET_NO_SQL, + field: + 'a duration reads TWO columns — name them in `from` (the earlier instant) and `to` (the later one); ' + + 'the measure itself carries no `field`.', + }, +}, { + from: z.string() + .regex(ANALYTICS_BASE_COLUMN, { error: () => DATASET_DURATION_COLUMN_NOT_BASE }) + .describe('The earlier instant: a datetime field of the dataset\'s own object, by its bare name.') + .meta({ title: 'From' }), + to: z.string() + .regex(ANALYTICS_BASE_COLUMN, { error: () => DATASET_DURATION_COLUMN_NOT_BASE }) + .describe('The later instant: a datetime field of the dataset\'s own object, by its bare name.') + .meta({ title: 'To' }), + unit: DatasetDurationUnit + .describe('Unit of the result: the measure answers a number of these (1.5 with "days" is a day and a half).') + .meta({ title: 'Unit' }), +}).superRefine((duration, ctx) => { + if (duration.from === duration.to) { + ctx.addIssue({ + code: 'custom', + path: ['to'], + message: + `\`from\` and \`to\` both name "${duration.from}", so every record's duration is zero. ` + + 'Name the two instants the duration spans — e.g. `from: \'submitted_at\'`, `to: \'activated_at\'`.', + }); + } +})); + /** * Measure — an aggregatable value (e.g. "revenue = sum(amount)"). Defined ONCE * here; every presentation references it by name. @@ -243,6 +346,9 @@ export const DatasetMeasureSchema = lazySchema(() => strictObject({ // Computed measures. calculated: 'derived', computed: 'derived', + // [#25778] The column-pair measure. + elapsed: 'duration', + timeBetween: 'duration', }, guidance: { sql: DATASET_NO_SQL, @@ -361,7 +467,72 @@ export const DatasetMeasureSchema = lazySchema(() => strictObject({ /** Names of other measures in this dataset (2+ for ratio/difference). */ of: z.array(SnakeCaseIdentifierSchema).min(1), }).optional().meta({ title: 'Derived From' }), + /** + * [#25778] Duration measure — the time between two `datetime` columns of + * each record, in `unit`, reduced by `aggregate` (`avg` / `min` / `max` / + * `sum`): e.g. `{ aggregate: 'avg', duration: { from: 'submitted_at', to: + * 'activated_at', unit: 'days' } }` is the average cycle time in days. See + * {@link DatasetMeasureDurationSchema}. Excludes `field` and `derived`: the + * two columns are the measure's source. + */ + duration: DatasetMeasureDurationSchema.optional().describe( + 'Time between two datetime columns of each record ({ from, to, unit }), reduced by `aggregate` ' + + '(avg / min / max / sum). Records with either instant null are excluded. Replaces `field`.', + ).meta({ title: 'Duration' }), }).superRefine((measure, ctx) => { + // [#25778] A duration measure's source is its column pair, and its value is + // a number of `unit`s: so no `field` beside it (two sources), no `derived` + // (two computed forms), only the aggregates that read a number, and no + // `currency` (a duration is no amount of money). + if (measure.duration) { + if (measure.field !== undefined) { + ctx.addIssue({ + code: 'custom', + path: ['field'], + message: + 'a duration measure reads the two columns named in `duration.from` / `duration.to` — it takes no ' + + '`field`. Delete `field`, or delete `duration` to aggregate one column.', + }); + } + if (measure.derived !== undefined) { + ctx.addIssue({ + code: 'custom', + path: ['derived'], + message: + 'a measure is either `derived` (combines other measures by name) or a `duration` (two columns of ' + + 'each record), never both. Declare them as two measures; a derived measure may name the duration one in `of`.', + }); + } + if (measure.aggregate === undefined) { + ctx.addIssue({ + code: 'custom', + path: ['aggregate'], + message: + `a duration measure needs \`aggregate\` — one of ${DATASET_DURATION_AGGREGATES.join(', ')} — to reduce ` + + 'the per-record durations to one number (e.g. `aggregate: \'avg\'` for the average cycle time).', + }); + } else if (!(DATASET_DURATION_AGGREGATES as readonly string[]).includes(measure.aggregate)) { + ctx.addIssue({ + code: 'custom', + path: ['aggregate'], + message: + `a duration measure takes \`aggregate\` ${DATASET_DURATION_AGGREGATES.join(' / ')}, not ` + + `"${measure.aggregate}": the per-record duration is a number to average, add up or pick the least or ` + + 'greatest of. To count records, declare a `count` measure (with a `filter` for the records whose ' + + 'two instants are set).', + }); + } + if (measure.currency !== undefined) { + ctx.addIssue({ + code: 'custom', + path: ['currency'], + message: + 'a duration measure answers a number of `duration.unit`s, not an amount of money — delete `currency`. ' + + 'Decimals and grouping go in `format` (e.g. "0.0").', + }); + } + } + // [#21409] `'*'` is the row wildcard a `count` aggregates (`COUNT(*)`), and // only a `count` consumes it: under any other aggregate — or none, as on a // `derived` measure — it names no column. Measured at @@ -463,6 +634,10 @@ export const DatasetSchema = lazySchema(() => strictObject({ } // Derived measures may only reference OTHER measures declared in this dataset. for (const m of ds.measures) { + // [#25778] A duration measure's aggregate and column pair are judged on the + // measure itself (`DatasetMeasureSchema`'s refinement); it carries no + // `field` and names no other measure, so neither check below is its. + if (m.duration) continue; if (!m.derived) { // A non-derived measure must declare an aggregate (a derived measure // omits it — it combines other measures by name instead). @@ -518,6 +693,8 @@ export function defineDataset(dataset: Dataset): Dataset { export type DatasetDimension = z.input; export type DatasetMeasure = z.input; +export type DatasetMeasureDuration = z.input; +export type DatasetDurationUnitValue = z.input; export type DerivedMeasureOpValue = z.input; export type Dataset = z.input; From f18374527a7f5b6334ed0f3607a3d966df99bbe6 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 25 Oct 2026 13:52:36 +0000 Subject: [PATCH 02/14] wip(analytics): compile dataset duration measures into the scope channel Co-Authored-By: Claude Claude-Session: https://claude.ai/code/session_01AKPixnkg91qCkQCVFDJuqW --- .../service-analytics/src/dataset-compiler.ts | 93 ++++++++++++++++++- .../service-analytics/src/strategies/types.ts | 30 ++++++ packages/spec/authorable-surface/ui.json | 4 + packages/spec/json-schema.manifest/ui.json | 2 + packages/spec/src/ui/dataset.zod.ts | 21 +++-- 5 files changed, 135 insertions(+), 15 deletions(-) diff --git a/packages/services/service-analytics/src/dataset-compiler.ts b/packages/services/service-analytics/src/dataset-compiler.ts index 170605d6319..05037854b93 100644 --- a/packages/services/service-analytics/src/dataset-compiler.ts +++ b/packages/services/service-analytics/src/dataset-compiler.ts @@ -10,10 +10,11 @@ import { virtualFieldAggregateRemedy, } from '@objectstack/spec/data'; import type { Dataset, DatasetMeasure, DatasetDimension } from '@objectstack/spec/ui'; -import { resolveI18nLabel } from '@objectstack/spec/ui'; +import { DATASET_DURATION_AGGREGATES, resolveI18nLabel } from '@objectstack/spec/ui'; import type { FilterCondition } from '@objectstack/spec/data'; import { datasetInvalidError } from './dataset-refusal.js'; import { TEMPORAL_SOURCE_FIELD_TYPES } from './measure-result-type.js'; +import type { DurationMeasureSpec } from './strategies/types.js'; /** * Dataset → Cube compiler (ADR-0021 D-A=(c), WS2). @@ -86,6 +87,12 @@ export interface CompiledDataset { filter?: FilterCondition; /** Per-measure scoped filters, keyed by measure name (applied by executor). */ measureFilters: Record; + /** + * [#25778] Duration measures, keyed by measure name — NOT in `cube.measures` + * (a cube metric reads one column, a duration two). Carried to the + * strategies on the dataset-scope channel; see `DatasetScope.measureDurations`. + */ + durations: Record; } /** @@ -260,8 +267,10 @@ const DIVERGENCE_BY_AGGREGATE = (aggregate: string, fieldType: string, column = * prescribes; it never re-states which pairs are accepted (that is the table's * line above it). * - * The temporal sentence is the one #16737 measured and shipped, kept verbatim - * for the class it was written about. The deriving sentence is #16099's + * The temporal sentence is the one #16737 measured and shipped; [#25778] its + * second half named a stored "days open" number as the only way to a duration, + * and now names the `duration` measure that answers one without a stored copy. + * The deriving sentence is #16099's * generalisation: counting is the aggregate that reads no arithmetic off a * value, so it is what an author who wanted "how much text is there" actually * wants. [#17560] The selecting sentence is the third: an author who wrote @@ -281,8 +290,9 @@ const REMEDY_BY_SOURCE_CLASS = (fieldType: string, aggregate: string): string => const virtual = virtualFieldAggregateRemedy(aggregate, fieldType); if (virtual !== undefined) return virtual; return TEMPORAL_SOURCE_FIELD_TYPES.has(fieldType) - ? 'For a temporal field, `min`/`max` return a real instant; a DURATION has to be ' - + 'stored as a number (a computed "days open" field) and aggregated as one.' + ? 'For a temporal field, `min`/`max` return a real instant; the time BETWEEN two `datetime` ' + + 'fields of a record is a `duration: { from, to, unit }` measure, aggregated by ' + + `${DATASET_DURATION_AGGREGATES.join(' / ')}.` : aggregate === 'count_distinct' ? 'For a JSON-stored field, `count` counts the rows; a distinct count has to be taken ' + 'over a field that stores one scalar value, so store the part you count in a field ' @@ -441,6 +451,62 @@ function assertAggregateFieldTypeCompatible( ); } +/** + * The one field type a duration measure's `from` / `to` may be declared as. + * A `datetime` is an instant (ADR-0053), and the time between two instants is + * a duration on every backend. A `date` is a calendar day with no instant and + * a `time` a wall clock with no day, so neither denotes a point a duration can + * be measured from. + */ +const DURATION_COLUMN_TYPE = 'datetime'; + +/** + * [#25778] Compile one dataset measure's `duration` — the column pair, the + * unit, and the measure's aggregate — after judging what the metadata can + * settle before any query runs. + * + * - The AGGREGATE is one of `DATASET_DURATION_AGGREGATES`. The spec refuses + * any other at parse; this is the guard for a dataset that reaches the + * compiler without meeting that parse (the same posture as every other + * `DATASET_INVALID` here). + * - Each COLUMN is a field the base object declares as `datetime`, judged + * through the same `declaredValueShape` hook the aggregate × field-type gate + * reads, with the same tiering: no hook, or a field the hook cannot resolve, + * is not judged. A column of another declared type is refused, naming the + * column, its type and the one type accepted — so a `text` column holding + * ISO strings, or a `date`, never reaches the SQL as an instant it is not. + * An undeclared column is the field gate's and the database's to refuse. + */ +function compileDurationMeasure( + datasetName: string, + objectName: string, + measure: DatasetMeasure & { duration: NonNullable }, + declaredValueShape?: (objectName: string, field: string) => ValueShapeFieldDef | undefined, +): DurationMeasureSpec { + const { from, to, unit } = measure.duration; + const aggregate = measure.aggregate; + if (!aggregate || !(DATASET_DURATION_AGGREGATES as readonly string[]).includes(aggregate)) { + throw datasetInvalidError( + `[dataset-compiler] dataset "${datasetName}" duration measure "${measure.name}" takes \`aggregate\` ` + + `${DATASET_DURATION_AGGREGATES.join(' / ')}; it declares ${aggregate ? `"${aggregate}"` : 'none'}.`, + ); + } + if (declaredValueShape) { + for (const [slot, column] of [['from', from], ['to', to]] as const) { + const fieldType = declaredValueShape(objectName, column)?.type; + if (!fieldType || fieldType === DURATION_COLUMN_TYPE) continue; + throw datasetInvalidError( + `[dataset-compiler] dataset "${datasetName}" duration measure "${measure.name}" reads ` + + `\`duration.${slot}\` "${column}", which object "${objectName}" declares as \`${fieldType}\`. ` + + `A duration spans two \`${DURATION_COLUMN_TYPE}\` fields of the same record, because an instant is ` + + `the one value every backend measures the time between alike; a \`${fieldType}\` field holds no ` + + `instant. Point \`duration.${slot}\` at a \`${DURATION_COLUMN_TYPE}\` field of "${objectName}".`, + ); + } + } + return { aggregate: aggregate as DurationMeasureSpec['aggregate'], from, to, unit }; +} + /** Map a dataset dimension type to the Cube dimension `type`. */ function dimensionType(d: DatasetDimension): CubeDimension['type'] { switch (d.type) { @@ -707,12 +773,28 @@ export function compileDataset( const measures: Record = {}; const derived: DerivedMeasureSpec[] = []; const measureFilters: Record = {}; + const durations: Record = {}; for (const m of dataset.measures) { if (m.derived) { derived.push({ name: m.name, op: m.derived.op, of: m.derived.of }); continue; } + // [#25778] A duration measure reads two columns, which no cube metric can + // carry: it goes to the dataset-scope sidecar and NOT into + // `cube.measures`, so any reader that does not know it refuses it rather + // than aggregating one of its columns (see `DatasetScope.measureDurations`). + // Its scoped `filter` rides the ordinary channel. + if (m.duration) { + durations[m.name] = compileDurationMeasure( + dataset.name, + dataset.object, + m as DatasetMeasure & { duration: NonNullable }, + options?.declaredValueShape, + ); + if (m.filter) measureFilters[m.name] = m.filter; + continue; + } if (m.field) assertDeclared(m.field, 'measure', m.name); // [#16737 / #16099] …and the aggregate ITSELF must be one the field's // declared type can carry. Placed after the join-declaration check so a @@ -755,5 +837,6 @@ export function compileDataset( derived, filter: dataset.filter, measureFilters, + durations, }; } diff --git a/packages/services/service-analytics/src/strategies/types.ts b/packages/services/service-analytics/src/strategies/types.ts index 8a840020c65..faa095ed33c 100644 --- a/packages/services/service-analytics/src/strategies/types.ts +++ b/packages/services/service-analytics/src/strategies/types.ts @@ -17,6 +17,7 @@ export type { import type { FilterCondition, ValueShapeFieldDef } from '@objectstack/spec/data'; import type { IObjectQLEngine, StrategyContext } from '@objectstack/spec/contracts'; +import type { DATASET_DURATION_AGGREGATES, DatasetDurationUnitValue } from '@objectstack/spec/ui'; /** * [#19995] The engine's judge-only `where` admission, as the contract declares @@ -65,6 +66,35 @@ export interface DatasetScope { filter?: FilterCondition; /** Per-measure scoped filters, keyed by measure name. */ measureFilters?: Record; + /** + * [#25778] The dataset's duration measures, keyed by measure name — the + * second half of a compiled dataset the Cube cannot carry. A cube metric + * names ONE column; a duration reads two, so a duration measure is + * deliberately ABSENT from `cube.measures` and present here only. + * + * Absent rather than approximated, so it fails closed: a reader that does + * not know this channel finds no such measure and refuses it (`INVALID_FIELD` + * / 400), where a cube metric standing in for it (an aggregate over one of + * its columns) would have answered an instant, or the average year (#16737), + * under the duration's name. The readers that DO serve it: the native-SQL + * strategy (`duration-measure-sql.ts`), the draft-preview evaluator, and the + * service's own gates (the field-level read gate, the measure mint, the + * strategy-route refusal). + */ + measureDurations?: Record; +} + +/** + * [#25778] One compiled duration measure: the time from `from` to `to` — two + * `datetime` columns of the dataset's base object — on each record, in `unit`, + * reduced by `aggregate`. The dataset's `duration` key (`@objectstack/spec/ui`, + * `DatasetMeasureDurationSchema`) with the measure's aggregate beside it. + */ +export interface DurationMeasureSpec { + aggregate: (typeof DATASET_DURATION_AGGREGATES)[number]; + from: string; + to: string; + unit: DatasetDurationUnitValue; } /** A {@link StrategyContext} that can answer for a compiled dataset (#10298). */ diff --git a/packages/spec/authorable-surface/ui.json b/packages/spec/authorable-surface/ui.json index 985a29bc6b4..e509a524ffc 100644 --- a/packages/spec/authorable-surface/ui.json +++ b/packages/spec/authorable-surface/ui.json @@ -416,11 +416,15 @@ "ui/DatasetMeasure:aggregate", "ui/DatasetMeasure:currency", "ui/DatasetMeasure:derived", + "ui/DatasetMeasure:duration", "ui/DatasetMeasure:field", "ui/DatasetMeasure:filter", "ui/DatasetMeasure:format", "ui/DatasetMeasure:label", "ui/DatasetMeasure:name", + "ui/DatasetMeasureDuration:from", + "ui/DatasetMeasureDuration:to", + "ui/DatasetMeasureDuration:unit", "ui/DocNavItem:badge", "ui/DocNavItem:badgeVariant", "ui/DocNavItem:book", diff --git a/packages/spec/json-schema.manifest/ui.json b/packages/spec/json-schema.manifest/ui.json index 7dd721b84a1..a7091bd7773 100644 --- a/packages/spec/json-schema.manifest/ui.json +++ b/packages/spec/json-schema.manifest/ui.json @@ -47,7 +47,9 @@ "ui/DashboardWidgetOptions", "ui/Dataset", "ui/DatasetDimension", + "ui/DatasetDurationUnit", "ui/DatasetMeasure", + "ui/DatasetMeasureDuration", "ui/DerivedMeasureOp", "ui/DocNavItem", "ui/ElementApiDataSource", diff --git a/packages/spec/src/ui/dataset.zod.ts b/packages/spec/src/ui/dataset.zod.ts index 21cdd43f74b..e13a7d1690a 100644 --- a/packages/spec/src/ui/dataset.zod.ts +++ b/packages/spec/src/ui/dataset.zod.ts @@ -298,16 +298,6 @@ export const DatasetMeasureDurationSchema = lazySchema(() => strictObject({ unit: DatasetDurationUnit .describe('Unit of the result: the measure answers a number of these (1.5 with "days" is a day and a half).') .meta({ title: 'Unit' }), -}).superRefine((duration, ctx) => { - if (duration.from === duration.to) { - ctx.addIssue({ - code: 'custom', - path: ['to'], - message: - `\`from\` and \`to\` both name "${duration.from}", so every record's duration is zero. ` - + 'Name the two instants the duration spans — e.g. `from: \'submitted_at\'`, `to: \'activated_at\'`.', - }); - } })); /** @@ -522,6 +512,17 @@ export const DatasetMeasureSchema = lazySchema(() => strictObject({ + 'two instants are set).', }); } + // Judged here rather than on the duration spec itself, so the published + // JSON Schema keeps ONE dropped-refinement site for the measure. + if (measure.duration.from === measure.duration.to) { + ctx.addIssue({ + code: 'custom', + path: ['duration', 'to'], + message: + `\`duration.from\` and \`duration.to\` both name "${measure.duration.from}", so every record's duration ` + + 'is zero. Name the two instants the duration spans — e.g. `from: \'submitted_at\'`, `to: \'activated_at\'`.', + }); + } if (measure.currency !== undefined) { ctx.addIssue({ code: 'custom', From c06b33cdb8e2fa2677308d3687dd97826f5e7afd Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 25 Oct 2026 14:08:32 +0000 Subject: [PATCH 03/14] wip(analytics): serve dataset duration measures on the native SQL face Co-Authored-By: Claude Claude-Session: https://claude.ai/code/session_01AKPixnkg91qCkQCVFDJuqW --- .../dataset-duration-measure.test.ts | 339 ++++++++++++++++++ .../native-sql-duration-measure.test.ts | 304 ++++++++++++++++ .../src/analytics-service.ts | 112 ++++-- .../service-analytics/src/dataset-compiler.ts | 29 +- .../service-analytics/src/dataset-executor.ts | 5 +- .../src/duration-measure-sql.ts | 95 +++++ .../src/preview-evaluator.ts | 29 ++ .../src/strategies/native-sql-strategy.ts | 80 ++++- 8 files changed, 962 insertions(+), 31 deletions(-) create mode 100644 packages/services/service-analytics/src/__tests__/dataset-duration-measure.test.ts create mode 100644 packages/services/service-analytics/src/__tests__/native-sql-duration-measure.test.ts create mode 100644 packages/services/service-analytics/src/duration-measure-sql.ts diff --git a/packages/services/service-analytics/src/__tests__/dataset-duration-measure.test.ts b/packages/services/service-analytics/src/__tests__/dataset-duration-measure.test.ts new file mode 100644 index 00000000000..a847ba193e2 --- /dev/null +++ b/packages/services/service-analytics/src/__tests__/dataset-duration-measure.test.ts @@ -0,0 +1,339 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * [#25778] A dataset DURATION measure — `duration: { from, to, unit }` — answers + * the aggregated time between two `datetime` columns of the same record, the + * same number on SQLite and on PostgreSQL. + * + * The fixture's durations are chosen to be exact in a double at every unit the + * assertions read: 90 minutes, a whole day, three quarters of a second (a + * millisecond-precision instant), and a NEGATIVE hour (a `to` before its + * `from`, aggregated as stored). Two rows miss one instant each and a whole + * group misses them all, so the null exclusion is measured, not assumed. Every + * expected value is computed from the rows with JS arithmetic, in the order the + * statement computes it (`duration-measure-sql.ts`), and asserted with `toBe`, + * so each cell equals the one expectation and the two cells equal each other. + * + * Driven through the plugin's own composition over a real engine + * (`AnalyticsServicePlugin` + `ObjectQL` + `SqlDriver`), the dataset door + * (`queryDataset`) and the cube door (`query` on a registered dataset), with + * the raw statements and the engine aggregates that read the object counted: + * the native strategy answers, one statement per pass, and no engine + * aggregate runs. + * + * ## The dialect axis + * + * The SQLite cell always runs. The PostgreSQL cell runs where + * `OS_TEST_POSTGRES_URL` is set and is a named skip otherwise (CI provisions + * it for this package, see `native-sql-measure-number-presentation.test.ts`). + * MySQL has no live cell in this package; its statement is pinned as text in + * `native-sql-duration-measure.test.ts`. + */ + +import { describe, it, expect, beforeAll, afterAll } from 'vitest'; +import { ObjectQL } from '@objectstack/objectql'; +import { SqlDriver } from '@objectstack/driver-sql'; +import { DatasetSchema, type Dataset } from '@objectstack/spec/ui'; +import type { AnalyticsService } from '../analytics-service.js'; +import { AnalyticsServicePlugin } from '../plugin.js'; +import { installDefaultOrganization } from './default-organization.testkit.js'; + +const OBJECT = 'os25778_ticket'; + +const TICKET = { + name: OBJECT, + label: 'Duration measure ticket', + fields: { + category: { name: 'category', type: 'text' as const }, + priority: { name: 'priority', type: 'text' as const }, + opened_at: { name: 'opened_at', type: 'datetime' as const }, + resolved_at: { name: 'resolved_at', type: 'datetime' as const }, + due_on: { name: 'due_on', type: 'date' as const }, + effort: { name: 'effort', type: 'number' as const }, + }, +}; + +interface Row { + id: string; + category: string; + priority: string; + opened_at: string | null; + resolved_at: string | null; + due_on: string; + effort: number; +} + +const ROWS: readonly Row[] = [ + // a: 90 minutes, a whole day, and one row never resolved (excluded). + { id: 't1', category: 'a', priority: 'high', opened_at: '2026-03-01T08:00:00.000Z', resolved_at: '2026-03-01T09:30:00.000Z', due_on: '2026-03-02', effort: 1 }, + { id: 't2', category: 'a', priority: 'low', opened_at: '2026-03-02T10:00:00.000Z', resolved_at: '2026-03-03T10:00:00.000Z', due_on: '2026-03-04', effort: 2 }, + { id: 't3', category: 'a', priority: 'high', opened_at: '2026-03-04T00:00:00.000Z', resolved_at: null, due_on: '2026-03-05', effort: 3 }, + // b: three quarters of a second (millisecond instants), and a negative hour. + { id: 't4', category: 'b', priority: 'high', opened_at: '2026-03-05T12:00:00.250Z', resolved_at: '2026-03-05T12:00:01.000Z', due_on: '2026-03-06', effort: 4 }, + { id: 't5', category: 'b', priority: 'low', opened_at: '2026-03-06T00:00:00.000Z', resolved_at: '2026-03-05T23:00:00.000Z', due_on: '2026-03-07', effort: 5 }, + // c: no row carries both instants. + { id: 't6', category: 'c', priority: 'high', opened_at: null, resolved_at: '2026-03-07T00:00:00.000Z', due_on: '2026-03-08', effort: 6 }, + { id: 't7', category: 'c', priority: 'low', opened_at: '2026-03-08T00:00:00.000Z', resolved_at: null, due_on: '2026-03-09', effort: 7 }, +]; + +const GROUPS = ['a', 'b', 'c'] as const; +const UNIT_SECONDS = { seconds: 1, minutes: 60, hours: 3600, days: 86400 } as const; +type Unit = keyof typeof UNIT_SECONDS; + +/** One row's duration in `unit`, as the statement computes it; `null` without both instants. */ +function durationOf(r: Row, unit: Unit): number | null { + if (r.opened_at === null || r.resolved_at === null) return null; + return Math.round(Date.parse(r.resolved_at) - Date.parse(r.opened_at)) / 1000 / UNIT_SECONDS[unit]; +} + +function aggregate(rows: readonly Row[], agg: 'avg' | 'min' | 'max' | 'sum', unit: Unit): number | null { + const values = rows.map((r) => durationOf(r, unit)).filter((v): v is number => v !== null); + if (agg === 'sum') return values.reduce((a, v) => a + v, 0); + if (values.length === 0) return null; + if (agg === 'avg') return values.reduce((a, v) => a + v, 0) / values.length; + return agg === 'min' ? Math.min(...values) : Math.max(...values); +} + +const byGroup = (g: string, rows: readonly Row[] = ROWS) => rows.filter((r) => r.category === g); + +const DURATION = { from: 'opened_at', to: 'resolved_at' } as const; + +const DATASET: Dataset = DatasetSchema.parse({ + name: 'os25778_resolution', + label: 'Resolution times', + object: OBJECT, + dimensions: [ + { name: 'category', field: 'category', type: 'string' }, + { name: 'opened', field: 'opened_at', type: 'date' }, + ], + measures: [ + { name: 'tickets', aggregate: 'count' }, + { name: 'avg_hours', aggregate: 'avg', duration: { ...DURATION, unit: 'hours' } }, + { name: 'total_seconds', aggregate: 'sum', duration: { ...DURATION, unit: 'seconds' } }, + { name: 'fastest_minutes', aggregate: 'min', duration: { ...DURATION, unit: 'minutes' } }, + { name: 'slowest_days', aggregate: 'max', duration: { ...DURATION, unit: 'days' } }, + { name: 'high_avg_hours', aggregate: 'avg', duration: { ...DURATION, unit: 'hours' }, filter: { priority: 'high' } }, + { name: 'seconds_per_ticket', derived: { op: 'ratio', of: ['total_seconds', 'tickets'] } }, + ], +}); + +const DURATION_MEASURES = ['avg_hours', 'total_seconds', 'fastest_minutes', 'slowest_days'] as const; + +function expected(g: string): Record<(typeof DURATION_MEASURES)[number] | 'high_avg_hours', number | null> { + const rows = byGroup(g); + return { + avg_hours: aggregate(rows, 'avg', 'hours'), + total_seconds: aggregate(rows, 'sum', 'seconds'), + fastest_minutes: aggregate(rows, 'min', 'minutes'), + slowest_days: aggregate(rows, 'max', 'days'), + high_avg_hours: aggregate(rows.filter((r) => r.priority === 'high'), 'avg', 'hours'), + }; +} + +interface Cell { + id: 'sqlite' | 'pg'; + label: string; + env: string | null; + config: () => Record | null; +} + +const CELLS: readonly Cell[] = [ + { id: 'sqlite', label: 'sqlite', env: null, config: () => ({ client: 'better-sqlite3', connection: { filename: ':memory:' }, useNullAsDefault: true }) }, + { + id: 'pg', + label: 'live postgres', + env: 'OS_TEST_POSTGRES_URL', + config: () => (process.env.OS_TEST_POSTGRES_URL ? { client: 'pg', connection: process.env.OS_TEST_POSTGRES_URL } : null), + }, +]; + +const quiet = { debug() {}, info() {}, warn() {}, error() {}, child() { return quiet; } }; + +type Answer = Record; +type Refusal = Error & { code?: string; status?: number; member?: string }; +const byCategory = (rows: readonly Answer[]) => new Map(rows.map((r) => [String(r.category), r])); + +for (const cell of CELLS) { + const config = cell.config(); + describe.skipIf(!config)( + `[#25778] dataset duration measure — the aggregated time between two datetime columns (${cell.label})${config ? '' : ` (skipped: set ${cell.env} to run this cell)`}`, + () => { + let driver: any; + let engine: ObjectQL; + const reads = { rawSql: 0, aggregate: 0, statements: [] as string[] }; + const services: Partial> = {}; + + const dropTable = async () => { + if (cell.id === 'pg') await driver?.execute(`drop table if exists ${OBJECT}`).catch(() => {}); + }; + + const counted = async (run: () => Promise) => { + const before = { rawSql: reads.rawSql, aggregate: reads.aggregate }; + const settled = await run().then((value) => ({ value, error: undefined }), (error: Refusal) => ({ value: undefined, error })); + return { ...settled, rawSql: reads.rawSql - before.rawSql, aggregate: reads.aggregate - before.aggregate }; + }; + + beforeAll(async () => { + driver = new SqlDriver(config as any); + await dropTable(); + engine = new ObjectQL({ logger: quiet } as any); + engine.registerDriver(driver, true); + await engine.init(); + engine.registry.registerObject(TICKET as any); + await engine.syncSchemas(); + await installDefaultOrganization(engine); + for (const row of ROWS) await engine.insert(OBJECT, { ...row } as any); + + const realExecute = (engine as any).execute.bind(engine); + (engine as any).execute = (sql: unknown, opts?: { object?: string }) => { + if (opts?.object === OBJECT) { + reads.rawSql += 1; + reads.statements.push(String((sql as { sql?: string })?.sql ?? sql)); + } + return realExecute(sql, opts); + }; + const realAggregate = engine.aggregate.bind(engine); + (engine as any).aggregate = (object: string, ...rest: unknown[]) => { + if (object === OBJECT) reads.aggregate += 1; + return (realAggregate as any)(object, ...rest); + }; + + for (const [face, caps] of [ + ['native', undefined], + ['objectql', () => ({ nativeSql: false, objectqlAggregate: true, inMemory: false })], + ] as const) { + const registered: Record = {}; + await new AnalyticsServicePlugin({ ...(caps ? { queryCapabilities: caps } : {}) } as any).init({ + getService: (name: string) => (name === 'data' ? engine : registered[name]), + registerService: (name: string, svc: unknown) => { registered[name] = svc; }, + replaceService: (name: string, svc: unknown) => { registered[name] = svc; }, + hook: () => {}, + logger: quiet, + } as never); + services[face] = registered.analytics as AnalyticsService; + // The configuration door, so the cube door can name the dataset. + services[face]!.registerDataset(DATASET); + } + }); + + afterAll(async () => { + await dropTable(); + try { await engine?.destroy(); } catch { /* noop */ } + }); + + it('avg / sum / min / max in each unit equal the rows, nulls excluded, on the native strategy', async () => { + const run = await counted(() => services.native!.queryDataset(DATASET, { measures: [...DURATION_MEASURES], dimensions: ['category'] } as any)); + expect(run.error, run.error?.message).toBeUndefined(); + expect(run.rawSql, 'one raw statement: NativeSQLStrategy answered').toBe(1); + expect(run.aggregate, 'no engine aggregate').toBe(0); + const answers = byCategory(run.value!.rows as Answer[]); + expect([...answers.keys()].sort()).toEqual([...GROUPS]); + for (const g of GROUPS) { + const want = expected(g); + for (const m of DURATION_MEASURES) { + expect(answers.get(g)![m], `${g} ${m}`).toBe(want[m]); + } + } + // The values the fixture was built to produce, stated once by hand. + expect(answers.get('a')!.avg_hours).toBe(12.75); + expect(answers.get('a')!.fastest_minutes).toBe(90); + expect(answers.get('b')!.total_seconds).toBe(-3599.25); + expect(answers.get('c')!.total_seconds, 'sum over no duration is 0').toBe(0); + expect(answers.get('c')!.avg_hours, 'avg over no duration has no answer').toBeNull(); + for (const m of DURATION_MEASURES) { + expect(run.value!.fields.find((f) => f.name === m)?.type, `${m} is declared a number`).toBe('number'); + } + }); + + it('a measure filter narrows only its own measure, and a derived measure reads a duration', async () => { + const run = await counted(() => + services.native!.queryDataset(DATASET, { measures: ['avg_hours', 'high_avg_hours', 'seconds_per_ticket'], dimensions: ['category'] } as any), + ); + expect(run.error, run.error?.message).toBeUndefined(); + expect(run.aggregate, 'no engine aggregate').toBe(0); + const answers = byCategory(run.value!.rows as Answer[]); + for (const g of GROUPS) { + const want = expected(g); + expect(answers.get(g)!.avg_hours, `${g} avg_hours`).toBe(want.avg_hours); + expect(answers.get(g)!.high_avg_hours, `${g} high_avg_hours`).toBe(want.high_avg_hours); + expect(answers.get(g)!.seconds_per_ticket, `${g} seconds_per_ticket`).toBe((want.total_seconds ?? 0) / byGroup(g).length); + } + }); + + it('the ungrouped total, and a page ordered and windowed by a duration measure', async () => { + const total = await services.native!.queryDataset(DATASET, { measures: ['avg_hours', 'total_seconds'] } as any); + expect(total.rows).toEqual([{ avg_hours: aggregate(ROWS, 'avg', 'hours'), total_seconds: aggregate(ROWS, 'sum', 'seconds') }]); + + // Group `c` is left out: its sum is SQL NULL until the strategy folds it + // to 0, and the dialects order a NULL at opposite ends. + const run = await counted(() => + services.native!.queryDataset(DATASET, { + measures: ['total_seconds'], + dimensions: ['category'], + runtimeFilter: { category: { $ne: 'c' } }, + order: { total_seconds: 'asc' }, + limit: 1, + } as any), + ); + expect(run.error, run.error?.message).toBeUndefined(); + expect(run.rawSql).toBe(1); + expect(reads.statements.at(-1)).toContain('ORDER BY "total_seconds" ASC'); + expect(run.value!.rows).toEqual([{ category: 'b', total_seconds: -3599.25 }]); + }); + + it('the cube door serves the registered dataset by name, with the same numbers', async () => { + const run = await counted(() => + services.native!.query({ cube: DATASET.name, measures: [...DURATION_MEASURES], dimensions: ['category'] } as any), + ); + expect(run.error, run.error?.message).toBeUndefined(); + expect(run.rawSql).toBe(1); + const answers = byCategory(run.value!.rows as Answer[]); + for (const g of GROUPS) { + for (const m of DURATION_MEASURES) expect(answers.get(g)![m], `${g} ${m}`).toBe(expected(g)[m]); + } + }); + + it('a granularity-bucketed trend routes off the native strategy: INVALID_FIELD / 400 naming the measure, nothing runs', async () => { + const run = await counted(() => + services.native!.queryDataset(DATASET, { + measures: ['avg_hours'], + dimensions: ['opened'], + timeDimensions: [{ dimension: 'opened', granularity: 'month' }], + } as any), + ); + expect(run.error?.code, run.error?.message).toBe('INVALID_FIELD'); + expect(run.error?.status).toBe(400); + expect(run.error?.message).toContain('Measure "avg_hours"'); + expect(run.error?.message).toContain('granularity'); + expect(run.rawSql + run.aggregate, 'no statement and no engine aggregate ran').toBe(0); + }); + + it('a deployment without native SQL refuses a duration measure the same way, and still serves the others', async () => { + const refused = await counted(() => services.objectql!.queryDataset(DATASET, { measures: ['tickets', 'avg_hours'], dimensions: ['category'] } as any)); + expect(refused.error?.code, refused.error?.message).toBe('INVALID_FIELD'); + expect(refused.error?.status).toBe(400); + expect(refused.error?.message).toContain('Measure "avg_hours"'); + expect(refused.rawSql + refused.aggregate).toBe(0); + + const served = await services.objectql!.queryDataset(DATASET, { measures: ['tickets'], dimensions: ['category'] } as any); + expect(Object.fromEntries((served.rows as Answer[]).map((r) => [r.category, r.tickets]))).toEqual({ a: 3, b: 2, c: 2 }); + }); + + it('a duration over a field that is not datetime is refused at compile: DATASET_INVALID / 400, naming the column and its type', async () => { + for (const [column, type] of [['due_on', 'date'], ['effort', 'number'], ['category', 'text']] as const) { + const dataset = DatasetSchema.parse({ + ...DATASET, + name: `os25778_bad_${column}`, + measures: [{ name: 'bad', aggregate: 'avg', duration: { from: column, to: 'resolved_at', unit: 'hours' } }], + }); + const run = await counted(() => services.native!.queryDataset(dataset, { measures: ['bad'], dimensions: ['category'] } as any)); + expect(run.error?.code, run.error?.message).toBe('DATASET_INVALID'); + expect(run.error?.status).toBe(400); + expect(run.error?.message).toContain(`\`duration.from\` "${column}"`); + expect(run.error?.message).toContain(`declares as \`${type}\``); + expect(run.rawSql + run.aggregate).toBe(0); + } + }); + }, + ); +} diff --git a/packages/services/service-analytics/src/__tests__/native-sql-duration-measure.test.ts b/packages/services/service-analytics/src/__tests__/native-sql-duration-measure.test.ts new file mode 100644 index 00000000000..f7c030ff13f --- /dev/null +++ b/packages/services/service-analytics/src/__tests__/native-sql-duration-measure.test.ts @@ -0,0 +1,304 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * [#25778] The dataset duration measure on the native-SQL face, as statements, + * and on the service's gates. + * + * `dataset-duration-measure.test.ts` measures the numbers on live SQLite and + * PostgreSQL. This file pins what that cannot reach: the statement per dialect + * (MySQL included, which has no live cell in this package), the routing on a + * dialect nobody named, the presentation of a client's string answer, the + * draft-preview evaluator, and the service gates a duration measure passes + * through — the field-level read gate, the measure mint, the inline-dataset + * door and the compile guards for a dataset that reaches the service unparsed. + */ + +import { describe, it, expect } from 'vitest'; +import type { Cube } from '@objectstack/spec/data'; +import type { AnalyticsQuery, StrategyContext } from '@objectstack/spec/contracts'; +import type { ExecutionContext } from '@objectstack/spec/kernel'; +import { DatasetSchema } from '@objectstack/spec/ui'; +import { AnalyticsService } from '../analytics-service.js'; +import { NativeSQLStrategy } from '../strategies/native-sql-strategy.js'; +import type { DatasetScope, DurationMeasureSpec } from '../strategies/types.js'; +import { durationMeasureIn } from '../duration-measure-sql.js'; +import { evaluateAnalyticsQueryOverRows } from '../preview-evaluator.js'; + +const OBJECT = 'ticket'; + +const CUBE: Cube = { + name: 'tickets', + sql: OBJECT, + measures: { n: { label: 'Rows', type: 'count', sql: '*' } }, + dimensions: { category: { label: 'Category', type: 'string', sql: 'category' } }, + joins: { owner: { name: 'owner_obj', relationship: 'many_to_one', sql: 'owner' } }, + public: true, +} as unknown as Cube; + +const AVG_HOURS: DurationMeasureSpec = { aggregate: 'avg', from: 'opened_at', to: 'resolved_at', unit: 'hours' }; +const TOTAL_SECONDS: DurationMeasureSpec = { aggregate: 'sum', from: 'opened_at', to: 'resolved_at', unit: 'seconds' }; + +const SCOPE: DatasetScope = { + measureDurations: { avg_hours: AVG_HOURS, total_seconds: TOTAL_SECONDS }, + measureFilters: { total_seconds: { priority: 'high' } }, +}; + +function nativeCtx(knobs: { + dialect?: string; + rows?: Record[]; + coerce?: (object: string, field: string, col: string) => string; +} = {}): StrategyContext { + return { + getCube: (name: string) => (name === CUBE.name ? CUBE : undefined), + queryCapabilities: () => ({ nativeSql: true, objectqlAggregate: true, inMemory: false }), + executeRawSql: async () => knobs.rows ?? [], + getDatasetScope: () => SCOPE, + ...(knobs.dialect ? { sqlDialect: () => knobs.dialect } : {}), + ...(knobs.coerce ? { coerceTemporalFilterColumn: knobs.coerce } : {}), + } as unknown as StrategyContext; +} + +const q = (extra: Partial = {}): AnalyticsQuery => + ({ cube: CUBE.name, measures: ['avg_hours', 'total_seconds'], dimensions: ['category'], ...extra }) as AnalyticsQuery; + +describe('[#25778] NativeSQLStrategy — the duration measure as a statement', () => { + const native = new NativeSQLStrategy(); + + it.each([ + [ + 'sqlite', + 'AVG(((ROUND((julianday(resolved_at) - julianday(opened_at)) * 86400000.0) / 1000.0) / 3600.0)) AS "avg_hours"', + 'SUM(CASE WHEN priority = $1 THEN (ROUND((julianday(resolved_at) - julianday(opened_at)) * 86400000.0) / 1000.0) END) AS "total_seconds"', + ], + [ + 'postgres', + 'AVG((CAST(EXTRACT(EPOCH FROM (resolved_at - opened_at)) AS DOUBLE PRECISION) / 3600.0)) AS "avg_hours"', + 'SUM(CASE WHEN priority = $1 THEN CAST(EXTRACT(EPOCH FROM (resolved_at - opened_at)) AS DOUBLE PRECISION) END) AS "total_seconds"', + ], + [ + 'mysql', + 'AVG(((TIMESTAMPDIFF(MICROSECOND, opened_at, resolved_at) / 1000000.0) / 3600.0)) AS "avg_hours"', + 'SUM(CASE WHEN priority = $1 THEN (TIMESTAMPDIFF(MICROSECOND, opened_at, resolved_at) / 1000000.0) END) AS "total_seconds"', + ], + ])('%s: the per-dialect difference, divided by the unit, under the aggregate and its measure filter', async (dialect, avg, filteredSum) => { + const { sql, params } = await native.generateSql(q(), nativeCtx({ dialect })); + expect(sql).toBe(`SELECT category AS "category", ${avg}, ${filteredSum} FROM "${OBJECT}" GROUP BY category`); + expect(params).toEqual(['high']); + }); + + it('a statement that joins qualifies both columns with the base table', async () => { + const { sql } = await native.generateSql(q({ measures: ['avg_hours'], dimensions: ['owner.region'] }), nativeCtx({ dialect: 'postgres' })); + expect(sql).toContain(`EXTRACT(EPOCH FROM ("${OBJECT}"."resolved_at" - "${OBJECT}"."opened_at"))`); + expect(sql).toContain('LEFT JOIN "owner_obj" "owner"'); + }); + + it('each column reads through the host\'s temporal repair, as a filter on it does', async () => { + const seen: string[] = []; + const coerce = (object: string, field: string, col: string) => { + seen.push(`${object}.${field}`); + return `canon(${col})`; + }; + const { sql } = await native.generateSql(q({ measures: ['avg_hours'] }), nativeCtx({ dialect: 'sqlite', coerce })); + expect(sql).toContain('julianday(canon(resolved_at)) - julianday(canon(opened_at))'); + expect(seen).toEqual([`${OBJECT}.opened_at`, `${OBJECT}.resolved_at`]); + }); + + it('declines a duration measure on a dialect nobody named, and only then', () => { + expect(native.canHandle(q(), nativeCtx())).toBe(false); + expect(native.canHandle(q(), nativeCtx({ dialect: 'oracle' }))).toBe(false); + expect(native.canHandle(q({ measures: ['n'] }), nativeCtx()), 'CONTROL a query with no duration measure').toBe(true); + for (const dialect of ['sqlite', 'postgres', 'mysql']) expect(native.canHandle(q(), nativeCtx({ dialect })), dialect).toBe(true); + }); + + it('presents a client string as a number, and folds an empty sum to 0 while an empty avg stays null', async () => { + const rows = [ + { category: 'a', avg_hours: '12.75', total_seconds: '91800' }, + { category: 'c', avg_hours: null, total_seconds: null }, + ]; + const res = await native.execute(q(), nativeCtx({ dialect: 'postgres', rows })); + expect(res.rows).toEqual([ + { category: 'a', avg_hours: 12.75, total_seconds: 91800 }, + { category: 'c', avg_hours: null, total_seconds: 0 }, + ]); + expect(res.fields).toEqual([ + { name: 'category', type: 'string' }, + { name: 'avg_hours', type: 'number' }, + { name: 'total_seconds', type: 'number' }, + ]); + }); + + it('a measure is a duration only by an own key: a name on Object.prototype is never one', () => { + expect(durationMeasureIn(SCOPE.measureDurations, 'avg_hours')).toBe(AVG_HOURS); + expect(durationMeasureIn(SCOPE.measureDurations, 'constructor')).toBeUndefined(); + expect(durationMeasureIn(SCOPE.measureDurations, 'toString')).toBeUndefined(); + expect(durationMeasureIn(undefined, 'avg_hours')).toBeUndefined(); + }); +}); + +describe('[#25778] the draft preview computes a duration measure from the rows', () => { + const rows = [ + { category: 'a', opened_at: '2026-03-01T08:00:00.000Z', resolved_at: '2026-03-01T09:30:00.000Z' }, + { category: 'a', opened_at: '2026-03-02T10:00:00.000Z', resolved_at: '2026-03-03T10:00:00.000Z' }, + { category: 'a', opened_at: '2026-03-04T00:00:00.000Z', resolved_at: null }, + { category: 'b', opened_at: '2026-03-05T12:00:00.250Z', resolved_at: '2026-03-05T12:00:01.000Z' }, + { category: 'c', opened_at: null, resolved_at: '2026-03-07T00:00:00.000Z' }, + ]; + const durations = { + avg_hours: AVG_HOURS, + total_seconds: TOTAL_SECONDS, + fastest_minutes: { ...AVG_HOURS, aggregate: 'min', unit: 'minutes' }, + slowest_days: { ...AVG_HOURS, aggregate: 'max', unit: 'days' }, + } satisfies Record; + + it('the native face\'s numbers over the same rows, nulls excluded', () => { + const res = evaluateAnalyticsQueryOverRows( + { cube: CUBE.name, measures: Object.keys(durations), dimensions: ['category'] } as AnalyticsQuery, + CUBE, + rows, + undefined, + durations, + ); + expect(res.rows).toEqual([ + { category: 'a', avg_hours: 12.75, total_seconds: 91800, fastest_minutes: 90, slowest_days: 1 }, + { category: 'b', avg_hours: 0.75 / 3600, total_seconds: 0.75, fastest_minutes: 0.75 / 60, slowest_days: 0.75 / 86400 }, + { category: 'c', avg_hours: null, total_seconds: 0, fastest_minutes: null, slowest_days: null }, + ]); + }); +}); + +// ── The service's gates ───────────────────────────────────────────────────── + +const FIELDS = ['id', 'category', 'priority', 'opened_at', 'resolved_at', 'effort']; +const MEMBER = { userId: 'u_member', tenantId: 'org_a' } as ExecutionContext; + +const DATASET_BASE = { + name: 'resolution', + label: 'Resolution', + object: OBJECT, + dimensions: [{ name: 'category', field: 'category', type: 'string' }], + measures: [ + { name: 'tickets', aggregate: 'count' }, + // A name the suffix mint would read as `sum(resolution_seconds)`. + { name: 'resolution_seconds_sum', aggregate: 'sum', duration: { from: 'opened_at', to: 'resolved_at', unit: 'seconds' } }, + ], +}; + +function makeService(opts: { + capabilities?: () => { nativeSql: boolean; objectqlAggregate: boolean; inMemory: boolean }; + readable?: readonly string[]; + draftRows?: Record[]; +} = {}) { + const executed: string[] = []; + const service = new AnalyticsService({ + queryCapabilities: opts.capabilities ?? (() => ({ nativeSql: true, objectqlAggregate: true, inMemory: false })), + getObjectFieldNames: (object: string) => (object === OBJECT ? FIELDS : undefined), + sourceFieldMeta: (object: string, field: string) => + object === OBJECT && FIELDS.includes(field) + ? { type: field.endsWith('_at') ? 'datetime' : field === 'effort' ? 'number' : 'text' } + : undefined, + sqlDialect: () => 'sqlite', + ...(opts.readable ? { getReadableFields: () => opts.readable } : {}), + ...(opts.draftRows ? { draftRowsResolver: async () => opts.draftRows } : {}), + executeRawSql: async (_object: string, sql: string) => { + executed.push(sql); + return []; + }, + executeAggregate: async (object: string) => { + executed.push(`aggregate:${object}`); + return []; + }, + } as never); + return { service, executed }; +} + +type Refusal = Error & { code?: string; status?: number; member?: string }; +const refusalOf = (run: Promise) => run.then(() => undefined, (e: Refusal) => e); + +describe('[#25778] AnalyticsService — the gates a duration measure passes through', () => { + const dataset = DatasetSchema.parse(DATASET_BASE); + + it('is served as a duration, never minted from its name by the suffix rule', async () => { + const { service, executed } = makeService(); + await service.queryDataset(dataset, { measures: ['resolution_seconds_sum'], dimensions: ['category'] } as never, MEMBER); + expect(executed).toHaveLength(1); + expect(executed[0]).toContain('SUM((ROUND((julianday(resolved_at) - julianday(opened_at))'); + expect(executed[0]).not.toContain('SUM(resolution_seconds)'); + }); + + it('the field-level read gate judges both columns: an unreadable one refuses the measure, nothing runs', async () => { + const { service, executed } = makeService({ readable: FIELDS.filter((f) => f !== 'resolved_at') }); + const err = await refusalOf(service.queryDataset(dataset, { measures: ['resolution_seconds_sum'], dimensions: ['category'] } as never, MEMBER)); + expect(err?.code, err?.message).toBe('PERMISSION_DENIED'); + expect(err?.status).toBe(403); + expect(err?.message).toContain('resolved_at'); + expect(executed).toEqual([]); + + const control = makeService({ readable: FIELDS }); + await control.service.queryDataset(dataset, { measures: ['resolution_seconds_sum'], dimensions: ['category'] } as never, MEMBER); + expect(control.executed).toHaveLength(1); + }); + + it('the dry-run door refuses it off the native strategy too', async () => { + const { service } = makeService({ capabilities: () => ({ nativeSql: false, objectqlAggregate: true, inMemory: false }) }); + service.registerDataset(dataset); + const err = await refusalOf(service.generateSql({ cube: dataset.name, measures: ['resolution_seconds_sum'], dimensions: ['category'] } as never, MEMBER)); + expect(err?.code, err?.message).toBe('INVALID_FIELD'); + expect(err?.status).toBe(400); + expect(err?.member).toBe('resolution_seconds_sum'); + }); + + it('the draft preview serves it over the drafted rows', async () => { + const draftRows = [ + { category: 'a', opened_at: '2026-03-01T08:00:00.000Z', resolved_at: '2026-03-01T09:30:00.000Z' }, + { category: 'a', opened_at: '2026-03-02T10:00:00.000Z', resolved_at: null }, + ]; + const { service, executed } = makeService({ draftRows }); + const res = await service.queryDataset(dataset, { measures: ['resolution_seconds_sum'], dimensions: ['category'] } as never, MEMBER, { previewDrafts: true }); + expect(res.rows).toEqual([{ category: 'a', resolution_seconds_sum: 5400 }]); + expect(executed).toEqual([]); + }); + + describe('a dataset that reaches the service without meeting the spec\'s parse', () => { + const stored = (duration: Record) => ({ + ...DATASET_BASE, + measures: [{ name: 'leaked', aggregate: 'avg', duration }], + }); + + it('the spec refuses each of these shapes at parse', () => { + for (const duration of [ + { from: '(SELECT secret FROM other)', to: 'resolved_at', unit: 'hours' }, + { from: 'owner.created_at', to: 'resolved_at', unit: 'hours' }, + { from: 'opened_at', to: 'resolved_at', unit: 'weeks' }, + ]) { + expect(DatasetSchema.safeParse(stored(duration)).success, JSON.stringify(duration)).toBe(false); + } + }); + + it('text that is no column reference is refused at the door: PERMISSION_DENIED / 403, nothing runs', async () => { + const { service, executed } = makeService(); + const err = await refusalOf( + service.queryDataset(stored({ from: '(SELECT secret FROM other)', to: 'resolved_at', unit: 'hours' }) as never, { measures: ['leaked'] } as never, MEMBER), + ); + expect(err?.code, err?.message).toBe('PERMISSION_DENIED'); + expect(err?.status).toBe(403); + expect(err?.member).toBe('leaked'); + expect(err?.message).not.toContain('SELECT'); + expect(executed).toEqual([]); + }); + + it.each([ + [{ from: 'owner.created_at', to: 'resolved_at', unit: 'hours' }, '`duration.from` "owner.created_at", which names no column'], + [{ from: 'opened_at', unit: 'hours' }, '`duration.to` undefined, which names no column'], + [{ from: 'opened_at', to: 'resolved_at', unit: 'weeks' }, '`duration.unit` seconds / minutes / hours / days; it declares "weeks"'], + ])('%j is refused at compile: DATASET_INVALID / 400, nothing runs', async (duration, words) => { + const { service, executed } = makeService(); + const err = await refusalOf(service.queryDataset(stored(duration) as never, { measures: ['leaked'] } as never, MEMBER)); + expect(err?.code, err?.message).toBe('DATASET_INVALID'); + expect(err?.status).toBe(400); + expect(err?.message).toContain(words); + expect(executed).toEqual([]); + // The configuration door compiles through the same guard. + expect(() => service.registerDataset(stored(duration) as never)).toThrow(words); + }); + }); +}); diff --git a/packages/services/service-analytics/src/analytics-service.ts b/packages/services/service-analytics/src/analytics-service.ts index f2f29c7b8bc..3a2cfdd78fb 100644 --- a/packages/services/service-analytics/src/analytics-service.ts +++ b/packages/services/service-analytics/src/analytics-service.ts @@ -111,6 +111,8 @@ import { type DimensionLabelDeps, } from './dimension-labels.js'; import { evaluateAnalyticsQueryOverRows } from './preview-evaluator.js'; +// [#25778] The one lookup of a dataset duration measure by name. +import { durationMeasureIn } from './duration-measure-sql.js'; // [#5918] The measure mint refuses a dotted member through the SAME constructor // the strategies' member-level refusals use — `INVALID_FIELD` / 400, naming the // member as the request spelled it (see `dataset-refusal.ts`'s header for why @@ -492,6 +494,7 @@ function fieldsOfColumnSql( * | `dimensions` | dimension | aggregate | * | `timeDimensions` | any | aggregate when bucketed, else predicate (a window) | * | `measures` | measure (the field it aggregates) | aggregate | + * | a duration measure, in `measures` or `order` (#25778) | its `from` and `to` columns | aggregate / predicate | * | `where` leaves | any | predicate | * | a requested measure's own filter | any | predicate | * | the dataset's own filter | any | predicate | @@ -542,6 +545,19 @@ function namedQueryFields( // (`entry` absent). The caller-supplied kind is refused in every tier. else out.push({ object: baseObject, member, expression: true, declared: !!entry }); }; + // [#25778] A dataset duration measure is absent from the cube (fail-closed, + // `DatasetScope.measureDurations`), so `name` would read nothing for it. It + // reads its two columns, and is judged on both. + const durationColumns = (member: string, role: FieldReadRole): boolean => { + const duration = durationMeasureIn(datasetScope?.measureDurations, member); + if (!duration || declaredMemberEntry(cube, member, 'any')) return false; + for (const column of [duration.from, duration.to]) { + const fields = fieldsOfColumnSql(cube, baseObject, column, role, referenceOf); + if (fields) out.push(...fields); + else out.push({ object: baseObject, member, expression: true, declared: true }); + } + return true; + }; const filterMembers = (where: unknown): string[] => { if (!where || typeof where !== 'object') return []; try { @@ -553,7 +569,7 @@ function namedQueryFields( for (const member of query.dimensions ?? []) name(member, 'dimension', 'aggregate'); for (const td of query.timeDimensions ?? []) name(td.dimension, 'any', td.granularity ? 'aggregate' : 'predicate'); - for (const member of query.measures ?? []) name(member, 'measure', 'aggregate'); + for (const member of query.measures ?? []) if (!durationColumns(member, 'aggregate')) name(member, 'measure', 'aggregate'); for (const member of filterMembers((query as { where?: unknown }).where)) name(member, 'any', 'predicate'); for (const measure of query.measures ?? []) { for (const member of filterMembers(datasetScope?.measureFilters?.[measure])) name(member, 'any', 'predicate'); @@ -561,7 +577,7 @@ function namedQueryFields( for (const member of filterMembers(datasetScope?.filter)) name(member, 'any', 'predicate'); const order = (query as { order?: unknown }).order; if (order && typeof order === 'object' && !Array.isArray(order)) { - for (const key of Object.keys(order)) name(key, 'any', 'predicate'); + for (const key of Object.keys(order)) if (!durationColumns(key, 'predicate')) name(key, 'any', 'predicate'); } return [...out.filter((f) => f.object === baseObject), ...out.filter((f) => f.object !== baseObject)]; } @@ -1716,7 +1732,8 @@ export class AnalyticsService implements IAnalyticsService { * - [#10298] `getDatasetScope` — the compiled dataset's definition-level * filter and its per-measure filters, the half of the declaration the Cube * model has no room for. Same source as the allowlist: answered for a cube - * that IS a compiled dataset, `undefined` for every other cube. + * that IS a compiled dataset, `undefined` for every other cube. [#25778] + * Its duration measures ride the same channel. */ private cubeReads(scope: CubeScope): CubeReads { return { @@ -1727,7 +1744,7 @@ export class AnalyticsService implements IAnalyticsService { getDatasetScope: (cubeName: string) => { const compiled = scope.getCompiledDataset(cubeName); if (!compiled) return undefined; - return { filter: compiled.filter, measureFilters: compiled.measureFilters }; + return { filter: compiled.filter, measureFilters: compiled.measureFilters, measureDurations: compiled.durations }; }, }; } @@ -1884,7 +1901,7 @@ export class AnalyticsService implements IAnalyticsService { const measureFilters = resolveFilterTokens(scope.measureFilters, tokenCtx); return filter === scope.filter && measureFilters === scope.measureFilters ? scope - : { filter, measureFilters }; + : { ...scope, filter, measureFilters }; }; } @@ -2137,6 +2154,7 @@ export class AnalyticsService implements IAnalyticsService { query: AnalyticsQuery, authorCube: Cube | undefined, context: ExecutionContext | undefined, + scope: CubeScope, ): void { const name = query.cube; if (!name) return; @@ -2150,8 +2168,10 @@ export class AnalyticsService implements IAnalyticsService { if (!baseObject || !BARE_IDENTIFIER.test(baseObject)) return; for (const measure of query.measures ?? []) { if (typeof measure !== 'string' || measure === '') continue; - // Author-declared measures are not this gate's business. + // Author-declared measures are not this gate's business. [#25778] A + // dataset's duration measure is one, declared beside the cube. if (authorCube && declaredMemberEntry(authorCube, measure, 'measure')) continue; + if (durationMeasureIn(scope.getCompiledDataset(name)?.durations, measure)) continue; const inferredSql = inferredCallerMeasureSql(measure, name); // `null` — a dotted non-qualifier measure #5918 already refused — and a // column reference (`'*'`, a bare identifier, a relationship path) both @@ -2278,6 +2298,15 @@ export class AnalyticsService implements IAnalyticsService { } for (const m of dataset.measures ?? []) { if ((m as { derived?: unknown }).derived) continue; + // [#25778] A duration measure reads its `from` and `to`. Text that is no + // column reference is caller content the statement would read unjudged; + // a column reference that is not one bare name is the compiler's to + // refuse (`DATASET_INVALID`), as it would be for any malformed key. + const duration = (m as { duration?: { from?: unknown; to?: unknown } }).duration; + if (duration && [duration.from, duration.to].some((c) => typeof c === 'string' && c !== '' && !isColumnReferenceSql(c))) { + caller.push({ object, member: m.name, expression: true, declared: false }); + continue; + } if (typeof m.field === 'string' && m.field !== '' && !isColumnReferenceSql(m.field)) { caller.push({ object, member: m.name, expression: true, declared: false }); } @@ -2409,11 +2438,14 @@ export class AnalyticsService implements IAnalyticsService { // [#21156] Refuse a caller-named member that is not a column reference and // names no declared member — in every tier, before a strategy compiles it. // After `ensureCube` so a non-existent cube/object still answers 404 first. - this.assertCallerMembersResolvable(query, authorCube, context); + this.assertCallerMembersResolvable(query, authorCube, context, scope); const ctx = await this.callCtx(query, context, tokenCtx, scope); let skip: Set | undefined; for (;;) { const strategy = this.resolveStrategy(query, ctx, skip); + // [#25778] Asked of every strategy the loop lands on, the runtime + // fallback's included. + this.assertDurationMeasuresServed(query, scope, strategy); this.logger.debug(`[Analytics] Query on cube "${query.cube}" → ${strategy.name}`); try { // [#8286] ONE gate, every strategy. This is the single seam a strategy @@ -2654,30 +2686,28 @@ export class AnalyticsService implements IAnalyticsService { // [#20917] …and the field-level gate, per query the executor issues, // over the same compiled dataset: a drafted seed row carries the same // fields the published ones do. + const previewScope: DatasetScope = { + filter: compiled.filter, + measureFilters: compiled.measureFilters, + measureDurations: compiled.durations, + }; const previewService = { query: async (q: AnalyticsQuery) => { // [#21120] The body-column refusal rides this preview path too — // a drafted seed row carries the same stored body the published // ones do. - this.assertStoredMetadataBodyNotQueried( - q, - compiled.cube, - { filter: compiled.filter, measureFilters: compiled.measureFilters }, - ); + this.assertStoredMetadataBodyNotQueried(q, compiled.cube, previewScope); // …and the generic-exit gate's field half: a drafted seed row // carries the same `internal` fields the published ones do. - this.assertNoInternalFieldsQueried( + this.assertNoInternalFieldsQueried(q, compiled.cube, previewScope); + await this.assertFieldsReadable(q, compiled.cube, previewScope, context); + return evaluateAnalyticsQueryOverRows( q, compiled.cube, - { filter: compiled.filter, measureFilters: compiled.measureFilters }, + seedRows!, + (field) => this.sourceFieldMeta?.(dataset.object, field)?.type, + compiled.durations, ); - await this.assertFieldsReadable( - q, - compiled.cube, - { filter: compiled.filter, measureFilters: compiled.measureFilters }, - context, - ); - return evaluateAnalyticsQueryOverRows(q, compiled.cube, seedRows!, (field) => this.sourceFieldMeta?.(dataset.object, field)?.type); }, } as IAnalyticsService; const previewResult = await new DatasetExecutor(previewService).execute(compiled, selection, context); @@ -3259,9 +3289,10 @@ export class AnalyticsService implements IAnalyticsService { // either. Pre-inference cube captured before `ensureCube`. const authorCube = scope.getCube(query.cube!); this.ensureCube(query, scope); - this.assertCallerMembersResolvable(query, authorCube, context); + this.assertCallerMembersResolvable(query, authorCube, context, scope); const ctx = await this.callCtx(query, context, tokenCtx, scope); const strategy = this.resolveStrategy(query, ctx); + this.assertDurationMeasuresServed(query, scope, strategy); this.logger.debug(`[Analytics] generateSql on cube "${query.cube}" → ${strategy.name}`); return strategy.generateSql(query, ctx); @@ -3269,6 +3300,37 @@ export class AnalyticsService implements IAnalyticsService { // ── Internal ───────────────────────────────────────────────────── + /** + * [#25778] Refuse a dataset duration measure the chosen strategy does not + * serve — `INVALID_FIELD` / 400, naming the measure, before anything runs. + * + * A duration measure reads two columns of a row, and only the native-SQL + * strategy computes it (`duration-measure-sql.ts`): the engine's aggregate + * contract names one column per aggregation, so the ObjectQL strategy has + * nothing to hand the engine for it. The native strategy declines a query it + * cannot compile as one statement on a SQL dialect it names — a time + * dimension bucketed by granularity, a field-to-field comparison, an object + * an engine middleware or an external datasource serves, a datasource that + * is not SQL — and such a query lands here. The measure is absent from the + * cube, so the strategy that did land would refuse it as unknown, in words + * that name nothing the author can change; this names the measure and why. + */ + private assertDurationMeasuresServed(query: AnalyticsQuery, scope: CubeScope, strategy: AnalyticsStrategy): void { + if (strategy instanceof NativeSQLStrategy) return; + const durations = scope.getCompiledDataset(query.cube!)?.durations; + const measure = (query.measures ?? []).find((m) => durationMeasureIn(durations, m)); + if (measure === undefined) return; + throw invalidMemberError( + `Measure "${measure}" of dataset "${query.cube}" is a duration, the time between two datetime fields ` + + `of one record, and it is computed only by a query that compiles to one SQL statement on a SQLite, ` + + `PostgreSQL or MySQL datasource. This query does not: a time dimension bucketed by granularity, a ` + + `field-to-field comparison, or an object served through engine middleware, an external datasource ` + + `or a non-SQL driver routes it to ${strategy.name}, which has no duration. Query the measure without ` + + `the granularity bucket or the field comparison.`, + { member: measure, param: 'measures', cube: query.cube! }, + ); + } + /** * Refuse a query against a cube declared `public: false` with the SAME * refusal an unknown name gets — `cubeNotFoundError`, byte for byte @@ -3398,9 +3460,15 @@ export class AnalyticsService implements IAnalyticsService { // rule — including a declared DOTTED key, which `lookupMember` resolves by // direct hit and which this loop must therefore check for verbatim FIRST or // it would refuse a cube's own authored vocabulary. + // + // [#25778] A dataset's duration measure is declared too, beside the cube + // rather than in it (`DatasetScope.measureDurations`), so it is never + // minted: the suffix rule would have read its name as a column to `sum`. const extraMeasures: Record = {}; + const durations = scope.getCompiledDataset(name)?.durations; for (const m of query.measures || []) { if (cube.measures[m] || extraMeasures[m]) continue; + if (durationMeasureIn(durations, m)) continue; const key = mintableMeasureKey(m, name); if (cube.measures[key] || extraMeasures[key]) continue; // [#21437] A spelling whose source names no field (`_sum`, `*`) is diff --git a/packages/services/service-analytics/src/dataset-compiler.ts b/packages/services/service-analytics/src/dataset-compiler.ts index 05037854b93..2165628bf4a 100644 --- a/packages/services/service-analytics/src/dataset-compiler.ts +++ b/packages/services/service-analytics/src/dataset-compiler.ts @@ -10,7 +10,7 @@ import { virtualFieldAggregateRemedy, } from '@objectstack/spec/data'; import type { Dataset, DatasetMeasure, DatasetDimension } from '@objectstack/spec/ui'; -import { DATASET_DURATION_AGGREGATES, resolveI18nLabel } from '@objectstack/spec/ui'; +import { DATASET_DURATION_AGGREGATES, DATASET_DURATION_UNIT_SECONDS, resolveI18nLabel } from '@objectstack/spec/ui'; import type { FilterCondition } from '@objectstack/spec/data'; import { datasetInvalidError } from './dataset-refusal.js'; import { TEMPORAL_SOURCE_FIELD_TYPES } from './measure-result-type.js'; @@ -460,15 +460,20 @@ function assertAggregateFieldTypeCompatible( */ const DURATION_COLUMN_TYPE = 'datetime'; +/** One column of the base object by its bare name — the spec's `duration.from` / `to` shape. */ +const DURATION_BASE_COLUMN = /^[A-Za-z_][A-Za-z0-9_]*$/; + /** * [#25778] Compile one dataset measure's `duration` — the column pair, the * unit, and the measure's aggregate — after judging what the metadata can * settle before any query runs. * - * - The AGGREGATE is one of `DATASET_DURATION_AGGREGATES`. The spec refuses - * any other at parse; this is the guard for a dataset that reaches the - * compiler without meeting that parse (the same posture as every other - * `DATASET_INVALID` here). + * - The SHAPE the spec holds at parse — each column one bare name, the unit + * one of `DATASET_DURATION_UNIT_SECONDS` — and the AGGREGATE, one of + * `DATASET_DURATION_AGGREGATES`. These are the guard for a dataset that + * reaches the compiler without meeting that parse (the same posture as every + * other `DATASET_INVALID` here): the native-SQL face writes the columns into + * its statement and divides by the unit's length. * - Each COLUMN is a field the base object declares as `datetime`, judged * through the same `declaredValueShape` hook the aggregate × field-type gate * reads, with the same tiering: no hook, or a field the hook cannot resolve, @@ -484,6 +489,20 @@ function compileDurationMeasure( declaredValueShape?: (objectName: string, field: string) => ValueShapeFieldDef | undefined, ): DurationMeasureSpec { const { from, to, unit } = measure.duration; + for (const [slot, column] of [['from', from], ['to', to]] as const) { + if (typeof column === 'string' && DURATION_BASE_COLUMN.test(column)) continue; + throw datasetInvalidError( + `[dataset-compiler] dataset "${datasetName}" duration measure "${measure.name}" reads ` + + `\`duration.${slot}\` ${JSON.stringify(column) ?? 'undefined'}, which names no column: it takes one ` + + `\`${DURATION_COLUMN_TYPE}\` field of "${objectName}" by its bare name.`, + ); + } + if (!Object.prototype.hasOwnProperty.call(DATASET_DURATION_UNIT_SECONDS, unit)) { + throw datasetInvalidError( + `[dataset-compiler] dataset "${datasetName}" duration measure "${measure.name}" takes \`duration.unit\` ` + + `${Object.keys(DATASET_DURATION_UNIT_SECONDS).join(' / ')}; it declares ${JSON.stringify(unit) ?? 'none'}.`, + ); + } const aggregate = measure.aggregate; if (!aggregate || !(DATASET_DURATION_AGGREGATES as readonly string[]).includes(aggregate)) { throw datasetInvalidError( diff --git a/packages/services/service-analytics/src/dataset-executor.ts b/packages/services/service-analytics/src/dataset-executor.ts index 96aed917cdf..f826b1da46a 100644 --- a/packages/services/service-analytics/src/dataset-executor.ts +++ b/packages/services/service-analytics/src/dataset-executor.ts @@ -24,6 +24,7 @@ import { import type { CompiledDataset, DerivedMeasureSpec } from './dataset-compiler.js'; import { explicitDateRangeWindow } from './date-range-array-arm.js'; import { datasetInvalidError } from './dataset-refusal.js'; +import { durationMeasureIn } from './duration-measure-sql.js'; import type { OrderLabelResolver } from './dimension-labels.js'; // Re-export the shared protocol shapes so existing importers keep working. @@ -1191,9 +1192,11 @@ export class DatasetExecutor { // // Derived measures are evaluated AFTER, so a ratio over a filled 0 computes // (0%) instead of being poisoned by an absent operand. + // [#25778] A duration measure is absent from the cube and carries its + // aggregate beside it. const fillColumns: Record = {}; for (const m of baseMeasures) { - const aggregate = compiled.cube.measures?.[m]?.type; + const aggregate = compiled.cube.measures?.[m]?.type ?? durationMeasureIn(compiled.durations, m)?.aggregate; fillColumns[m] = aggregate; if (selection.compareTo) fillColumns[`${m}__compare`] = aggregate; } diff --git a/packages/services/service-analytics/src/duration-measure-sql.ts b/packages/services/service-analytics/src/duration-measure-sql.ts new file mode 100644 index 00000000000..df8f0e370df --- /dev/null +++ b/packages/services/service-analytics/src/duration-measure-sql.ts @@ -0,0 +1,95 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * [#25778] The per-row operand of a dataset DURATION measure — + * `duration: { from, to, unit }` (`DatasetMeasureDurationSchema`, + * `@objectstack/spec/ui`) — on the native-SQL strategy, the one face that + * answers it. + * + * A duration measure aggregates, per group, the time between two `datetime` + * columns of the same row: `to − from`, in the declared unit. ADR-0021 holds + * the author to names (no expression, no SQL), so the difference is spelled + * here, once per dialect, over the storage form each SQL driver writes: + * + * | dialect | a `datetime` column holds | seconds from `from` to `to` | + * |---|---|---| + * | `sqlite` | canonical UTC ISO-8601 TEXT | `ROUND((julianday(to) - julianday(from)) * 86400000) / 1000` | + * | `postgres` | `timestamptz` | `EXTRACT(EPOCH FROM (to - from))`, as double precision | + * | `mysql` | `DATETIME(3)`, the UTC wall clock | `TIMESTAMPDIFF(MICROSECOND, from, to) / 1000000` | + * + * SQLite's `julianday` is a double counted from 4714 BC, so its difference + * carries float noise below the millisecond. Rounded to the millisecond, the + * resolution every driver stores a datetime at, the three dialects answer the + * same number for the same instants. The column references arrive already + * passed through the host's temporal repair (`coerceTemporalFilterColumn`), so + * a legacy SQLite column still holding epoch integers reads as canonical text + * here too. + * + * A NULL in either column makes the difference NULL, and every aggregate skips + * a NULL, so a row missing either instant takes no part: `avg` is the mean over + * the rows that carry both. A `to` earlier than its `from` is a negative + * duration, aggregated as stored. The measure reports the data and never + * clamps it. + * + * The seconds are divided by the unit's length (`DATASET_DURATION_UNIT_SECONDS`), + * so `hours` answers a fractional hour count. + * + * `'unknown'` has no arm: no difference of two instants parses alike on every + * SQL dialect. `NativeSQLStrategy.canHandle` declines a duration measure on a + * datasource whose dialect the host cannot name, and the service refuses it. + * + * {@link durationMeasureIn} is here too: the one lookup of a duration measure by + * name, for this face and for the service's gates. + */ + +import { DATASET_DURATION_UNIT_SECONDS, type DatasetDurationUnitValue } from '@objectstack/spec/ui'; +import type { DurationMeasureSpec } from './strategies/types.js'; +import type { AnalyticsSqlDialect } from './text-match-sql.js'; + +/** + * The duration measure `name` names in a compiled dataset's `durations` (the + * dataset scope's `measureDurations`), or `undefined`. The one lookup every + * reader uses, own keys only, so a measure named after an `Object.prototype` + * member (`constructor`) is never read as a duration it is not. + */ +export function durationMeasureIn( + durations: Readonly> | undefined, + name: string, +): DurationMeasureSpec | undefined { + return durations && Object.prototype.hasOwnProperty.call(durations, name) ? durations[name] : undefined; +} + +/** The dialects {@link durationOperandSql} has an arm for. */ +export type DurationSqlDialect = Exclude; + +/** Whether {@link durationOperandSql} can spell a duration on `dialect`. */ +export function hasDurationSql(dialect: AnalyticsSqlDialect): dialect is DurationSqlDialect { + return dialect !== 'unknown'; +} + +/** Seconds from `fromSql` to `toSql`, both column references, on `dialect`. */ +function durationSecondsSql(dialect: DurationSqlDialect, fromSql: string, toSql: string): string { + switch (dialect) { + case 'sqlite': + return `(ROUND((julianday(${toSql}) - julianday(${fromSql})) * 86400000.0) / 1000.0)`; + case 'postgres': + return `CAST(EXTRACT(EPOCH FROM (${toSql} - ${fromSql})) AS DOUBLE PRECISION)`; + case 'mysql': + return `(TIMESTAMPDIFF(MICROSECOND, ${fromSql}, ${toSql}) / 1000000.0)`; + } +} + +/** + * The per-row duration from `fromSql` to `toSql` in `unit`: the operand an + * aggregate (`AVG(…)`, or its conditional form under a measure filter) wraps. + */ +export function durationOperandSql( + dialect: DurationSqlDialect, + fromSql: string, + toSql: string, + unit: DatasetDurationUnitValue, +): string { + const seconds = durationSecondsSql(dialect, fromSql, toSql); + const length = DATASET_DURATION_UNIT_SECONDS[unit]; + return length === 1 ? seconds : `(${seconds} / ${length}.0)`; +} diff --git a/packages/services/service-analytics/src/preview-evaluator.ts b/packages/services/service-analytics/src/preview-evaluator.ts index 4a8656492a5..2273c3ea1d9 100644 --- a/packages/services/service-analytics/src/preview-evaluator.ts +++ b/packages/services/service-analytics/src/preview-evaluator.ts @@ -41,6 +41,9 @@ import { type TemporalComparandKind, } from '@objectstack/core'; import { explicitDateRangeWindow } from './date-range-array-arm.js'; +import { DATASET_DURATION_UNIT_SECONDS } from '@objectstack/spec/ui'; +import { durationMeasureIn } from './duration-measure-sql.js'; +import type { DurationMeasureSpec } from './strategies/types.js'; // [#19810] The `where` door's refusal envelope — `INVALID_FILTER` / 400, // EXPORTED by `filter-normalizer` precisely so a sibling in this package cannot // invent a second spelling of it. A draft-preview filter is a `where`-door @@ -729,6 +732,24 @@ function previewStorageRow(row: Row, kindOf: PreviewTemporalKind): Row { return out ?? row; } +/** [#25778] The key a row's duration is carried under to {@link aggregate}. */ +const DURATION_OPERAND = 'duration'; + +/** + * [#25778] One row's duration measure: the time from `duration.from` to + * `duration.to` in `duration.unit`, as the native-SQL face computes it + * (`duration-measure-sql.ts`), rounded to the millisecond — or `null` when + * either column holds no instant, which {@link aggregate} skips as SQL skips a + * NULL. Each column reads through `utcInstantMs`, the primitive this face + * orders instants by. + */ +function durationOf(row: Row, duration: DurationMeasureSpec): number | null { + const from = utcInstantMs(row[duration.from]); + const to = utcInstantMs(row[duration.to]); + if (from === null || to === null) return null; + return Math.round(to - from) / 1000 / DATASET_DURATION_UNIT_SECONDS[duration.unit]; +} + /** * Evaluate `query` over `rows` using the cube's measure/dimension specs. * Mirrors the engine strategies' output contract: rows keyed by bare @@ -745,6 +766,7 @@ export function evaluateAnalyticsQueryOverRows( cube: Cube, rows: Row[], declaredType?: (field: string) => string | undefined, + durations?: Readonly>, ): AnalyticsResult { const lowering = declaredPreviewLowering(declaredType); // 1. Row-level filters: `where`, then timeDimension dateRanges. @@ -842,6 +864,13 @@ export function evaluateAnalyticsQueryOverRows( for (const g of groups.values()) { const row: Row = { ...g.values }; for (const m of query.measures) { + // [#25778] A dataset duration measure, absent from the cube: its + // aggregate over each row's duration, through the same arms. + const duration = durationMeasureIn(durations, m); + if (duration) { + row[m] = aggregate(g.rows.map((r) => ({ [DURATION_OPERAND]: durationOf(r, duration) })), duration.aggregate, DURATION_OPERAND); + continue; + } const metric = cube.measures?.[m]; row[m] = aggregate(g.rows, String(metric?.type ?? 'count'), String(metric?.sql ?? '*')); } diff --git a/packages/services/service-analytics/src/strategies/native-sql-strategy.ts b/packages/services/service-analytics/src/strategies/native-sql-strategy.ts index d255ba69917..dcf7ad5fed3 100644 --- a/packages/services/service-analytics/src/strategies/native-sql-strategy.ts +++ b/packages/services/service-analytics/src/strategies/native-sql-strategy.ts @@ -3,7 +3,7 @@ import type { AnalyticsQuery, AnalyticsResult } from '@objectstack/spec/contracts'; import type { Cube } from '@objectstack/spec/data'; import { AggregationMetricType, NUMERIC_VALUE_TYPES, type AggregationFunction } from '@objectstack/spec/data'; -import type { AnalyticsStrategy, StrategyContext, DatasetScopedStrategyContext } from './types.js'; +import type { AnalyticsStrategy, StrategyContext, DatasetScopedStrategyContext, DurationMeasureSpec } from './types.js'; import { declaredDatetimeLowering, findNestedRelationCondition, @@ -45,6 +45,8 @@ import { datasetInvalidError, invalidMemberError } from '../dataset-refusal.js'; import { type LikeShape } from '../like-pattern.js'; import { textMatchPredicateSql, sqlDialectFor, type AnalyticsSqlDialect } from '../text-match-sql.js'; import { whereContainsMembershipSql } from '../contains-membership-sql.js'; +// [#25778] The per-row operand of a dataset duration measure, per dialect. +import { durationMeasureIn, durationOperandSql, hasDurationSql } from '../duration-measure-sql.js'; import { isJsonStoredShape } from '../contains-membership-sql.js'; import { expandEmptyOperator } from '@objectstack/spec/data'; import { resolveAnalyticsDateRangeString } from '@objectstack/core'; @@ -754,6 +756,15 @@ export class NativeSQLStrategy implements AnalyticsStrategy { // reason: one owner of the composition, no second copy here. Every object // the statement scopes is read, as the cross-field decline reads them. if (this.readScopeCarriesReadableIds(query, ctx)) return false; + // ── [#25778] DECLINE a duration measure on a dialect nobody named ────── + // + // A dataset duration measure is the time between two `datetime` columns of + // a row, and the difference of two instants is spelled per dialect + // (`duration-measure-sql.ts`); on `'unknown'` there is no spelling to + // emit. Declined here, before anything compiles, as the JSON-function + // decline above is: the query routes on, and the service refuses a + // duration measure no native statement serves, with the reason. + if (this.durationMeasureOnUnknownDialectIn(query, ctx)) return false; const caps = ctx.queryCapabilities(query.cube); return caps.nativeSql && typeof ctx.executeRawSql === 'function'; } @@ -777,6 +788,20 @@ export class NativeSQLStrategy implements AnalyticsStrategy { return objects.some((object) => findReadableIdsLeaf(ctx.getReadScope!(object)) !== null); } + /** + * [#25778] Does this query request a dataset duration measure on a base + * object whose SQL dialect the host cannot name? See the decline at + * {@link canHandle}. A duration reads two columns of the base object, so + * the base object's dialect is the one asked. + */ + private durationMeasureOnUnknownDialectIn(query: AnalyticsQuery, ctx: StrategyContext): boolean { + const cube = query.cube ? ctx.getCube(query.cube) : undefined; + if (!cube) return false; + const durations = (ctx as DatasetScopedStrategyContext).getDatasetScope?.(query.cube!)?.measureDurations; + if (!(query.measures ?? []).some((measure) => durationMeasureIn(durations, measure))) return false; + return !hasDurationSql(sqlDialectFor(ctx, this.extractObjectName(cube))); + } + /** * [#21080] Does this query read an object the engine holds a middleware * for — or one it cannot answer about? See the decline at {@link canHandle}. @@ -1101,9 +1126,14 @@ export class NativeSQLStrategy implements AnalyticsStrategy { // never projected, a different defect that must stay visible. The dataset // door's `DatasetExecutor` fill still runs after this and is idempotent on // a folded row. + // [#25778] A dataset duration measure is absent from the cube and carries + // its aggregate in the dataset scope; it folds by that aggregate, and it + // answers a number whatever its aggregate. + const durations = (ctx as DatasetScopedStrategyContext).getDatasetScope?.(query.cube!)?.measureDurations; + const durationOf = (member: string) => durationMeasureIn(durations, member); const folds: Array = []; for (const member of query.measures ?? []) { - const identity = emptyGroupValueFor(this.lookupMember(cube, member, 'measure')?.type); + const identity = emptyGroupValueFor(durationOf(member)?.aggregate ?? this.lookupMember(cube, member, 'measure')?.type); if (identity !== undefined) folds.push([member, identity]); } if (folds.length > 0 && Array.isArray(rows)) { @@ -1139,6 +1169,7 @@ export class NativeSQLStrategy implements AnalyticsStrategy { const declaredType = (ctx as DatasetScopedStrategyContext).declaredFieldType; const referenceOf = relationshipReferenceOf(ctx); const numberMeasures = (query.measures ?? []).filter((member) => { + if (durationOf(member)) return true; const measure = this.lookupMember(cube, member, 'measure'); if (!measure?.type || !Object.prototype.hasOwnProperty.call(AGGREGATE_ANSWER_KIND, measure.type)) return false; if (AGGREGATE_ANSWER_KIND[measure.type as AggregationFunction] === 'number') return true; @@ -1274,7 +1305,10 @@ export class NativeSQLStrategy implements AnalyticsStrategy { ctx, ) : null; - const aggExpr = this.resolveMeasureSql(cube, measure, tableName, joins, predicate, ctx); + const duration = durationMeasureIn(datasetScope?.measureDurations, measure); + const aggExpr = duration + ? this.resolveDurationMeasureSql(cube, measure, duration, tableName, joins, predicate, ctx) + : this.resolveMeasureSql(cube, measure, tableName, joins, predicate, ctx); selectClauses.push(`${aggExpr} AS "${measure}"`); } } @@ -1766,6 +1800,46 @@ export class NativeSQLStrategy implements AnalyticsStrategy { return AGGREGATE_SQL[aggregate](col); } + /** + * [#25778] A dataset duration measure: `aggregate` over the per-row time + * from `duration.from` to `duration.to`, two `datetime` columns of the base + * object, in `duration.unit` (`duration-measure-sql.ts`). Each column reads + * through the host's temporal repair, as a filter on it does, so a legacy + * SQLite column that still holds epoch integers measures the same instants. + * Under a measure filter the operand takes the conditional aggregate, as + * every other measure's does. + */ + private resolveDurationMeasureSql( + cube: Cube, + member: string, + duration: DurationMeasureSpec, + parentTable: string, + joins: StatementJoins, + predicate: string | null, + ctx: StrategyContext, + ): string { + const dialect = sqlDialectFor(ctx, parentTable); + // Unreachable by construction: {@link canHandle} declines a duration + // measure on `'unknown'`. Deliberately BARE, an undeclared 500, the tier + // of {@link assertNoCrossFieldComparison}: arriving here is drift between + // our own routing gate and this emitter, never the author's mistake. + if (!hasDurationSql(dialect)) { + throw new Error( + `[native-sql-strategy] duration measure "${member}" on cube "${cube.name}" reached the SQL ` + + `compiler on a datasource whose dialect the host does not name; \`canHandle\` declines it there.`, + ); + } + const column = (field: string) => this.temporalColumn( + ctx, + { object: parentTable, field }, + this.qualifyAndRegisterJoin(field, parentTable, joins, cube), + ); + const operand = durationOperandSql(dialect, column(duration.from), column(duration.to), duration.unit); + return predicate !== null + ? CONDITIONAL_AGGREGATE_SQL[duration.aggregate](operand, predicate) + : AGGREGATE_SQL[duration.aggregate](operand); + } + private resolveFieldSql( cube: Cube, member: string, From 4edb2dea594f2e64fd995f3c55e25a5c184d339d Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 25 Oct 2026 14:24:43 +0000 Subject: [PATCH 04/14] feat(lint,spec): os validate refuses a non-datetime duration column; docs, changeset, ledger - lint: `measure-duration-field-type-refused` at `duration.from` / `.to`, with an `os explain` entry; `dataset-field-unknown` checks both columns. - spec: `DATASET_DURATION_FIELD_TYPE`; the unit's type alias shares its schema's name; liveness drills `duration.{from,to,unit}`. - docs: the duration measure in analytics.mdx and validating-metadata.mdx. - changeset: minor on spec, service-analytics and lint. Co-Authored-By: Claude Claude-Session: https://claude.ai/code/session_01AKPixnkg91qCkQCVFDJuqW --- .changeset/25778-dataset-duration-measure.md | 16 ++ content/docs/data-modeling/analytics.mdx | 34 ++++ .../docs/deployment/validating-metadata.mdx | 10 ++ content/docs/references/index.mdx | 10 +- content/docs/references/ui/dataset.mdx | 41 ++++- .../ui.md | 10 +- packages/lint/src/index.ts | 1 + packages/lint/src/rule-explanations.ts | 23 ++- .../validate-dataset-measure-aggregates.ts | 94 ++++++++++- .../validate-dataset-measure-duration.test.ts | 154 ++++++++++++++++++ .../lint/src/validate-dataset-references.ts | 13 ++ .../service-analytics/src/dataset-compiler.ts | 24 ++- .../src/duration-measure-sql.ts | 4 +- .../service-analytics/src/strategies/types.ts | 4 +- packages/spec/api-surface/ui.json | 7 + packages/spec/declaration-map/ui.json | 3 + packages/spec/export-origins/ui.json | 7 + packages/spec/liveness/dataset.json | 23 +++ .../spec/liveness/state-counts/dataset.md | 2 +- .../src/ui/dataset-measure-duration.test.ts | 131 +++++++++++++++ packages/spec/src/ui/dataset.zod.ts | 13 +- 21 files changed, 585 insertions(+), 39 deletions(-) create mode 100644 .changeset/25778-dataset-duration-measure.md create mode 100644 packages/lint/src/validate-dataset-measure-duration.test.ts create mode 100644 packages/spec/src/ui/dataset-measure-duration.test.ts diff --git a/.changeset/25778-dataset-duration-measure.md b/.changeset/25778-dataset-duration-measure.md new file mode 100644 index 00000000000..16af085802a --- /dev/null +++ b/.changeset/25778-dataset-duration-measure.md @@ -0,0 +1,16 @@ +--- +'@objectstack/spec': minor +'@objectstack/service-analytics': minor +'@objectstack/lint': minor +--- + +feat(spec,analytics,lint): a dataset measure can aggregate the time between two datetime fields of each record + +Clause-②: yes (widening) +Metadata-protocol: added + +- **`@objectstack/spec`**: `DatasetMeasureSchema` gains an optional `duration: { from, to, unit }`, in place of `field`. `from` and `to` name two `datetime` fields of the dataset's own object by their bare names, `unit` is `seconds`, `minutes`, `hours` or `days`, and the measure's `aggregate` is `avg`, `min`, `max` or `sum` of `to` minus `from`. A relationship path, an expression or the row wildcard in either column, the same column twice, another unit, a counting aggregate, or a `field`, `derived` or `currency` beside it is refused at parse. New exports: `DatasetMeasureDurationSchema`, `DatasetDurationUnit`, `DATASET_DURATION_UNIT_SECONDS`, `DATASET_DURATION_AGGREGATES`, `DATASET_DURATION_FIELD_TYPE` and the `DatasetMeasureDuration` / `DatasetDurationUnit` types. +- **`@objectstack/service-analytics`**: the dataset compile refuses a duration column declared anything but `datetime` (`400 DATASET_INVALID`, naming the column and its type). The native-SQL strategy computes the duration as one statement on SQLite, PostgreSQL and MySQL, with the same number on each: a record missing either instant is left out, a `to` before its `from` counts as negative, and over a group with no such record `sum` answers `0` and the others `null`. Measure filters, derived measures, ordering and the draft preview serve it. A query the native strategy does not serve (a time dimension bucketed by `granularity`, a field-to-field filter, an object served through engine middleware or an external datasource, a non-SQL datasource) is refused with `400 INVALID_FIELD` naming the measure. Its two columns pass the field-level read gate like any measure's field. The temporal half of the aggregate × field-type refusal now names the duration measure instead of a stored "days open" number. +- **`@objectstack/lint`**: `os validate` refuses a duration column not declared `datetime` with the new gating rule `measure-duration-field-type-refused` (at `datasets[N].measures[M].duration.from` / `.to`), with an `os explain` entry. `dataset-field-unknown` now also reports a duration column the object does not declare. + +What to do: nothing. A dataset without `duration` parses, compiles and answers exactly as before. diff --git a/content/docs/data-modeling/analytics.mdx b/content/docs/data-modeling/analytics.mdx index 4541d873fa0..254627cb0a8 100644 --- a/content/docs/data-modeling/analytics.mdx +++ b/content/docs/data-modeling/analytics.mdx @@ -64,6 +64,8 @@ export const SalesDataset = defineDataset({ { name: 'won_amount', aggregate: 'sum', field: 'amount', filter: { stage: 'closed_won' } }, // Derived measure — references OTHER measures by name only (no raw fields/SQL). { name: 'win_rate', derived: { op: 'ratio', of: ['won_amount', 'revenue'] }, format: '0.0%' }, + // Duration measure — the time between two datetime fields of each record. + { name: 'avg_days_to_close', aggregate: 'avg', duration: { from: 'created_at', to: 'closed_at', unit: 'days' } }, ], }); ``` @@ -94,6 +96,38 @@ export default defineStack({ - **RLS / tenant scoping is enforced by the runtime**, per joined object — never declared in the dataset. There is one place to reason about access. +### Duration measures + +A measure can aggregate the time between two `datetime` fields of the same +record (time to resolve, time to first response, cycle time) with `duration` +in place of `field`: + +```ts +{ name: 'avg_hours_to_resolve', aggregate: 'avg', duration: { from: 'created_at', to: 'resolved_at', unit: 'hours' } } +``` + +- `from` and `to` name two `datetime` fields of the dataset's own object, by + their bare names. A relationship path, an expression, or a `date`, `time`, + number or text field is refused: `os validate` reports + `measure-duration-field-type-refused`, and the dataset compile answers + `400 DATASET_INVALID`. +- `unit` is `seconds`, `minutes`, `hours` or `days`, and the measure answers a + number of them (`1.5` hours). `aggregate` is `avg`, `min`, `max` or `sum`. +- A record missing either instant is left out, as a null is left out of an + `avg`. A `to` earlier than its `from` counts as a negative duration. Over a + group with no record carrying both, `sum` answers `0` and the others answer + nothing. +- A measure `filter`, and a `derived` measure naming the duration measure in + `of`, work as they do for any measure. +- It runs as one SQL statement on SQLite, PostgreSQL and MySQL, and answers the + same number on each. A query the service cannot compile to one such + statement is refused with `400 INVALID_FIELD` naming the measure, rather than + answered another way: a time dimension bucketed by `granularity` (a monthly + trend), a field-to-field comparison in a filter, an object served through + engine middleware or an external datasource, or a datasource that is not SQL. +- Comparing each record's duration against a value held on another object (an + SLA target per ticket type, say) is not part of it. + ## Binding a dashboard widget A widget selects dimensions/measures by name. Its presentation-scope `filter` diff --git a/content/docs/deployment/validating-metadata.mdx b/content/docs/deployment/validating-metadata.mdx index 20c5abb4e95..22363b9c267 100644 --- a/content/docs/deployment/validating-metadata.mdx +++ b/content/docs/deployment/validating-metadata.mdx @@ -249,6 +249,16 @@ on the object the last hop reaches — the join the cube declares for that hop, else the lookup's `reference`. A measure of an expression type (`number`, `string`, `boolean`) and the row wildcard `'*'` are not judged by this check. +A **duration measure** (`duration: { from, to, unit }`, the time between two +fields of each record) is judged by the same check under a third id. Each of +`from` and `to` must name a field declared `datetime`; one declared anything +else (a `date`, a `time`, a number, text) is refused +(`measure-duration-field-type-refused`) at +`datasets[N].measures[M].duration.from` or `.to`, because it holds no instant +to measure from. The analytics service refuses the same column with +`400 DATASET_INVALID` when a query is built. A name the object does not declare +is reported as `dataset-field-unknown` instead. + ### 7. Navigation exposing objects nobody can read Navigation and permissions are separate metadata, each valid on its own — so an diff --git a/content/docs/references/index.mdx b/content/docs/references/index.mdx index fb20c56a6a1..2061295d4ac 100644 --- a/content/docs/references/index.mdx +++ b/content/docs/references/index.mdx @@ -1,7 +1,7 @@ --- title: Protocol reference — every schema by module navTitle: Protocol Reference -description: Every schema published by @objectstack/spec — 1549 schemas across 14 protocol modules +description: Every schema published by @objectstack/spec — 1551 schemas across 14 protocol modules --- {/* ⚠️ AUTO-GENERATED — DO NOT EDIT. Run build-docs.ts to regenerate. Hand-written docs live in the module folders under content/docs/. */} @@ -33,8 +33,8 @@ counts are sums of the rows they head. Regenerate with | [Shared Protocol](/docs/references/shared) | 10 | 31 | Primitives used across every protocol — identifiers, HTTP, expressions, error maps, enums. | | [Studio Protocol](/docs/references/studio) | 3 | 35 | Studio designer metadata — the authoring surfaces for the protocols above. | | [System Protocol](/docs/references/system) | 36 | 285 | The runtime environment — logging, jobs, cache, metrics, notifications, i18n and compliance. | -| [UI Protocol](/docs/references/ui) | 16 | 169 | Apps, pages, views, dashboards, reports, actions and themes — the ObjectUI layer. | -| **Total** | **198** | **1549** | 14 protocol modules | +| [UI Protocol](/docs/references/ui) | 16 | 171 | Apps, pages, views, dashboards, reports, actions and themes — the ObjectUI layer. | +| **Total** | **198** | **1551** | 14 protocol modules | --- @@ -365,7 +365,7 @@ The runtime environment — logging, jobs, cache, metrics, notifications, i18n a ## UI Protocol -**Source:** `packages/spec/src/ui/` · **Import:** `@objectstack/spec/ui` · **16 pages, 169 schemas** +**Source:** `packages/spec/src/ui/` · **Import:** `@objectstack/spec/ui` · **16 pages, 171 schemas** Apps, pages, views, dashboards, reports, actions and themes — the ObjectUI layer. @@ -378,7 +378,7 @@ Apps, pages, views, dashboards, reports, actions and themes — the ObjectUI lay | [`chart.zod.ts`](/docs/references/ui/chart) | `ChartAggregate`, `ChartAggregateFunction`, `ChartAnnotation`, `ChartAxis`, `ChartConfig`, `ChartDrillDown`, `ChartGroupBy`, `ChartInteraction`, `ChartSeries`, `ChartType` | | [`component.zod.ts`](/docs/references/ui/component) | `ActionButtonProps`, `ActionGroupProps`, `ActionIconProps`, `ActionMenuProps`, `ElementButtonProps`, `ElementDefinitionListProps`, `ElementFilterProps`, `ElementFormProps`, `ElementImageProps`, `ElementMetadataViewerProps`, `ElementNumberProps`, `ElementRecordPickerProps`, `ElementRepeaterProps`, `ElementTextInputProps`, `ElementTextProps`, `ObjectCalendarProps`, `ObjectFormProps`, `ObjectGanttProps`, `ObjectGridProps`, `ObjectKanbanProps`, `ObjectMapProps`, `ObjectMasterDetailFormProps`, `ObjectMetricProps`, `ObjectTimelineProps`, `ObjectTreeProps`, `PageAccordionProps`, `PageCardProps`, `PageContainerProps`, `PageHeaderProps`, `PageTabsProps`, `RecordActivityProps`, `RecordAlertAction`, `RecordAlertProps`, `RecordChatterProps`, `RecordDetailsProps`, `RecordHighlightsField`, `RecordHighlightsProps`, `RecordHistoryProps`, `RecordLineItemsProps`, `RecordPathProps`, `RecordQuickActionsProps`, `RecordReferenceRailProps`, `RecordRelatedListProps`, `ReferenceRailEntry` | | [`dashboard.zod.ts`](/docs/references/ui/dashboard) | `Dashboard`, `DashboardHeader`, `DashboardHeaderAction`, `DashboardWidget`, `DashboardWidgetChartConfig`, `DashboardWidgetOptions`, `GlobalFilter`, `GlobalFilterOptionsFrom`, `WidgetActionType`, `WidgetColorVariant` | -| [`dataset.zod.ts`](/docs/references/ui/dataset) | `Dataset`, `DatasetDimension`, `DatasetMeasure`, `DerivedMeasureOp` | +| [`dataset.zod.ts`](/docs/references/ui/dataset) | `Dataset`, `DatasetDimension`, `DatasetDurationUnit`, `DatasetMeasure`, `DatasetMeasureDuration`, `DerivedMeasureOp` | | [`expression-bindable-text-keys.zod.ts`](/docs/references/ui/expression-bindable-text-keys) | `ExpressionBindableTextKey` | | [`i18n.zod.ts`](/docs/references/ui/i18n) | `AriaProps`, `I18nLabel`, `InlineLocaleMap` | | [`notification.zod.ts`](/docs/references/ui/notification) | `NotificationPosition`, `NotificationSeverity`, `NotificationType` | diff --git a/content/docs/references/ui/dataset.mdx b/content/docs/references/ui/dataset.mdx index 14b3ab55887..f5c5b3ff0f9 100644 --- a/content/docs/references/ui/dataset.mdx +++ b/content/docs/references/ui/dataset.mdx @@ -1,7 +1,7 @@ --- title: Dataset schema — UI Protocol reference navTitle: Dataset -description: "Analytics Dataset — the one semantic layer. Reference for Dataset, DatasetDimension, DatasetMeasure, DerivedMeasureOp: every property with its type and default." +description: "Analytics Dataset — the one semantic layer. Reference for Dataset, DatasetDimension, DatasetDurationUnit and 3 more: every property with its type and default." --- {/* ⚠️ AUTO-GENERATED — DO NOT EDIT. Run build-docs.ts to regenerate. Hand-written docs live in the module folders under content/docs/. */} @@ -34,8 +34,8 @@ layer is absorbed/retired in a later phase (D-A). ## TypeScript Usage ```typescript -import { DatasetSchema, DatasetDimensionSchema, DatasetMeasureSchema, DerivedMeasureOp } from '@objectstack/spec/ui'; -import type { Dataset, DatasetDimension, DatasetMeasure } from '@objectstack/spec/ui'; +import { DatasetSchema, DatasetDimensionSchema, DatasetDurationUnit, DatasetMeasureSchema, DatasetMeasureDurationSchema, DerivedMeasureOp } from '@objectstack/spec/ui'; +import type { Dataset, DatasetDimension, DatasetMeasure, DatasetMeasureDuration } from '@objectstack/spec/ui'; // Validate data const result = DatasetSchema.parse(data); @@ -89,6 +89,7 @@ const result = DatasetSchema.parse(data); | **format** | `string` | optional | Numeral pattern for a NUMERIC measure — grouping, decimals, percent; e.g. "0,0.00", "0.0%". An amount takes its symbol from `currency`, not from a "$" in the pattern. A DATE-valued measure never reads a date pattern: `"YYYY-MM-DD"` renders that arm's default face. A date or datetime value reads `format` as a display style — `short` or `relative`, honoured on both. | | **currency** | `string` | optional | Display currency code (ISO 4217) | | **derived** | `{ op: Enum<'ratio' \| 'sum' \| 'difference' \| 'product'>; of: string[] }` | optional | | +| **duration** | `{ from: string; to: string; unit: Enum<'seconds' \| 'minutes' \| 'hours' \| 'days'> }` | optional | Time between two datetime columns of each record (`{ from, to, unit }`), reduced by `aggregate` (avg / min / max / sum). Records with either instant null are excluded. Replaces `field`. | ### Nested Shape: `Dataset.protection` @@ -115,6 +116,18 @@ const result = DatasetSchema.parse(data); | **nullLabel** | `string \| Record` | optional | Label for the bucket of rows whose value for this dimension is null (e.g. "System" on a user axis). Display copy only: grouping is unchanged. When absent, consumers fall back to their own default label for that bucket. | +--- + +## DatasetDurationUnit + +### Allowed Values + +* `seconds` +* `minutes` +* `hours` +* `days` + + --- ## DatasetMeasure @@ -131,6 +144,28 @@ const result = DatasetSchema.parse(data); | **format** | `string` | optional | Numeral pattern for a NUMERIC measure — grouping, decimals, percent; e.g. "0,0.00", "0.0%". An amount takes its symbol from `currency`, not from a "$" in the pattern. A DATE-valued measure never reads a date pattern: `"YYYY-MM-DD"` renders that arm's default face. A date or datetime value reads `format` as a display style — `short` or `relative`, honoured on both. | | **currency** | `string` | optional | Display currency code (ISO 4217) | | **derived** | `{ op: Enum<'ratio' \| 'sum' \| 'difference' \| 'product'>; of: string[] }` | optional | | +| **duration** | `{ from: string; to: string; unit: Enum<'seconds' \| 'minutes' \| 'hours' \| 'days'> }` | optional | Time between two datetime columns of each record (`{ from, to, unit }`), reduced by `aggregate` (avg / min / max / sum). Records with either instant null are excluded. Replaces `field`. | + +### Nested Shape: `DatasetMeasure.duration` + +| Property | Type | Required | Description | +| :--- | :--- | :--- | :--- | +| **from** | `string` | ✅ | The earlier instant: a datetime field of the dataset's own object, by its bare name. | +| **to** | `string` | ✅ | The later instant: a datetime field of the dataset's own object, by its bare name. | +| **unit** | `Enum<'seconds' \| 'minutes' \| 'hours' \| 'days'>` | ✅ | Unit of the result: the measure answers a number of these (1.5 with "days" is a day and a half). | + + +--- + +## DatasetMeasureDuration + +### Properties + +| Property | Type | Required | Description | +| :--- | :--- | :--- | :--- | +| **from** | `string` | ✅ | The earlier instant: a datetime field of the dataset's own object, by its bare name. | +| **to** | `string` | ✅ | The later instant: a datetime field of the dataset's own object, by its bare name. | +| **unit** | `Enum<'seconds' \| 'minutes' \| 'hours' \| 'days'>` | ✅ | Unit of the result: the measure answers a number of these (1.5 with "days" is a day and a half). | --- diff --git a/docs/audits/2026-07-unknown-key-strictness-ledger.counts/ui.md b/docs/audits/2026-07-unknown-key-strictness-ledger.counts/ui.md index 9a3c1047a2a..35f51238c07 100644 --- a/docs/audits/2026-07-unknown-key-strictness-ledger.counts/ui.md +++ b/docs/audits/2026-07-unknown-key-strictness-ledger.counts/ui.md @@ -21,7 +21,7 @@ The `strict` column is the one the campaign schedules against; it counts both th | Dir | Sites | strict | passthrough | catchall | strip | |---|---|---|---|---|---| -| `ui/` | 215 | 204 | 4 | 0 | 7 | +| `ui/` | 216 | 205 | 4 | 0 | 7 | ## `ui/` — sites @@ -38,7 +38,7 @@ classify and is not listed (it becomes reportable the day it grows its first sit | `chart.zod.ts` | 8 | | `component.zod.ts` | 81 | | `dashboard.zod.ts` | 11 | -| `dataset.zod.ts` | 4 | +| `dataset.zod.ts` | 5 | | `i18n.zod.ts` | 1 | | `page.zod.ts` | 12 | | `report.zod.ts` | 3 | @@ -46,7 +46,7 @@ classify and is not listed (it becomes reportable the day it grows its first sit | `sharing.zod.ts` | 1 | | `view.zod.ts` | 59 | | `widget.zod.ts` | 1 | -| **total** | **215** | +| **total** | **216** | ## `ui/` — open @@ -54,7 +54,7 @@ Per file, how many of its sites still silently discard unknown keys. The `Class` column that decides the bucket split is hand-written in the ledger; the arithmetic over it is here. -**7 strip of 215**, in 4 file(s). +**7 strip of 216**, in 4 file(s). | File | Strip | Sites | |---|---|---| @@ -62,7 +62,7 @@ over it is here. | `app.zod.ts` | 1 | 19 | | `view.zod.ts` | 4 | 59 | | `widget.zod.ts` | 1 | 1 | -| **total** | **7** | **215** | +| **total** | **7** | **216** | | Bucket | Sites | |---|---| diff --git a/packages/lint/src/index.ts b/packages/lint/src/index.ts index 211e012a8a1..b7a3c3cd5de 100644 --- a/packages/lint/src/index.ts +++ b/packages/lint/src/index.ts @@ -60,6 +60,7 @@ export { validateDatasetMeasureAggregates, MEASURE_AGGREGATE_FIELD_TYPE_REFUSED, DIMENSION_JSON_STORED_FIELD_REFUSED, + MEASURE_DURATION_FIELD_TYPE_REFUSED, // [#23235] The dimension leg's group-key verdict on its own, so a consumer // deriving dataset dimensions (cloud's AI build) asks the door's two // predicates in the door's order here rather than composing them itself. diff --git a/packages/lint/src/rule-explanations.ts b/packages/lint/src/rule-explanations.ts index 1ed3de20a02..db0572f0dee 100644 --- a/packages/lint/src/rule-explanations.ts +++ b/packages/lint/src/rule-explanations.ts @@ -2060,7 +2060,8 @@ const MEASURE_AGGREGATE_FIELD_TYPE_REFUSED_EXPLANATION: RuleExplanation = { 'datetime, time, boolean, toggle.', 'So the fix is a field of an accepted type, or an aggregate the field\'s declaration accepts. A ' + 'quantity that must be added up or averaged has to be STORED as a numeric field and aggregated as ' + - 'one.', + 'one. The time BETWEEN two `datetime` fields of a record is not an aggregate over either one: it is ' + + 'a `duration: { from, to, unit }` measure (`measure-duration-field-type-refused`).', // Held equal to `VIRTUAL_FIELD_AGGREGATE_REMEDY` (`@objectstack/spec/data`) — the `fix:` line, and // the compile leg's remedy, for this pair — by `validate-dataset-measure-aggregates.test.ts`. 'A `formula` field stores no value, whatever its `returnType` says, so `sum`, `avg`, `min` and `max` ' + @@ -2104,6 +2105,25 @@ const DIMENSION_JSON_STORED_FIELD_REFUSED_EXPLANATION: RuleExplanation = { ], }; +const MEASURE_DURATION_FIELD_TYPE_REFUSED_EXPLANATION: RuleExplanation = { + rule: 'measure-duration-field-type-refused', + covers: 'which fields a duration measure measures between', + paragraphs: [ + 'A duration measure, `duration: { from, to, unit }` on a dataset measure, aggregates the time from one ' + + 'field to another on the same record: `avg`, `min`, `max` or `sum` of `to` minus `from`, as a number ' + + 'of `unit`s (seconds, minutes, hours or days). The time between two INSTANTS is the same number on ' + + 'every backend, so both fields must be declared `datetime`. A `date` is a calendar day and a `time` ' + + 'a wall clock, neither one an instant; a `text` holding ISO strings or a number holding epoch values ' + + 'is no instant the database knows, and each SQL family would read it its own way.', + 'So the fix is a `datetime` field of the dataset\'s own object in each slot, by its bare name. A ' + + 'record missing either instant takes no part in the aggregate, and a `to` earlier than its `from` ' + + 'counts as a negative duration.', + 'The dataset compile refuses the same column later, with `400 DATASET_INVALID` when a query is built. ' + + 'This rule makes the same refusal where the author is standing. A name the object does not declare ' + + 'is `dataset-field-unknown`\'s finding, and a field with no declared type is not judged.', + ], +}; + // ── React page props (`validate-react-page-props.ts`) ─────────────────────── // Two ids forward `@objectstack/spec`'s own refusal of a chart prop. The // verdict quotes it to its head — the key or value, and the schema's rename @@ -4965,6 +4985,7 @@ export const RULE_EXPLANATIONS: Readonly> = Obje [VALIDATION_RULE_JSON_SCHEMA_UNKNOWN_FORMAT_EXPLANATION.rule]: VALIDATION_RULE_JSON_SCHEMA_UNKNOWN_FORMAT_EXPLANATION, [MEASURE_AGGREGATE_FIELD_TYPE_REFUSED_EXPLANATION.rule]: MEASURE_AGGREGATE_FIELD_TYPE_REFUSED_EXPLANATION, [DIMENSION_JSON_STORED_FIELD_REFUSED_EXPLANATION.rule]: DIMENSION_JSON_STORED_FIELD_REFUSED_EXPLANATION, + [MEASURE_DURATION_FIELD_TYPE_REFUSED_EXPLANATION.rule]: MEASURE_DURATION_FIELD_TYPE_REFUSED_EXPLANATION, [REACT_CHART_DRILLDOWN_INVALID_EXPLANATION.rule]: REACT_CHART_DRILLDOWN_INVALID_EXPLANATION, [REACT_CHART_AGGREGATE_INVALID_EXPLANATION.rule]: REACT_CHART_AGGREGATE_INVALID_EXPLANATION, [REACT_CHART_FIELD_UNPROVISIONED_EXPLANATION.rule]: REACT_CHART_FIELD_UNPROVISIONED_EXPLANATION, diff --git a/packages/lint/src/validate-dataset-measure-aggregates.ts b/packages/lint/src/validate-dataset-measure-aggregates.ts index e0f84182c23..62e68da86a5 100644 --- a/packages/lint/src/validate-dataset-measure-aggregates.ts +++ b/packages/lint/src/validate-dataset-measure-aggregates.ts @@ -185,13 +185,29 @@ * The same skips 1, 3 and 4 hold, plus a member that writes no `sql`; and on * a measure, skip 5 and a measure that writes no `type`. * - * Both legs ride the one registry entry: gating, on all three commands. At the + * ## [#25778] The duration leg — `measure-duration-field-type-refused` + * + * A dataset measure written `duration: { from, to, unit }` aggregates the time + * between two columns of the same record, and the dataset compile + * (`dataset-compiler.ts`) refuses it with `400 DATASET_INVALID` when either + * column is declared anything but `DATASET_DURATION_FIELD_TYPE` (`datetime`, + * `@objectstack/spec/ui`): a `date`, a `time`, a number or a text holds no + * instant to measure from. This leg makes the same refusal where the author + * is standing, one finding per refused column, at + * `datasets[N].measures[M].duration.from` / `.to`. Each column is a bare name + * on the dataset's own object (the spec refuses anything else at parse), read + * on the object graph under the same skips 1, 3 and 4: a name that does not + * resolve is `dataset-field-unknown`'s finding, and an untyped one is not + * judged. A duration measure has no `field`, so the aggregate leg never + * reads it. + * + * The legs ride the one registry entry: gating, on all three commands. At the * runtime write door that entry is dispatched for a `dataset` write, whose * per-write snapshot carries no `analyticsCubes`, so the cube leg reads nothing * there; an `analytics_cube` write is dispatched to no rule today. * * The function keeps its name: it is the registry's key (`authoring-rules.ts`), - * and the dimension and cube legs ride on that one entry. + * and the dimension, duration and cube legs ride on that one entry. */ import { @@ -202,6 +218,7 @@ import { virtualFieldAggregateRemedy, type ValueShapeFieldDef, } from '@objectstack/spec/data'; +import { DATASET_DURATION_FIELD_TYPE } from '@objectstack/spec/ui'; import { collectionEntries } from './collection-entries.js'; import { @@ -232,6 +249,14 @@ export const MEASURE_AGGREGATE_FIELD_TYPE_REFUSED = 'measure-aggregate-field-typ */ export const DIMENSION_JSON_STORED_FIELD_REFUSED = 'dimension-json-stored-field-refused'; +/** + * [#25778] Stable diagnostic id for a dataset duration measure whose `from` or + * `to` names a field declared anything but `datetime`. Named for the same axis + * as {@link MEASURE_AGGREGATE_FIELD_TYPE_REFUSED}, the field's declared type, + * on the duration measure's own two slots. + */ +export const MEASURE_DURATION_FIELD_TYPE_REFUSED = 'measure-duration-field-type-refused'; + export interface DatasetMeasureAggregateFinding { /** * Always `error`. The pair is decidable from the author's own declarations — @@ -243,12 +268,16 @@ export interface DatasetMeasureAggregateFinding { * analytics door answers every query grouping by it with `400 INVALID_FIELD`. */ severity: 'error'; - rule: typeof MEASURE_AGGREGATE_FIELD_TYPE_REFUSED | typeof DIMENSION_JSON_STORED_FIELD_REFUSED; + rule: + | typeof MEASURE_AGGREGATE_FIELD_TYPE_REFUSED + | typeof DIMENSION_JSON_STORED_FIELD_REFUSED + | typeof MEASURE_DURATION_FIELD_TYPE_REFUSED; /** Human-readable location, e.g. `dataset "sales" › measure "avg_closed"` or `cube "sales" › dimension "meta"`. */ where: string; /** * Config path, e.g. `datasets[0].measures[2].aggregate`, `datasets[0].dimensions[1].field`, - * `analyticsCubes[0].dimensions.meta.sql` or `analyticsCubes[0].measures.distinct_meta.type`. + * `datasets[0].measures[3].duration.from`, `analyticsCubes[0].dimensions.meta.sql` or + * `analyticsCubes[0].measures.distinct_meta.type`. */ path: string; message: string; @@ -455,6 +484,31 @@ function refusedMeasureFinding( }; } +/** + * [#25778] The finding for a duration measure's `from` or `to` over a field + * declared anything but {@link DATASET_DURATION_FIELD_TYPE}: one verdict + * sentence, and one remedy line. Why a `date` or a `time` holds no instant, + * and which door refuses it later, are `os explain + * measure-duration-field-type-refused`. + */ +function durationColumnFinding( + site: MemberSite, + slot: 'from' | 'to', + shape: ValueShapeFieldDef, + baseObject: string, +): DatasetMeasureAggregateFinding { + return { + severity: 'error', + rule: MEASURE_DURATION_FIELD_TYPE_REFUSED, + where: site.where, + path: site.path, + message: + `\`duration.${slot}\` reads field "${site.column}" (${declaredAs(shape)} on ${site.declarer}), which holds ` + + `no instant, so the dataset compile refuses it (400 DATASET_INVALID)`, + hint: `Point \`duration.${slot}\` at a \`${DATASET_DURATION_FIELD_TYPE}\` field of "${baseObject}".`, + }; +} + /** * [#21082] The column a cube member's `sql` reads, resolved the way the * analytics door resolves it (`hop-object.ts`): a hop the cube declares a @@ -580,7 +634,8 @@ function cubeMemberFindings(cube: AnyRec, cubePath: string, graph: ObjectGraph): /** * Refuse every dataset measure whose `aggregate` the field's declaration - * cannot carry, and every dataset dimension whose field is JSON-stored; then + * cannot carry, [#25778] every duration column not declared `datetime`, and + * every dataset dimension whose field is JSON-stored; then * [#21082] every cube dimension whose `sql` column is JSON-stored, and * [#21419] every cube measure whose `type` the column's declaration cannot * carry. Returns findings (empty = @@ -642,6 +697,35 @@ export function validateDatasetMeasureAggregates(stack: unknown): DatasetMeasure }); recordsOf(ds.measures).forEach((measure, k) => { + // ── [#25778] The duration leg: each of the two columns, on the base object ── + if (isRec(measure.duration)) { + const measureName = strName(measure.name) ?? `#${k}`; + for (const slot of ['from', 'to'] as const) { + const column = strName(measure.duration[slot]); + if (!column) continue; + // ── Skip 3: the name does not resolve — `dataset-field-unknown`'s finding ── + const verdict = resolveFieldPath(graph, object, column); + if (!verdict || isUnjudgeable(verdict) || verdict.kind !== 'ok') continue; + // ── Skip 4: the field declares no type ── + const fieldType = verdict.meta?.type; + if (!fieldType || fieldType === DATASET_DURATION_FIELD_TYPE) continue; + findings.push( + durationColumnFinding( + { + where: `dataset "${dsName}" › measure "${measureName}"`, + path: `datasets[${di}].measures[${k}].duration.${slot}`, + column, + declarer: declarerOf(verdict, object), + }, + slot, + shapeOf(verdict, fieldType), + object, + ), + ); + } + return; + } + // ── Skip 2: nothing written in one of the two positions ── const aggregate = strName(measure.aggregate); const field = strName(measure.field); diff --git a/packages/lint/src/validate-dataset-measure-duration.test.ts b/packages/lint/src/validate-dataset-measure-duration.test.ts new file mode 100644 index 00000000000..9b683ec48d3 --- /dev/null +++ b/packages/lint/src/validate-dataset-measure-duration.test.ts @@ -0,0 +1,154 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. +// +// [#25778] `measure-duration-field-type-refused` — the authoring-time leg of the +// dataset duration measure (`duration: { from, to, unit }`). The dataset compile +// refuses a duration column not declared `datetime` with `400 DATASET_INVALID`; +// this leg refuses the same column where the author is standing. The negative +// controls matter as much as the refusals: a `datetime` pair, the registry's own +// `created_at`, and the skips (a name the object does not declare, an untyped +// field) must stay silent, and the duration measure must never reach the +// aggregate × field-type leg, which reads a `field` it does not have. +import { describe, expect, it } from 'vitest'; +import { DatasetSchema, DATASET_DURATION_FIELD_TYPE } from '@objectstack/spec/ui'; + +import { runAuthoringRules } from './authoring-rules.js'; +import { MEASURE_DURATION_FIELD_TYPE_REFUSED as BARREL_ID } from './index.js'; +import { explainRule } from './rule-explanations.js'; +import { DATASET_FIELD_UNKNOWN, validateDatasetReferences } from './validate-dataset-references.js'; +import { + MEASURE_AGGREGATE_FIELD_TYPE_REFUSED, + MEASURE_DURATION_FIELD_TYPE_REFUSED, + validateDatasetMeasureAggregates, +} from './validate-dataset-measure-aggregates.js'; +import { FIELD_NO_CONSUMERS, validateFieldConsumers } from './validate-field-consumers.js'; + +const RULE = MEASURE_DURATION_FIELD_TYPE_REFUSED; + +/** One object with a field of every class the leg distinguishes, and one dataset whose duration reads `from` → `to`. */ +const stackWith = (from: unknown, to: unknown, extra: { fields?: Record } = {}): Record => ({ + name: 'duration_probe', + objects: [ + { + name: 'support_ticket', + label: 'Ticket', + sharingModel: 'private', + fields: extra.fields ?? { + subject: { type: 'text' }, + opened_at: { type: 'datetime' }, + resolved_at: { type: 'datetime' }, + due_on: { type: 'date' }, + cutoff: { type: 'time' }, + effort: { type: 'number' }, + untyped: { label: 'Untyped' }, + }, + }, + ], + datasets: [ + { + name: 'resolution', + label: 'Resolution', + object: 'support_ticket', + dimensions: [{ name: 'subject', field: 'subject', type: 'string' }], + measures: [{ name: 'avg_hours', aggregate: 'avg', duration: { from, to, unit: 'hours' } }], + }, + ], +}); + +const findings = (stack: unknown) => validateDatasetMeasureAggregates(stack).filter((f) => f.rule === RULE); + +describe('[#25778] measure-duration-field-type-refused — a duration column that holds no instant', () => { + it.each([ + ['due_on', 'date'], + ['cutoff', 'time'], + ['effort', 'number'], + ['subject', 'text'], + ])('refuses `from` over %s (`%s`), naming the column, its type and the remedy', (column, type) => { + const found = findings(stackWith(column, 'resolved_at')); + expect(found).toEqual([ + { + severity: 'error', + rule: RULE, + where: 'dataset "resolution" › measure "avg_hours"', + path: 'datasets[0].measures[0].duration.from', + message: + `\`duration.from\` reads field "${column}" (\`${type}\` on object "support_ticket"), which holds no ` + + 'instant, so the dataset compile refuses it (400 DATASET_INVALID)', + hint: 'Point `duration.from` at a `datetime` field of "support_ticket".', + }, + ]); + }); + + it('one finding per refused column, `to` at its own path', () => { + const found = findings(stackWith('due_on', 'effort')); + expect(found.map((f) => f.path)).toEqual(['datasets[0].measures[0].duration.from', 'datasets[0].measures[0].duration.to']); + expect(findings(stackWith('opened_at', 'cutoff')).map((f) => f.path)).toEqual(['datasets[0].measures[0].duration.to']); + }); + + it('accepts a `datetime` pair, and the registry\'s own `created_at` — the spec\'s one accepted type', () => { + expect(DATASET_DURATION_FIELD_TYPE).toBe('datetime'); + expect(findings(stackWith('opened_at', 'resolved_at'))).toEqual([]); + expect(findings(stackWith('created_at', 'resolved_at'))).toEqual([]); + }); + + it('stays silent on what it cannot judge: a name the object does not declare (dataset-field-unknown\'s), an untyped field', () => { + expect(findings(stackWith('opend_at', 'resolved_at'))).toEqual([]); + expect(findings(stackWith('untyped', 'resolved_at'))).toEqual([]); + // …and the name that does not resolve is reported where it belongs. + const unknown = validateDatasetReferences(stackWith('opend_at', 'resolved_at')).filter((f) => f.rule === DATASET_FIELD_UNKNOWN); + expect(unknown.map((f) => f.path)).toEqual(['datasets[0].measures[0].duration.from']); + expect(unknown[0].message).toContain('opend_at'); + expect(validateDatasetReferences(stackWith('opened_at', 'resolved_at')).filter((f) => f.rule === DATASET_FIELD_UNKNOWN)).toEqual([]); + }); + + it('a duration measure never reaches the aggregate × field-type leg', () => { + const all = validateDatasetMeasureAggregates(stackWith('opened_at', 'resolved_at')); + expect(all.filter((f) => f.rule === MEASURE_AGGREGATE_FIELD_TYPE_REFUSED)).toEqual([]); + }); + + it('the fixtures are documents the spec accepts — the leg judges types, not shapes', () => { + for (const [from, to] of [['due_on', 'resolved_at'], ['opened_at', 'resolved_at'], ['effort', 'cutoff']]) { + const { datasets } = stackWith(from, to) as { datasets: unknown[] }; + expect(DatasetSchema.safeParse(datasets[0]).success, `${from} → ${to}`).toBe(true); + } + }); + + it('fires through runAuthoringRules on all three commands, gating, and is exported from the barrel', () => { + expect(BARREL_ID).toBe(RULE); + const stack = stackWith('due_on', 'resolved_at'); + for (const command of ['validate', 'build', 'lint'] as const) { + const found = runAuthoringRules(command, { normalized: stack, parsed: stack }).filter((f) => f.rule === RULE); + expect(found, command).toHaveLength(1); + expect(found[0].severity, command).toBe('error'); + } + const accepted = stackWith('opened_at', 'resolved_at'); + expect(runAuthoringRules('lint', { normalized: accepted, parsed: accepted }).filter((f) => f.rule === RULE)).toEqual([]); + }); + + it('a field read only by a duration measure has a consumer', () => { + const stack = stackWith('opened_at', 'resolved_at'); + const unread = validateFieldConsumers(stack).filter((f) => f.rule === FIELD_NO_CONSUMERS).map((f) => f.message); + expect(unread.some((m) => /"opened_at"/.test(m)), unread.join('\n')).toBe(false); + expect(unread.some((m) => /"resolved_at"/.test(m)), unread.join('\n')).toBe(false); + }); + + it('the `fix:` line is one line of at most 200 characters, and `os explain` carries the rest', () => { + for (const f of findings(stackWith('due_on', 'effort'))) { + expect(f.hint).not.toContain('\n'); + expect(f.hint.length).toBeLessThanOrEqual(200); + } + const text = explainRule(RULE)?.paragraphs.join('\n') ?? ''; + for (const fact of [ + 'so both fields must be declared `datetime`', + 'A `date` is a calendar day and a `time` a wall clock', + 'A record missing either instant takes no part in the aggregate', + '`400 DATASET_INVALID`', + "`dataset-field-unknown`'s finding", + ]) { + expect(text, fact).toContain(fact); + } + // The aggregate rule's explanation points the time-between question here. + expect(explainRule(MEASURE_AGGREGATE_FIELD_TYPE_REFUSED)?.paragraphs.join('\n')).toContain( + 'a `duration: { from, to, unit }` measure (`measure-duration-field-type-refused`)', + ); + }); +}); diff --git a/packages/lint/src/validate-dataset-references.ts b/packages/lint/src/validate-dataset-references.ts index 3395ee4702a..de831ff2c0b 100644 --- a/packages/lint/src/validate-dataset-references.ts +++ b/packages/lint/src/validate-dataset-references.ts @@ -343,6 +343,19 @@ export function validateDatasetReferences(stack: AnyRec): DatasetRefFinding[] { `Bind the measure to a field on "${object}", or to a ` + `\`relationship[.relationship].field\` path whose prefix is declared in \`include\`.`, ); + // [#25778] A duration measure's two columns, each a field of the base + // object by its bare name (the spec refuses a path at parse). + const duration = isRec(measure.duration) ? measure.duration : undefined; + for (const slot of ['from', 'to'] as const) { + checkFieldPath( + duration?.[slot], + `${where} › measure "${name}"`, + `${dsPath}.measures[${i}].duration.${slot}`, + `duration ${slot}`, + DATASET_FIELD_UNKNOWN, + `Name a \`datetime\` field on "${object}" by its bare name.`, + ); + } }); // ── (4) filter KEYS, on the dataset scope filter and on every measure ── diff --git a/packages/services/service-analytics/src/dataset-compiler.ts b/packages/services/service-analytics/src/dataset-compiler.ts index 2165628bf4a..04edd0b6517 100644 --- a/packages/services/service-analytics/src/dataset-compiler.ts +++ b/packages/services/service-analytics/src/dataset-compiler.ts @@ -10,7 +10,12 @@ import { virtualFieldAggregateRemedy, } from '@objectstack/spec/data'; import type { Dataset, DatasetMeasure, DatasetDimension } from '@objectstack/spec/ui'; -import { DATASET_DURATION_AGGREGATES, DATASET_DURATION_UNIT_SECONDS, resolveI18nLabel } from '@objectstack/spec/ui'; +import { + DATASET_DURATION_AGGREGATES, + DATASET_DURATION_FIELD_TYPE, + DATASET_DURATION_UNIT_SECONDS, + resolveI18nLabel, +} from '@objectstack/spec/ui'; import type { FilterCondition } from '@objectstack/spec/data'; import { datasetInvalidError } from './dataset-refusal.js'; import { TEMPORAL_SOURCE_FIELD_TYPES } from './measure-result-type.js'; @@ -451,15 +456,6 @@ function assertAggregateFieldTypeCompatible( ); } -/** - * The one field type a duration measure's `from` / `to` may be declared as. - * A `datetime` is an instant (ADR-0053), and the time between two instants is - * a duration on every backend. A `date` is a calendar day with no instant and - * a `time` a wall clock with no day, so neither denotes a point a duration can - * be measured from. - */ -const DURATION_COLUMN_TYPE = 'datetime'; - /** One column of the base object by its bare name — the spec's `duration.from` / `to` shape. */ const DURATION_BASE_COLUMN = /^[A-Za-z_][A-Za-z0-9_]*$/; @@ -494,7 +490,7 @@ function compileDurationMeasure( throw datasetInvalidError( `[dataset-compiler] dataset "${datasetName}" duration measure "${measure.name}" reads ` + `\`duration.${slot}\` ${JSON.stringify(column) ?? 'undefined'}, which names no column: it takes one ` + - `\`${DURATION_COLUMN_TYPE}\` field of "${objectName}" by its bare name.`, + `\`${DATASET_DURATION_FIELD_TYPE}\` field of "${objectName}" by its bare name.`, ); } if (!Object.prototype.hasOwnProperty.call(DATASET_DURATION_UNIT_SECONDS, unit)) { @@ -513,13 +509,13 @@ function compileDurationMeasure( if (declaredValueShape) { for (const [slot, column] of [['from', from], ['to', to]] as const) { const fieldType = declaredValueShape(objectName, column)?.type; - if (!fieldType || fieldType === DURATION_COLUMN_TYPE) continue; + if (!fieldType || fieldType === DATASET_DURATION_FIELD_TYPE) continue; throw datasetInvalidError( `[dataset-compiler] dataset "${datasetName}" duration measure "${measure.name}" reads ` + `\`duration.${slot}\` "${column}", which object "${objectName}" declares as \`${fieldType}\`. ` + - `A duration spans two \`${DURATION_COLUMN_TYPE}\` fields of the same record, because an instant is ` + + `A duration spans two \`${DATASET_DURATION_FIELD_TYPE}\` fields of the same record, because an instant is ` + `the one value every backend measures the time between alike; a \`${fieldType}\` field holds no ` + - `instant. Point \`duration.${slot}\` at a \`${DURATION_COLUMN_TYPE}\` field of "${objectName}".`, + `instant. Point \`duration.${slot}\` at a \`${DATASET_DURATION_FIELD_TYPE}\` field of "${objectName}".`, ); } } diff --git a/packages/services/service-analytics/src/duration-measure-sql.ts b/packages/services/service-analytics/src/duration-measure-sql.ts index df8f0e370df..437f2fdd5e9 100644 --- a/packages/services/service-analytics/src/duration-measure-sql.ts +++ b/packages/services/service-analytics/src/duration-measure-sql.ts @@ -42,7 +42,7 @@ * name, for this face and for the service's gates. */ -import { DATASET_DURATION_UNIT_SECONDS, type DatasetDurationUnitValue } from '@objectstack/spec/ui'; +import { DATASET_DURATION_UNIT_SECONDS, type DatasetDurationUnit } from '@objectstack/spec/ui'; import type { DurationMeasureSpec } from './strategies/types.js'; import type { AnalyticsSqlDialect } from './text-match-sql.js'; @@ -87,7 +87,7 @@ export function durationOperandSql( dialect: DurationSqlDialect, fromSql: string, toSql: string, - unit: DatasetDurationUnitValue, + unit: DatasetDurationUnit, ): string { const seconds = durationSecondsSql(dialect, fromSql, toSql); const length = DATASET_DURATION_UNIT_SECONDS[unit]; diff --git a/packages/services/service-analytics/src/strategies/types.ts b/packages/services/service-analytics/src/strategies/types.ts index faa095ed33c..0962509965e 100644 --- a/packages/services/service-analytics/src/strategies/types.ts +++ b/packages/services/service-analytics/src/strategies/types.ts @@ -17,7 +17,7 @@ export type { import type { FilterCondition, ValueShapeFieldDef } from '@objectstack/spec/data'; import type { IObjectQLEngine, StrategyContext } from '@objectstack/spec/contracts'; -import type { DATASET_DURATION_AGGREGATES, DatasetDurationUnitValue } from '@objectstack/spec/ui'; +import type { DATASET_DURATION_AGGREGATES, DatasetDurationUnit } from '@objectstack/spec/ui'; /** * [#19995] The engine's judge-only `where` admission, as the contract declares @@ -94,7 +94,7 @@ export interface DurationMeasureSpec { aggregate: (typeof DATASET_DURATION_AGGREGATES)[number]; from: string; to: string; - unit: DatasetDurationUnitValue; + unit: DatasetDurationUnit; } /** A {@link StrategyContext} that can answer for a compiled dataset (#10298). */ diff --git a/packages/spec/api-surface/ui.json b/packages/spec/api-surface/ui.json index 3dccc6b976c..a8f3bb9dd41 100644 --- a/packages/spec/api-surface/ui.json +++ b/packages/spec/api-surface/ui.json @@ -126,6 +126,9 @@ "ComponentPropsInput (type)", "ComponentPropsMap (const)", "DASHBOARD_WIDGET_MULTI_MEASURE_TYPES (const)", + "DATASET_DURATION_AGGREGATES (const)", + "DATASET_DURATION_FIELD_TYPE (const)", + "DATASET_DURATION_UNIT_SECONDS (const)", "DATE_RANGE_DEFAULT_RANGES (const)", "DATE_RANGE_PRESETS (const)", "Dashboard (const)", @@ -151,7 +154,11 @@ "Dataset (type)", "DatasetDimension (type)", "DatasetDimensionSchema (const)", + "DatasetDurationUnit (const)", + "DatasetDurationUnitValue (type)", "DatasetMeasure (type)", + "DatasetMeasureDuration (type)", + "DatasetMeasureDurationSchema (const)", "DatasetMeasureSchema (const)", "DatasetSchema (const)", "DateRangeDefaultRange (type)", diff --git a/packages/spec/declaration-map/ui.json b/packages/spec/declaration-map/ui.json index 7d55b40b417..ebce25a2a04 100644 --- a/packages/spec/declaration-map/ui.json +++ b/packages/spec/declaration-map/ui.json @@ -90,7 +90,10 @@ "Dataset": "ui/Dataset", "DatasetDimension": "ui/DatasetDimension", "DatasetDimensionSchema": "ui/DatasetDimension", + "DatasetDurationUnit": "ui/DatasetDurationUnit", "DatasetMeasure": "ui/DatasetMeasure", + "DatasetMeasureDuration": "ui/DatasetMeasureDuration", + "DatasetMeasureDurationSchema": "ui/DatasetMeasureDuration", "DatasetMeasureSchema": "ui/DatasetMeasure", "DatasetSchema": "ui/Dataset", "DerivedMeasureOp": "ui/DerivedMeasureOp", diff --git a/packages/spec/export-origins/ui.json b/packages/spec/export-origins/ui.json index f8853aa66fb..ce8613de6e2 100644 --- a/packages/spec/export-origins/ui.json +++ b/packages/spec/export-origins/ui.json @@ -123,6 +123,9 @@ "ComponentPropsInput": "src/ui/component.zod.ts#ComponentPropsInput (type)", "ComponentPropsMap": "src/ui/component.zod.ts#ComponentPropsMap (const)", "DASHBOARD_WIDGET_MULTI_MEASURE_TYPES": "src/ui/dashboard.zod.ts#DASHBOARD_WIDGET_MULTI_MEASURE_TYPES (const)", + "DATASET_DURATION_AGGREGATES": "src/ui/dataset.zod.ts#DATASET_DURATION_AGGREGATES (const)", + "DATASET_DURATION_FIELD_TYPE": "src/ui/dataset.zod.ts#DATASET_DURATION_FIELD_TYPE (const)", + "DATASET_DURATION_UNIT_SECONDS": "src/ui/dataset.zod.ts#DATASET_DURATION_UNIT_SECONDS (const)", "DATE_RANGE_DEFAULT_RANGES": "src/ui/dashboard.zod.ts#DATE_RANGE_DEFAULT_RANGES (const)", "DATE_RANGE_PRESETS": "src/data/date-range-presets.ts#DATE_RANGE_PRESETS (const)", "Dashboard": "src/ui/dashboard.zod.ts#Dashboard (type)", @@ -147,7 +150,11 @@ "Dataset": "src/ui/dataset.zod.ts#Dataset (type)", "DatasetDimension": "src/ui/dataset.zod.ts#DatasetDimension (type)", "DatasetDimensionSchema": "src/ui/dataset.zod.ts#DatasetDimensionSchema (const)", + "DatasetDurationUnit": "src/ui/dataset.zod.ts#DatasetDurationUnit (const)", + "DatasetDurationUnitValue": "src/ui/dataset.zod.ts#DatasetDurationUnitValue (type)", "DatasetMeasure": "src/ui/dataset.zod.ts#DatasetMeasure (type)", + "DatasetMeasureDuration": "src/ui/dataset.zod.ts#DatasetMeasureDuration (type)", + "DatasetMeasureDurationSchema": "src/ui/dataset.zod.ts#DatasetMeasureDurationSchema (const)", "DatasetMeasureSchema": "src/ui/dataset.zod.ts#DatasetMeasureSchema (const)", "DatasetSchema": "src/ui/dataset.zod.ts#DatasetSchema (const)", "DateRangeDefaultRange": "src/ui/dashboard.zod.ts#DateRangeDefaultRange (type)", diff --git a/packages/spec/liveness/dataset.json b/packages/spec/liveness/dataset.json index 8164d63bc5a..f17cc307af2 100644 --- a/packages/spec/liveness/dataset.json +++ b/packages/spec/liveness/dataset.json @@ -128,6 +128,29 @@ "evidence": "packages/services/service-analytics/src/dataset-compiler.ts#compileDataset (`derived.push({ name: m.name, op: m.derived.op, of: m.derived.of })` — both members read here, and a derived measure is deliberately NOT compiled into a Cube metric); packages/services/service-analytics/src/dataset-executor.ts#evaluateDerivedMeasures (post-aggregation arithmetic row by row, driven from `compiled.derived`)", "note": "post-aggregation arithmetic (ratio/sum/difference/product) over base measures; {op, of} both consumed. 2026-08-28: RE-ANCHORED (commit 9ee2dcfbd) and REPOINTED — `:247` had rotted onto a BLANK LINE, and the compiler leg (where `{op, of}` are actually read off the authored key) was uncited. Re-closed by hand against c459da6bc; DATED for the first time (this file carried no `verifiedAt` anywhere, which the review accepting that re-anchoring sorts as OLDEST).", "verifiedAt": "2026-08-28" + }, + "duration": { + "status": "live", + "evidence": "packages/services/service-analytics/src/dataset-compiler.ts#compileDurationMeasure (`from`, `to`, `unit` and the measure's `aggregate` read off the authored key into the dataset scope's `measureDurations`; each column judged `datetime` against the declaration, `400 DATASET_INVALID` otherwise); packages/services/service-analytics/src/duration-measure-sql.ts#durationOperandSql (the per-row `to - from` per SQL dialect, divided by the unit's seconds, which `NativeSQLStrategy` wraps in the aggregate)", + "note": "[#25778] aggregated time between two datetime columns of one record; {from, to, unit} all consumed. Served by the native-SQL strategy and the draft preview; the service refuses it `400 INVALID_FIELD` on any other strategy. Live on SQLite and PostgreSQL in dataset-duration-measure.test.ts.", + "verifiedAt": "2026-10-25", + "children": { + "from": { + "status": "live", + "evidence": "packages/services/service-analytics/src/dataset-compiler.ts#compileDurationMeasure (`duration.from` checked as one bare column name and judged `datetime` against the object's declaration, `400 DATASET_INVALID` otherwise, then carried into `measureDurations`); packages/services/service-analytics/src/strategies/native-sql-strategy.ts#resolveDurationMeasureSql (the column qualified, temporal-repaired and passed to `durationOperandSql` as the start instant)", + "verifiedAt": "2026-10-25" + }, + "to": { + "status": "live", + "evidence": "packages/services/service-analytics/src/dataset-compiler.ts#compileDurationMeasure (`duration.to` checked as one bare column name and judged `datetime` against the object's declaration, `400 DATASET_INVALID` otherwise, then carried into `measureDurations`); packages/services/service-analytics/src/strategies/native-sql-strategy.ts#resolveDurationMeasureSql (the column qualified, temporal-repaired and passed to `durationOperandSql` as the end instant)", + "verifiedAt": "2026-10-25" + }, + "unit": { + "status": "live", + "evidence": "packages/services/service-analytics/src/duration-measure-sql.ts#durationOperandSql (divides the per-row seconds by `DATASET_DURATION_UNIT_SECONDS[unit]`); packages/services/service-analytics/src/preview-evaluator.ts#durationOf (the same division on the draft preview)", + "verifiedAt": "2026-10-25" + } + } } } } diff --git a/packages/spec/liveness/state-counts/dataset.md b/packages/spec/liveness/state-counts/dataset.md index 4cc522e6e84..2065c8e6415 100644 --- a/packages/spec/liveness/state-counts/dataset.md +++ b/packages/spec/liveness/state-counts/dataset.md @@ -12,4 +12,4 @@ committed anywhere: `check:liveness` sums the shards when it reads them. | Type | live | exp | elsewhere | dead | planned | classified | |---|---|---|---|---|---|---| -| `dataset` | 28 | 0 | 0 | 0 | 0 | 28 | +| `dataset` | 31 | 0 | 0 | 0 | 0 | 31 | diff --git a/packages/spec/src/ui/dataset-measure-duration.test.ts b/packages/spec/src/ui/dataset-measure-duration.test.ts new file mode 100644 index 00000000000..65012848a26 --- /dev/null +++ b/packages/spec/src/ui/dataset-measure-duration.test.ts @@ -0,0 +1,131 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * [#25778] `DatasetMeasureSchema.duration` — a measure over the time between two + * `datetime` columns of the same record (`{ from, to, unit }`), reduced by the + * measure's `aggregate`. + * + * What parses: two bare column names of the dataset's own object, a + * fixed-length unit, and one of the four aggregates that read a number. What is + * refused, each with the slot it names: a relationship path, an expression or + * the row wildcard in either column (ADR-0021: no raw SQL or expressions), a + * unit outside the four, the same column twice, a `field` / `derived` / + * `currency` beside it, and an aggregate that is missing or counts. The key is + * optional: a measure without it parses exactly as before. + */ + +import { describe, it, expect } from 'vitest'; +import { + DATASET_DURATION_AGGREGATES, + DATASET_DURATION_FIELD_TYPE, + DATASET_DURATION_UNIT_SECONDS, + DatasetDurationUnit, + DatasetMeasureSchema, + DatasetSchema, +} from './dataset.zod'; + +const DURATION = { from: 'opened_at', to: 'resolved_at', unit: 'hours' } as const; +const measure = (extra: Record = {}) => ({ name: 'avg_hours', aggregate: 'avg', duration: DURATION, ...extra }); + +const issuesOf = (value: unknown) => { + const parsed = DatasetMeasureSchema.safeParse(value); + return parsed.success ? [] : parsed.error.issues.map((i) => ({ path: i.path.join('.'), message: i.message })); +}; + +describe('[#25778] DatasetMeasureSchema.duration', () => { + it('accepts each of the four aggregates in each unit, and keeps the value', () => { + for (const aggregate of DATASET_DURATION_AGGREGATES) { + for (const unit of DatasetDurationUnit.options) { + const parsed = DatasetMeasureSchema.safeParse(measure({ aggregate, duration: { ...DURATION, unit } })); + expect(parsed.success, `${aggregate} ${unit}`).toBe(true); + expect(parsed.data?.duration).toEqual({ ...DURATION, unit }); + } + } + }); + + it('a measure-scoped filter and a format ride beside it', () => { + expect(issuesOf(measure({ filter: { priority: 'high' }, format: '0.0' }))).toEqual([]); + }); + + it('the constants every consumer reads: the units in seconds, the aggregates, the one field type', () => { + expect(DATASET_DURATION_UNIT_SECONDS).toEqual({ seconds: 1, minutes: 60, hours: 3600, days: 86400 }); + expect(Object.isFrozen(DATASET_DURATION_UNIT_SECONDS)).toBe(true); + expect(Object.keys(DATASET_DURATION_UNIT_SECONDS)).toEqual(DatasetDurationUnit.options); + expect([...DATASET_DURATION_AGGREGATES]).toEqual(['avg', 'min', 'max', 'sum']); + expect(DATASET_DURATION_FIELD_TYPE).toBe('datetime'); + }); + + it.each([ + ['a relationship path', 'account.created_at'], + ['an expression', 'resolved_at - opened_at'], + ['raw SQL', "julianday('now')"], + ['the row wildcard', '*'], + ['an empty string', ''], + ['a padded name', ' opened_at'], + ])('refuses %s in a column, naming the slot', (_label, column) => { + const found = issuesOf(measure({ duration: { ...DURATION, from: column } })); + expect(found).toHaveLength(1); + expect(found[0].path).toBe('duration.from'); + expect(found[0].message).toContain('a duration reads two columns of the SAME record'); + expect(found[0].message).toContain('ADR-0021'); + }); + + it('refuses a unit with no one length in seconds', () => { + for (const unit of ['weeks', 'months', 'years', 'ms']) { + expect(issuesOf(measure({ duration: { ...DURATION, unit } })).map((i) => i.path), unit).toEqual(['duration.unit']); + } + }); + + it('refuses the same column twice', () => { + const found = issuesOf(measure({ duration: { ...DURATION, to: 'opened_at' } })); + expect(found).toEqual([{ path: 'duration.to', message: expect.stringContaining('every record\'s duration is zero') }]); + }); + + it.each([ + ['a `field`', { field: 'resolved_at' }, 'field', 'it takes no `field`'], + ['`derived`', { derived: { op: 'ratio', of: ['a', 'b'] } }, 'derived', 'never both'], + ['`currency`', { currency: 'USD' }, 'currency', 'not an amount of money'], + ])('refuses %s beside it', (_label, extra, path, words) => { + const found = issuesOf(measure(extra)); + expect(found.map((i) => i.path)).toContain(path); + expect(found.find((i) => i.path === path)?.message).toContain(words); + }); + + it('refuses a missing aggregate, and one that counts', () => { + expect(issuesOf({ name: 'avg_hours', duration: DURATION })).toEqual([ + { path: 'aggregate', message: expect.stringContaining('a duration measure needs `aggregate`') }, + ]); + for (const aggregate of ['count', 'count_distinct']) { + expect(issuesOf(measure({ aggregate })), aggregate).toEqual([ + { path: 'aggregate', message: expect.stringContaining('declare a `count` measure') }, + ]); + } + }); + + it('refuses an unknown key inside it, and maps the near spellings onto the real ones', () => { + expect(issuesOf(measure({ duration: { ...DURATION, bucket: 'month' } })).map((i) => i.path)).toEqual(['duration']); + const parsed = DatasetMeasureSchema.safeParse(measure({ duration: { start: 'opened_at', end: 'resolved_at', unit: 'hours' } })); + expect(parsed.success).toBe(false); + expect(parsed.error?.issues.map((i) => i.message).join('\n')).toMatch(/`from`[\s\S]*`to`|`to`[\s\S]*`from`/); + }); + + it('a measure without it parses exactly as before', () => { + expect(issuesOf({ name: 'total', aggregate: 'sum', field: 'amount' })).toEqual([]); + expect(DatasetMeasureSchema.parse({ name: 'tickets', aggregate: 'count' }).duration).toBeUndefined(); + }); + + it('a dataset accepts a duration measure with no `field`, and a derived measure over it', () => { + const parsed = DatasetSchema.safeParse({ + name: 'resolution', + label: 'Resolution', + object: 'ticket', + dimensions: [{ name: 'category', field: 'category', type: 'string' }], + measures: [ + { name: 'tickets', aggregate: 'count' }, + measure({ aggregate: 'sum', duration: { ...DURATION, unit: 'seconds' }, name: 'total_seconds' }), + { name: 'seconds_per_ticket', derived: { op: 'ratio', of: ['total_seconds', 'tickets'] } }, + ], + }); + expect(parsed.success, JSON.stringify(parsed.error?.issues)).toBe(true); + }); +}); diff --git a/packages/spec/src/ui/dataset.zod.ts b/packages/spec/src/ui/dataset.zod.ts index e13a7d1690a..c38a745d969 100644 --- a/packages/spec/src/ui/dataset.zod.ts +++ b/packages/spec/src/ui/dataset.zod.ts @@ -235,6 +235,17 @@ export const DATASET_DURATION_UNIT_SECONDS: Readonly; export type DatasetMeasure = z.input; export type DatasetMeasureDuration = z.input; -export type DatasetDurationUnitValue = z.input; +export type DatasetDurationUnit = z.input; export type DerivedMeasureOpValue = z.input; export type Dataset = z.input; From eda01f1f98e356dd803687b69aa737f19cb5f722 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 25 Oct 2026 14:28:48 +0000 Subject: [PATCH 05/14] chore(spec): regenerate api-surface, export-origins and reference docs for the duration unit type Co-Authored-By: Claude Claude-Session: https://claude.ai/code/session_01AKPixnkg91qCkQCVFDJuqW --- content/docs/references/ui/dataset.mdx | 2 +- packages/spec/api-surface/ui.json | 2 +- packages/spec/export-origins/ui.json | 3 +-- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/content/docs/references/ui/dataset.mdx b/content/docs/references/ui/dataset.mdx index f5c5b3ff0f9..9b42a2f47c5 100644 --- a/content/docs/references/ui/dataset.mdx +++ b/content/docs/references/ui/dataset.mdx @@ -35,7 +35,7 @@ layer is absorbed/retired in a later phase (D-A). ```typescript import { DatasetSchema, DatasetDimensionSchema, DatasetDurationUnit, DatasetMeasureSchema, DatasetMeasureDurationSchema, DerivedMeasureOp } from '@objectstack/spec/ui'; -import type { Dataset, DatasetDimension, DatasetMeasure, DatasetMeasureDuration } from '@objectstack/spec/ui'; +import type { Dataset, DatasetDimension, DatasetDurationUnit, DatasetMeasure, DatasetMeasureDuration } from '@objectstack/spec/ui'; // Validate data const result = DatasetSchema.parse(data); diff --git a/packages/spec/api-surface/ui.json b/packages/spec/api-surface/ui.json index a8f3bb9dd41..78713509d22 100644 --- a/packages/spec/api-surface/ui.json +++ b/packages/spec/api-surface/ui.json @@ -155,7 +155,7 @@ "DatasetDimension (type)", "DatasetDimensionSchema (const)", "DatasetDurationUnit (const)", - "DatasetDurationUnitValue (type)", + "DatasetDurationUnit (type)", "DatasetMeasure (type)", "DatasetMeasureDuration (type)", "DatasetMeasureDurationSchema (const)", diff --git a/packages/spec/export-origins/ui.json b/packages/spec/export-origins/ui.json index ce8613de6e2..8a023456510 100644 --- a/packages/spec/export-origins/ui.json +++ b/packages/spec/export-origins/ui.json @@ -150,8 +150,7 @@ "Dataset": "src/ui/dataset.zod.ts#Dataset (type)", "DatasetDimension": "src/ui/dataset.zod.ts#DatasetDimension (type)", "DatasetDimensionSchema": "src/ui/dataset.zod.ts#DatasetDimensionSchema (const)", - "DatasetDurationUnit": "src/ui/dataset.zod.ts#DatasetDurationUnit (const)", - "DatasetDurationUnitValue": "src/ui/dataset.zod.ts#DatasetDurationUnitValue (type)", + "DatasetDurationUnit": "src/ui/dataset.zod.ts#DatasetDurationUnit (type)", "DatasetMeasure": "src/ui/dataset.zod.ts#DatasetMeasure (type)", "DatasetMeasureDuration": "src/ui/dataset.zod.ts#DatasetMeasureDuration (type)", "DatasetMeasureDurationSchema": "src/ui/dataset.zod.ts#DatasetMeasureDurationSchema (const)", From b2ddd351536bce09aee2658ae06dc2ab69e5a372 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 25 Oct 2026 14:56:16 +0000 Subject: [PATCH 06/14] fix(analytics): review round 1 on the duration measure - The field read gate resolves a duration measure first, as every other seam does: a dimension sharing its name no longer hides its two columns. - MySQL divides by a DOUBLE literal, so the arm is double precision like the other two; the docs and changeset claim live parity for SQLite and PostgreSQL only. - getMeta lists a registered dataset's duration measures with their aggregate, label and format, and every seam accepts the `.` spelling it lists them under, its scoped filter included. - Docs: path and expression refusals happen at parse, not in the lint rule. Co-Authored-By: Claude Claude-Session: https://claude.ai/code/session_01AKPixnkg91qCkQCVFDJuqW --- .changeset/25778-dataset-duration-measure.md | 2 +- content/docs/data-modeling/analytics.mdx | 12 +-- .../dataset-duration-measure.test.ts | 2 +- .../native-sql-duration-measure.test.ts | 86 ++++++++++++++++++- .../src/analytics-service.ts | 57 ++++++++---- .../service-analytics/src/dataset-compiler.ts | 17 +++- .../service-analytics/src/dataset-executor.ts | 2 +- .../src/duration-measure-sql.ts | 49 ++++++++--- .../src/preview-evaluator.ts | 2 +- .../src/strategies/native-sql-strategy.ts | 13 +-- .../service-analytics/src/strategies/types.ts | 7 ++ .../src/data/analytics-column-reference.ts | 2 +- 12 files changed, 205 insertions(+), 46 deletions(-) diff --git a/.changeset/25778-dataset-duration-measure.md b/.changeset/25778-dataset-duration-measure.md index 16af085802a..a2b6fa8d934 100644 --- a/.changeset/25778-dataset-duration-measure.md +++ b/.changeset/25778-dataset-duration-measure.md @@ -10,7 +10,7 @@ Clause-②: yes (widening) Metadata-protocol: added - **`@objectstack/spec`**: `DatasetMeasureSchema` gains an optional `duration: { from, to, unit }`, in place of `field`. `from` and `to` name two `datetime` fields of the dataset's own object by their bare names, `unit` is `seconds`, `minutes`, `hours` or `days`, and the measure's `aggregate` is `avg`, `min`, `max` or `sum` of `to` minus `from`. A relationship path, an expression or the row wildcard in either column, the same column twice, another unit, a counting aggregate, or a `field`, `derived` or `currency` beside it is refused at parse. New exports: `DatasetMeasureDurationSchema`, `DatasetDurationUnit`, `DATASET_DURATION_UNIT_SECONDS`, `DATASET_DURATION_AGGREGATES`, `DATASET_DURATION_FIELD_TYPE` and the `DatasetMeasureDuration` / `DatasetDurationUnit` types. -- **`@objectstack/service-analytics`**: the dataset compile refuses a duration column declared anything but `datetime` (`400 DATASET_INVALID`, naming the column and its type). The native-SQL strategy computes the duration as one statement on SQLite, PostgreSQL and MySQL, with the same number on each: a record missing either instant is left out, a `to` before its `from` counts as negative, and over a group with no such record `sum` answers `0` and the others `null`. Measure filters, derived measures, ordering and the draft preview serve it. A query the native strategy does not serve (a time dimension bucketed by `granularity`, a field-to-field filter, an object served through engine middleware or an external datasource, a non-SQL datasource) is refused with `400 INVALID_FIELD` naming the measure. Its two columns pass the field-level read gate like any measure's field. The temporal half of the aggregate × field-type refusal now names the duration measure instead of a stored "days open" number. +- **`@objectstack/service-analytics`**: the dataset compile refuses a duration column declared anything but `datetime` (`400 DATASET_INVALID`, naming the column and its type). The native-SQL strategy computes the duration as one statement on SQLite, PostgreSQL and MySQL, in double precision on each; SQLite and PostgreSQL are tested live to answer the same number: a record missing either instant is left out, a `to` before its `from` counts as negative, and over a group with no such record `sum` answers `0` and the others `null`. Measure filters, derived measures, ordering and the draft preview serve it. A query the native strategy does not serve (a time dimension bucketed by `granularity`, a field-to-field filter, an object served through engine middleware or an external datasource, a non-SQL datasource) is refused with `400 INVALID_FIELD` naming the measure. Its two columns pass the field-level read gate like any measure's field. The temporal half of the aggregate × field-type refusal now names the duration measure instead of a stored "days open" number. - **`@objectstack/lint`**: `os validate` refuses a duration column not declared `datetime` with the new gating rule `measure-duration-field-type-refused` (at `datasets[N].measures[M].duration.from` / `.to`), with an `os explain` entry. `dataset-field-unknown` now also reports a duration column the object does not declare. What to do: nothing. A dataset without `duration` parses, compiles and answers exactly as before. diff --git a/content/docs/data-modeling/analytics.mdx b/content/docs/data-modeling/analytics.mdx index 254627cb0a8..735612e6bf6 100644 --- a/content/docs/data-modeling/analytics.mdx +++ b/content/docs/data-modeling/analytics.mdx @@ -107,10 +107,10 @@ in place of `field`: ``` - `from` and `to` name two `datetime` fields of the dataset's own object, by - their bare names. A relationship path, an expression, or a `date`, `time`, - number or text field is refused: `os validate` reports - `measure-duration-field-type-refused`, and the dataset compile answers - `400 DATASET_INVALID`. + their bare names. A relationship path or an expression is refused when the + dataset is parsed. A `date`, `time`, number or text field is refused too: + `os validate` reports `measure-duration-field-type-refused`, and the dataset + compile answers `400 DATASET_INVALID`. - `unit` is `seconds`, `minutes`, `hours` or `days`, and the measure answers a number of them (`1.5` hours). `aggregate` is `avg`, `min`, `max` or `sum`. - A record missing either instant is left out, as a null is left out of an @@ -119,8 +119,8 @@ in place of `field`: nothing. - A measure `filter`, and a `derived` measure naming the duration measure in `of`, work as they do for any measure. -- It runs as one SQL statement on SQLite, PostgreSQL and MySQL, and answers the - same number on each. A query the service cannot compile to one such +- It runs as one SQL statement on SQLite, PostgreSQL and MySQL, computed in + double precision on each. A query the service cannot compile to one such statement is refused with `400 INVALID_FIELD` naming the measure, rather than answered another way: a time dimension bucketed by `granularity` (a monthly trend), a field-to-field comparison in a filter, an object served through diff --git a/packages/services/service-analytics/src/__tests__/dataset-duration-measure.test.ts b/packages/services/service-analytics/src/__tests__/dataset-duration-measure.test.ts index a847ba193e2..5cc9c3f4d82 100644 --- a/packages/services/service-analytics/src/__tests__/dataset-duration-measure.test.ts +++ b/packages/services/service-analytics/src/__tests__/dataset-duration-measure.test.ts @@ -277,7 +277,7 @@ for (const cell of CELLS) { ); expect(run.error, run.error?.message).toBeUndefined(); expect(run.rawSql).toBe(1); - expect(reads.statements.at(-1)).toContain('ORDER BY "total_seconds" ASC'); + expect(reads.statements[reads.statements.length - 1]).toContain('ORDER BY "total_seconds" ASC'); expect(run.value!.rows).toEqual([{ category: 'b', total_seconds: -3599.25 }]); }); diff --git a/packages/services/service-analytics/src/__tests__/native-sql-duration-measure.test.ts b/packages/services/service-analytics/src/__tests__/native-sql-duration-measure.test.ts index f7c030ff13f..b2884cf9100 100644 --- a/packages/services/service-analytics/src/__tests__/native-sql-duration-measure.test.ts +++ b/packages/services/service-analytics/src/__tests__/native-sql-duration-measure.test.ts @@ -77,8 +77,8 @@ describe('[#25778] NativeSQLStrategy — the duration measure as a statement', ( ], [ 'mysql', - 'AVG(((TIMESTAMPDIFF(MICROSECOND, opened_at, resolved_at) / 1000000.0) / 3600.0)) AS "avg_hours"', - 'SUM(CASE WHEN priority = $1 THEN (TIMESTAMPDIFF(MICROSECOND, opened_at, resolved_at) / 1000000.0) END) AS "total_seconds"', + 'AVG(((TIMESTAMPDIFF(MICROSECOND, opened_at, resolved_at) / 1.0E6) / 3600.0)) AS "avg_hours"', + 'SUM(CASE WHEN priority = $1 THEN (TIMESTAMPDIFF(MICROSECOND, opened_at, resolved_at) / 1.0E6) END) AS "total_seconds"', ], ])('%s: the per-dialect difference, divided by the unit, under the aggregate and its measure filter', async (dialect, avg, filteredSum) => { const { sql, params } = await native.generateSql(q(), nativeCtx({ dialect })); @@ -258,6 +258,88 @@ describe('[#25778] AnalyticsService — the gates a duration measure passes thro expect(executed).toEqual([]); }); + it('a dimension sharing its name does not hide its columns from the read gate', async () => { + const shadowed = DatasetSchema.parse({ + ...DATASET_BASE, + dimensions: [...DATASET_BASE.dimensions, { name: 'resolution_seconds_sum', field: 'category', type: 'string' }], + }); + const { service, executed } = makeService({ readable: FIELDS.filter((f) => f !== 'resolved_at') }); + const err = await refusalOf(service.queryDataset(shadowed, { measures: ['resolution_seconds_sum'], dimensions: ['category'] } as never, MEMBER)); + expect(err?.code, err?.message).toBe('PERMISSION_DENIED'); + expect(err?.status).toBe(403); + expect(err?.message).toContain('resolved_at'); + expect(executed).toEqual([]); + }); + + describe('on the cube door, under the `.` spelling the discovery surface lists', () => { + const listed = DatasetSchema.parse({ + ...DATASET_BASE, + measures: [ + ...DATASET_BASE.measures, + { + name: 'high_hours', + label: 'High-priority hours', + aggregate: 'avg', + format: '0.0', + filter: { priority: 'high' }, + duration: { from: 'opened_at', to: 'resolved_at', unit: 'hours' }, + }, + ], + }); + + it('getMeta lists each duration measure with its aggregate, label and format', async () => { + const { service } = makeService(); + service.registerDataset(listed); + const [meta] = await service.getMeta(listed.name); + expect(meta.measures.filter((m) => m.name !== `${listed.name}.tickets`)).toEqual([ + { name: 'resolution.resolution_seconds_sum', type: 'sum', title: 'resolution_seconds_sum' }, + { name: 'resolution.high_hours', type: 'avg', title: 'High-priority hours', format: '0.0' }, + ]); + }); + + it('the listed spelling is served as the bare one, its scoped filter included, under the name asked', async () => { + const { service, executed } = makeService(); + service.registerDataset(listed); + await service.query({ cube: listed.name, measures: ['resolution.high_hours'], dimensions: ['category'] } as never, MEMBER); + expect(executed).toHaveLength(1); + expect(executed[0]).toContain('AVG(CASE WHEN priority = $1 THEN ((ROUND((julianday(resolved_at) - julianday(opened_at))'); + expect(executed[0]).toContain('AS "resolution.high_hours"'); + }); + + it('the read gate and the strategy refusal know the listed spelling too', async () => { + const gated = makeService({ readable: FIELDS.filter((f) => f !== 'opened_at') }); + gated.service.registerDataset(listed); + const denied = await refusalOf(gated.service.query({ cube: listed.name, measures: ['resolution.high_hours'] } as never, MEMBER)); + expect(denied?.code, denied?.message).toBe('PERMISSION_DENIED'); + expect(denied?.status).toBe(403); + expect(gated.executed).toEqual([]); + + const offNative = makeService({ capabilities: () => ({ nativeSql: false, objectqlAggregate: true, inMemory: false }) }); + offNative.service.registerDataset(listed); + const refused = await refusalOf(offNative.service.query({ cube: listed.name, measures: ['resolution.high_hours'] } as never, MEMBER)); + expect(refused?.code, refused?.message).toBe('INVALID_FIELD'); + expect(refused?.status).toBe(400); + expect(refused?.member).toBe('resolution.high_hours'); + expect(offNative.executed).toEqual([]); + }); + }); + + it('a field-to-field filter routes the query off the native strategy, and the measure is refused by name', async () => { + const { service, executed } = makeService(); + service.registerDataset(dataset); + const err = await refusalOf( + service.query({ + cube: dataset.name, + measures: ['resolution_seconds_sum'], + where: { resolved_at: { $gt: { $field: 'opened_at' } } }, + } as never, MEMBER), + ); + expect(err?.code, err?.message).toBe('INVALID_FIELD'); + expect(err?.status).toBe(400); + expect(err?.member).toBe('resolution_seconds_sum'); + expect(executed).toEqual([]); + }); + describe('a dataset that reaches the service without meeting the spec\'s parse', () => { const stored = (duration: Record) => ({ ...DATASET_BASE, diff --git a/packages/services/service-analytics/src/analytics-service.ts b/packages/services/service-analytics/src/analytics-service.ts index 3a2cfdd78fb..870896c8505 100644 --- a/packages/services/service-analytics/src/analytics-service.ts +++ b/packages/services/service-analytics/src/analytics-service.ts @@ -112,7 +112,7 @@ import { } from './dimension-labels.js'; import { evaluateAnalyticsQueryOverRows } from './preview-evaluator.js'; // [#25778] The one lookup of a dataset duration measure by name. -import { durationMeasureIn } from './duration-measure-sql.js'; +import { durationMeasureIn, durationMeasureKey } from './duration-measure-sql.js'; // [#5918] The measure mint refuses a dotted member through the SAME constructor // the strategies' member-level refusals use — `INVALID_FIELD` / 400, naming the // member as the request spelled it (see `dataset-refusal.ts`'s header for why @@ -547,10 +547,13 @@ function namedQueryFields( }; // [#25778] A dataset duration measure is absent from the cube (fail-closed, // `DatasetScope.measureDurations`), so `name` would read nothing for it. It - // reads its two columns, and is judged on both. + // reads its two columns, and is judged on both. Resolved FIRST, as the mint, + // `assertCallerMembersResolvable` and `NativeSQLStrategy` resolve it: a + // dimension may share the measure's name, and that dimension's column is not + // what the measure aggregates. const durationColumns = (member: string, role: FieldReadRole): boolean => { - const duration = durationMeasureIn(datasetScope?.measureDurations, member); - if (!duration || declaredMemberEntry(cube, member, 'any')) return false; + const duration = durationMeasureIn(datasetScope?.measureDurations, member, cube.name); + if (!duration) return false; for (const column of [duration.from, duration.to]) { const fields = fieldsOfColumnSql(cube, baseObject, column, role, referenceOf); if (fields) out.push(...fields); @@ -572,12 +575,17 @@ function namedQueryFields( for (const member of query.measures ?? []) if (!durationColumns(member, 'aggregate')) name(member, 'measure', 'aggregate'); for (const member of filterMembers((query as { where?: unknown }).where)) name(member, 'any', 'predicate'); for (const measure of query.measures ?? []) { - for (const member of filterMembers(datasetScope?.measureFilters?.[measure])) name(member, 'any', 'predicate'); + const filterKey = durationMeasureKey(datasetScope?.measureDurations, measure, cube.name) ?? measure; + for (const member of filterMembers(datasetScope?.measureFilters?.[filterKey])) name(member, 'any', 'predicate'); } for (const member of filterMembers(datasetScope?.filter)) name(member, 'any', 'predicate'); const order = (query as { order?: unknown }).order; if (order && typeof order === 'object' && !Array.isArray(order)) { - for (const key of Object.keys(order)) if (!durationColumns(key, 'predicate')) name(key, 'any', 'predicate'); + // An order key naming both a duration measure and a declared member is + // judged as both. + for (const key of Object.keys(order)) { + if (!durationColumns(key, 'predicate') || declaredMemberEntry(cube, key, 'any')) name(key, 'any', 'predicate'); + } } return [...out.filter((f) => f.object === baseObject), ...out.filter((f) => f.object !== baseObject)]; } @@ -2171,7 +2179,7 @@ export class AnalyticsService implements IAnalyticsService { // Author-declared measures are not this gate's business. [#25778] A // dataset's duration measure is one, declared beside the cube. if (authorCube && declaredMemberEntry(authorCube, measure, 'measure')) continue; - if (durationMeasureIn(scope.getCompiledDataset(name)?.durations, measure)) continue; + if (durationMeasureIn(scope.getCompiledDataset(name)?.durations, measure, name)) continue; const inferredSql = inferredCallerMeasureSql(measure, name); // `null` — a dotted non-qualifier measure #5918 already refused — and a // column reference (`'*'`, a bare identifier, a relationship path) both @@ -3238,13 +3246,24 @@ export class AnalyticsService implements IAnalyticsService { name: cube.name, title: cube.title, ...(cube.description === undefined ? {} : { description: cube.description }), - measures: Object.entries(cube.measures).map(([key, measure]) => ({ - name: `${cube.name}.${key}`, - type: measure.type, - title: measure.label, - ...(measure.description === undefined ? {} : { description: measure.description }), - ...(measure.format === undefined ? {} : { format: measure.format }), - })), + measures: [ + ...Object.entries(cube.measures).map(([key, measure]) => ({ + name: `${cube.name}.${key}`, + type: measure.type, + title: measure.label, + ...(measure.description === undefined ? {} : { description: measure.description }), + ...(measure.format === undefined ? {} : { format: measure.format }), + })), + // [#25778] A registered dataset's duration measures are absent from + // `cube.measures` (fail-closed) yet answer on the cube door, so the + // discovery surface lists them from the compiled dataset of THIS cube. + ...Object.entries(this.compiledDatasetOf(cube)?.durations ?? {}).map(([key, duration]) => ({ + name: `${cube.name}.${key}`, + type: duration.aggregate, + title: duration.label ?? key, + ...(duration.format === undefined ? {} : { format: duration.format }), + })), + ], dimensions: Object.entries(cube.dimensions).map(([key, dimension]) => ({ name: `${cube.name}.${key}`, type: dimension.type, @@ -3300,6 +3319,12 @@ export class AnalyticsService implements IAnalyticsService { // ── Internal ───────────────────────────────────────────────────── + /** The compiled dataset a registered cube came from, or nothing when another cube took the name. */ + private compiledDatasetOf(cube: Cube): CompiledDataset | undefined { + const compiled = this.datasetRegistry.get(cube.name); + return compiled?.cube === cube ? compiled : undefined; + } + /** * [#25778] Refuse a dataset duration measure the chosen strategy does not * serve — `INVALID_FIELD` / 400, naming the measure, before anything runs. @@ -3318,7 +3343,7 @@ export class AnalyticsService implements IAnalyticsService { private assertDurationMeasuresServed(query: AnalyticsQuery, scope: CubeScope, strategy: AnalyticsStrategy): void { if (strategy instanceof NativeSQLStrategy) return; const durations = scope.getCompiledDataset(query.cube!)?.durations; - const measure = (query.measures ?? []).find((m) => durationMeasureIn(durations, m)); + const measure = (query.measures ?? []).find((m) => durationMeasureIn(durations, m, query.cube)); if (measure === undefined) return; throw invalidMemberError( `Measure "${measure}" of dataset "${query.cube}" is a duration, the time between two datetime fields ` + @@ -3468,7 +3493,7 @@ export class AnalyticsService implements IAnalyticsService { const durations = scope.getCompiledDataset(name)?.durations; for (const m of query.measures || []) { if (cube.measures[m] || extraMeasures[m]) continue; - if (durationMeasureIn(durations, m)) continue; + if (durationMeasureIn(durations, m, name)) continue; const key = mintableMeasureKey(m, name); if (cube.measures[key] || extraMeasures[key]) continue; // [#21437] A spelling whose source names no field (`_sum`, `*`) is diff --git a/packages/services/service-analytics/src/dataset-compiler.ts b/packages/services/service-analytics/src/dataset-compiler.ts index 04edd0b6517..e938f74a3e7 100644 --- a/packages/services/service-analytics/src/dataset-compiler.ts +++ b/packages/services/service-analytics/src/dataset-compiler.ts @@ -456,7 +456,12 @@ function assertAggregateFieldTypeCompatible( ); } -/** One column of the base object by its bare name — the spec's `duration.from` / `to` shape. */ +/** + * One column of the base object by its bare name — the spec's `duration.from` / + * `to` shape (`ANALYTICS_BASE_COLUMN`, which `@objectstack/spec` does not + * export). Judged again here because a dataset can reach the compile without + * passing the schema. + */ const DURATION_BASE_COLUMN = /^[A-Za-z_][A-Za-z0-9_]*$/; /** @@ -519,7 +524,15 @@ function compileDurationMeasure( ); } } - return { aggregate: aggregate as DurationMeasureSpec['aggregate'], from, to, unit }; + return { + aggregate: aggregate as DurationMeasureSpec['aggregate'], + from, + to, + unit, + // [#6761] The same registry-locale label a cube metric gets. + label: resolveI18nLabel(measure.label, REGISTRY_LOCALE) ?? measure.name, + ...(typeof measure.format === 'string' ? { format: measure.format } : {}), + }; } /** Map a dataset dimension type to the Cube dimension `type`. */ diff --git a/packages/services/service-analytics/src/dataset-executor.ts b/packages/services/service-analytics/src/dataset-executor.ts index f826b1da46a..4f55161ea53 100644 --- a/packages/services/service-analytics/src/dataset-executor.ts +++ b/packages/services/service-analytics/src/dataset-executor.ts @@ -1196,7 +1196,7 @@ export class DatasetExecutor { // aggregate beside it. const fillColumns: Record = {}; for (const m of baseMeasures) { - const aggregate = compiled.cube.measures?.[m]?.type ?? durationMeasureIn(compiled.durations, m)?.aggregate; + const aggregate = compiled.cube.measures?.[m]?.type ?? durationMeasureIn(compiled.durations, m, compiled.cube.name)?.aggregate; fillColumns[m] = aggregate; if (selection.compareTo) fillColumns[`${m}__compare`] = aggregate; } diff --git a/packages/services/service-analytics/src/duration-measure-sql.ts b/packages/services/service-analytics/src/duration-measure-sql.ts index 437f2fdd5e9..b3e5d3ccc2d 100644 --- a/packages/services/service-analytics/src/duration-measure-sql.ts +++ b/packages/services/service-analytics/src/duration-measure-sql.ts @@ -15,12 +15,20 @@ * |---|---|---| * | `sqlite` | canonical UTC ISO-8601 TEXT | `ROUND((julianday(to) - julianday(from)) * 86400000) / 1000` | * | `postgres` | `timestamptz` | `EXTRACT(EPOCH FROM (to - from))`, as double precision | - * | `mysql` | `DATETIME(3)`, the UTC wall clock | `TIMESTAMPDIFF(MICROSECOND, from, to) / 1000000` | + * | `mysql` | `DATETIME(3)`, the UTC wall clock | `TIMESTAMPDIFF(MICROSECOND, from, to) / 1.0E6` | + * + * Every arm computes in double precision, so the unit division below keeps + * the same digits on each. MySQL's divisor is written `1.0E6`, a DOUBLE + * literal: `/ 1000000.0` would be exact-value DECIMAL arithmetic, rounded to + * `div_precision_increment` places at each division (5000 s in hours: MySQL + * `1.38888889`, a double `1.3888888888888888`). * * SQLite's `julianday` is a double counted from 4714 BC, so its difference * carries float noise below the millisecond. Rounded to the millisecond, the - * resolution every driver stores a datetime at, the three dialects answer the - * same number for the same instants. The column references arrive already + * resolution every driver stores a datetime at, SQLite and PostgreSQL answer + * the same number for the same instants, measured live on both + * (`dataset-duration-measure.test.ts`). MySQL's arm is pinned as text only; + * this package runs no live MySQL. The column references arrive already * passed through the host's temporal repair (`coerceTemporalFilterColumn`), so * a legacy SQLite column still holding epoch integers reads as canonical text * here too. @@ -39,7 +47,9 @@ * datasource whose dialect the host cannot name, and the service refuses it. * * {@link durationMeasureIn} is here too: the one lookup of a duration measure by - * name, for this face and for the service's gates. + * name, for this face and for the service's gates, and {@link durationMeasureKey} + * beside it for the readers that key something else (the measure's scoped + * filter) by the same name. */ import { DATASET_DURATION_UNIT_SECONDS, type DatasetDurationUnit } from '@objectstack/spec/ui'; @@ -47,16 +57,35 @@ import type { DurationMeasureSpec } from './strategies/types.js'; import type { AnalyticsSqlDialect } from './text-match-sql.js'; /** - * The duration measure `name` names in a compiled dataset's `durations` (the - * dataset scope's `measureDurations`), or `undefined`. The one lookup every - * reader uses, own keys only, so a measure named after an `Object.prototype` - * member (`constructor`) is never read as a duration it is not. + * The key in a compiled dataset's `durations` (the dataset scope's + * `measureDurations`) that the member `name` names, or `undefined`. A member + * names a duration measure by its bare name or as `.`, the spelling + * the discovery surface lists it under (`AnalyticsService.getMeta`), when the + * caller passes the cube's name. Own keys only, so a measure named after an + * `Object.prototype` member (`constructor`) is never read as a duration it is + * not. */ +export function durationMeasureKey( + durations: Readonly> | undefined, + name: string, + cubeName?: string, +): string | undefined { + if (!durations || typeof name !== 'string') return undefined; + const own = (key: string) => Object.prototype.hasOwnProperty.call(durations, key); + if (own(name)) return name; + const prefix = cubeName ? `${cubeName}.` : undefined; + if (prefix && name.startsWith(prefix) && own(name.slice(prefix.length))) return name.slice(prefix.length); + return undefined; +} + +/** The duration measure {@link durationMeasureKey} finds for `name`, or `undefined`. The one lookup every reader uses. */ export function durationMeasureIn( durations: Readonly> | undefined, name: string, + cubeName?: string, ): DurationMeasureSpec | undefined { - return durations && Object.prototype.hasOwnProperty.call(durations, name) ? durations[name] : undefined; + const key = durationMeasureKey(durations, name, cubeName); + return key === undefined ? undefined : durations![key]; } /** The dialects {@link durationOperandSql} has an arm for. */ @@ -75,7 +104,7 @@ function durationSecondsSql(dialect: DurationSqlDialect, fromSql: string, toSql: case 'postgres': return `CAST(EXTRACT(EPOCH FROM (${toSql} - ${fromSql})) AS DOUBLE PRECISION)`; case 'mysql': - return `(TIMESTAMPDIFF(MICROSECOND, ${fromSql}, ${toSql}) / 1000000.0)`; + return `(TIMESTAMPDIFF(MICROSECOND, ${fromSql}, ${toSql}) / 1.0E6)`; } } diff --git a/packages/services/service-analytics/src/preview-evaluator.ts b/packages/services/service-analytics/src/preview-evaluator.ts index 2273c3ea1d9..894e77b3a80 100644 --- a/packages/services/service-analytics/src/preview-evaluator.ts +++ b/packages/services/service-analytics/src/preview-evaluator.ts @@ -866,7 +866,7 @@ export function evaluateAnalyticsQueryOverRows( for (const m of query.measures) { // [#25778] A dataset duration measure, absent from the cube: its // aggregate over each row's duration, through the same arms. - const duration = durationMeasureIn(durations, m); + const duration = durationMeasureIn(durations, m, cube.name); if (duration) { row[m] = aggregate(g.rows.map((r) => ({ [DURATION_OPERAND]: durationOf(r, duration) })), duration.aggregate, DURATION_OPERAND); continue; diff --git a/packages/services/service-analytics/src/strategies/native-sql-strategy.ts b/packages/services/service-analytics/src/strategies/native-sql-strategy.ts index dcf7ad5fed3..eee1560c6e0 100644 --- a/packages/services/service-analytics/src/strategies/native-sql-strategy.ts +++ b/packages/services/service-analytics/src/strategies/native-sql-strategy.ts @@ -46,7 +46,7 @@ import { type LikeShape } from '../like-pattern.js'; import { textMatchPredicateSql, sqlDialectFor, type AnalyticsSqlDialect } from '../text-match-sql.js'; import { whereContainsMembershipSql } from '../contains-membership-sql.js'; // [#25778] The per-row operand of a dataset duration measure, per dialect. -import { durationMeasureIn, durationOperandSql, hasDurationSql } from '../duration-measure-sql.js'; +import { durationMeasureIn, durationMeasureKey, durationOperandSql, hasDurationSql } from '../duration-measure-sql.js'; import { isJsonStoredShape } from '../contains-membership-sql.js'; import { expandEmptyOperator } from '@objectstack/spec/data'; import { resolveAnalyticsDateRangeString } from '@objectstack/core'; @@ -798,7 +798,7 @@ export class NativeSQLStrategy implements AnalyticsStrategy { const cube = query.cube ? ctx.getCube(query.cube) : undefined; if (!cube) return false; const durations = (ctx as DatasetScopedStrategyContext).getDatasetScope?.(query.cube!)?.measureDurations; - if (!(query.measures ?? []).some((measure) => durationMeasureIn(durations, measure))) return false; + if (!(query.measures ?? []).some((measure) => durationMeasureIn(durations, measure, cube.name))) return false; return !hasDurationSql(sqlDialectFor(ctx, this.extractObjectName(cube))); } @@ -1130,7 +1130,7 @@ export class NativeSQLStrategy implements AnalyticsStrategy { // its aggregate in the dataset scope; it folds by that aggregate, and it // answers a number whatever its aggregate. const durations = (ctx as DatasetScopedStrategyContext).getDatasetScope?.(query.cube!)?.measureDurations; - const durationOf = (member: string) => durationMeasureIn(durations, member); + const durationOf = (member: string) => durationMeasureIn(durations, member, cube.name); const folds: Array = []; for (const member of query.measures ?? []) { const identity = emptyGroupValueFor(durationOf(member)?.aggregate ?? this.lookupMember(cube, member, 'measure')?.type); @@ -1294,7 +1294,10 @@ export class NativeSQLStrategy implements AnalyticsStrategy { // loop, so its bound values are pushed onto `params` in the order the // placeholders appear in the statement — the SELECT list precedes the // WHERE clause, and `$n` is positional. - const measureFilter = datasetScope?.measureFilters?.[measure]; + // [#25778] A duration measure named `.` reads the filter + // its dataset scoped under ``. + const durationKey = durationMeasureKey(datasetScope?.measureDurations, measure, cube.name); + const measureFilter = datasetScope?.measureFilters?.[durationKey ?? measure]; const predicate = measureFilter ? this.compileFilterNode( normalizeAnalyticsFilterTree(judgedComparands({ where: measureFilter }, comparandTypeOf), lowering), @@ -1305,7 +1308,7 @@ export class NativeSQLStrategy implements AnalyticsStrategy { ctx, ) : null; - const duration = durationMeasureIn(datasetScope?.measureDurations, measure); + const duration = durationKey === undefined ? undefined : datasetScope!.measureDurations![durationKey]; const aggExpr = duration ? this.resolveDurationMeasureSql(cube, measure, duration, tableName, joins, predicate, ctx) : this.resolveMeasureSql(cube, measure, tableName, joins, predicate, ctx); diff --git a/packages/services/service-analytics/src/strategies/types.ts b/packages/services/service-analytics/src/strategies/types.ts index 0962509965e..606fd6a36f0 100644 --- a/packages/services/service-analytics/src/strategies/types.ts +++ b/packages/services/service-analytics/src/strategies/types.ts @@ -95,6 +95,13 @@ export interface DurationMeasureSpec { from: string; to: string; unit: DatasetDurationUnit; + /** + * Display only, read by the discovery surface (`AnalyticsService.getMeta`), + * which lists the measure though the cube does not carry it: the measure's + * resolved `label` and its `format`. No strategy reads either. + */ + label?: string; + format?: string; } /** A {@link StrategyContext} that can answer for a compiled dataset (#10298). */ diff --git a/packages/spec/src/data/analytics-column-reference.ts b/packages/spec/src/data/analytics-column-reference.ts index 9666aef5a56..81e1303ea29 100644 --- a/packages/spec/src/data/analytics-column-reference.ts +++ b/packages/spec/src/data/analytics-column-reference.ts @@ -96,7 +96,7 @@ export const ANALYTICS_COLUMN_PATH = new RegExp(`^${COLUMN_PATH}$`); * column of the dataset's OWN object, by its bare name: the slot of a duration * measure's `from` / `to` (`DatasetMeasureDurationSchema`, * `../ui/dataset.zod.ts`), which reads two columns of the same row. Built from - * the same identifier as {@link COLUMN_PATH}, never a second spelling of it. + * the same identifier as {@link COLUMN_PATH}. */ export const ANALYTICS_BASE_COLUMN = new RegExp(`^${IDENTIFIER}$`); From 78be554647e13ddf3334eb15d669916ae0b8b742 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 25 Oct 2026 15:00:11 +0000 Subject: [PATCH 07/14] chore(platform-objects): translate the dataset form's Duration row Co-Authored-By: Claude Claude-Session: https://claude.ai/code/session_01AKPixnkg91qCkQCVFDJuqW --- .changeset/25778-dataset-duration-measure.md | 4 +++- .../src/apps/translations/en.metadata-forms.generated.ts | 3 +++ .../src/apps/translations/es-ES.metadata-forms.generated.ts | 3 +++ .../src/apps/translations/ja-JP.metadata-forms.generated.ts | 3 +++ .../src/apps/translations/zh-CN.metadata-forms.generated.ts | 3 +++ 5 files changed, 15 insertions(+), 1 deletion(-) diff --git a/.changeset/25778-dataset-duration-measure.md b/.changeset/25778-dataset-duration-measure.md index a2b6fa8d934..5fc4eda0bee 100644 --- a/.changeset/25778-dataset-duration-measure.md +++ b/.changeset/25778-dataset-duration-measure.md @@ -2,9 +2,10 @@ '@objectstack/spec': minor '@objectstack/service-analytics': minor '@objectstack/lint': minor +'@objectstack/platform-objects': minor --- -feat(spec,analytics,lint): a dataset measure can aggregate the time between two datetime fields of each record +feat(spec,analytics,lint,platform-objects): a dataset measure can aggregate the time between two datetime fields of each record Clause-②: yes (widening) Metadata-protocol: added @@ -12,5 +13,6 @@ Metadata-protocol: added - **`@objectstack/spec`**: `DatasetMeasureSchema` gains an optional `duration: { from, to, unit }`, in place of `field`. `from` and `to` name two `datetime` fields of the dataset's own object by their bare names, `unit` is `seconds`, `minutes`, `hours` or `days`, and the measure's `aggregate` is `avg`, `min`, `max` or `sum` of `to` minus `from`. A relationship path, an expression or the row wildcard in either column, the same column twice, another unit, a counting aggregate, or a `field`, `derived` or `currency` beside it is refused at parse. New exports: `DatasetMeasureDurationSchema`, `DatasetDurationUnit`, `DATASET_DURATION_UNIT_SECONDS`, `DATASET_DURATION_AGGREGATES`, `DATASET_DURATION_FIELD_TYPE` and the `DatasetMeasureDuration` / `DatasetDurationUnit` types. - **`@objectstack/service-analytics`**: the dataset compile refuses a duration column declared anything but `datetime` (`400 DATASET_INVALID`, naming the column and its type). The native-SQL strategy computes the duration as one statement on SQLite, PostgreSQL and MySQL, in double precision on each; SQLite and PostgreSQL are tested live to answer the same number: a record missing either instant is left out, a `to` before its `from` counts as negative, and over a group with no such record `sum` answers `0` and the others `null`. Measure filters, derived measures, ordering and the draft preview serve it. A query the native strategy does not serve (a time dimension bucketed by `granularity`, a field-to-field filter, an object served through engine middleware or an external datasource, a non-SQL datasource) is refused with `400 INVALID_FIELD` naming the measure. Its two columns pass the field-level read gate like any measure's field. The temporal half of the aggregate × field-type refusal now names the duration measure instead of a stored "days open" number. - **`@objectstack/lint`**: `os validate` refuses a duration column not declared `datetime` with the new gating rule `measure-duration-field-type-refused` (at `datasets[N].measures[M].duration.from` / `.to`), with an `os explain` entry. `dataset-field-unknown` now also reports a duration column the object does not declare. +- **`@objectstack/platform-objects`**: the dataset form's new **Duration** row is translated in every shipped locale. What to do: nothing. A dataset without `duration` parses, compiles and answers exactly as before. diff --git a/packages/platform-objects/src/apps/translations/en.metadata-forms.generated.ts b/packages/platform-objects/src/apps/translations/en.metadata-forms.generated.ts index 3a1aea12bda..8a8bd3d8db4 100644 --- a/packages/platform-objects/src/apps/translations/en.metadata-forms.generated.ts +++ b/packages/platform-objects/src/apps/translations/en.metadata-forms.generated.ts @@ -2069,6 +2069,9 @@ export const enMetadataForms: NonNullable = { }, "measures.derived": { label: "Derived From" + }, + "measures.duration": { + label: "Duration" } } }, diff --git a/packages/platform-objects/src/apps/translations/es-ES.metadata-forms.generated.ts b/packages/platform-objects/src/apps/translations/es-ES.metadata-forms.generated.ts index 80fca2de789..409018c8da6 100644 --- a/packages/platform-objects/src/apps/translations/es-ES.metadata-forms.generated.ts +++ b/packages/platform-objects/src/apps/translations/es-ES.metadata-forms.generated.ts @@ -2069,6 +2069,9 @@ export const esESMetadataForms: NonNullable = }, "measures.derived": { label: "Derivada de" + }, + "measures.duration": { + label: "Duración" } } }, diff --git a/packages/platform-objects/src/apps/translations/ja-JP.metadata-forms.generated.ts b/packages/platform-objects/src/apps/translations/ja-JP.metadata-forms.generated.ts index 52ae3927fcd..1e23c195d45 100644 --- a/packages/platform-objects/src/apps/translations/ja-JP.metadata-forms.generated.ts +++ b/packages/platform-objects/src/apps/translations/ja-JP.metadata-forms.generated.ts @@ -2069,6 +2069,9 @@ export const jaJPMetadataForms: NonNullable = }, "measures.derived": { label: "派生元" + }, + "measures.duration": { + label: "所要時間" } } }, diff --git a/packages/platform-objects/src/apps/translations/zh-CN.metadata-forms.generated.ts b/packages/platform-objects/src/apps/translations/zh-CN.metadata-forms.generated.ts index 80387ad938f..bddb993a46a 100644 --- a/packages/platform-objects/src/apps/translations/zh-CN.metadata-forms.generated.ts +++ b/packages/platform-objects/src/apps/translations/zh-CN.metadata-forms.generated.ts @@ -2069,6 +2069,9 @@ export const zhCNMetadataForms: NonNullable = }, "measures.derived": { label: "派生自" + }, + "measures.duration": { + label: "时长" } } }, From 634fe6bd013812443d9a0eab4f3692ed7eb61f5e Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 25 Oct 2026 15:04:31 +0000 Subject: [PATCH 08/14] test(spec): pin the duration schema and unit enum as isomorphic (ADR-0122) Co-Authored-By: Claude Claude-Session: https://claude.ai/code/session_01AKPixnkg91qCkQCVFDJuqW --- .../spec/src/type-alias-convention.pin.test.ts | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/packages/spec/src/type-alias-convention.pin.test.ts b/packages/spec/src/type-alias-convention.pin.test.ts index a9971f38565..fc720ccf228 100644 --- a/packages/spec/src/type-alias-convention.pin.test.ts +++ b/packages/spec/src/type-alias-convention.pin.test.ts @@ -279,7 +279,7 @@ import type * as M190 from './kernel/execution-context.zod.js'; import type * as M191 from './automation/api-trigger-secret-ref.zod.js'; // --------------------------------------------------------------------------- -// 790 isomorphic aliases: `z.input` === `z.infer`, so no `XParsed` is declared. +// 792 isomorphic aliases: `z.input` === `z.infer`, so no `XParsed` is declared. // // That number is machine-checked, not hand-kept. The runtime companion at the // bottom of this file recomputes the pin count from the source and asserts that @@ -1548,6 +1548,10 @@ export type Iso_ui_dashboard__WidgetColorVariantSchema = Assert, z.infer< typeof M160.DatasetDimensionSchema > >>; +// [#25778] The duration measure's `{ from, to, unit }` and its unit enum: two +// bare names and an enum, no default, transform, catch or pipe. +export type Iso_ui_dataset__DatasetDurationUnit = Assert, z.infer< typeof M160.DatasetDurationUnit > >>; +export type Iso_ui_dataset__DatasetMeasureDurationSchema = Assert, z.infer< typeof M160.DatasetMeasureDurationSchema > >>; export type Iso_ui_dataset__DatasetMeasureSchema = Assert, z.infer< typeof M160.DatasetMeasureSchema > >>; export type Iso_ui_dataset__DatasetSchema = Assert, z.infer< typeof M160.DatasetSchema > >>; export type Iso_ui_dataset__DerivedMeasureOp = Assert, z.infer< typeof M160.DerivedMeasureOp > >>; @@ -1708,7 +1712,7 @@ describe('ADR-0122 type-alias convention', () => { // this title and the section header above the pin list — are now asserted // against the recomputed count below, so neither can go stale without a red // test naming it. - it('still declares all 790 isomorphic pins', () => { + it('still declares all 792 isomorphic pins', () => { // The truth of each pin is proved by tsc, not here — an `Assert>` // that stops holding is a compile error with the alias named. What tsc // cannot notice is a pin that was DELETED: removing the assertion removes @@ -2573,7 +2577,14 @@ describe('ADR-0122 type-alias convention', () => { // start node gives its hook secret — a plain string, so it is pinned here // (Iso_automation_apiTriggerSecretRef__ApiTriggerSecretRefSchema) rather // than given a `…Parsed` synonym. +1 added. - expect(pins).toHaveLength(790); + // + // 790 -> 792 is #25778: the dataset duration measure's + // `DatasetMeasureDurationSchema` (`{ from, to, unit }`, two bare names and + // an enum) and its `DatasetDurationUnit` enum, pinned here + // (Iso_ui_dataset__DatasetMeasureDurationSchema, + // Iso_ui_dataset__DatasetDurationUnit) rather than given `…Parsed` + // synonyms. +2 added. + expect(pins).toHaveLength(792); // The count is stated in PROSE twice as well — this case's title and the // section header above the pin list — and until commit c6b05c76a nothing read either From 7caa5668709d087617e37dd1a4eee169405031b2 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 26 Oct 2026 02:16:58 +0000 Subject: [PATCH 09/14] chore(spec): regenerate the four artifacts the main merge deferred Co-Authored-By: Claude Claude-Session: https://claude.ai/code/session_01AKPixnkg91qCkQCVFDJuqW --- content/docs/references/index.mdx | 9 ++++----- packages/spec/api-surface/ui.json | 1 + packages/spec/authorable-surface/ui.json | 1 + packages/spec/export-origins/ui.json | 1 + 4 files changed, 7 insertions(+), 5 deletions(-) diff --git a/content/docs/references/index.mdx b/content/docs/references/index.mdx index 2061295d4ac..3e3f154402a 100644 --- a/content/docs/references/index.mdx +++ b/content/docs/references/index.mdx @@ -1,7 +1,7 @@ --- title: Protocol reference — every schema by module navTitle: Protocol Reference -description: Every schema published by @objectstack/spec — 1551 schemas across 14 protocol modules +description: Every schema published by @objectstack/spec — 1541 schemas across 14 protocol modules --- {/* ⚠️ AUTO-GENERATED — DO NOT EDIT. Run build-docs.ts to regenerate. Hand-written docs live in the module folders under content/docs/. */} @@ -26,7 +26,7 @@ counts are sums of the rows they head. Regenerate with | [Data Protocol](/docs/references/data) | 29 | 178 | Objects, fields, queries, filters, datasources and drivers — the ObjectQL layer. | | [Identity Protocol](/docs/references/identity) | 5 | 27 | Users and accounts, organizations, positions, SCIM provisioning. | | [Integration Protocol](/docs/references/integration) | 1 | 12 | The single connector protocol (ADR-0097) — catalog descriptors and provider-bound instances. | -| [Kernel Protocol](/docs/references/kernel) | 30 | 161 | Plugin lifecycle and manifests, capabilities and security, metadata loading, service registry. | +| [Kernel Protocol](/docs/references/kernel) | 29 | 151 | Plugin lifecycle and manifests, capabilities and security, metadata loading, service registry. | | [Marketplace Protocol](/docs/references/marketplace) | 4 | 30 | The package & marketplace format — package identity and versions, listing, publish, review, search, install, template manifests. | | [QA Protocol](/docs/references/qa) | 1 | 8 | Declarative test suites — scenarios, steps, actions and assertions. | | [Security Protocol](/docs/references/security) | 5 | 34 | Permission sets, row-level security, sharing rules, tenancy posture. | @@ -34,7 +34,7 @@ counts are sums of the rows they head. Regenerate with | [Studio Protocol](/docs/references/studio) | 3 | 35 | Studio designer metadata — the authoring surfaces for the protocols above. | | [System Protocol](/docs/references/system) | 36 | 285 | The runtime environment — logging, jobs, cache, metrics, notifications, i18n and compliance. | | [UI Protocol](/docs/references/ui) | 16 | 171 | Apps, pages, views, dashboards, reports, actions and themes — the ObjectUI layer. | -| **Total** | **198** | **1551** | 14 protocol modules | +| **Total** | **197** | **1541** | 14 protocol modules | --- @@ -199,7 +199,7 @@ The single connector protocol (ADR-0097) — catalog descriptors and provider-bo ## Kernel Protocol -**Source:** `packages/spec/src/kernel/` · **Import:** `@objectstack/spec/kernel` · **30 pages, 161 schemas** +**Source:** `packages/spec/src/kernel/` · **Import:** `@objectstack/spec/kernel` · **29 pages, 151 schemas** Plugin lifecycle and manifests, capabilities and security, metadata loading, service registry. @@ -222,7 +222,6 @@ Plugin lifecycle and manifests, capabilities and security, metadata loading, ser | [`metadata-protection.zod.ts`](/docs/references/kernel/metadata-protection) | `MetadataLock`, `MetadataLockSource`, `MetadataProvenance` | | [`package-artifact.zod.ts`](/docs/references/kernel/package-artifact) | `ArtifactChecksum`, `ArtifactFileEntry`, `ArtifactSignature`, `MetadataCategoryEnum`, `PackageArtifact` | | [`package-registry.zod.ts`](/docs/references/kernel/package-registry) | `DisablePackageRequest`, `DisablePackageResponse`, `EnablePackageRequest`, `EnablePackageResponse`, `GetPackageRequest`, `GetPackageResponse`, `InstallPackageRequest`, `InstallPackageResponse`, `InstalledPackage`, `InstalledPackageSource`, `ListPackagesRequest`, `ListPackagesResponse`, `NamespaceConflictError`, `NamespaceRegistryEntry`, `PackageStatusEnum`, `UninstallPackageRequest`, `UninstallPackageResponse` | -| [`package-upgrade.zod.ts`](/docs/references/kernel/package-upgrade) | `MetadataChangeType`, `MetadataDiffItem`, `RollbackPackageRequest`, `RollbackPackageResponse`, `UpgradeImpactLevel`, `UpgradePackageRequest`, `UpgradePackageResponse`, `UpgradePhase`, `UpgradePlan`, `UpgradeSnapshot` | | [`plugin.zod.ts`](/docs/references/kernel/plugin) | `Plugin` | | [`plugin-capability.zod.ts`](/docs/references/kernel/plugin-capability) | `CapabilityConformanceLevel`, `ExtensionPoint`, `PluginCapability`, `PluginCapabilityManifest`, `PluginDependency`, `PluginInterface`, `ProtocolFeature`, `ProtocolReference`, `ProtocolVersion` | | [`plugin-lifecycle-advanced.zod.ts`](/docs/references/kernel/plugin-lifecycle-advanced) | `HotReloadConfig`, `PluginHealthCheck`, `PluginHealthReport`, `PluginHealthStatus`, `PluginStateSnapshot` | diff --git a/packages/spec/api-surface/ui.json b/packages/spec/api-surface/ui.json index 78713509d22..42b9df6e45d 100644 --- a/packages/spec/api-surface/ui.json +++ b/packages/spec/api-surface/ui.json @@ -555,6 +555,7 @@ "isRecordContextBlockType (function)", "isViewContainerShaped (function)", "listViewGroupKeyPredicate (function)", + "listViewGroupOrderColumn (function)", "misplacedDefaultPageRefusal (function)", "navItemVisibleDataNeedsDataSource (function)", "normalizeFilterOperator (function)", diff --git a/packages/spec/authorable-surface/ui.json b/packages/spec/authorable-surface/ui.json index e509a524ffc..3529ea0da68 100644 --- a/packages/spec/authorable-surface/ui.json +++ b/packages/spec/authorable-surface/ui.json @@ -668,6 +668,7 @@ "ui/GroupingField:collapsed", "ui/GroupingField:field", "ui/GroupingField:order", + "ui/GroupingField:orderBy", "ui/HttpRequest:body", "ui/HttpRequest:headers", "ui/HttpRequest:method", diff --git a/packages/spec/export-origins/ui.json b/packages/spec/export-origins/ui.json index 8a023456510..19a4416f98c 100644 --- a/packages/spec/export-origins/ui.json +++ b/packages/spec/export-origins/ui.json @@ -539,6 +539,7 @@ "isRecordContextBlockType": "src/ui/react-blocks.ts#isRecordContextBlockType (function)", "isViewContainerShaped": "src/ui/assembled-views.zod.ts#isViewContainerShaped (function)", "listViewGroupKeyPredicate": "src/ui/view-grouping-query.ts#listViewGroupKeyPredicate (function)", + "listViewGroupOrderColumn": "src/ui/view-grouping-query.ts#listViewGroupOrderColumn (function)", "misplacedDefaultPageRefusal": "src/ui/default-record-page.ts#misplacedDefaultPageRefusal (function)", "navItemVisibleDataNeedsDataSource": "src/ui/app.zod.ts#navItemVisibleDataNeedsDataSource (function)", "normalizeFilterOperator": "src/ui/view.zod.ts#normalizeFilterOperator (function)", From 24e5933a25849404f1d364a52f345b629cd9ceec Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 26 Oct 2026 02:17:17 +0000 Subject: [PATCH 10/14] fix(app-shell): the dataset inspector clears a duration measure when the base object changes A duration measure has no `field`, so `measureNamesNoField` read it as a plain row count and kept it across a base-object change, though its `from` / `to` name two of the old object's fields. Co-Authored-By: Claude Claude-Session: https://claude.ai/code/session_01AKPixnkg91qCkQCVFDJuqW --- .changeset/25778-dataset-duration-measure.md | 4 +++- ...tInspector.firstPickKeepsCount-12848.test.tsx | 2 ++ .../inspectors/DatasetDefaultInspector.tsx | 16 +++++++++++++--- 3 files changed, 18 insertions(+), 4 deletions(-) diff --git a/.changeset/25778-dataset-duration-measure.md b/.changeset/25778-dataset-duration-measure.md index 5fc4eda0bee..ee39dfe19c7 100644 --- a/.changeset/25778-dataset-duration-measure.md +++ b/.changeset/25778-dataset-duration-measure.md @@ -3,9 +3,10 @@ '@objectstack/service-analytics': minor '@objectstack/lint': minor '@objectstack/platform-objects': minor +'@object-ui/app-shell': minor --- -feat(spec,analytics,lint,platform-objects): a dataset measure can aggregate the time between two datetime fields of each record +feat(spec,analytics,lint,platform-objects,app-shell): a dataset measure can aggregate the time between two datetime fields of each record Clause-②: yes (widening) Metadata-protocol: added @@ -14,5 +15,6 @@ Metadata-protocol: added - **`@objectstack/service-analytics`**: the dataset compile refuses a duration column declared anything but `datetime` (`400 DATASET_INVALID`, naming the column and its type). The native-SQL strategy computes the duration as one statement on SQLite, PostgreSQL and MySQL, in double precision on each; SQLite and PostgreSQL are tested live to answer the same number: a record missing either instant is left out, a `to` before its `from` counts as negative, and over a group with no such record `sum` answers `0` and the others `null`. Measure filters, derived measures, ordering and the draft preview serve it. A query the native strategy does not serve (a time dimension bucketed by `granularity`, a field-to-field filter, an object served through engine middleware or an external datasource, a non-SQL datasource) is refused with `400 INVALID_FIELD` naming the measure. Its two columns pass the field-level read gate like any measure's field. The temporal half of the aggregate × field-type refusal now names the duration measure instead of a stored "days open" number. - **`@objectstack/lint`**: `os validate` refuses a duration column not declared `datetime` with the new gating rule `measure-duration-field-type-refused` (at `datasets[N].measures[M].duration.from` / `.to`), with an `os explain` entry. `dataset-field-unknown` now also reports a duration column the object does not declare. - **`@objectstack/platform-objects`**: the dataset form's new **Duration** row is translated in every shipped locale. +- **`@object-ui/app-shell`**: changing a dataset's base object in the console's dataset inspector now clears a duration measure along with the other measures that name the old object's fields, instead of keeping it as if it were a plain row count. What to do: nothing. A dataset without `duration` parses, compiles and answers exactly as before. diff --git a/ui/packages/app-shell/src/views/metadata-admin/inspectors/DatasetDefaultInspector.firstPickKeepsCount-12848.test.tsx b/ui/packages/app-shell/src/views/metadata-admin/inspectors/DatasetDefaultInspector.firstPickKeepsCount-12848.test.tsx index 56c951e92c7..86854c664d7 100644 --- a/ui/packages/app-shell/src/views/metadata-admin/inspectors/DatasetDefaultInspector.firstPickKeepsCount-12848.test.tsx +++ b/ui/packages/app-shell/src/views/metadata-admin/inspectors/DatasetDefaultInspector.firstPickKeepsCount-12848.test.tsx @@ -170,6 +170,8 @@ describe('objectChangePatch moves a dataset between objects (objectui#12848)', ( { name: 'amount_sum', aggregate: 'sum', field: 'amount' }, { name: 'won_count', aggregate: 'count', filter: { stage: 'won' } }, { name: 'won_rate', derived: { op: 'ratio', of: ['won_count', 'deal_count'] } }, + // objectstack#25778: no `field`, yet `from` / `to` name two of the old object's fields. + { name: 'avg_days_to_close', aggregate: 'avg', duration: { from: 'created_at', to: 'closed_at', unit: 'days' } }, ], }; diff --git a/ui/packages/app-shell/src/views/metadata-admin/inspectors/DatasetDefaultInspector.tsx b/ui/packages/app-shell/src/views/metadata-admin/inspectors/DatasetDefaultInspector.tsx index 19e4b2ef5bf..3d069001596 100644 --- a/ui/packages/app-shell/src/views/metadata-admin/inspectors/DatasetDefaultInspector.tsx +++ b/ui/packages/app-shell/src/views/metadata-admin/inspectors/DatasetDefaultInspector.tsx @@ -129,6 +129,7 @@ const CURRENCY_OPTIONS = [ type Dimension = { name?: string; label?: string; field?: string; type?: string; dateGranularity?: string }; type DerivedSpec = { op?: string; of?: string[] }; +type DurationSpec = { from?: string; to?: string; unit?: string }; type Measure = { name?: string; label?: string; @@ -137,6 +138,8 @@ type Measure = { format?: string; currency?: string; derived?: DerivedSpec; + /** Time between two `datetime` fields of a record (`DatasetMeasureSchema.duration`, objectstack#25778). */ + duration?: DurationSpec; filter?: FilterCondition; }; @@ -361,11 +364,18 @@ function DatasetFilterField({ label, help, value, onCommit, fields, disabled, lo /** * Whether a measure names none of its object's fields: a plain row count — no * `field` (or the row wildcard `'*'`), no measure-scoped `filter` (its keys are - * fields), and not `derived` (a derived measure combines other measures, which - * may name fields, and one left pointing at a cleared measure is refused). + * fields), not `derived` (a derived measure combines other measures, which + * may name fields, and one left pointing at a cleared measure is refused), and + * not a `duration` (its `from` / `to` are two fields, with no `field` beside + * them — objectstack#25778). */ function measureNamesNoField(m: Measure): boolean { - return (isBlankField(m.field) || m.field === '*') && m.filter === undefined && m.derived === undefined; + return ( + (isBlankField(m.field) || m.field === '*') && + m.filter === undefined && + m.derived === undefined && + m.duration === undefined + ); } /** From 76866719f70f24b3ac7471c378bd7cf678bcf007 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 26 Oct 2026 03:51:14 +0000 Subject: [PATCH 11/14] test(platform-objects): the two form-catalog count pins move by the dataset form's Duration row The measures repeater's `duration` row adds one leaf to the dataset form and one authored row label per translated locale: the positive control goes 679 -> 680 and the dataset panel's leaf count 38 -> 39. Co-Authored-By: Claude Claude-Session: https://claude.ai/code/session_01AKPixnkg91qCkQCVFDJuqW --- .../object-lifecycle-panel-echo-decisions.test.ts | 5 ++++- .../src/apps/translations/report-form-echo-decisions.test.ts | 5 +++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/packages/platform-objects/src/apps/translations/object-lifecycle-panel-echo-decisions.test.ts b/packages/platform-objects/src/apps/translations/object-lifecycle-panel-echo-decisions.test.ts index 74d267a263f..04d0e0c837c 100644 --- a/packages/platform-objects/src/apps/translations/object-lifecycle-panel-echo-decisions.test.ts +++ b/packages/platform-objects/src/apps/translations/object-lifecycle-panel-echo-decisions.test.ts @@ -1217,7 +1217,10 @@ describe('#19403 round 10 — the verdicts, on the live bundles', () => { // authored in all three locales. // 679 since the flow form offers `requiredPermissions` (who may start the // flow) — one row label, authored in all three locales. - expect(translated.length, `${locale} positive control`).toBe(679); + // 680 since the dataset form's measures repeater offers `duration` (the + // time between two datetime fields) — one row label, authored in all + // three locales. + expect(translated.length, `${locale} positive control`).toBe(680); } // ⭐ DARK — the blindness, executable. On a synthetic two-locale catalog the // all-three predicate returns 0 while the per-locale one returns 1, so the diff --git a/packages/platform-objects/src/apps/translations/report-form-echo-decisions.test.ts b/packages/platform-objects/src/apps/translations/report-form-echo-decisions.test.ts index 02cd8e88391..b94b6f932b4 100644 --- a/packages/platform-objects/src/apps/translations/report-form-echo-decisions.test.ts +++ b/packages/platform-objects/src/apps/translations/report-form-echo-decisions.test.ts @@ -59,7 +59,7 @@ // echoes in two locales TODAY. It is OUT of this population, asserted by // name — a walk that over-reached into the object form would sweep in 32 // undecided leaves, and the assertion is what notices. `datasetForm` — -// the sibling semantic-layer panel, 38 leaves, 0 echoing — is excluded +// the sibling semantic-layer panel, 39 leaves, 0 echoing — is excluded // too, and that exclusion is asserted rather than trusted precisely // because a wrong sweep of it would NOT go red. // (2) INWARD, AUTHORED. 38 of this population's own 44 leaves come back @@ -889,7 +889,8 @@ describe('#19403 round 9 — the population, DERIVED from `reportForm`', () => { // It is fully authored, so a wrong sweep of it would NOT go red — which is // exactly why the exclusion is asserted instead of trusted. // 38 since #25330: the dimensions repeater offers `nullLabel`. - expect(DATASET_LEAVES.length).toBe(38); + // 39 since the measures repeater offers `duration`. + expect(DATASET_LEAVES.length).toBe(39); const datasetEchoes = DATASET_LEAVES.filter((l) => TRANSLATED_LOCALES.some(([, forms]) => forms.dataset?.[l.kind === 'section' ? 'sections' : 'fields']?.[l.path]?.[l.prop] === l.en), ); From a5c3e1f6080a05207ab001a5b37f89a93aa7a0a2 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 26 Oct 2026 04:21:25 +0000 Subject: [PATCH 12/14] chore(spec): regenerate the references index the main merge deferred The merge brought main's system/demo-staff.zod.ts page; the index now counts it beside this branch's dataset duration schemas (1542 schemas, 198 pages). Co-Authored-By: Claude Claude-Session: https://claude.ai/code/session_01AKPixnkg91qCkQCVFDJuqW --- content/docs/references/index.mdx | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/content/docs/references/index.mdx b/content/docs/references/index.mdx index 3e3f154402a..fd661e65017 100644 --- a/content/docs/references/index.mdx +++ b/content/docs/references/index.mdx @@ -1,7 +1,7 @@ --- title: Protocol reference — every schema by module navTitle: Protocol Reference -description: Every schema published by @objectstack/spec — 1541 schemas across 14 protocol modules +description: Every schema published by @objectstack/spec — 1542 schemas across 14 protocol modules --- {/* ⚠️ AUTO-GENERATED — DO NOT EDIT. Run build-docs.ts to regenerate. Hand-written docs live in the module folders under content/docs/. */} @@ -32,9 +32,9 @@ counts are sums of the rows they head. Regenerate with | [Security Protocol](/docs/references/security) | 5 | 34 | Permission sets, row-level security, sharing rules, tenancy posture. | | [Shared Protocol](/docs/references/shared) | 10 | 31 | Primitives used across every protocol — identifiers, HTTP, expressions, error maps, enums. | | [Studio Protocol](/docs/references/studio) | 3 | 35 | Studio designer metadata — the authoring surfaces for the protocols above. | -| [System Protocol](/docs/references/system) | 36 | 285 | The runtime environment — logging, jobs, cache, metrics, notifications, i18n and compliance. | +| [System Protocol](/docs/references/system) | 37 | 286 | The runtime environment — logging, jobs, cache, metrics, notifications, i18n and compliance. | | [UI Protocol](/docs/references/ui) | 16 | 171 | Apps, pages, views, dashboards, reports, actions and themes — the ObjectUI layer. | -| **Total** | **197** | **1541** | 14 protocol modules | +| **Total** | **198** | **1542** | 14 protocol modules | --- @@ -317,7 +317,7 @@ Studio designer metadata — the authoring surfaces for the protocols above. ## System Protocol -**Source:** `packages/spec/src/system/` · **Import:** `@objectstack/spec/system` · **36 pages, 285 schemas** +**Source:** `packages/spec/src/system/` · **Import:** `@objectstack/spec/system` · **37 pages, 286 schemas** The runtime environment — logging, jobs, cache, metrics, notifications, i18n and compliance. @@ -329,6 +329,7 @@ The runtime environment — logging, jobs, cache, metrics, notifications, i18n a | [`cache.zod.ts`](/docs/references/system/cache) | `CacheAvalanchePrevention`, `CacheConfig`, `CacheConsistency`, `CacheInvalidation`, `CacheStrategy`, `CacheTier`, `CacheWarmup`, `DistributedCacheConfig` | | [`collaboration.zod.ts`](/docs/references/system/collaboration) | `AwarenessEvent`, `AwarenessSession`, `AwarenessUpdate`, `AwarenessUserState`, `CRDTMergeResult`, `CRDTState`, `CRDTType`, `CollaborationMode`, `CollaborationSession`, `CollaborationSessionConfig`, `CollaborativeCursor`, `CounterOperation`, `CursorColorPreset`, `CursorSelection`, `CursorStyle`, `CursorUpdate`, `GCounter`, `LWWRegister`, `ORSet`, `ORSetElement`, `OTComponent`, `OTOperation`, `OTOperationType`, `OTTransformResult`, `PNCounter`, `TextCRDTOperation`, `TextCRDTState`, `UserActivityStatus`, `VectorClock` | | [`core-services.zod.ts`](/docs/references/system/core-services) | `CoreServiceName`, `KernelServiceMap`, `KernelServiceStatus`, `ServiceConfig`, `ServiceCriticality` | +| [`demo-staff.zod.ts`](/docs/references/system/demo-staff) | `DemoStaff` | | [`deploy-bundle.zod.ts`](/docs/references/system/deploy-bundle) | `DeployBundle`, `DeployDiff`, `DeployManifest`, `DeployStatusEnum`, `DeployValidationIssue`, `DeployValidationResult`, `MigrationPlan`, `MigrationStatement`, `SchemaChange` | | [`dev-login.zod.ts`](/docs/references/system/dev-login) | `DevLogin` | | [`disaster-recovery.zod.ts`](/docs/references/system/disaster-recovery) | `BackupConfig`, `BackupRetention`, `BackupStrategy`, `DisasterRecoveryPlan`, `FailoverConfig`, `FailoverMode`, `RPO`, `RTO` | From f4f3e4108e1bd9f2c23deee956b9f14098a4cd6f Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 26 Oct 2026 05:32:32 +0000 Subject: [PATCH 13/14] chore(spec): regenerate the references index the main merge deferred MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The merge of main at 493a6c4a5 (#25813) deferred content/docs/references/index.mdx. Regenerated from the merged tree: main's column-apply-on page and SeedColumn beside this branch's dataset duration schemas — 1544 schemas, 199 pages. Co-Authored-By: Claude Claude-Session: https://claude.ai/code/session_01AKPixnkg91qCkQCVFDJuqW --- content/docs/references/index.mdx | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/content/docs/references/index.mdx b/content/docs/references/index.mdx index fd661e65017..1767273fd9c 100644 --- a/content/docs/references/index.mdx +++ b/content/docs/references/index.mdx @@ -1,7 +1,7 @@ --- title: Protocol reference — every schema by module navTitle: Protocol Reference -description: Every schema published by @objectstack/spec — 1542 schemas across 14 protocol modules +description: Every schema published by @objectstack/spec — 1544 schemas across 14 protocol modules --- {/* ⚠️ AUTO-GENERATED — DO NOT EDIT. Run build-docs.ts to regenerate. Hand-written docs live in the module folders under content/docs/. */} @@ -23,7 +23,7 @@ counts are sums of the rows they head. Regenerate with | [AI Protocol](/docs/references/ai) | 12 | 68 | Agents, tools, skills, RAG and knowledge sources, model registry, conversations. | | [API Protocol](/docs/references/api) | 32 | 438 | REST contracts, endpoints, routing, realtime, batch, discovery. | | [Automation Protocol](/docs/references/automation) | 14 | 73 | Flows and their nodes, approvals, ETL pipelines, webhooks, state machines, execution records. | -| [Data Protocol](/docs/references/data) | 29 | 178 | Objects, fields, queries, filters, datasources and drivers — the ObjectQL layer. | +| [Data Protocol](/docs/references/data) | 30 | 180 | Objects, fields, queries, filters, datasources and drivers — the ObjectQL layer. | | [Identity Protocol](/docs/references/identity) | 5 | 27 | Users and accounts, organizations, positions, SCIM provisioning. | | [Integration Protocol](/docs/references/integration) | 1 | 12 | The single connector protocol (ADR-0097) — catalog descriptors and provider-bound instances. | | [Kernel Protocol](/docs/references/kernel) | 29 | 151 | Plugin lifecycle and manifests, capabilities and security, metadata loading, service registry. | @@ -34,7 +34,7 @@ counts are sums of the rows they head. Regenerate with | [Studio Protocol](/docs/references/studio) | 3 | 35 | Studio designer metadata — the authoring surfaces for the protocols above. | | [System Protocol](/docs/references/system) | 37 | 286 | The runtime environment — logging, jobs, cache, metrics, notifications, i18n and compliance. | | [UI Protocol](/docs/references/ui) | 16 | 171 | Apps, pages, views, dashboards, reports, actions and themes — the ObjectUI layer. | -| **Total** | **198** | **1542** | 14 protocol modules | +| **Total** | **199** | **1544** | 14 protocol modules | --- @@ -131,13 +131,14 @@ Flows and their nodes, approvals, ETL pipelines, webhooks, state machines, execu ## Data Protocol -**Source:** `packages/spec/src/data/` · **Import:** `@objectstack/spec/data` · **29 pages, 178 schemas** +**Source:** `packages/spec/src/data/` · **Import:** `@objectstack/spec/data` · **30 pages, 180 schemas** Objects, fields, queries, filters, datasources and drivers — the ObjectQL layer. | File | Schemas | | :--- | :--- | | [`analytics.zod.ts`](/docs/references/data/analytics) | `AggregationMetricType`, `AnalyticsDateRange`, `AnalyticsDateRangePreset`, `AnalyticsQuery`, `Cube`, `CubeJoin`, `Dimension`, `DimensionType`, `Metric`, `TimeUpdateInterval` | +| [`column-apply-on.zod.ts`](/docs/references/data/column-apply-on) | `ColumnApplyOn` | | [`context-tokens.zod.ts`](/docs/references/data/context-tokens) | `ContextToken`, `ContextTokenPlaceholder` | | [`data-engine.zod.ts`](/docs/references/data/data-engine) | `BaseEngineOptions`, `DataEngineAggregateOptions`, `DataEngineAggregateRequest`, `DataEngineCountOptions`, `DataEngineCountRequest`, `DataEngineDeleteOptions`, `DataEngineDeleteRequest`, `DataEngineExecuteRequest`, `DataEngineFilter`, `DataEngineFindOneRequest`, `DataEngineFindRequest`, `DataEngineInsertOptions`, `DataEngineInsertRequest`, `DataEngineQueryOptions`, `DataEngineRequest`, `DataEngineSort`, `DataEngineUpdateOptions`, `DataEngineUpdateRequest`, `DataEngineVectorFindRequest`, `DroppedFieldsEvent`, `EngineAggregateOptions`, `EngineCountOptions`, `EngineDeleteOptions`, `EngineQueryOptions`, `EngineUpdateOptions`, `QueryPageWindow`, `QueryPageWindowParam`, `QueryTransportParams`, `QueryWithTransport`, `ValidationAdvisoryEvent` | | [`datasource.zod.ts`](/docs/references/data/datasource) | `Datasource`, `DriverDefinition`, `DriverType`, `ExternalDatasourceSettings`, `SchemaMode` | @@ -163,7 +164,7 @@ Objects, fields, queries, filters, datasources and drivers — the ObjectQL laye | [`object.zod.ts`](/docs/references/data/object) | `ApiMethod`, `ApiOperation`, `Index`, `Lifecycle`, `LifecycleClass`, `Object`, `ObjectAccessConfig`, `ObjectCapabilities`, `ObjectExtension`, `ObjectExternalBinding`, `ObjectFieldGroup`, `ObjectOwnershipEnum`, `ObjectRequiredPermissions`, `PerOperationRequiredPermissions`, `RowCrudActionOverride`, `TenancyConfig` | | [`picklist.zod.ts`](/docs/references/data/picklist) | `Picklist`, `PicklistExtension`, `PicklistServedField` | | [`query.zod.ts`](/docs/references/data/query) | `AggregationFunction`, `AggregationNode`, `DateGranularity`, `FieldNode`, `FullTextSearch`, `GroupByNode`, `Query`, `SortNode` | -| [`seed.zod.ts`](/docs/references/data/seed) | `Seed`, `SeedMode` | +| [`seed.zod.ts`](/docs/references/data/seed) | `Seed`, `SeedColumn`, `SeedMode` | | [`seed-loader.zod.ts`](/docs/references/data/seed-loader) | `ObjectDependencyGraph`, `ObjectDependencyNode`, `ReferenceResolution`, `ReferenceResolutionError`, `SeedIdentity`, `SeedLoadResult`, `SeedLoaderConfig`, `SeedLoaderRequest`, `SeedLoaderResult` | | [`validation.zod.ts`](/docs/references/data/validation) | `ConditionalValidation`, `CrossFieldValidation`, `FormatValidation`, `JSONValidation`, `ScriptValidation`, `StateMachineValidation`, `ValidationRule` | From 8821a8d40e7c4e93f7353433648dbb386b3123fa Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 26 Oct 2026 06:52:21 +0000 Subject: [PATCH 14/14] test(service-analytics): a duration sum's empty-group 0 is answered in the statement, and the all-NULL group sorts and pages as it Main now answers a measure's empty-group identity in the native statement (COALESCE) instead of folding it after the rows come back, and orders with NULLS LAST. The duration measure's statement takes the same wrap, so: - the exact-SQL cases expect COALESCE(SUM(...), 0) on the filtered sum; - the presentation case's client hands back the statement's 0; - the live ordering case no longer leaves out group c (no record carrying both columns): ascending it pages as b (-3599.25), c (0), a (91800) on SQLite and PostgreSQL alike. Co-Authored-By: Claude Claude-Session: https://claude.ai/code/session_01AKPixnkg91qCkQCVFDJuqW --- .../dataset-duration-measure.test.ts | 36 +++++++++++-------- .../native-sql-duration-measure.test.ts | 14 +++++--- 2 files changed, 31 insertions(+), 19 deletions(-) diff --git a/packages/services/service-analytics/src/__tests__/dataset-duration-measure.test.ts b/packages/services/service-analytics/src/__tests__/dataset-duration-measure.test.ts index 5cc9c3f4d82..c34b5e59407 100644 --- a/packages/services/service-analytics/src/__tests__/dataset-duration-measure.test.ts +++ b/packages/services/service-analytics/src/__tests__/dataset-duration-measure.test.ts @@ -264,21 +264,29 @@ for (const cell of CELLS) { const total = await services.native!.queryDataset(DATASET, { measures: ['avg_hours', 'total_seconds'] } as any); expect(total.rows).toEqual([{ avg_hours: aggregate(ROWS, 'avg', 'hours'), total_seconds: aggregate(ROWS, 'sum', 'seconds') }]); - // Group `c` is left out: its sum is SQL NULL until the strategy folds it - // to 0, and the dialects order a NULL at opposite ends. - const run = await counted(() => - services.native!.queryDataset(DATASET, { - measures: ['total_seconds'], - dimensions: ['category'], - runtimeFilter: { category: { $ne: 'c' } }, - order: { total_seconds: 'asc' }, - limit: 1, - } as any), - ); - expect(run.error, run.error?.message).toBeUndefined(); - expect(run.rawSql).toBe(1); + // [#25907] Group `c` has no record carrying both columns. Its sum is + // answered as 0 in the statement, so it sorts and pages as 0 on both + // dialects: ascending it is b (-3599.25), c (0), a (91800). Folded after + // the rows came back, SQLite ordered the NULL first and PostgreSQL last. + const page = (offset: number) => + counted(() => + services.native!.queryDataset(DATASET, { + measures: ['total_seconds'], + dimensions: ['category'], + order: { total_seconds: 'asc' }, + limit: 1, + offset, + } as any), + ); + const first = await page(0); + expect(first.error, first.error?.message).toBeUndefined(); + expect(first.rawSql).toBe(1); expect(reads.statements[reads.statements.length - 1]).toContain('ORDER BY "total_seconds" ASC'); - expect(run.value!.rows).toEqual([{ category: 'b', total_seconds: -3599.25 }]); + expect(reads.statements[reads.statements.length - 1]).toContain('COALESCE(SUM('); + expect(first.value!.rows).toEqual([{ category: 'b', total_seconds: -3599.25 }]); + const second = await page(1); + expect(second.error, second.error?.message).toBeUndefined(); + expect(second.value!.rows).toEqual([{ category: 'c', total_seconds: 0 }]); }); it('the cube door serves the registered dataset by name, with the same numbers', async () => { diff --git a/packages/services/service-analytics/src/__tests__/native-sql-duration-measure.test.ts b/packages/services/service-analytics/src/__tests__/native-sql-duration-measure.test.ts index b2884cf9100..4cd2efccbba 100644 --- a/packages/services/service-analytics/src/__tests__/native-sql-duration-measure.test.ts +++ b/packages/services/service-analytics/src/__tests__/native-sql-duration-measure.test.ts @@ -68,19 +68,21 @@ describe('[#25778] NativeSQLStrategy — the duration measure as a statement', ( [ 'sqlite', 'AVG(((ROUND((julianday(resolved_at) - julianday(opened_at)) * 86400000.0) / 1000.0) / 3600.0)) AS "avg_hours"', - 'SUM(CASE WHEN priority = $1 THEN (ROUND((julianday(resolved_at) - julianday(opened_at)) * 86400000.0) / 1000.0) END) AS "total_seconds"', + 'COALESCE(SUM(CASE WHEN priority = $1 THEN (ROUND((julianday(resolved_at) - julianday(opened_at)) * 86400000.0) / 1000.0) END), 0) AS "total_seconds"', ], [ 'postgres', 'AVG((CAST(EXTRACT(EPOCH FROM (resolved_at - opened_at)) AS DOUBLE PRECISION) / 3600.0)) AS "avg_hours"', - 'SUM(CASE WHEN priority = $1 THEN CAST(EXTRACT(EPOCH FROM (resolved_at - opened_at)) AS DOUBLE PRECISION) END) AS "total_seconds"', + 'COALESCE(SUM(CASE WHEN priority = $1 THEN CAST(EXTRACT(EPOCH FROM (resolved_at - opened_at)) AS DOUBLE PRECISION) END), 0) AS "total_seconds"', ], [ 'mysql', 'AVG(((TIMESTAMPDIFF(MICROSECOND, opened_at, resolved_at) / 1.0E6) / 3600.0)) AS "avg_hours"', - 'SUM(CASE WHEN priority = $1 THEN (TIMESTAMPDIFF(MICROSECOND, opened_at, resolved_at) / 1.0E6) END) AS "total_seconds"', + 'COALESCE(SUM(CASE WHEN priority = $1 THEN (TIMESTAMPDIFF(MICROSECOND, opened_at, resolved_at) / 1.0E6) END), 0) AS "total_seconds"', ], ])('%s: the per-dialect difference, divided by the unit, under the aggregate and its measure filter', async (dialect, avg, filteredSum) => { + // [#25907] The `sum` answers its empty-group identity `0` in the statement, + // where the order and the window read it; the `avg` has none and keeps NULL. const { sql, params } = await native.generateSql(q(), nativeCtx({ dialect })); expect(sql).toBe(`SELECT category AS "category", ${avg}, ${filteredSum} FROM "${OBJECT}" GROUP BY category`); expect(params).toEqual(['high']); @@ -110,10 +112,12 @@ describe('[#25778] NativeSQLStrategy — the duration measure as a statement', ( for (const dialect of ['sqlite', 'postgres', 'mysql']) expect(native.canHandle(q(), nativeCtx({ dialect })), dialect).toBe(true); }); - it('presents a client string as a number, and folds an empty sum to 0 while an empty avg stays null', async () => { + it('presents a client string as a number; an empty sum arrives as the statement\'s 0 and an empty avg stays null', async () => { + // [#25907] Category `c` has no record carrying both columns: the statement + // answered `COALESCE(SUM(…), 0)` (above), so the client hands back `'0'`. const rows = [ { category: 'a', avg_hours: '12.75', total_seconds: '91800' }, - { category: 'c', avg_hours: null, total_seconds: null }, + { category: 'c', avg_hours: null, total_seconds: '0' }, ]; const res = await native.execute(q(), nativeCtx({ dialect: 'postgres', rows })); expect(res.rows).toEqual([