From e71e9fca7b2038057cb8837e4fd8fc77433c5d09 Mon Sep 17 00:00:00 2001 From: Mattias Fornander Date: Mon, 14 Sep 2026 12:12:50 -0600 Subject: [PATCH 01/10] chore(dbml-parse): add peggy and a conformance test entry point Add peggy as a devDependency of @dbml/parse only, plus a `test:conformance` script backed by vitest.conformance.config.ts. The default `vitest run` excludes __tests__/conformance so the spec tests can be non-blocking in CI. Co-Authored-By: Claude Fable 5.1 --- packages/dbml-parse/eslint.config.ts | 1 + packages/dbml-parse/package.json | 4 ++- packages/dbml-parse/vite.config.ts | 3 ++ .../dbml-parse/vitest.conformance.config.ts | 21 +++++++++++++ yarn.lock | 31 +++++++++++++++++++ 5 files changed, 59 insertions(+), 1 deletion(-) create mode 100644 packages/dbml-parse/vitest.conformance.config.ts diff --git a/packages/dbml-parse/eslint.config.ts b/packages/dbml-parse/eslint.config.ts index 3e802863b..d73db3b38 100644 --- a/packages/dbml-parse/eslint.config.ts +++ b/packages/dbml-parse/eslint.config.ts @@ -21,6 +21,7 @@ export default defineConfig( 'dist-profile/*', 'vite.config.ts', 'vite.profile.config.ts', + 'vitest.conformance.config.ts', 'eslint.config.ts', ], }, diff --git a/packages/dbml-parse/package.json b/packages/dbml-parse/package.json index 1543bc892..d38b73add 100644 --- a/packages/dbml-parse/package.json +++ b/packages/dbml-parse/package.json @@ -36,11 +36,13 @@ "lint:fix": "eslint --fix .", "bench": "npx tsx __benchmarks__/compiler.benchmark.ts", "profile:build": "vite build --config vite.profile.config.mts && cp -r __benchmarks__/input dist-profile/input", - "profile": "yarn profile:build && node --cpu-prof --cpu-prof-dir __benchmarks__/output/ dist-profile/__benchmarks__/compiler.profile.mjs" + "profile": "yarn profile:build && node --cpu-prof --cpu-prof-dir __benchmarks__/output/ dist-profile/__benchmarks__/compiler.profile.mjs", + "test:conformance": "vitest run --config vitest.conformance.config.ts" }, "devDependencies": { "@types/luxon": "^3.7.1", "monaco-editor-core": "^0.44.0", + "peggy": "^5.1.0", "vite-plugin-no-bundle": "^4.0.0" }, "dependencies": { diff --git a/packages/dbml-parse/vite.config.ts b/packages/dbml-parse/vite.config.ts index 7f4c2c896..0c3875d08 100644 --- a/packages/dbml-parse/vite.config.ts +++ b/packages/dbml-parse/vite.config.ts @@ -3,6 +3,7 @@ import path from 'path'; import { defineConfig } from 'vite'; import dts from 'vite-plugin-dts'; +import { configDefaults } from 'vitest/config'; export default defineConfig({ plugins: [ @@ -33,6 +34,8 @@ export default defineConfig({ }, test: { globals: true, + // Spec conformance tests run separately via vitest.conformance.config.ts + exclude: [...configDefaults.exclude, '__tests__/conformance/**'], coverage: { provider: 'v8', reporter: ['json-summary', 'text'], diff --git a/packages/dbml-parse/vitest.conformance.config.ts b/packages/dbml-parse/vitest.conformance.config.ts new file mode 100644 index 000000000..bfd68f8d3 --- /dev/null +++ b/packages/dbml-parse/vitest.conformance.config.ts @@ -0,0 +1,21 @@ +/// + +// Runs only the spec conformance tests (see spec/README.md). They are kept out +// of the default `vitest run` so that they can be non-blocking in CI while the +// spec is being brought into agreement with the parser. +import path from 'path'; +import { defineConfig } from 'vitest/config'; + +export default defineConfig({ + resolve: { + alias: { + '@': path.resolve(__dirname, 'src/'), + '@tests': path.resolve(__dirname, '__tests__/'), + }, + }, + test: { + globals: true, + include: ['__tests__/conformance/**/*.test.ts'], + testTimeout: 120000, + }, +}); diff --git a/yarn.lock b/yarn.lock index 0508fc5bb..9248d6048 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1837,6 +1837,13 @@ node-addon-api "^3.2.1" node-gyp-build "^4.3.0" +"@peggyjs/from-mem@3.1.3": + version "3.1.3" + resolved "https://registry.yarnpkg.com/@peggyjs/from-mem/-/from-mem-3.1.3.tgz#2956c3c68ec2e08e55178049a327009ee61541b8" + integrity sha512-LLlgtfXIaeYXoOYovOI0spLM8ZXaqkAlmcRRrLzHJzLMqkU6Sw0R4KMoCoHx1PjaP815pSCBlS+BN6aD8t1Jgg== + dependencies: + semver "7.7.4" + "@phosphor-icons/vue@^2.2.0": version "2.2.1" resolved "https://registry.npmjs.org/@phosphor-icons/vue/-/vue-2.2.1.tgz" @@ -4165,6 +4172,11 @@ commander@^11.0.0: resolved "https://registry.npmjs.org/commander/-/commander-11.1.0.tgz" integrity sha512-yPVavfyCcRhmorC7rWlkHn15b4wDVgVmBA7kV4QVBsF7kv/9TKJAbAXVTxvTnwP8HHKjRCJDClKbciiYS7p0DQ== +commander@^14.0.3: + version "14.0.3" + resolved "https://registry.yarnpkg.com/commander/-/commander-14.0.3.tgz#425d79b48f9af82fcd9e4fc1ea8af6c5ec07bbc2" + integrity sha512-H+y0Jo/T1RZ9qPP4Eh1pkcQcLRglraJaSLoyOtHxu6AapkjWVCy2Sit1QQ4x3Dng8qDlSsZEet7g5Pq06MvTgw== + commander@^2.20.0, commander@~2.20.3: version "2.20.3" resolved "https://registry.npmjs.org/commander/-/commander-2.20.3.tgz" @@ -7904,6 +7916,15 @@ pathe@^2.0.1, pathe@^2.0.3: resolved "https://registry.npmjs.org/pathe/-/pathe-2.0.3.tgz" integrity sha512-WUjGcAqP1gQacoQe+OBJsFA7Ld4DyXuUIjZ5cc75cLHvJ7dtNsTugphxIADwspS+AraAUePCKrSVtPLFj/F88w== +peggy@^5.1.0: + version "5.1.0" + resolved "https://registry.yarnpkg.com/peggy/-/peggy-5.1.0.tgz#8e2a681128cf776648417f60394a40b4a712f503" + integrity sha512-IEo5aYRZ2kXH4Qby06cjtL114PZnwLoTiA41vUmg2vPZgANn+c87m5BUurhuDr5/cu758ZlpgsAfBVx+hhO5+w== + dependencies: + "@peggyjs/from-mem" "3.1.3" + commander "^14.0.3" + source-map-generator "2.0.6" + pegjs-require-import@^0.0.2: version "0.0.2" resolved "https://registry.npmjs.org/pegjs-require-import/-/pegjs-require-import-0.0.2.tgz" @@ -8549,6 +8570,11 @@ semver@7.5.3: dependencies: lru-cache "^6.0.0" +semver@7.7.4: + version "7.7.4" + resolved "https://registry.yarnpkg.com/semver/-/semver-7.7.4.tgz#28464e36060e991fa7a11d0279d2d3f3b57a7e8a" + integrity sha512-vFKC2IEtQnVhpT78h1Yp8wzwrf8CM+MzKMHGJZfBtzhZNycRFnXsHk6E5TxIkkMsgNS7mdX3AGB7x2QM2di4lA== + semver@^6.0.0: version "6.3.1" resolved "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz" @@ -8724,6 +8750,11 @@ sort-keys@^2.0.0: dependencies: is-plain-obj "^1.0.0" +source-map-generator@2.0.6: + version "2.0.6" + resolved "https://registry.yarnpkg.com/source-map-generator/-/source-map-generator-2.0.6.tgz#284776dff4bd5efdefead67aaed0c0f85f5a42ca" + integrity sha512-IlassDs1Ve8nV6uyQZXF9kdkJpVKnMte2JZQXu13M0A5zwc+vu6+LNHfmxsHBMDtoZE21RHiKI0/xvpecZRCNg== + source-map-js@^1.2.1: version "1.2.1" resolved "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.1.tgz" From bbf181b5075604d9ef111ca09081f3c2d0b67a99 Mon Sep 17 00:00:00 2001 From: Mattias Fornander Date: Mon, 14 Sep 2026 12:13:27 -0600 Subject: [PATCH 02/10] docs(spec): describe the layered DBML spec and the Layer 1 generic syntax Co-Authored-By: Claude Fable 5.1 --- spec/README.md | 87 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 87 insertions(+) create mode 100644 spec/README.md diff --git a/spec/README.md b/spec/README.md new file mode 100644 index 000000000..f5a12d3f6 --- /dev/null +++ b/spec/README.md @@ -0,0 +1,87 @@ +# DBML formal specification + +This directory holds the formal specification of DBML. It is organised in the same three layers as the reference implementation in [`packages/dbml-parse`](../packages/dbml-parse), so that every statement in the spec can be checked mechanically against the code. + +| Layer | What it specifies | Spec artefact | Reference implementation | +| --- | --- | --- | --- | +| 1. Generic syntax | Tokens and the element-agnostic syntax tree (`SyntaxNodeKind`) | [`dbml-syntax.peggy`](./dbml-syntax.peggy) | `src/core/lexer/lexer.ts`, `src/core/parser/parser.ts` | +| 2. Element semantics | Per-element rules: name shape, body form, allowed settings and sub-elements | `elements/.schema.json` (future PRs) | `src/core/local_modules//validate.ts` | +| 3. Interpreted model | The `Database` JSON model produced by `@dbml/core` | `output/database.schema.json` (future PR) | `packages/dbml-core/src/model_structure/*.ts` | + +## The authority rule + +The reference parser is the authority on what DBML *is* today. The spec is the authority on what DBML *should be*. Whenever the two disagree, the disagreement is recorded in [`DISAGREEMENTS.md`](./DISAGREEMENTS.md) with one of three verdicts: + +- **spec bug**: the spec is wrong and must be changed. +- **parser bug**: the parser is wrong; the spec stands and the entry stays until the parser is fixed. +- **intentional leniency**: the parser accepts more than the spec on purpose (for example, to give better error recovery). The spec documents the strict form. + +Conformance tests in `packages/dbml-parse/__tests__/conformance/` run both the reference parser and a parser generated from the spec over the same inputs and fail on any disagreement that is not recorded with a verdict. "Accept" means the reference lexer and parser report no error, and the spec parser parses without throwing. When both accept, the trees must also match: both are flattened to the same list of node kinds and token kinds in source order. The tests are currently non-blocking in CI; they become blocking once the corpus is clean. + +Nothing under `packages/dbml-parse/src` changes because of the spec. The spec restates behaviour; it does not alter it. + +## Layer 1 in one page + +DBML source is parsed in two passes. A lexer turns UTF-16 code units into tokens and attaches whitespace and comments to neighbouring tokens as *trivia*. A recursive-descent parser then builds a small, element-agnostic tree. Element keywords such as `Table` or `Ref` are ordinary identifiers at this layer; nothing in Layer 1 knows what a table is. + +### Tokens + +- **Whitespace and comments** are trivia: space, tab, newline, `// ...` to end of line, and `/* ... */`. A carriage return is dropped entirely. Trivia after a token on the same line, up to and including the first newline, is that token's *trailing trivia*; trivia at the start of a line is the next token's *leading trivia*. The parser consults trivia in exactly three places, described below. +- **Identifiers** start with a Unicode letter, combining mark, or underscore and continue with those or digits. A token that starts with a digit is an identifier when it contains letters or underscores and is not a well-formed number (`12_abc`, `3a`). +- **Numbers** are `digits`, `digits . digits?`, optionally followed by an exponent `e[+-]?digits`. A number followed directly by a letter is an identifier (`1e5x`); a number with two dots or with a dot followed by letters is a lexical error (`1.2.3`, `1.a`). +- **Strings** are single-quoted `'...'` on one line or triple-quoted `'''...'''` across lines. Backslash escapes are processed, including `\uHHHH` and backslash-newline for line continuation. A newline inside a single-quoted string is an error. +- **Quoted variables** `"..."` are identifiers with arbitrary content; they follow the single-quoted string rules for escapes and newlines. +- **Function expressions** `` `...` `` are raw: no escapes, newlines allowed, ends at the first backtick. +- **Colour literals** are `#` followed by alphanumerics. Layer 1 does not check that the digits are hexadecimal. +- **Wildcard** is `*`. +- **Punctuation**: `( ) [ ] { } , :`. A semicolon is also a token, but no rule accepts it, so `;` is always an error. +- **Operators**, scanned greedily as a prefix-closed set: `. / % + - = == != ! ~ & | ?` and the relationship family `< <= > >= <> <- -> -? ?- ?-? ? ?> ?? <>? ?<> ?<>?`. + +Anything else is a lexical error. + +### Program structure + +A program is a sequence of top-level statements with no separators required between them. + +- **Use declaration**: `use` or `reuse` (case-insensitive), then either `*` or a braced list of newline-separated specifiers ` [as ]`, then `from` and a string literal. +- **Element declaration**: ` [] [] [as ] [] (: | )`. `` is an identifier; `` is present only when `` is `metadata`. `` and `` are normal expressions. A simple body is a function application and must not itself resolve to an element declaration. + +### Blocks + +A block `{ ... }` contains body items. An item that starts with ` :` is a *field declaration*: an element declaration with a simple body (`Note: 'text'`). Any other item is a *function application*: a callee expression followed by zero or more argument expressions on the same line, each separated by at least one space or tab. When a function application has the shape ` [ [as ]] [] `, it is reinterpreted as a nested element declaration. Otherwise it stays a function application; `id integer [pk]` is the callee `id` applied to `integer` and `[pk]`. + +A function application stops at a newline, at end of input, or before `}`, `]`, `)`, `,`, or `:`. + +### Expressions + +Each callee and argument is a *comma expression*: one or more normal expressions separated by commas, where an empty slot (`a, , c` or a trailing comma) yields an empty node. A comma continues the list only on the same line and not before a bracket or colon. + +A *normal expression* is a Pratt-style operator expression over these operands: + +- primary: a literal (number, string, colour) or a variable (identifier, quoted variable) +- function expression, wildcard +- list `[ attribute, ... ]`, block `{ ... }`, tuple `( expr, ... )`; a tuple with exactly one element is a *group* + +Binding powers, tightest first: member access `.` (its right operand is a bare operand, never a prefix expression), prefix operators (`+ - ! ~` and every relationship operator), call `( ... )`, multiplicative `/ %`, additive `+ - -? ?- ?-?`, comparison (all relationship operators, including `->` and `<-`), equality `== !=`, assignment `=`. All infix operators are left-associative. Newlines are permitted around infix operators. + +The three trivia-sensitive rules: + +1. `[` directly after an operand is array indexing (`int[]`) only when no space or tab separates them on that line; otherwise it starts a new argument (`int [pk]`). +2. `(` directly after an operand is a call unless it begins a new line. Inside an unclosed `(` or `[` the newline rule is suspended, so `(f\n(1))` is a call. +3. Function-application arguments must be separated by at least one space or tab; `1 **2` is an error. + +### Attribute lists + +`[ attr, attr ]` where each attribute is ` [: ]`. A name is a stream of identifiers (`primary key`) or a single literal or quoted variable. A value is an identifier stream when it starts with two identifiers (`no action`), otherwise a normal expression (`> users.id`, `'text'`, `` `now()` ``). Empty names and trailing commas are errors. + +### Syntax node kinds + +The grammar produces exactly the `SyntaxNodeKind` set from `src/core/types/nodes.ts`: program, element-declaration, use-declaration, use-specifier, use-specifier-list, attribute, identifier-stream, literal, variable, primary-expression, prefix-expression, infix-expression, postfix-expression, function-expression, function-application, block-expression, list-expression, tuple-expression, group-expression, call-expression, comma-expression, array, wildcard, and the empty node. Postfix-expression is reachable only through error recovery today and has no grammar rule. + +## Running the conformance tests + +```bash +yarn workspace @dbml/parse test:conformance +``` + +The peggy grammar is compiled in memory at test time; no generated parser is committed. From 23dfe41bde31c1774416d489ba734754af6ba09a Mon Sep 17 00:00:00 2001 From: Mattias Fornander Date: Mon, 14 Sep 2026 12:13:27 -0600 Subject: [PATCH 03/10] feat(spec): add Layer 1 generic syntax grammar A peggy grammar restating packages/dbml-parse's lexer and recursive-descent parser: tokens, trivia-sensitive rules (index vs argument, call vs new line, argument spacing), Pratt precedence, comma expressions, attribute lists, use declarations and element declarations, producing the SyntaxNodeKind tree. Co-Authored-By: Claude Fable 5.1 --- spec/dbml-syntax.peggy | 551 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 551 insertions(+) create mode 100644 spec/dbml-syntax.peggy diff --git a/spec/dbml-syntax.peggy b/spec/dbml-syntax.peggy new file mode 100644 index 000000000..dba5c94c5 --- /dev/null +++ b/spec/dbml-syntax.peggy @@ -0,0 +1,551 @@ +// DBML formal specification, Layer 1: generic syntax. +// +// This grammar restates the lexer and parser in packages/dbml-parse +// (src/core/lexer/lexer.ts, src/core/parser/parser.ts). It accepts exactly +// the inputs the reference parser accepts without reporting an error, and it +// produces the same tree of SyntaxNodeKind values (src/core/types/nodes.ts). +// Element keywords such as Table or Ref are ordinary identifiers here. +// +// Conventions +// - Rules never consume trailing trivia. Trivia is consumed at the start of +// the construct that needs it, or inspected with lookahead. +// - Node shape: { kind, children }, children in source order, tokens as +// { token, value }. This mirrors getMemberChain() in src/core/parser/utils.ts. +// - Comments of the form "parser.ts: foo()" name the reference method a rule +// corresponds to. +// +// Grammar is written for peggy >= 5 (unicode character classes need the u flag). + +{{ + const PREFIX_OPS = new Set([ + '+', '-', '<', '>', '<>', '-?', '?-', '?-?', '?>', '>?', '?<', '?', '?', '<>?', '?<>?', '->', '<-', '!', '~', + ]); + const ADDITIVE_OPS = new Set(['+', '-', '-?', '?-', '?-?']); + const COMPARISON_OPS = new Set([ + '<', '<=', '>', '>=', '<>', '?>', '>?', '?<', '?', '?', + '<>?', '?<>?', '->', '<-', + ]); + const MULTIPLICATIVE_OPS = new Set(['/', '%']); + const EQUALITY_OPS = new Set(['==', '!=']); + + function tok (kind, value) { + return { token: kind, value }; + } + + function node (kind, ...children) { + return { + kind, + children: children.flat(Infinity).filter((c) => c !== null && c !== undefined), + }; + } + + const empty = () => node(''); + + // Trivia helpers. A gap is an array of trivia units: ' ', '\t', '\r', '\n', + // or the full text of a comment. A multi-line comment is a single unit, so a + // newline inside a comment is not a line break for the parser. + + // lexer/utils.ts: hasTrailingNewLines() + function hasNewline (gap) { + return gap.includes('\n'); + } + + // lexer/utils.ts: hasTrailingSpaces(), restricted to the trivia before the + // first line break (that is what the lexer attaches as trailing trivia). + function hasTrailingSpaces (gap) { + for (const unit of gap) { + if (unit === '\n') return false; + if (unit === ' ' || unit === '\t') return true; + } + return false; + } + + function isIdentifierPrimary (n) { + return n && n.kind === '' + && n.children[0].kind === '' + && n.children[0].children[0].token === ''; + } + + function identifierValue (n) { + return n.children[0].children[0].value; + } + + function isKeyword (identifierToken, word) { + return identifierToken.token === '' + && identifierToken.value.toLowerCase() === word; + } + + function foldInfix (head, tail) { + return tail.reduce((left, [op, right]) => node('', left, op, right), head); + } + + function foldPostfix (head, tail) { + return tail.reduce((left, step) => { + switch (step.step) { + case 'call': return node('', left, step.arg); + case 'index': return node('', left, step.arg); + default: return node('', left, step.op, step.arg); + } + }, head); + } + + function interleave (items, separators) { + const out = []; + items.forEach((item, i) => { + out.push(item); + if (i < separators.length) out.push(separators[i]); + }); + return out; + } + + // parser/utils.ts: convertFuncAppToElem() + // A function application whose shape is + // [ [as ]] [] + // is reinterpreted as a nested element declaration. + function convertFuncAppToElem (callee, args) { + let type = callee; + let rest = args; + if (type.kind === '') { + rest = [type.children[1], ...rest]; + type = type.children[0]; + } + if (!isIdentifierPrimary(type) || rest.length === 0) return null; + const typeToken = type.children[0].children[0]; + rest = [...rest]; + const body = rest.pop(); + if (body.kind !== '') return null; + const attributeList = rest.length > 0 && rest[rest.length - 1].kind === '' + ? rest.pop() + : null; + if (rest.length === 3) { + if (!isIdentifierPrimary(rest[1]) || identifierValue(rest[1]).toLowerCase() !== 'as') return null; + const asToken = rest[1].children[0].children[0]; + return node('', typeToken, rest[0], asToken, rest[2], attributeList, body); + } + if (rest.length === 1) { + return node('', typeToken, rest[0], attributeList, body); + } + if (rest.length === 0) { + return node('', typeToken, attributeList, body); + } + return null; + } +}} + +{ + // parser/contextStack.ts: number of unclosed '(' and '['. Inside them a '(' + // that starts a new line still continues a call expression. + let depth = 0; +} + +// --------------------------------------------------------------------------- +// Program +// --------------------------------------------------------------------------- + +// parser.ts: program() +Program + = body:Statement* GapAny EOF + { return node('', body, tok('', '')); } + +// A statement starting with the identifier `use` or `reuse` is always a use +// declaration; it never falls back to an element declaration. +Statement + = GapAny &UseKeyword d:UseDeclaration { return d; } + / GapAny !UseKeyword d:ElementDeclaration { return d; } + +// --------------------------------------------------------------------------- +// Use declarations +// --------------------------------------------------------------------------- + +// parser.ts: useDeclaration() +// use * from '' +// use { \n ... } from '' +UseDeclaration + = kw:UseKeyword GapAny specifiers:(WildcardNode / UseSpecifierList) + GapAny from:FromKeyword GapAny path:StringLiteral + { return node('', kw, specifiers, from, path); } + +// parser.ts: useSpecifierList(); specifiers are separated by line breaks. +UseSpecifierList + = open:LBrace first:(GapAny s:UseSpecifier { return s; })? + rest:(GapNewline s:UseSpecifier { return s; })* + GapAny close:RBrace + { return node('', open, first, rest, close); } + +// parser.ts: useSpecifier() +// [as ] +UseSpecifier + = kind:IdentifierToken GapAny name:UseName + alias:(GapAny as:AsKeyword GapAny a:UseName { return [as, a]; })? + { return node('', kind, name, alias); } + +// The name must start with an identifier or quoted variable; it is then read +// as a full normal expression. +UseName + = &(IdentifierToken / QuotedVariable) e:NormalExpression { return e; } + +// --------------------------------------------------------------------------- +// Element declarations +// --------------------------------------------------------------------------- + +// parser.ts: elementDeclaration() +// [] [] [as ] [] (: | { }) +// is present exactly when is `metadata`. +ElementDeclaration + = type:IdentifierToken + targetKind:( + &{ return isKeyword(type, 'metadata'); } GapAny t:IdentifierToken { return t; } + / !{ return isKeyword(type, 'metadata'); } { return null; } + ) + name:(&(GapAny !BodyStart) GapAny n:NormalExpression { return n; })? + alias:(GapAny as:AsKeyword GapAny !BodyStart a:NormalExpression { return [as, a]; })? + attributeList:(GapAny l:ListExpression { return l; })? + GapAny body:ElementBody + { return node('', type, targetKind, name, alias, attributeList, body); } + +BodyStart = ":" / "{" / "[" + +ElementBody + = colon:Colon GapAny body:SimpleBody { return [colon, body]; } + / BlockExpression + +// The simple body is a function application that must not itself resolve to +// an element declaration (parser.ts: UNEXPECTED_ELEMENT_DECLARATION). +SimpleBody + = e:Expression &{ return e.kind !== ''; } { return e; } + +// --------------------------------------------------------------------------- +// Blocks +// --------------------------------------------------------------------------- + +// parser.ts: blockExpression() +BlockExpression + = open:LBrace body:BodyItem* GapAny close:RBrace + { return node('', open, body, close); } + +// parser.ts: canBeField() decides between fieldDeclaration() and expression(). +BodyItem + = GapAny &FieldStart d:FieldDeclaration { return d; } + / GapAny !FieldStart e:Expression { return e; } + +FieldStart = IdentifierToken GapAny ":" + +// parser.ts: fieldDeclaration() +// : +FieldDeclaration + = type:IdentifierToken GapAny colon:Colon GapAny body:SimpleBody + { return node('', type, colon, body); } + +// --------------------------------------------------------------------------- +// Function application +// --------------------------------------------------------------------------- + +// parser.ts: expression() +// A callee followed by arguments on the same line, each preceded by at least +// one space or tab. The application ends at a line break, end of input, or +// before } ] ) , : -- anything else there is an error (MISSING_SPACES). +Expression + = callee:CommaExpression + args:(gap:GapInline &{ return hasTrailingSpaces(gap); } !"," a:CommaExpression { return a; })* + &FunctionApplicationStop + { return convertFuncAppToElem(callee, args) ?? node('', callee, args); } + +// parser.ts: shouldStopFunctionApplication() +FunctionApplicationStop = GapInline ("\n" / EOF / [}\]):,]) + +// parser.ts: commaExpression() +// CSV-like list without delimiters. Empty slots produce nodes. +CommaExpression + = first:NormalExpression tail:CommaTail? + { return tail ? node('', first, tail) : first; } + / &"," tail:CommaTail + { return node('', empty(), tail); } + +CommaTail + = GapAny comma:Comma rest:CommaRest { return [comma, rest]; } + +// After a comma: a line break, end of input, bracket or colon ends the list +// with an empty slot; another comma is an empty slot; otherwise a normal +// expression, which continues only if a comma follows on the same line. +// parser.ts: shouldStopCommaExpression() +CommaRest + = &CommaStop { return [empty()]; } + / GapInline comma:Comma rest:CommaRest { return [empty(), comma, rest]; } + / GapInline e:NormalExpression more:(GapInline comma:Comma rest:CommaRest { return [comma, rest]; })? + { return [e, more]; } + +CommaStop = GapInline ("\n" / EOF / [{}\[\]():]) + +// --------------------------------------------------------------------------- +// Normal expressions (parser.ts: expression_bp(), Pratt precedence climbing) +// --------------------------------------------------------------------------- +// Binding powers from parser.ts, tightest first: +// . 16 right operand is a bare operand +// prefix operators 15 +// call ( ... ) 14 +// / % 11 +// + - -? ?- ?-? 9 +// relationship ops 7 < <= > >= <> and the ? variants, -> <- +// == != 4 +// = 2 +// All infix operators are left-associative. Trivia, including line breaks, +// may surround infix operators. + +NormalExpression = AssignmentExpression + +AssignmentExpression + = head:EqualityExpression tail:(GapAny op:AssignOp GapAny rhs:EqualityExpression { return [op, rhs]; })* + { return foldInfix(head, tail); } + +EqualityExpression + = head:ComparisonExpression tail:(GapAny op:EqualityOp GapAny rhs:ComparisonExpression { return [op, rhs]; })* + { return foldInfix(head, tail); } + +ComparisonExpression + = head:AdditiveExpression tail:(GapAny op:ComparisonOp GapAny rhs:AdditiveExpression { return [op, rhs]; })* + { return foldInfix(head, tail); } + +AdditiveExpression + = head:MultiplicativeExpression tail:(GapAny op:AdditiveOp GapAny rhs:MultiplicativeExpression { return [op, rhs]; })* + { return foldInfix(head, tail); } + +MultiplicativeExpression + = head:PostfixExpression tail:(GapAny op:MultiplicativeOp GapAny rhs:PostfixExpression { return [op, rhs]; })* + { return foldInfix(head, tail); } + +// Calls, indexing and member access applied to a unary expression. +PostfixExpression + = head:UnaryExpression tail:(CallStep / IndexStep / MemberStep)* + { return foldPostfix(head, tail); } + +// parser.ts: leftExpression_bp() +// A prefix operator binds tighter than a call: -2() is (-2)(). +UnaryExpression + = op:PrefixOp GapAny e:UnaryExpression { return node('', op, e); } + / MemberChain + +// Member access and indexing on a bare operand. The right side of `.` is a +// bare operand (parser.ts: extractOperand()), never a prefix expression. +MemberChain + = head:Operand tail:(IndexStep / MemberStep)* + { return foldPostfix(head, tail); } + +MemberStep + = GapAny op:MemberOp GapAny rhs:Operand { return { step: 'member', op, arg: rhs }; } + +// A '(' continues a call unless it starts a new line; inside an unclosed '(' +// or '[' the line-break rule is suspended. +CallStep + = gap:GapAny &"(" &{ return depth > 0 || !hasNewline(gap); } arg:TupleExpression + { return { step: 'call', arg }; } + +// A '[' is array indexing only when no space or tab separates it from the +// operand on the same line; otherwise it starts a new argument. +IndexStep + = gap:GapAny &"[" &{ return !hasTrailingSpaces(gap); } arg:ListExpression + { return { step: 'index', arg }; } + +// parser.ts: extractOperand() +Operand + = WildcardNode + / PrimaryExpression + / FunctionExpression + / ListExpression + / BlockExpression + / TupleExpression + +// parser.ts: primaryExpression() +PrimaryExpression + = t:(StringLiteral / NumericLiteral / ColorLiteral) + { return node('', node('', t)); } + / t:(QuotedVariable / IdentifierToken) + { return node('', node('', t)); } + +FunctionExpression + = t:FunctionExpressionToken { return node('', t); } + +WildcardNode + = t:Wildcard { return node('', t); } + +// parser.ts: tupleExpression(); one element without a comma is a group. +TupleExpression + = open:TupleOpen GapAny elements:TupleElements? GapAny close:TupleClose + { + if (elements && elements.items.length === 1) { + return node('', open, elements.items[0], close); + } + return node('', open, elements ? interleave(elements.items, elements.commas) : [], close); + } + +TupleElements + = head:NormalExpression tail:(GapAny c:Comma GapAny e:NormalExpression { return [c, e]; })* + { return { items: [head, ...tail.map((t) => t[1])], commas: tail.map((t) => t[0]) }; } + +// --------------------------------------------------------------------------- +// Attribute lists +// --------------------------------------------------------------------------- + +// parser.ts: listExpression() +// [ , , ... ] +ListExpression + = open:ListOpen GapAny attributes:ListAttributes? GapAny close:ListClose + { return node('', open, attributes ? interleave(attributes.items, attributes.commas) : [], close); } + +ListAttributes + = head:Attribute tail:(GapAny c:Comma GapAny a:Attribute { return [c, a]; })* + { return { items: [head, ...tail.map((t) => t[1])], commas: tail.map((t) => t[0]) }; } + +// parser.ts: attribute() +// [: ] +Attribute + = name:AttributeName value:(GapAny c:Colon GapAny v:AttributeValue { return [c, v]; })? + { return node('', name, value); } + +// parser.ts: attributeName() +// A stream of identifiers (primary key), or a single literal or quoted variable. +AttributeName + = IdentifierStream + / t:(StringLiteral / NumericLiteral / ColorLiteral) + { return node('', node('', t)); } + / t:QuotedVariable + { return node('', node('', t)); } + +IdentifierStream + = head:IdentifierToken tail:(GapAny t:IdentifierToken { return t; })* + { return node('', head, tail); } + +// parser.ts: attributeValue() +// Two leading identifiers make the value an identifier stream (no action); +// anything else is a normal expression. +AttributeValue + = &(IdentifierToken GapAny IdentifierToken) s:IdentifierStream { return s; } + / NormalExpression + +// --------------------------------------------------------------------------- +// Operators +// --------------------------------------------------------------------------- + +// lexer.ts: operator(). Scanned greedily; the set is prefix-closed so longest +// match reproduces the lexer. +OpToken + = $("?<>?" / "?<>" / "??" / "?>" / "?-?" / "?-" / "?" + / "<>?" / "<>" / "<=" / "=" / ">?" / ">" + / "->" / "-?" / "-" + / "==" / "=" / "!=" / "!" + / "+" / "/" ![/*] / "%" / "." / "&" / "|" / "~") + +PrefixOp = op:OpToken &{ return PREFIX_OPS.has(op); } { return tok('', op); } +MemberOp = op:OpToken &{ return op === '.'; } { return tok('', op); } +MultiplicativeOp = op:OpToken &{ return MULTIPLICATIVE_OPS.has(op); } { return tok('', op); } +AdditiveOp = op:OpToken &{ return ADDITIVE_OPS.has(op); } { return tok('', op); } +ComparisonOp = op:OpToken &{ return COMPARISON_OPS.has(op); } { return tok('', op); } +EqualityOp = op:OpToken &{ return EQUALITY_OPS.has(op); } { return tok('', op); } +AssignOp = op:OpToken &{ return op === '='; } { return tok('', op); } + +// --------------------------------------------------------------------------- +// Keywords (case-insensitive identifiers) +// --------------------------------------------------------------------------- + +UseKeyword = t:IdentifierToken &{ return isKeyword(t, 'use') || isKeyword(t, 'reuse'); } { return t; } +FromKeyword = t:IdentifierToken &{ return isKeyword(t, 'from'); } { return t; } +AsKeyword = t:IdentifierToken &{ return isKeyword(t, 'as'); } { return t; } + +// --------------------------------------------------------------------------- +// Punctuation +// --------------------------------------------------------------------------- + +// text() is used here because peggy balances braces inside action code +// without understanding JavaScript string literals. +LBrace = "{" { return tok('', text()); } +RBrace = "}" { return tok('', text()); } +Comma = "," { return tok('', ','); } +Colon = ":" { return tok('', ':'); } +Wildcard = "*" { return tok('', '*'); } + +TupleOpen = "(" { depth += 1; return tok('', '('); } +TupleClose = ")" { depth -= 1; return tok('', ')'); } +ListOpen = "[" { depth += 1; return tok('', '['); } +ListClose = "]" { depth -= 1; return tok('', ']'); } + +// --------------------------------------------------------------------------- +// Tokens (lexer.ts) +// --------------------------------------------------------------------------- + +// lexer.ts: identifier(), numericLiteralOrIdentifier() +IdentifierToken + = t:$(IdentifierStart AlnumChar*) { return tok('', t); } + / DigitIdentifier + +IdentifierStart = [\p{L}\p{M}_]u +AlnumChar = [\p{L}\p{M}_0-9]u +Digits = $[0-9]+ + +// A token starting with a digit that is not a well-formed number is an +// identifier: 12_abc, 3a, 1e5x. It never contains a dot. +DigitIdentifier + = !NumericLiteral t:$([0-9] !"." AlnumChar*) { return tok('', t); } + +// lexer.ts: numericLiteralOrIdentifier() +// 123 1.5 5. 1e5 1.5e-3 +// A number followed by a letter is not a number (1e5x is an identifier); a +// number with two dots or a dot followed by letters is a lexical error. +NumericLiteral + = t:$(Digits Exponent) !AlnumChar { return tok('', t); } + / t:$(Digits "." Digits? Exponent) !AlnumChar { return tok('', t); } + / t:$(Digits "." Digits?) !(AlnumChar / ".") { return tok('', t); } + / t:$(Digits) !(AlnumChar / ".") { return tok('', t); } + +Exponent = [eE] [+-]? Digits + +// lexer.ts: singleLineStringLiteral(), multilineStringLiteral() +StringLiteral + = t:$("'''" (Escape / !("'''" / "\\") .)* "'''") { return tok('', t); } + / t:$("'" (Escape / !("'" / "\n" / "\\") .)* "'") { return tok('', t); } + +// lexer.ts: quotedVariable() +QuotedVariable + = t:$('"' (Escape / !('"' / "\n" / "\\") .)* '"') { return tok('', t); } + +// lexer.ts: escapedString(). A backslash always starts an escape. Any escaped +// character is accepted; \uHHHH needs four hexadecimal digits; a backslash +// before a line break continues the line. +Escape + = "\\u" Hex Hex Hex Hex + / "\\\r\n" + / "\\" !"u" . + +Hex = [0-9a-fA-F] + +// lexer.ts: functionExpression(); raw text, no escapes, may span lines. +FunctionExpressionToken + = t:$("`" (!"`" .)* "`") { return tok('', t); } + +// lexer.ts: colorLiteral(); '#' followed by alphanumerics. Hex validation is a +// Layer 2 rule (utils/validate.ts: isValidHexColor). +ColorLiteral + = t:$("#" AlnumChar*) { return tok('', t); } + +// --------------------------------------------------------------------------- +// Trivia (lexer.ts: gatherTrivia()) +// --------------------------------------------------------------------------- +// A carriage return is discarded by the lexer, so it is neither whitespace nor +// a line break. A single-line comment does not include its terminating +// newline. A multi-line comment is one unit even when it spans lines. + +GapAny = (InlineTrivia / Newline)* +GapInline = InlineTrivia* +GapNewline = GapInline Newline GapAny + +InlineTrivia + = [ \t\r] + / Comment + +Newline = "\n" + +Comment + = $("//" (!"\n" .)*) + / $("/*" (!"*/" .)* "*/") + +EOF = !. From a2df569e34c438943e5d3660758ce3969790fcbb Mon Sep 17 00:00:00 2001 From: Mattias Fornander Date: Mon, 14 Sep 2026 12:13:27 -0600 Subject: [PATCH 04/10] test(dbml-parse): add spec conformance tests Run the reference parser and the peggy-generated spec parser over every snapshot input and over the fast-check arbitraries, asserting agreement on accept/reject and on the normalised node-kind tree. Known disagreements are pinned so that a change on either side fails a test. Co-Authored-By: Claude Fable 5.1 --- .../__tests__/conformance/harness.ts | 140 +++++++++++++++ .../__tests__/conformance/properties.test.ts | 115 ++++++++++++ .../__tests__/conformance/syntax.test.ts | 164 ++++++++++++++++++ 3 files changed, 419 insertions(+) create mode 100644 packages/dbml-parse/__tests__/conformance/harness.ts create mode 100644 packages/dbml-parse/__tests__/conformance/properties.test.ts create mode 100644 packages/dbml-parse/__tests__/conformance/syntax.test.ts diff --git a/packages/dbml-parse/__tests__/conformance/harness.ts b/packages/dbml-parse/__tests__/conformance/harness.ts new file mode 100644 index 000000000..b7f04a027 --- /dev/null +++ b/packages/dbml-parse/__tests__/conformance/harness.ts @@ -0,0 +1,140 @@ +// Shared machinery for the Layer 1 conformance tests. +// +// Both parsers run over the same source text: +// - the reference parser (`@dbml/parse` lexer + parser from src/) +// - a parser generated at test time from spec/dbml-syntax.peggy +// +// They are compared on two things: +// 1. acceptance: the reference "accepts" when lexer and parser report no error; +// the spec parser accepts when peggy does not throw. +// 2. tree shape: both trees are flattened to the same S-expression of node +// kinds and token kinds, in source order, mirroring getMemberChain(). +import { readFileSync } from 'node:fs'; +import path from 'node:path'; +import peggy from 'peggy'; +import { getMemberChain } from '@/core/parser/utils'; +import { SyntaxNode } from '@/core/types/nodes'; +import { SyntaxToken } from '@/core/types/tokens'; +import { parse } from '@tests/utils'; + +export const SPEC_DIR = path.resolve(__dirname, '../../../../spec'); +export const GRAMMAR_PATH = path.join(SPEC_DIR, 'dbml-syntax.peggy'); + +export interface SpecToken { + token: string; + value: string; +} + +export interface SpecNode { + kind: string; + children: (SpecNode | SpecToken)[]; +} + +export interface SpecParser { + parse (input: string): SpecNode; +} + +export type Verdict = 'accept' | 'reject'; + +export interface Outcome { + verdict: Verdict; + // Normalised tree, only when the verdict is `accept`. + tree?: string; + // Human-readable reason for a `reject`. + detail?: string; +} + +export interface Comparison { + reference: Outcome; + spec: Outcome; + agreeOnVerdict: boolean; + // Undefined when at least one side rejected. + agreeOnTree?: boolean; +} + +export function loadSpecParser (): SpecParser { + const grammar = readFileSync(GRAMMAR_PATH, 'utf-8'); + return peggy.generate(grammar, { + output: 'parser', + grammarSource: GRAMMAR_PATH, + }) as unknown as SpecParser; +} + +function stripAngles (kind: string): string { + return kind.replace(/^<|>$/g, ''); +} + +export function normalizeReference (node: SyntaxNode): string { + const children = getMemberChain(node).map((child) => ( + child instanceof SyntaxToken + ? `#${stripAngles(child.kind)}` + : normalizeReference(child as SyntaxNode) + )); + return `(${[stripAngles(node.kind), ...children].join(' ')})`; +} + +export function normalizeSpec (node: SpecNode): string { + const children = node.children.map((child) => ( + 'token' in child + ? `#${stripAngles(child.token)}` + : normalizeSpec(child) + )); + return `(${[stripAngles(node.kind), ...children].join(' ')})`; +} + +export function runReference (source: string): Outcome { + const report = parse(source); + const errors = report.getErrors(); + if (errors.length > 0) { + return { + verdict: 'reject', + detail: errors.map((e) => `${e.code} ${e.diagnostic} @${e.start}`).join('; '), + }; + } + return { + verdict: 'accept', + tree: normalizeReference(report.getValue().ast), + }; +} + +export function runSpec (parser: SpecParser, source: string): Outcome { + try { + return { + verdict: 'accept', + tree: normalizeSpec(parser.parse(source)), + }; + } catch (e) { + return { + verdict: 'reject', + detail: e instanceof Error ? e.message.split('\n')[0] : String(e), + }; + } +} + +export function compare (parser: SpecParser, source: string): Comparison { + const reference = runReference(source); + const spec = runSpec(parser, source); + const agreeOnVerdict = reference.verdict === spec.verdict; + const agreeOnTree = reference.verdict === 'accept' && spec.verdict === 'accept' + ? reference.tree === spec.tree + : undefined; + return { + reference, + spec, + agreeOnVerdict, + agreeOnTree, + }; +} + +// Pretty-print a comparison for assertion messages. +export function describeComparison (source: string, c: Comparison): string { + const lines = [ + `reference: ${c.reference.verdict}${c.reference.detail ? ` (${c.reference.detail})` : ''}`, + `spec: ${c.spec.verdict}${c.spec.detail ? ` (${c.spec.detail})` : ''}`, + ]; + if (c.agreeOnTree === false) { + lines.push(`reference tree: ${c.reference.tree}`, `spec tree: ${c.spec.tree}`); + } + lines.push('source:', source); + return lines.join('\n'); +} diff --git a/packages/dbml-parse/__tests__/conformance/properties.test.ts b/packages/dbml-parse/__tests__/conformance/properties.test.ts new file mode 100644 index 000000000..1f5c220b7 --- /dev/null +++ b/packages/dbml-parse/__tests__/conformance/properties.test.ts @@ -0,0 +1,115 @@ +// Layer 1 conformance: property-based agreement between the spec grammar and +// the reference parser over the generators in __tests__/utils/arbitraries. +// +// Every generated source is parsed by both sides; they must agree on +// accept/reject and, when both accept, on the shape of the syntax tree. +import { describe, expect, it } from 'vitest'; +import * as fc from 'fast-check'; +import { + anyRefArbitrary, + binaryGarbageArbitrary, + crlfSchemaArbitrary, + dbmlSchemaArbitrary, + enumArbitrary, + malformedEnumArbitrary, + malformedRefArbitrary, + malformedTableArbitrary, + mismatchedBracketsArbitrary, + partialInjectionArbitrary, + projectArbitrary, + standaloneNoteArbitrary, + tableArbitrary, + tableGroupArbitrary, + tablePartialArbitrary, + tokenStreamArbitrary, + truncatedInputArbitrary, + unclosedBracketArbitrary, + unclosedStringArbitrary, +} from '../utils/arbitraries'; +import { + type SpecParser, + compare, + describeComparison, + loadSpecParser, +} from './harness'; + +const SCHEMA_RUNS = 500; +const ELEMENT_RUNS = 100; +const MALFORMED_RUNS = 100; + +function expectAgreement (parser: SpecParser, source: string): void { + const result = compare(parser, source); + expect(result.agreeOnVerdict, describeComparison(source, result)).toBe(true); + if (result.agreeOnTree !== undefined) { + expect(result.agreeOnTree, describeComparison(source, result)).toBe(true); + } +} + +describe('[conformance] properties', () => { + const parser = loadSpecParser(); + + it('agrees on whole schemas', { + timeout: 300000, + }, () => { + fc.assert( + fc.property(dbmlSchemaArbitrary, (source: string) => expectAgreement(parser, source)), + { + numRuns: SCHEMA_RUNS, + }, + ); + }); + + it('agrees on schemas with CRLF line endings', () => { + fc.assert( + fc.property(crlfSchemaArbitrary, (source: string) => expectAgreement(parser, source)), + { + numRuns: ELEMENT_RUNS, + }, + ); + }); + + const elementArbitraries: [string, fc.Arbitrary][] = [ + ['tables', tableArbitrary], + ['enums', enumArbitrary], + ['refs', anyRefArbitrary], + ['table groups', tableGroupArbitrary], + ['table partials', tablePartialArbitrary], + ['partial injections', partialInjectionArbitrary], + ['projects', projectArbitrary], + ['standalone notes', standaloneNoteArbitrary], + ]; + + elementArbitraries.forEach(([name, arbitrary]) => { + it(`agrees on ${name}`, () => { + fc.assert( + fc.property(arbitrary, (source: string) => expectAgreement(parser, source)), + { + numRuns: ELEMENT_RUNS, + }, + ); + }); + }); + + const malformedArbitraries: [string, fc.Arbitrary][] = [ + ['malformed tables', malformedTableArbitrary], + ['malformed enums', malformedEnumArbitrary], + ['malformed refs', malformedRefArbitrary], + ['unclosed brackets', unclosedBracketArbitrary], + ['mismatched brackets', mismatchedBracketsArbitrary], + ['unclosed strings', unclosedStringArbitrary], + ['truncated input', truncatedInputArbitrary], + ['binary garbage', binaryGarbageArbitrary], + ['raw token streams', tokenStreamArbitrary], + ]; + + malformedArbitraries.forEach(([name, arbitrary]) => { + it(`agrees on ${name}`, () => { + fc.assert( + fc.property(arbitrary, (source: string) => expectAgreement(parser, source)), + { + numRuns: MALFORMED_RUNS, + }, + ); + }); + }); +}); diff --git a/packages/dbml-parse/__tests__/conformance/syntax.test.ts b/packages/dbml-parse/__tests__/conformance/syntax.test.ts new file mode 100644 index 000000000..e3e22ea7f --- /dev/null +++ b/packages/dbml-parse/__tests__/conformance/syntax.test.ts @@ -0,0 +1,164 @@ +// Layer 1 conformance: the spec grammar and the reference parser must agree on +// every snapshot input in __tests__/snapshots/**/input/*.dbml. +// +// Disagreements are not silently tolerated. Each one is pinned below with the +// outcome of both sides and documented in spec/DISAGREEMENTS.md. If either +// side changes behaviour, the pinned expectation fails and the entry has to be +// revisited. +import { readdirSync, readFileSync } from 'node:fs'; +import path from 'node:path'; +import { describe, expect, it } from 'vitest'; +import { + type Verdict, + compare, + describeComparison, + loadSpecParser, +} from './harness'; + +interface KnownDisagreement { + reference: Verdict; + spec: Verdict; + // Entry in spec/DISAGREEMENTS.md + see: string; +} + +// Snapshot inputs that disagree, keyed by `/`. +const KNOWN_DISAGREEMENTS: Record = { + 'parser/last_invalid_number': { + reference: 'reject', + spec: 'accept', + see: 'D1', + }, +}; + +// Inline inputs for the disagreements that the snapshot corpus does not reach. +// `treesAgree: false` pins a case where both sides accept but build different +// trees. +const PINNED_DISAGREEMENTS: (KnownDisagreement & { source: string; treesAgree?: boolean })[] = [ + { + see: 'D1', + source: 'Note: 12.', + reference: 'reject', + spec: 'accept', + }, + { + see: 'D2', + source: 'Note: 1a', + reference: 'accept', + spec: 'accept', + treesAgree: false, + }, + { + see: 'D3', + source: 'Note: 1.a', + reference: 'accept', + spec: 'reject', + }, + { + see: 'D4', + source: "Note: '\\uzzzz'", + reference: 'accept', + spec: 'reject', + }, + { + see: 'D5', + source: 'Table \u{2000B} { id int }', + reference: 'reject', + spec: 'accept', + }, +]; + +// Behaviour that both sides share and that spec/DISAGREEMENTS.md calls out as +// surprising. Pinned so that the description stays truthful. +const AGREED_QUIRKS: [string, string][] = [ + ['operator continues previous line', 'Table t {\n (a, b) [pk]\n -2()\n}'], + ['index across a line break', 'Table t {\n a int\n[note: 1]\n}'], + ['trailing space defeats index across a line break', 'Table t {\n a int \n[note: 1]\n}'], + ['semicolon is never valid', 'Table t { id int; }'], + ['comment alone does not separate arguments', 'Table t { a/**/int }'], + ['comment with space separates arguments', 'Table t { a /**/int }'], + ['carriage return does not separate arguments', 'Table t { a\rint }'], + ['non-hex colour literal', 'Table t [headercolor: #zzz] { id int }'], + ['bare colour literal', 'Table t [headercolor: #] { id int }'], + ['use never falls back to an element', 'use x {}'], + ['simple body swallows the line', "Note: 'x' Table y {}"], + ['trailing-dot number mid-file', 'Note: 12.\n'], + ['digit-leading identifier mid-file', 'Note: 1a\n'], + ['BMP letters in identifiers', 'Table \u65E5\u672C { id int }'], +]; + +const SNAPSHOT_ROOT = path.resolve(__dirname, '../snapshots'); + +function listInputs (): { key: string; file: string }[] { + return readdirSync(SNAPSHOT_ROOT, { withFileTypes: true }) + .filter((d) => d.isDirectory()) + .flatMap((dir) => { + const inputDir = path.join(SNAPSHOT_ROOT, dir.name, 'input'); + return readdirSync(inputDir) + .filter((f) => f.endsWith('.in.dbml')) + .map((f) => ({ + key: `${dir.name}/${f.replace(/\.in\.dbml$/, '')}`, + file: path.join(inputDir, f), + })); + }) + .sort((a, b) => a.key.localeCompare(b.key)); +} + +describe('[conformance] syntax', () => { + const parser = loadSpecParser(); + const inputs = listInputs(); + + it('covers every snapshot input', () => { + expect(inputs.length).toBeGreaterThan(0); + }); + + it('pins only disagreements that still have an input file', () => { + const keys = new Set(inputs.map((i) => i.key)); + Object.keys(KNOWN_DISAGREEMENTS).forEach((key) => expect(keys.has(key), key).toBe(true)); + }); + + inputs.forEach(({ key, file }) => { + it(key, () => { + const source = readFileSync(file, 'utf-8'); + const result = compare(parser, source); + const known = KNOWN_DISAGREEMENTS[key]; + + if (known) { + expect(result.reference.verdict, `reference verdict changed; see DISAGREEMENTS.md ${known.see}\n${describeComparison(source, result)}`).toBe(known.reference); + expect(result.spec.verdict, `spec verdict changed; see DISAGREEMENTS.md ${known.see}\n${describeComparison(source, result)}`).toBe(known.spec); + return; + } + + expect(result.agreeOnVerdict, describeComparison(source, result)).toBe(true); + if (result.agreeOnTree !== undefined) { + expect(result.agreeOnTree, describeComparison(source, result)).toBe(true); + } + }); + }); +}); + +describe('[conformance] recorded disagreements', () => { + const parser = loadSpecParser(); + + PINNED_DISAGREEMENTS.forEach((pinned) => { + it(`${pinned.see}: ${JSON.stringify(pinned.source)}`, () => { + const result = compare(parser, pinned.source); + const message = `behaviour changed; revisit DISAGREEMENTS.md ${pinned.see}\n${describeComparison(pinned.source, result)}`; + expect(result.reference.verdict, message).toBe(pinned.reference); + expect(result.spec.verdict, message).toBe(pinned.spec); + if (pinned.treesAgree !== undefined) { + expect(result.agreeOnTree, message).toBe(pinned.treesAgree); + } + }); + }); + + AGREED_QUIRKS.forEach(([name, source]) => { + it(`agreed: ${name}`, () => { + const result = compare(parser, source); + expect(result.agreeOnVerdict, describeComparison(source, result)).toBe(true); + if (result.agreeOnTree !== undefined) { + expect(result.agreeOnTree, describeComparison(source, result)).toBe(true); + } + }); + }); +}); From 637658f5e6932aab84220bb824f19a441eb47d5b Mon Sep 17 00:00:00 2001 From: Mattias Fornander Date: Mon, 14 Sep 2026 12:13:28 -0600 Subject: [PATCH 05/10] docs(spec): record disagreements between the spec and the parser Co-Authored-By: Claude Fable 5.1 --- spec/DISAGREEMENTS.md | Bin 0 -> 5121 bytes 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 spec/DISAGREEMENTS.md diff --git a/spec/DISAGREEMENTS.md b/spec/DISAGREEMENTS.md new file mode 100644 index 0000000000000000000000000000000000000000..330d5dbc064ebd872f37f63432b092f4721e3d1c GIT binary patch literal 5121 zcma)A-EteZ749{k;>_s6k|HiGCC-IZrSIk4S*HNMxN-CRneNdUMtx{{h1$3{Zz^74)4;!J0@K8@ARtQ_*xys@3J4u1(C_%Py1K+hk&jx3`nhyO|FQjqoP@ z(fp`LQ5{kA*uR`>R~zz<3=TAbqOA~% zjsBYBjHh&XDD|0k_2D7y&+KZb*})MW<|@&X$NAGULakA|nDjbQ=aOysWhXCMm+;hj zryFXFv#9x+Pv|$)WocS8YT;2Jq| z^=R}!mluQ|uI&s zZ5_R1opHL2bDya6q3t4>l?kPd;2|*OU9$#IHfv&5kT7XS$Xcat(( zb|v9xFrYU0)`YaC+BlFn`5Q08Di+Mq4dju1^!g*0%g^(9CZxp_r*nOuAGx% zPbtbSz(iBwXS0ZOU%q^ackwy*tbp3rFvBq&D^S5j5FKY`V=@bA$`c_27w@I(zN_cF zRMaMuxuA2mi;}1oxdIDYjV0fy>EU@wsxMN}SL;uY zv=24upXcb%5sq~?mhFY6PKnV?YRqWN^D`W8+b*QHDQu!JKz{u(DN z=N1rKThrhmb)~Nq8$Ky^k<8QxnQ#eG+VcR*{-DNRRFVYb3t(B&p>@wUz~0r_PabU z(?IY=OWmMFH}Ae5AAb#+Vv6h>b8}w}On#r~{WFYD#%II?SJP+_FC|^WcM+zVd}x1t z^YH;c-+p|!opcJErP#G?BhxsBv~kOpyRqlkVGCG^V%NoRaP!5<^XJcB9Wa&HULw@R z*O0WGxL9WM#0#CuU0CP@pyk@-`oO_}(yZmYk^02KNq`aI`#m;)2sf@WA;Pl_gS>=K z2H;D^SJWOJ+D~#~Ty*2C6f4tT`Zl0|VG%4Q5Jw&^m!oezwql8>2GZdLHojqo`8;eC zip)pd?krSW-uH74qXXf-di&!eQsOj3(>Xnx$pn+}a?Tf|>Il<31OtvYr#t7Dz?%RA zG4XB1!B#g0F7tOmuRNCKfrqbF&`OLOjD055Qm?n}HV7#nk3P1+M#i&oBJDCAuyI0& zSx!(SOiW#a%MouBa81hmt%K(2p1YY1uLqt7iXK}YJntL>F+P{gaTj4H&(|^b0lv|& zK)Mg_U>oarrl{k zVo>7~Mv>GCP0s?u6~K(SbDZK0;E=qSUHhEWdk9>wD4YV1W|(I=FUbYsAmauK1xd{1 zN3^App-$FKQPDfDyADA!f-A@gYTq0CklRii`wHMx!vv$=qxpRAL2}rKA8tQ6U{(*q z*C)a3gpj)6#PRVyUs^FB!H&!#XBsgBlpb~`q4x7ABN8V%CK=_yN%yvY2JYbIeAfGL zz87a6^M>F0MZP6gUZ5KdiqG1y5&E%)5ZO|T=<87DOC=OQPYw?!l0VlBH)?ig+Fw{e z0%em756-ZTb{EdjlAK71I$O4vU^rurzrt|W=J9Z=NKAdnE$bJ=3!^%Zj))i2S#Gj4 zkAwQ-w?BRJ|ANYSH|)gyRgjww^V)j?N)+zNWs4z(U>dzP0lDoB&Z=p;*@AydnPHS{ z*6ZAnf?mV1q&c~mb5z!FryafF{ph6cFvc5Ee*R`4h@T0Dxg=XLBh^Kti2Xen`7gJj zftCqWF@yXexq+33Wh z27;f+vX{4Kkzx@;7EV>zuJ RRXT^oBN4t0>v-rJ{TFqY4Wa-5 literal 0 HcmV?d00001 From c00bd62267016a6cbc7989cbc2a37b413b8245cf Mon Sep 17 00:00:00 2001 From: Mattias Fornander Date: Mon, 14 Sep 2026 12:13:28 -0600 Subject: [PATCH 06/10] ci: run spec conformance tests as a non-blocking job Co-Authored-By: Claude Fable 5.1 --- .github/workflows/test.yml | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index f0c3c1798..43a62df8c 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -114,6 +114,27 @@ jobs: path: ./coverage-report.md include-hidden-files: true + # Spec conformance (see spec/README.md). Non-blocking until the corpus in + # spec/DISAGREEMENTS.md is empty; then drop continue-on-error. + spec-conformance: + runs-on: blacksmith-2vcpu-ubuntu-2204 + timeout-minutes: 15 + continue-on-error: true + strategy: + matrix: + node-version: [22.x] + steps: + - uses: actions/checkout@v3 + - name: Use Node.js ${{ matrix.node-version }} + uses: actions/setup-node@v3 + with: + node-version: ${{ matrix.node-version }} + cache: "yarn" + - name: Install dependencies + run: yarn install --frozen-lockfile + - name: Run spec conformance tests + run: yarn workspace @dbml/parse test:conformance + comment-coverage: runs-on: blacksmith-2vcpu-ubuntu-2204 timeout-minutes: 5 From f59a4153a2d7f806ba4be28655c02d7c273920a2 Mon Sep 17 00:00:00 2001 From: Mattias Fornander Date: Mon, 14 Sep 2026 12:13:28 -0600 Subject: [PATCH 07/10] docs: point CONTRIBUTING at dbml-parse and spec instead of PEG.js Co-Authored-By: Claude Fable 5.1 --- CONTRIBUTING.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 5a42dccc0..64b60e236 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -6,8 +6,8 @@ The following is a set of guidelines for contributing to DBML and its packages, ## Not sure where to start ? -- [PEG.js](https://pegjs.org/): DBML uses PEG.js library to parse different database languages and structure them as plain Javascript object. -- Check out the `parse` folder in `@dbml/core` source code to learn more about how to structure and import different grammars from multiple PEG.js files. +- The DBML parser lives in [`packages/dbml-parse`](./packages/dbml-parse). It is a hand-written lexer and recursive-descent parser (`src/core/lexer`, `src/core/parser`) followed by per-element validation in `src/core/local_modules` and binding and interpretation in `src/core/global_modules`. The formal grammar and the conformance tests that keep it honest live in [`spec/`](./spec). +- Importing other database languages (PostgreSQL, MySQL, MSSQL, ...) is done in `@dbml/core` under `packages/dbml-core/src/parse`, using ANTLR grammars. The [PEG.js](https://pegjs.org/) grammars under `parse/deprecated` are no longer maintained and are kept for reference only. - When you're ready to jump to the source code, look for issues tagged with [help wanted](https://github.com/holistics/dbml/labels/help%20wanted) label to get list of current issues you can make a contribution to or create a new issue for suggesting enhancement. - Check out DBML [website](https://www.dbml-lang.org/home/) for more documentation on DBML syntax. From 872d0cc6bc142fc24fa6b8b1222a4f679413cbd4 Mon Sep 17 00:00:00 2001 From: Mattias Fornander Date: Tue, 15 Sep 2026 17:50:50 -0600 Subject: [PATCH 08/10] feat(spec): add an ANTLR rendering of the Layer 1 grammar Express the same Layer 1 specification as a two-stage ANTLR 4 grammar (DbmlLexer.g4 + DbmlParser.g4) so that the notation question raised in the RFC can be judged on the same conformance corpus. Trivia lives on the hidden channel; predicates carry the trivia-sensitive rules and the one-token commitments the reference parser makes. Co-Authored-By: Claude Fable 5.1 --- spec/antlr/DbmlLexer.g4 | 159 +++++++++++++++++++ spec/antlr/DbmlParser.g4 | 335 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 494 insertions(+) create mode 100644 spec/antlr/DbmlLexer.g4 create mode 100644 spec/antlr/DbmlParser.g4 diff --git a/spec/antlr/DbmlLexer.g4 b/spec/antlr/DbmlLexer.g4 new file mode 100644 index 000000000..82657b266 --- /dev/null +++ b/spec/antlr/DbmlLexer.g4 @@ -0,0 +1,159 @@ +// DBML formal specification, Layer 1: generic syntax. ANTLR notation, lexer. +// +// This is the same specification as ../dbml-syntax.peggy expressed as a +// two-stage ANTLR 4 grammar (DbmlLexer.g4 + DbmlParser.g4), so that the two +// notations can be compared on the same conformance corpus. It restates +// packages/dbml-parse/src/core/lexer/lexer.ts. +// +// Trivia (whitespace and comments) goes to the hidden channel; the parser +// inspects it with getHiddenTokensToLeft(), which is the ANTLR equivalent of +// the reference lexer's trailing/leading trivia (lexer.ts: gatherTrivia()). +// The character stream must be opened with code-point decoding so that +// letters outside the Basic Multilingual Plane match \p{L}. + +lexer grammar DbmlLexer; + +// --------------------------------------------------------------------------- +// Trivia (lexer.ts: scanTokens(), gatherTrivia()) +// --------------------------------------------------------------------------- +// A carriage return is discarded outright: it is neither whitespace nor a line +// break. A single-line comment does not include its terminating newline. + +NEWLINE: '\n' -> channel(HIDDEN); +SPACES: [ \t]+ -> channel(HIDDEN); +CR: '\r' -> skip; +LINE_COMMENT: '//' ~[\n]* -> channel(HIDDEN); +BLOCK_COMMENT: '/*' .*? '*/' -> channel(HIDDEN); + +// lexer.ts: multilineComment() reports UNEXPECTED_EOF. Without this rule the +// lexer would fall back to SLASH WILDCARD and the parser could accept the file. +UNTERMINATED_BLOCK_COMMENT: '/*' (~'*' | '*'+ ~[*/])* '*'* EOF; + +// --------------------------------------------------------------------------- +// Keywords (utils/tokens.ts: isUseKeyword() etc.) +// --------------------------------------------------------------------------- +// Case-insensitive, and ordinary identifiers everywhere the parser does not +// look for them: the parser's `identifier` rule includes all of them. They must +// precede IDENTIFIER so that they win the longest-match tie. + +USE: [uU] [sS] [eE]; +REUSE: [rR] [eE] [uU] [sS] [eE]; +FROM: [fF] [rR] [oO] [mM]; +AS: [aA] [sS]; +METADATA: [mM] [eE] [tT] [aA] [dD] [aA] [tT] [aA]; + +// --------------------------------------------------------------------------- +// Punctuation +// --------------------------------------------------------------------------- + +LPAREN: '('; +RPAREN: ')'; +LBRACKET: '['; +RBRACKET: ']'; +LBRACE: '{'; +RBRACE: '}'; +COMMA: ','; +COLON: ':'; +// Lexed by the reference but accepted nowhere; any ';' is a syntax error. +SEMICOLON: ';'; +WILDCARD: '*'; + +// --------------------------------------------------------------------------- +// Operators (lexer.ts: operator()) +// --------------------------------------------------------------------------- +// The set is prefix-closed, so ANTLR's longest match reproduces the reference +// lexer's greedy scan. One token per operator lets the parser classify them +// without predicates. + +Q_LTGT_Q: '?<>?'; +Q_LTGT: '?<>'; +Q_LT_Q: '??'; +Q_GT: '?>'; +Q_MINUS_Q: '?-?'; +Q_MINUS: '?-'; +QUESTION: '?'; +LTGT_Q: '<>?'; +LTGT: '<>'; +LTE: '<='; +LT_Q: '='; +GT_Q: '>?'; +GT: '>'; +ARROW_RIGHT: '->'; +MINUS_Q: '-?'; +MINUS: '-'; +EQ: '=='; +ASSIGN: '='; +NEQ: '!='; +BANG: '!'; +PLUS: '+'; +SLASH: '/'; +PERCENT: '%'; +DOT: '.'; +AMP: '&'; +PIPE: '|'; +TILDE: '~'; + +// --------------------------------------------------------------------------- +// Literals and identifiers +// --------------------------------------------------------------------------- + +// lexer.ts: numericLiteralOrIdentifier(). Longest match decides between +// NUMBER, INVALID_NUMBER and IDENTIFIER; on a tie the first rule wins, so the +// order below matters: 1e5 is a NUMBER, 1e5x is an IDENTIFIER, 1.5x is an +// INVALID_NUMBER. +NUMBER + : DIGITS EXPONENT + | DIGITS '.' DIGITS? EXPONENT? + | DIGITS + ; + +// A dot part followed by letters, or a second dot, is a lexical error in the +// reference (UNKNOWN_TOKEN "Invalid number"). The parser has no rule for this +// token, so its presence rejects the file. +INVALID_NUMBER + : DIGITS '.' DIGITS? IDENT_START ALNUM_DOT* + | DIGITS '.' DIGITS? '.' ALNUM_DOT* + ; + +// lexer.ts: identifier(). A token that starts with a digit is an identifier +// when it contains a letter or underscore and is not a well-formed number. +IDENTIFIER + : IDENT_START ALNUM* + | DIGITS IDENT_START ALNUM* + ; + +// lexer.ts: singleLineStringLiteral(), multilineStringLiteral() +STRING + : '\'\'\'' (ESCAPE | ~'\\')*? '\'\'\'' + | '\'' (ESCAPE | ~['\\\n])* '\'' + ; + +// lexer.ts: quotedVariable() +QUOTED_VARIABLE: '"' (ESCAPE | ~["\\\n])* '"'; + +// lexer.ts: functionExpression(); raw text, no escapes, may span lines. +FUNCTION_EXPRESSION: '`' ~'`'* '`'; + +// lexer.ts: colorLiteral(); hex validation is a Layer 2 rule. +COLOR: '#' ALNUM*; + +// lexer.ts: escapedString(). A backslash always starts an escape. Any escaped +// character is accepted; \uHHHH needs four hexadecimal digits; a backslash +// before a line break continues the line. +fragment ESCAPE + : '\\u' HEX HEX HEX HEX + | '\\\r\n' + | '\\' ~'u' + ; + +fragment HEX: [0-9a-fA-F]; +fragment DIGITS: [0-9]+; +fragment EXPONENT: [eE] [+-]? DIGITS; +fragment IDENT_START: [\p{L}\p{M}_]; +fragment ALNUM: [\p{L}\p{M}_0-9]; +fragment ALNUM_DOT: [\p{L}\p{M}_0-9.]; diff --git a/spec/antlr/DbmlParser.g4 b/spec/antlr/DbmlParser.g4 new file mode 100644 index 000000000..8a3e5217a --- /dev/null +++ b/spec/antlr/DbmlParser.g4 @@ -0,0 +1,335 @@ +// DBML formal specification, Layer 1: generic syntax. ANTLR notation, parser. +// +// Restates packages/dbml-parse/src/core/parser/parser.ts. Rule comments name +// the reference method they correspond to. The parser is generated for the +// JavaScript target (the runtime @dbml/core already uses); the members block +// is therefore JavaScript and is emitted into the generated constructor. +// +// Three groups of semantic predicates carry what the notation cannot: +// 1. trivia-sensitive rules (index vs new argument, call vs new line, argument +// spacing, line-separated use specifiers), read from the hidden channel; +// 2. one-token commitments the reference makes with a single lookahead, so +// that ALL(*) does not find a second parse the reference never tries; +// 3. the bracket-depth context that suspends the call/new-line rule. +// One embedded action rejects a simple body that resolves to an element +// declaration (parser.ts: UNEXPECTED_ELEMENT_DECLARATION). + +parser grammar DbmlParser; + +options { tokenVocab = DbmlLexer; } + +@parser::members { + // parser/contextStack.ts: number of unclosed '(' and '['. Inside them a '(' + // that starts a new line still continues a call expression. + this.depth = 0; + + // Hidden-channel tokens between the previous token and the next one: the + // reference lexer's trailing/leading trivia (lexer.ts: gatherTrivia()). + this.gapAhead = () => { + const next = this._input.LT(1); + return this._input.getHiddenTokensToLeft(next.tokenIndex, antlr4.Token.HIDDEN_CHANNEL) ?? []; + }; + // lexer/utils.ts: hasTrailingNewLines(), isAtStartOfLine() + this.newlineBefore = () => this.gapAhead().some((t) => t.type === DbmlParser.NEWLINE); + // lexer/utils.ts: hasTrailingSpaces(): a space or tab before the first line break + this.spaceBefore = () => { + for (const t of this.gapAhead()) { + if (t.type === DbmlParser.NEWLINE) return false; + if (t.type === DbmlParser.SPACES) return true; + } + return false; + }; + + this.ahead = (k) => this._input.LT(k).type; + this.isIdentifierToken = (type) => [ + DbmlParser.IDENTIFIER, DbmlParser.USE, DbmlParser.REUSE, DbmlParser.FROM, DbmlParser.AS, DbmlParser.METADATA, + ].includes(type); + + // parser.ts: elementDeclaration(): a name or alias is read unless ':' '{' or '[' follows + this.bodyStartAhead = () => [DbmlParser.COLON, DbmlParser.LBRACE, DbmlParser.LBRACKET].includes(this.ahead(1)); + // parser.ts: useSpecifier(): a name must start with an identifier or quoted variable + this.nameStartAhead = () => this.isIdentifierToken(this.ahead(1)) || this.ahead(1) === DbmlParser.QUOTED_VARIABLE; + // parser.ts: canBeField() + this.fieldAhead = () => this.isIdentifierToken(this.ahead(1)) && this.ahead(2) === DbmlParser.COLON; + // parser.ts: attributeValue(): two leading identifiers make an identifier stream + this.twoIdentifiersAhead = () => this.isIdentifierToken(this.ahead(1)) && this.isIdentifierToken(this.ahead(2)); + // parser.ts: expression(): arguments sit on the same line, after a space, and never start with a comma + this.argAhead = () => this.spaceBefore() && !this.newlineBefore() && this.ahead(1) !== DbmlParser.COMMA; + // parser.ts: shouldStopFunctionApplication() + this.functionApplicationStops = () => this.ahead(1) === antlr4.Token.EOF || this.newlineBefore() + || [DbmlParser.RBRACE, DbmlParser.RBRACKET, DbmlParser.RPAREN, DbmlParser.COMMA, DbmlParser.COLON].includes(this.ahead(1)); + // parser.ts: shouldStopCommaExpression() + this.commaStops = () => this.ahead(1) === antlr4.Token.EOF || this.newlineBefore() + || [DbmlParser.LBRACE, DbmlParser.RBRACE, DbmlParser.LBRACKET, DbmlParser.RBRACKET, DbmlParser.LPAREN, DbmlParser.RPAREN, DbmlParser.COLON].includes(this.ahead(1)); + + // parser/utils.ts: convertFuncAppToElem(), shape test only. A function + // application of the form [ [as ]] [] + // is an element declaration and may not be a simple body. + this.unwrap = (ctx) => { + while (ctx && ctx.children && ctx.children.length === 1 && ctx.children[0].children) ctx = ctx.children[0]; + return ctx; + }; + this.isElementShaped = (exprCtx) => { + const parts = exprCtx.commaExpression(); + let callee = this.unwrap(parts[0]); + let args = parts.slice(1); + if (callee instanceof DbmlParser.PostfixExpressionContext) { + if (callee.callStep().length !== 1 || callee.chainStep().length !== 0) return false; + args = [callee.callStep(0).tupleExpression(), ...args]; + callee = this.unwrap(callee.unaryExpression()); + } + if (!(callee instanceof DbmlParser.IdentifierContext) || args.length === 0) return false; + const rest = args.slice(); + if (!(this.unwrap(rest.pop()) instanceof DbmlParser.BlockExpressionContext)) return false; + if (rest.length > 0 && this.unwrap(rest[rest.length - 1]) instanceof DbmlParser.ListExpressionContext) rest.pop(); + if (rest.length === 3) { + const as = this.unwrap(rest[1]); + return as instanceof DbmlParser.IdentifierContext && as.getText().toLowerCase() === 'as'; + } + return rest.length <= 1; + }; +} + +// --------------------------------------------------------------------------- +// Program +// --------------------------------------------------------------------------- + +// parser.ts: program() +program: statement* EOF; + +// A statement starting with `use` or `reuse` is always a use declaration; +// elementDeclaration does not accept those tokens as a type. +statement + : useDeclaration + | elementDeclaration + ; + +// --------------------------------------------------------------------------- +// Use declarations +// --------------------------------------------------------------------------- + +// parser.ts: useDeclaration() +// use * from '' +// use { \n ... } from '' +useDeclaration: (USE | REUSE) (wildcard | useSpecifierList) FROM STRING; + +// parser.ts: useSpecifierList(); specifiers are separated by line breaks. +useSpecifierList: LBRACE useSpecifier? ({this.newlineBefore()}? useSpecifier)* RBRACE; + +// parser.ts: useSpecifier() +// [as ] +useSpecifier: identifier useName (AS useName)?; + +// The name must start with an identifier or quoted variable; it is then read +// as a full normal expression. +useName: {this.nameStartAhead()}? normalExpression; + +// --------------------------------------------------------------------------- +// Element declarations +// --------------------------------------------------------------------------- + +// parser.ts: elementDeclaration() +// [] [] [as ] [] (: | { }) +// is present exactly when is `metadata`. +elementDeclaration + : METADATA identifier elementTail + | (IDENTIFIER | FROM | AS) elementTail + ; + +// The reference decides on one token whether a name or alias is present; the +// predicates keep ALL(*) from choosing the other reading. +elementTail + : ({!this.bodyStartAhead()}? normalExpression | {this.bodyStartAhead()}?) + (AS {!this.bodyStartAhead()}? normalExpression)? + listExpression? + elementBody + ; + +elementBody + : COLON simpleBody + | blockExpression + ; + +// parser.ts: UNEXPECTED_ELEMENT_DECLARATION / INVALID_ELEMENT_IN_SIMPLE_BODY +simpleBody + : expression + { if (this.isElementShaped(localctx.expression())) this.notifyErrorListeners('a simple body must not be an element declaration'); } + ; + +// --------------------------------------------------------------------------- +// Blocks +// --------------------------------------------------------------------------- + +// parser.ts: blockExpression() +blockExpression: LBRACE bodyItem* RBRACE; + +// parser.ts: canBeField() decides between fieldDeclaration() and expression(). +bodyItem + : {this.fieldAhead()}? fieldDeclaration + | {!this.fieldAhead()}? expression + ; + +// parser.ts: fieldDeclaration() +// : +fieldDeclaration: identifier COLON simpleBody; + +// --------------------------------------------------------------------------- +// Function application +// --------------------------------------------------------------------------- + +// parser.ts: expression() +// A callee followed by arguments on the same line, each preceded by at least +// one space or tab. The application must end at a line break, end of input, or +// before } ] ) , : (parser.ts: MISSING_SPACES otherwise). +expression + : commaExpression ({this.argAhead()}? commaExpression)* {this.functionApplicationStops()}? + ; + +// parser.ts: commaExpression() +// CSV-like list without delimiters. Empty slots are recorded by the tree +// converter as nodes. +commaExpression + : normalExpression (COMMA commaRest)? + | COMMA commaRest + ; + +// After a comma: a line break, end of input, bracket or colon ends the list +// with an empty slot; another comma is an empty slot; otherwise a normal +// expression, which continues only if a comma follows on the same line. +// parser.ts: shouldStopCommaExpression() +commaRest + : {this.commaStops()}? + | {!this.commaStops()}? COMMA commaRest + | {!this.commaStops()}? normalExpression ({!this.newlineBefore()}? COMMA commaRest)? + ; + +// --------------------------------------------------------------------------- +// Normal expressions (parser.ts: expression_bp(), Pratt precedence climbing) +// --------------------------------------------------------------------------- +// Binding powers from parser.ts, tightest first: +// . 16 right operand is a bare operand +// prefix operators 15 +// call ( ... ) 14 +// / % 11 +// + - -? ?- ?-? 9 +// relationship ops 7 < <= > >= <> and the ? variants, -> <- +// == != 4 +// = 2 +// All infix operators are left-associative. Trivia, including line breaks, +// may surround infix operators. + +normalExpression: assignmentExpression; + +assignmentExpression: equalityExpression (ASSIGN equalityExpression)*; +equalityExpression: comparisonExpression ((EQ | NEQ) comparisonExpression)*; +comparisonExpression: additiveExpression (comparisonOp additiveExpression)*; +additiveExpression: multiplicativeExpression (additiveOp multiplicativeExpression)*; +multiplicativeExpression: postfixExpression ((SLASH | PERCENT) postfixExpression)*; + +// Calls applied to a unary expression; after a call, indexing and member +// access may continue the chain. +postfixExpression: unaryExpression (callStep chainStep*)*; + +// parser.ts: leftExpression_bp() +// A prefix operator binds tighter than a call: -2() is (-2)(). +unaryExpression + : prefixOp unaryExpression + | memberChain + ; + +// Member access and indexing on a bare operand. The right side of `.` is a +// bare operand (parser.ts: extractOperand()), never a prefix expression. +memberChain: operand chainStep*; + +chainStep + : indexStep + | memberStep + ; + +memberStep: DOT operand; + +// A '(' continues a call unless it starts a new line; inside an unclosed '(' +// or '[' the line-break rule is suspended. +callStep: {this.depth > 0 || !this.newlineBefore()}? tupleExpression; + +// A '[' is array indexing only when no space or tab separates it from the +// operand on the same line; otherwise it starts a new argument. +indexStep: {!this.spaceBefore()}? listExpression; + +// parser.ts: extractOperand() +operand + : wildcard + | primaryExpression + | functionExpression + | listExpression + | blockExpression + | tupleExpression + ; + +// parser.ts: primaryExpression() +primaryExpression + : literal + | variable + ; + +literal: STRING | NUMBER | COLOR; +variable: identifier | QUOTED_VARIABLE; +functionExpression: FUNCTION_EXPRESSION; +wildcard: WILDCARD; + +// parser.ts: tupleExpression(); one element without a comma is a group. +tupleExpression + : LPAREN {this.depth += 1;} (normalExpression (COMMA normalExpression)*)? RPAREN {this.depth -= 1;} + ; + +// --------------------------------------------------------------------------- +// Attribute lists +// --------------------------------------------------------------------------- + +// parser.ts: listExpression() +// [ , , ... ] +listExpression + : LBRACKET {this.depth += 1;} (attribute (COMMA attribute)*)? RBRACKET {this.depth -= 1;} + ; + +// parser.ts: attribute() +// [: ] +attribute: attributeName (COLON attributeValue)?; + +// parser.ts: attributeName() +// A stream of identifiers (primary key), or a single literal or quoted variable. +attributeName + : identifierStream + | literal + | QUOTED_VARIABLE + ; + +identifierStream: identifier+; + +// parser.ts: attributeValue() +attributeValue + : {this.twoIdentifiersAhead()}? identifierStream + | {!this.twoIdentifiersAhead()}? normalExpression + ; + +// --------------------------------------------------------------------------- +// Operators and identifiers +// --------------------------------------------------------------------------- + +// parser.ts: prefixBpMap +prefixOp + : PLUS | MINUS | LT | GT | LTGT | MINUS_Q | Q_MINUS | Q_MINUS_Q | Q_GT | GT_Q | Q_LT | LT_Q + | Q_GT_Q | Q_LT_Q | Q_LTGT | LTGT_Q | Q_LTGT_Q | ARROW_RIGHT | ARROW_LEFT | BANG | TILDE + ; + +// parser.ts: infixBpMap, Prec.Additive +additiveOp: PLUS | MINUS | MINUS_Q | Q_MINUS | Q_MINUS_Q; + +// parser.ts: infixBpMap, Prec.Comparison +comparisonOp + : LT | LTE | GT | GTE | LTGT | Q_GT | GT_Q | Q_LT | LT_Q | Q_GT_Q | Q_LT_Q | Q_LTGT | LTGT_Q | Q_LTGT_Q + | ARROW_RIGHT | ARROW_LEFT + ; + +// Keywords are ordinary identifiers wherever the parser is not looking for them. +identifier: IDENTIFIER | USE | REUSE | FROM | AS | METADATA; From ed569125aa0f716481a55012492e208417abeb95 Mon Sep 17 00:00:00 2001 From: Mattias Fornander Date: Tue, 15 Sep 2026 17:51:13 -0600 Subject: [PATCH 09/10] test(dbml-parse): run conformance tests against both spec notations Generate the ANTLR parser at test time with antlr-ng (no Java) for the antlr4 JavaScript runtime, convert its parse tree to the shared spec node shape, and run the snapshot corpus, pinned disagreements and property tests against the peggy and ANTLR parsers alike. Generated code is ignored by git. Co-Authored-By: Claude Fable 5.1 --- .gitignore | 3 + .../__tests__/conformance/antlr.setup.ts | 7 + .../dbml-parse/__tests__/conformance/antlr.ts | 300 ++++++++++++++++++ .../__tests__/conformance/harness.ts | 85 ++++- .../__tests__/conformance/properties.test.ts | 6 +- .../__tests__/conformance/syntax.test.ts | 11 +- packages/dbml-parse/eslint.config.ts | 1 + packages/dbml-parse/package.json | 2 + .../dbml-parse/vitest.conformance.config.ts | 2 + yarn.lock | 75 ++++- 10 files changed, 475 insertions(+), 17 deletions(-) create mode 100644 packages/dbml-parse/__tests__/conformance/antlr.setup.ts create mode 100644 packages/dbml-parse/__tests__/conformance/antlr.ts diff --git a/.gitignore b/.gitignore index b44693f4c..7c546a82a 100644 --- a/.gitignore +++ b/.gitignore @@ -34,3 +34,6 @@ coverage/ __benchmarks__/output/ .gitnexus + +# spec conformance: parsers generated from spec/antlr at test time +packages/dbml-parse/__tests__/conformance/generated/ diff --git a/packages/dbml-parse/__tests__/conformance/antlr.setup.ts b/packages/dbml-parse/__tests__/conformance/antlr.setup.ts new file mode 100644 index 000000000..27ad14b32 --- /dev/null +++ b/packages/dbml-parse/__tests__/conformance/antlr.setup.ts @@ -0,0 +1,7 @@ +// vitest global setup: compile spec/antlr/*.g4 once before the conformance +// test files are collected. +import { generateAntlrParser } from './antlr'; + +export default function setup (): void { + generateAntlrParser(); +} diff --git a/packages/dbml-parse/__tests__/conformance/antlr.ts b/packages/dbml-parse/__tests__/conformance/antlr.ts new file mode 100644 index 000000000..4e84f850d --- /dev/null +++ b/packages/dbml-parse/__tests__/conformance/antlr.ts @@ -0,0 +1,300 @@ +// ANTLR notation of the Layer 1 spec (spec/antlr/*.g4). +// +// The grammars are compiled with antlr-ng (a Node port of the ANTLR tool, no +// Java needed) for the JavaScript target, which runs on the same `antlr4` +// runtime @dbml/core uses. Generation happens in the vitest global setup +// (antlr.setup.ts) into ./generated/antlr, which is not committed. +// +// ANTLR produces a parse tree shaped like the grammar rules, so this file also +// converts that tree into the same { kind, children } spec nodes the peggy +// grammar builds directly. The element reinterpretation of a function +// application (parser/utils.ts: convertFuncAppToElem) is applied here. +import { execFileSync } from 'node:child_process'; +import { existsSync, mkdirSync, statSync } from 'node:fs'; +import path from 'node:path'; +import { pathToFileURL } from 'node:url'; +import antlr4 from 'antlr4'; +import { + type SpecNode, + type SpecParser, + type SpecToken, + SPEC_DIR, + convertFuncAppToElem, + node, + tok, +} from './harness'; + +export const ANTLR_GRAMMAR_DIR = path.join(SPEC_DIR, 'antlr'); +export const ANTLR_OUTPUT_DIR = path.resolve(__dirname, 'generated/antlr'); + +const GRAMMARS = ['DbmlLexer.g4', 'DbmlParser.g4']; + +function isStale (): boolean { + const outputs = ['DbmlLexer.js', 'DbmlParser.js'].map((f) => path.join(ANTLR_OUTPUT_DIR, f)); + if (!outputs.every(existsSync)) return true; + const newestGrammar = Math.max(...GRAMMARS.map((g) => statSync(path.join(ANTLR_GRAMMAR_DIR, g)).mtimeMs)); + const oldestOutput = Math.min(...outputs.map((o) => statSync(o).mtimeMs)); + return newestGrammar > oldestOutput; +} + +// antlr-ng only exposes an ESM entry point, so `require.resolve` cannot find +// it; locate its CLI by walking up the node_modules chain instead. +function findAntlrNgRunner (): string { + for (let dir = __dirname; ; dir = path.dirname(dir)) { + const candidate = path.join(dir, 'node_modules/antlr-ng/dist/cli/runner.js'); + if (existsSync(candidate)) return candidate; + if (path.dirname(dir) === dir) throw new Error('antlr-ng is not installed'); + } +} + +// Compile spec/antlr/*.g4 to JavaScript with antlr-ng. Idempotent: skipped +// when the generated files are newer than the grammars. +export function generateAntlrParser (): void { + if (!isStale()) return; + mkdirSync(ANTLR_OUTPUT_DIR, { recursive: true }); + const runner = findAntlrNgRunner(); + const run = (grammar: string) => execFileSync(process.execPath, [ + runner, + '-Dlanguage=JavaScript', + '-o', + ANTLR_OUTPUT_DIR, + '--lib', + ANTLR_OUTPUT_DIR, + '--generate-listener', + 'false', + path.join(ANTLR_GRAMMAR_DIR, grammar), + ], { stdio: 'pipe' }); + // The parser needs the lexer's .tokens file, so generate in order. + GRAMMARS.forEach(run); +} + +// The bundled runtime keeps the error strategies under `antlr4.error`, which +// its type definitions do not declare. +const runtime = antlr4 as unknown as typeof antlr4 & { error: { BailErrorStrategy: new () => unknown } }; + +// Minimal view of the antlr4 JavaScript runtime's tree types. +interface Terminal { + symbol: { type: number; text: string }; +} +interface RuleNode { + ruleIndex: number; + children?: (RuleNode | Terminal)[] | null; + getText (): string; +} +type Tree = RuleNode | Terminal; + +function isTerminal (t: Tree): t is Terminal { + return 'symbol' in t; +} + +// Map ANTLR token types onto the reference SyntaxTokenKind names. +const TOKEN_KINDS: Record = { + IDENTIFIER: '', + USE: '', + REUSE: '', + FROM: '', + AS: '', + METADATA: '', + QUOTED_VARIABLE: '', + STRING: '', + NUMBER: '', + COLOR: '', + FUNCTION_EXPRESSION: '', + WILDCARD: '', + LPAREN: '', + RPAREN: '', + LBRACKET: '', + RBRACKET: '', + LBRACE: '', + RBRACE: '', + COMMA: '', + COLON: '', + EOF: '', +}; + +class TreeConverter { + constructor (private ruleNames: string[], private symbolicNames: (string | null)[]) {} + + token (t: Terminal): SpecToken { + const name = t.symbol.type === antlr4.Token.EOF ? 'EOF' : this.symbolicNames[t.symbol.type] ?? ''; + return tok(TOKEN_KINDS[name] ?? '', t.symbol.text); + } + + kids (ctx: RuleNode): Tree[] { + return ctx.children ?? []; + } + + // Convert a child that is either a token or a rule wrapping a single token. + leafToken (t: Tree): SpecToken { + if (isTerminal(t)) return this.token(t); + return this.leafToken(this.kids(t)[0]); + } + + // Fold `head (op operand)*` into left-associative infix nodes. + foldInfix (ctx: RuleNode): SpecNode { + const kids = this.kids(ctx); + let left = this.convert(kids[0]) as SpecNode; + for (let i = 1; i < kids.length; i += 2) { + left = node('', left, this.leafToken(kids[i]), this.convert(kids[i + 1])); + } + return left; + } + + // Fold postfix steps (callStep, chainStep, indexStep, memberStep) onto a head. + foldSteps (head: SpecNode, steps: Tree[]): SpecNode { + return steps.reduce((left, step) => { + const rule = this.ruleNames[(step as RuleNode).ruleIndex]; + const inner = this.kids(step as RuleNode); + switch (rule) { + case 'chainStep': return this.foldSteps(left, inner); + case 'callStep': return node('', left, this.convert(inner[0])); + case 'indexStep': return node('', left, this.convert(inner[0])); + case 'memberStep': return node('', left, this.leafToken(inner[0]), this.convert(inner[1])); + default: throw new Error(`unexpected postfix step ${rule}`); + } + }, head); + } + + convertAll (trees: Tree[]): (SpecNode | SpecToken)[] { + return trees.map((t) => this.convert(t)); + } + + convert (tree: Tree): SpecNode | SpecToken { + if (isTerminal(tree)) return this.token(tree); + const ctx = tree; + const kids = this.kids(ctx); + const rule = this.ruleNames[ctx.ruleIndex]; + switch (rule) { + case 'program': + return node('', this.convertAll(kids)); + case 'useDeclaration': + return node('', this.convertAll(kids)); + case 'useSpecifierList': + return node('', this.convertAll(kids)); + case 'useSpecifier': + return node('', this.convertAll(kids)); + case 'elementDeclaration': { + // METADATA identifier elementTail | (IDENTIFIER | FROM | AS) elementTail + const tail = kids[kids.length - 1] as RuleNode; + const head = kids.slice(0, -1).map((k) => this.leafToken(k)); + return node('', head, this.convertAll(this.kids(tail))); + } + case 'elementBody': + // COLON simpleBody | blockExpression: splice into the parent + return node('', this.convertAll(kids)); + case 'fieldDeclaration': + return node('', this.convertAll(kids)); + case 'blockExpression': + return node('', this.convertAll(kids)); + case 'expression': { + const [callee, ...args] = kids.map((k) => this.convert(k) as SpecNode); + return convertFuncAppToElem(callee, args) ?? node('', callee, args); + } + case 'commaExpression': { + // normalExpression (COMMA commaRest)? | COMMA commaRest + if (kids.length === 1) return this.convert(kids[0]); + const first = isTerminal(kids[0]) ? node('') : this.convert(kids[0]); + const rest = kids.filter((k) => !isTerminal(k) || k.symbol.type !== antlr4.Token.EOF).slice(isTerminal(kids[0]) ? 0 : 1); + return node('', first, this.convertAll(rest)); + } + case 'commaRest': { + // empty | COMMA commaRest | normalExpression (COMMA commaRest)? + if (kids.length === 0) return node('', node('')); + if (isTerminal(kids[0])) return node('', node(''), this.convertAll(kids)); + return node('', this.convertAll(kids)); + } + case 'assignmentExpression': + case 'equalityExpression': + case 'comparisonExpression': + case 'additiveExpression': + case 'multiplicativeExpression': + return this.foldInfix(ctx); + case 'postfixExpression': + case 'memberChain': + return this.foldSteps(this.convert(kids[0]) as SpecNode, kids.slice(1)); + case 'unaryExpression': + if (kids.length === 2) return node('', this.leafToken(kids[0]), this.convert(kids[1])); + return this.convert(kids[0]); + case 'primaryExpression': { + const inner = kids[0] as RuleNode; + const kind = this.ruleNames[inner.ruleIndex] === 'literal' ? '' : ''; + return node('', node(kind, this.leafToken(inner))); + } + case 'functionExpression': + return node('', this.leafToken(kids[0])); + case 'wildcard': + return node('', this.leafToken(kids[0])); + case 'tupleExpression': { + const converted = this.convertAll(kids); + const kind = kids.length === 3 ? '' : ''; + return node(kind, converted); + } + case 'listExpression': + return node('', this.convertAll(kids)); + case 'attribute': + return node('', this.convertAll(kids)); + case 'attributeName': { + const inner = kids[0]; + if (!isTerminal(inner) && this.ruleNames[inner.ruleIndex] === 'identifierStream') return this.convert(inner); + const kind = !isTerminal(inner) && this.ruleNames[inner.ruleIndex] === 'literal' ? '' : ''; + return node('', node(kind, this.leafToken(inner))); + } + case 'identifierStream': + return node('', kids.map((k) => this.leafToken(k))); + case 'identifier': + case 'prefixOp': + case 'additiveOp': + case 'comparisonOp': + return this.leafToken(kids[0]); + default: + // statement, useName, elementTail, simpleBody, bodyItem, normalExpression, + // operand, attributeValue, literal, variable: transparent wrappers + if (kids.length !== 1) throw new Error(`unexpected shape for rule ${rule}`); + return this.convert(kids[0]); + } + } +} + +// `` nodes are transparent lists produced by the converter for +// sub-rules whose children belong to the parent node; flatten them. +function flattenSplices (n: SpecNode | SpecToken): (SpecNode | SpecToken)[] { + if ('token' in n) return [n]; + const children = n.children.flatMap(flattenSplices); + if (n.kind === '') return children; + return [{ kind: n.kind, children }]; +} + +export async function loadAntlrParser (): Promise { + generateAntlrParser(); + const importGenerated = (file: string) => import(/* @vite-ignore */ pathToFileURL(path.join(ANTLR_OUTPUT_DIR, file)).href); + const DbmlLexer = (await importGenerated('DbmlLexer.js')).default; + const DbmlParser = (await importGenerated('DbmlParser.js')).default; + const converter = new TreeConverter(DbmlParser.ruleNames, DbmlParser.symbolicNames); + + return { + name: 'antlr', + parse (source: string): SpecNode { + const errors: string[] = []; + const listener = { + syntaxError: (_r: unknown, _s: unknown, line: number, column: number, msg: string) => { + errors.push(`${line}:${column} ${msg}`); + }, + reportAmbiguity: () => {}, + reportAttemptingFullContext: () => {}, + reportContextSensitivity: () => {}, + }; + // Decode to code points so that \p{L} sees supplementary-plane letters. + const lexer = new DbmlLexer(new antlr4.CharStream(source, true)); + lexer.removeErrorListeners(); + lexer.addErrorListener(listener); + const parser = new DbmlParser(new antlr4.CommonTokenStream(lexer)); + parser.removeErrorListeners(); + parser.addErrorListener(listener); + parser._errHandler = new runtime.error.BailErrorStrategy(); + const tree = parser.program(); + if (errors.length > 0) throw new Error(errors.join('; ')); + const [program] = flattenSplices(converter.convert(tree)); + return program as SpecNode; + }, + }; +} diff --git a/packages/dbml-parse/__tests__/conformance/harness.ts b/packages/dbml-parse/__tests__/conformance/harness.ts index b7f04a027..95fa0fcc1 100644 --- a/packages/dbml-parse/__tests__/conformance/harness.ts +++ b/packages/dbml-parse/__tests__/conformance/harness.ts @@ -1,8 +1,9 @@ // Shared machinery for the Layer 1 conformance tests. // -// Both parsers run over the same source text: -// - the reference parser (`@dbml/parse` lexer + parser from src/) -// - a parser generated at test time from spec/dbml-syntax.peggy +// The reference parser (`@dbml/parse` lexer + parser from src/) runs over the +// same source text as each spec parser: +// - `peggy`: generated at test time from spec/dbml-syntax.peggy +// - `antlr`: generated at test time from spec/antlr/*.g4 (see antlr.ts) // // They are compared on two things: // 1. acceptance: the reference "accepts" when lexer and parser report no error; @@ -31,9 +32,70 @@ export interface SpecNode { } export interface SpecParser { + // Short notation name used in test titles: 'peggy' or 'antlr'. + name: string; parse (input: string): SpecNode; } +export function tok (kind: string, value: string): SpecToken { + return { token: kind, value }; +} + +type Child = SpecNode | SpecToken | Child[] | null | undefined; + +function flattenChildren (children: Child[]): (SpecNode | SpecToken)[] { + return children.flatMap((c) => { + if (c === null || c === undefined) return []; + if (Array.isArray(c)) return flattenChildren(c); + return [c]; + }); +} + +export function node (kind: string, ...children: Child[]): SpecNode { + return { + kind, + children: flattenChildren(children), + }; +} + +function isIdentifierPrimary (n: SpecNode | SpecToken): n is SpecNode { + if ('token' in n || n.kind !== '') return false; + const variable = n.children[0]; + if ('token' in variable || variable.kind !== '') return false; + const token = variable.children[0]; + return 'token' in token && token.token === ''; +} + +function identifierToken (n: SpecNode): SpecToken { + return (n.children[0] as SpecNode).children[0] as SpecToken; +} + +// parser/utils.ts: convertFuncAppToElem() +// A function application of the form [ [as ]] [] +// is reinterpreted as a nested element declaration. The peggy grammar carries +// its own copy of this rewrite in its actions; the ANTLR converter uses this one. +export function convertFuncAppToElem (callee: SpecNode, args: SpecNode[]): SpecNode | null { + let type = callee; + let rest = args; + if (type.kind === '') { + rest = [type.children[1] as SpecNode, ...rest]; + type = type.children[0] as SpecNode; + } + if (!isIdentifierPrimary(type) || rest.length === 0) return null; + const typeToken = identifierToken(type); + rest = [...rest]; + const body = rest.pop()!; + if (body.kind !== '') return null; + const attributeList = rest.length > 0 && rest[rest.length - 1].kind === '' ? rest.pop() : null; + if (rest.length === 3) { + if (!isIdentifierPrimary(rest[1]) || identifierToken(rest[1]).value.toLowerCase() !== 'as') return null; + return node('', typeToken, rest[0], identifierToken(rest[1]), rest[2], attributeList, body); + } + if (rest.length === 1) return node('', typeToken, rest[0], attributeList, body); + if (rest.length === 0) return node('', typeToken, attributeList, body); + return null; +} + export type Verdict = 'accept' | 'reject'; export interface Outcome { @@ -52,12 +114,23 @@ export interface Comparison { agreeOnTree?: boolean; } -export function loadSpecParser (): SpecParser { +export function loadPeggyParser (): SpecParser { const grammar = readFileSync(GRAMMAR_PATH, 'utf-8'); - return peggy.generate(grammar, { + const generated = peggy.generate(grammar, { output: 'parser', grammarSource: GRAMMAR_PATH, - }) as unknown as SpecParser; + }) as unknown as { parse (input: string): SpecNode }; + return { + name: 'peggy', + parse: (input) => generated.parse(input), + }; +} + +// Both notations of the spec. Kept in a separate module so that the ANTLR +// toolchain is only loaded when needed. +export async function loadSpecParsers (): Promise { + const { loadAntlrParser } = await import('./antlr'); + return [loadPeggyParser(), await loadAntlrParser()]; } function stripAngles (kind: string): string { diff --git a/packages/dbml-parse/__tests__/conformance/properties.test.ts b/packages/dbml-parse/__tests__/conformance/properties.test.ts index 1f5c220b7..ab5cf1c80 100644 --- a/packages/dbml-parse/__tests__/conformance/properties.test.ts +++ b/packages/dbml-parse/__tests__/conformance/properties.test.ts @@ -30,7 +30,7 @@ import { type SpecParser, compare, describeComparison, - loadSpecParser, + loadSpecParsers, } from './harness'; const SCHEMA_RUNS = 500; @@ -45,9 +45,9 @@ function expectAgreement (parser: SpecParser, source: string): void { } } -describe('[conformance] properties', () => { - const parser = loadSpecParser(); +const parsers = await loadSpecParsers(); +describe.each(parsers)('[conformance] properties ($name)', (parser) => { it('agrees on whole schemas', { timeout: 300000, }, () => { diff --git a/packages/dbml-parse/__tests__/conformance/syntax.test.ts b/packages/dbml-parse/__tests__/conformance/syntax.test.ts index e3e22ea7f..259561e78 100644 --- a/packages/dbml-parse/__tests__/conformance/syntax.test.ts +++ b/packages/dbml-parse/__tests__/conformance/syntax.test.ts @@ -12,7 +12,7 @@ import { type Verdict, compare, describeComparison, - loadSpecParser, + loadSpecParsers, } from './harness'; interface KnownDisagreement { @@ -104,8 +104,9 @@ function listInputs (): { key: string; file: string }[] { .sort((a, b) => a.key.localeCompare(b.key)); } -describe('[conformance] syntax', () => { - const parser = loadSpecParser(); +const parsers = await loadSpecParsers(); + +describe.each(parsers)('[conformance] syntax ($name)', (parser) => { const inputs = listInputs(); it('covers every snapshot input', () => { @@ -137,9 +138,7 @@ describe('[conformance] syntax', () => { }); }); -describe('[conformance] recorded disagreements', () => { - const parser = loadSpecParser(); - +describe.each(parsers)('[conformance] recorded disagreements ($name)', (parser) => { PINNED_DISAGREEMENTS.forEach((pinned) => { it(`${pinned.see}: ${JSON.stringify(pinned.source)}`, () => { const result = compare(parser, pinned.source); diff --git a/packages/dbml-parse/eslint.config.ts b/packages/dbml-parse/eslint.config.ts index d73db3b38..35849c899 100644 --- a/packages/dbml-parse/eslint.config.ts +++ b/packages/dbml-parse/eslint.config.ts @@ -22,6 +22,7 @@ export default defineConfig( 'vite.config.ts', 'vite.profile.config.ts', 'vitest.conformance.config.ts', + '__tests__/conformance/generated/*', 'eslint.config.ts', ], }, diff --git a/packages/dbml-parse/package.json b/packages/dbml-parse/package.json index d38b73add..bd42ac340 100644 --- a/packages/dbml-parse/package.json +++ b/packages/dbml-parse/package.json @@ -41,6 +41,8 @@ }, "devDependencies": { "@types/luxon": "^3.7.1", + "antlr-ng": "^1.0.10", + "antlr4": "^4.13.2", "monaco-editor-core": "^0.44.0", "peggy": "^5.1.0", "vite-plugin-no-bundle": "^4.0.0" diff --git a/packages/dbml-parse/vitest.conformance.config.ts b/packages/dbml-parse/vitest.conformance.config.ts index bfd68f8d3..b17586d8e 100644 --- a/packages/dbml-parse/vitest.conformance.config.ts +++ b/packages/dbml-parse/vitest.conformance.config.ts @@ -16,6 +16,8 @@ export default defineConfig({ test: { globals: true, include: ['__tests__/conformance/**/*.test.ts'], + // Compiles spec/antlr/*.g4 into __tests__/conformance/generated (not committed). + globalSetup: ['__tests__/conformance/antlr.setup.ts'], testTimeout: 120000, }, }); diff --git a/yarn.lock b/yarn.lock index 9248d6048..86a30b6e5 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3567,11 +3567,31 @@ ansi-styles@^6.1.0: resolved "https://registry.npmjs.org/ansi-styles/-/ansi-styles-6.2.1.tgz" integrity sha512-bN798gFfQX+viw3R7yrGWRqnrN2oRkEkUjjl4JNn4E8GxxbjtG3FbrEIIY3l8/hrwUwIeCZvi4QuOTP4MErVug== -antlr4@^4.13.1: +antlr-ng@^1.0.10: + version "1.0.10" + resolved "https://registry.yarnpkg.com/antlr-ng/-/antlr-ng-1.0.10.tgz#fdca4f793bc2ff71dc16607f11a4286a8b895ed4" + integrity sha512-fw3NdsQP3dabuZrDhKAMewrBsY5KSAcMrvhWBVDmHYegv5D51pypzCYK1PpjaRVKcVeP/5xKfqJY31TvXACOdA== + dependencies: + antlr4ng "3.0.16" + commander "13.1.0" + stringtemplate4ts "1.0.9" + unicode-properties "1.4.1" + +antlr4@^4.13.1, antlr4@^4.13.2: version "4.13.2" resolved "https://registry.npmjs.org/antlr4/-/antlr4-4.13.2.tgz" integrity sha512-QiVbZhyy4xAZ17UPEuG3YTOt8ZaoeOR1CvEAqrEsDBsOqINslaB147i9xqljZqoyf5S+EUlGStaj+t22LT9MOg== +antlr4ng@3.0.15: + version "3.0.15" + resolved "https://registry.yarnpkg.com/antlr4ng/-/antlr4ng-3.0.15.tgz#8ed2ac92e4b604974775dbac3eeaf6d9ef1965aa" + integrity sha512-VELFqTfcpGI2bj6ScMWuxM3FI6HOsojrgmnw3cCbUtsQ1DNOq32wJsjOt7vLvfIniyyuE1DIYegGcuFmn+jgyw== + +antlr4ng@3.0.16: + version "3.0.16" + resolved "https://registry.yarnpkg.com/antlr4ng/-/antlr4ng-3.0.16.tgz#2e9db4fde07293a4d48c6537a90f9c1a917cbaca" + integrity sha512-DQuJkC7kX3xunfF4K2KsWTSvoxxslv+FQp/WHQZTJSsH2Ec3QfFmrxC3Nky2ok9yglXn6nHM4zUaVDxcN5f6kA== + any-promise@^1.0.0: version "1.3.0" resolved "https://registry.npmjs.org/any-promise/-/any-promise-1.3.0.tgz" @@ -4167,6 +4187,11 @@ combined-stream@^1.0.8: dependencies: delayed-stream "~1.0.0" +commander@13.1.0: + version "13.1.0" + resolved "https://registry.yarnpkg.com/commander/-/commander-13.1.0.tgz#776167db68c78f38dcce1f9b8d7b8b9a488abf46" + integrity sha512-/rFeCpNJQbhSZjGVwO9RFV3xPqbnERS8MmIQzCtD/zl6gpJuV/bMLuN92oG3F7d8oDEHHRrujSXNUr8fpjntKw== + commander@^11.0.0: version "11.1.0" resolved "https://registry.npmjs.org/commander/-/commander-11.1.0.tgz" @@ -4951,6 +4976,11 @@ fast-levenshtein@^2.0.6: resolved "https://registry.npmjs.org/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz" integrity sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw== +fast-printf@1.6.10: + version "1.6.10" + resolved "https://registry.yarnpkg.com/fast-printf/-/fast-printf-1.6.10.tgz#c44ad871726152159d7a903a5af0d65cf3d75875" + integrity sha512-GwTgG9O4FVIdShhbVF3JxOgSBY2+ePGsu2V/UONgoCPzF9VY6ZdBMKsHKCYQHZwNk3qNouUolRDsgVxcVA5G1w== + fast-xml-builder@^1.1.4: version "1.2.0" resolved "https://registry.npmjs.org/fast-xml-builder/-/fast-xml-builder-1.2.0.tgz" @@ -5553,7 +5583,7 @@ hasown@^2.0.4: dependencies: function-bind "^1.1.2" -he@^1.2.0: +he@1.2.0, he@^1.2.0: version "1.2.0" resolved "https://registry.npmjs.org/he/-/he-1.2.0.tgz" integrity sha512-F/1DnUGPopORZi0ni+CvrCgHQ5FyEAHRLSApuYWMmrbSwoN2Mn/7k+Gl38gJnR7yyDZk6WLXwiGod1JOWNDKGw== @@ -6701,6 +6731,11 @@ lru-cache@^8.0.0: resolved "https://registry.npmjs.org/lru-cache/-/lru-cache-8.0.5.tgz" integrity sha512-MhWWlVnuab1RG5/zMRRcVGXZLCXrZTgfwMikgzCegsPnG62yDQo5JnqKkrK4jO5iKqDAZGItAqN5CtKBCBWRUA== +luxon@3.5.0: + version "3.5.0" + resolved "https://registry.yarnpkg.com/luxon/-/luxon-3.5.0.tgz#6b6f65c5cd1d61d1fd19dbf07ee87a50bf4b8e20" + integrity sha512-rh+Zjr6DNfUYR3bPwJEnuwDdqMbxZW7LOQfUN4B54+Cl+0o5zaU9RJ6bcidfDtC1cWCZXQ+nvX8bf6bAji37QQ== + luxon@^3.4.4, luxon@^3.7.2: version "3.7.2" resolved "https://registry.npmjs.org/luxon/-/luxon-3.7.2.tgz" @@ -7782,6 +7817,11 @@ pacote@^15.2.0: ssri "^10.0.0" tar "^6.1.11" +pako@^0.2.5: + version "0.2.9" + resolved "https://registry.yarnpkg.com/pako/-/pako-0.2.9.tgz#f3f7522f4ef782348da8161bad9ecfd51bf83a75" + integrity sha512-NUcwaKxUxWrZLpDG+z/xZaCgQITkA/Dv4V/T6bw7VON6l1Xz/VnrBqrYjZQ12TamKHzITTfOEIYUj48y2KXImA== + parent-module@^1.0.0: version "1.0.1" resolved "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz" @@ -8938,6 +8978,16 @@ string_decoder@~1.1.1: dependencies: safe-buffer "~5.1.0" +stringtemplate4ts@1.0.9: + version "1.0.9" + resolved "https://registry.yarnpkg.com/stringtemplate4ts/-/stringtemplate4ts-1.0.9.tgz#b48c10f10218a6729b32de188d879439959f9b53" + integrity sha512-KYZm2bJlSjynG5Y+L46fkaKBQG6mhV6hb2RBA8dpx3/Vj6G4u7gwXNKYvaN9+QD5sj68/1srtSNDvqEso7MwsQ== + dependencies: + antlr4ng "3.0.15" + fast-printf "1.6.10" + he "1.2.0" + luxon "3.5.0" + "strip-ansi-cjs@npm:strip-ansi@^6.0.1": version "6.0.1" resolved "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz" @@ -9167,6 +9217,11 @@ through@2, "through@>=2.2.7 <3", through@^2.3.4, through@^2.3.6: resolved "https://registry.npmjs.org/through/-/through-2.3.8.tgz" integrity sha1-DdTJ/6q8NXlgsbckEV1+Doai4fU= +tiny-inflate@^1.0.0: + version "1.0.3" + resolved "https://registry.yarnpkg.com/tiny-inflate/-/tiny-inflate-1.0.3.tgz#122715494913a1805166aaf7c93467933eea26c4" + integrity sha512-pkY1fj1cKHb2seWDy0B16HeWyczlJA9/WW3u3c4z/NiWDsO3DOU5D7nhTLE9CF0yXv/QZFY7sEJmj24dK+Rrqw== + tinybench@^2.9.0: version "2.9.0" resolved "https://registry.npmjs.org/tinybench/-/tinybench-2.9.0.tgz" @@ -9391,6 +9446,22 @@ undici-types@~7.8.0: resolved "https://registry.npmjs.org/undici-types/-/undici-types-7.8.0.tgz" integrity sha512-9UJ2xGDvQ43tYyVMpuHlsgApydB8ZKfVYTsLDhXkFL/6gfkp+U8xTGdh8pMJv1SpZna0zxG1DwsKZsreLbXBxw== +unicode-properties@1.4.1: + version "1.4.1" + resolved "https://registry.yarnpkg.com/unicode-properties/-/unicode-properties-1.4.1.tgz#96a9cffb7e619a0dc7368c28da27e05fc8f9be5f" + integrity sha512-CLjCCLQ6UuMxWnbIylkisbRj31qxHPAurvena/0iwSVbQ2G1VY5/HjV0IRabOEbDHlzZlRdCrD4NhB0JtU40Pg== + dependencies: + base64-js "^1.3.0" + unicode-trie "^2.0.0" + +unicode-trie@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/unicode-trie/-/unicode-trie-2.0.0.tgz#8fd8845696e2e14a8b67d78fa9e0dd2cad62fec8" + integrity sha512-x7bc76x0bm4prf1VLg79uhAzKw8DVboClSN5VxJuQ+LKDOVEW9CdH+VY7SP+vX7xCYQqzzgQpFqz15zeLvAtZQ== + dependencies: + pako "^0.2.5" + tiny-inflate "^1.0.0" + unique-filename@^1.1.1: version "1.1.1" resolved "https://registry.npmjs.org/unique-filename/-/unique-filename-1.1.1.tgz" From 93ac0f22e402aaee9b7b9830eb900dbf74874a47 Mon Sep 17 00:00:00 2001 From: Mattias Fornander Date: Tue, 15 Sep 2026 17:51:13 -0600 Subject: [PATCH 10/10] docs(spec): compare the peggy and ANTLR notations Also replace a stray NUL byte in DISAGREEMENTS.md that made git treat the file as binary. Co-Authored-By: Claude Fable 5.1 --- spec/DISAGREEMENTS.md | Bin 5121 -> 5523 bytes spec/README.md | 20 ++++++++++++++++++-- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/spec/DISAGREEMENTS.md b/spec/DISAGREEMENTS.md index 330d5dbc064ebd872f37f63432b092f4721e3d1c..4b9bd9cbc24e256c74bcf7e1f668192124c7b5b8 100644 GIT binary patch delta 464 zcmX|8y-veG43^m13Go2PVnBudF|c)Fk3g!*j(c&EgXA3f97-f4#1lZplOP@rMx4-c z+kIz$fA>227{4cnS`fsV9OQykg|bbjC#yY1Pd;h{A3!$@ffdzrh%nkF$4R#}$DpOm z&u97N-Pr=xtOcMDxPsgRrGO#*18d>5aRj{_Rs`Irs;uboc2|9CRK?3I^@bSP};Ykg^!T3^<}>Rm}T9H M<6xatXxg*kFa0v61ONa4 delta 67 zcmbQN-Ka6aXX0wPiAm~{#Tf&{brg#86%rMaGZORCQx)>_6jC!wGE$2+_cAhZ3M4Qj X=;mZ*rz)f;79}O7r*2-z^_>#{_|O*8 diff --git a/spec/README.md b/spec/README.md index f5a12d3f6..fd408b329 100644 --- a/spec/README.md +++ b/spec/README.md @@ -4,7 +4,7 @@ This directory holds the formal specification of DBML. It is organised in the sa | Layer | What it specifies | Spec artefact | Reference implementation | | --- | --- | --- | --- | -| 1. Generic syntax | Tokens and the element-agnostic syntax tree (`SyntaxNodeKind`) | [`dbml-syntax.peggy`](./dbml-syntax.peggy) | `src/core/lexer/lexer.ts`, `src/core/parser/parser.ts` | +| 1. Generic syntax | Tokens and the element-agnostic syntax tree (`SyntaxNodeKind`) | [`dbml-syntax.peggy`](./dbml-syntax.peggy), with an ANTLR rendering in [`antlr/`](./antlr) | `src/core/lexer/lexer.ts`, `src/core/parser/parser.ts` | | 2. Element semantics | Per-element rules: name shape, body form, allowed settings and sub-elements | `elements/.schema.json` (future PRs) | `src/core/local_modules//validate.ts` | | 3. Interpreted model | The `Database` JSON model produced by `@dbml/core` | `output/database.schema.json` (future PR) | `packages/dbml-core/src/model_structure/*.ts` | @@ -78,10 +78,26 @@ The three trivia-sensitive rules: The grammar produces exactly the `SyntaxNodeKind` set from `src/core/types/nodes.ts`: program, element-declaration, use-declaration, use-specifier, use-specifier-list, attribute, identifier-stream, literal, variable, primary-expression, prefix-expression, infix-expression, postfix-expression, function-expression, function-application, block-expression, list-expression, tuple-expression, group-expression, call-expression, comma-expression, array, wildcard, and the empty node. Postfix-expression is reachable only through error recovery today and has no grammar rule. +## Notations + +The Layer 1 grammar exists in two notations that are kept in agreement by running both through the same conformance tests. This is deliberate: which notation the spec should standardise on is an open question, and the corpus gives an objective way to compare them. + +| | peggy ([`dbml-syntax.peggy`](./dbml-syntax.peggy)) | ANTLR 4 ([`antlr/DbmlLexer.g4`](./antlr/DbmlLexer.g4), [`antlr/DbmlParser.g4`](./antlr/DbmlParser.g4)) | +| --- | --- | --- | +| Model | Scannerless PEG; ordered choice mirrors the reference's recursive descent | Separate lexer and parser; ALL(*) prediction with predicates where the reference commits on one token | +| Trivia | Threaded explicitly through rules (`GapAny`, `GapInline`, `GapNewline`) | Hidden channel, inspected with `getHiddenTokensToLeft()`, close to the reference lexer's trivia model | +| Embedded code | About 116 lines of JavaScript in the initializer plus one-line tree-building actions on most rules; 15 semantic predicates | 71-line members block (50 lines of JavaScript), 17 predicates, 6 actions; no tree-building code | +| Tree shaping | In the grammar | In the test harness (`__tests__/conformance/antlr.ts`, about 170 lines), because ANTLR yields a rule-shaped parse tree | +| Toolchain | `peggy` (dev dependency), compiled in memory at test time | `antlr-ng` (Node port of the ANTLR tool, no Java) generating JavaScript for the `antlr4` runtime that `@dbml/core` already uses; generated in vitest global setup, not committed | +| Unicode | Code points via `[\p{L}]u` classes | Code points via `CharStream(source, true)` | +| Corpus result | 121 of 122 snapshot inputs agree, all property tests pass | Identical | + +Both notations reproduce the same recorded disagreements (see [`DISAGREEMENTS.md`](./DISAGREEMENTS.md)); the pinned cases run against each. + ## Running the conformance tests ```bash yarn workspace @dbml/parse test:conformance ``` -The peggy grammar is compiled in memory at test time; no generated parser is committed. +Both grammars are compiled at test time. The peggy parser is built in memory; the ANTLR parser is generated into `packages/dbml-parse/__tests__/conformance/generated/`, which is ignored by git. No generated parser is committed.