From 79186ce0b9d0e38e0e420c08b7b7420537a7c9cb Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 21 Sep 2026 08:12:49 +0000 Subject: [PATCH 01/15] fix(pages): validate the widgets inside a placeholder block MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A page's widgets live in two AST fields. `Widgets` is the bare body; content addressed to a named layout placeholder is held apart in `Placeholders` (#532). validate.go passed `s.Widgets` alone to all three page validators, so every reference inside a `placeholder X { … }` block — microflow, nanoflow, page, snippet, entity, image — was validated by nothing. That is not an edge shape: `placeholder Main { … }` is what mxcli's own skills, its bug-test examples and its DESCRIBE output write. Measured on a blank Mendix 11.14.0 project, the same button in the two positions: placeholder Main { actionbutton btn (Action: MICROFLOW Mod.NoSuch) } -> ✓ All references valid actionbutton btn (Action: MICROFLOW Mod.NoSuch) (bare body) -> microflow not found: Mod.NoSuch It is the third copy of one walk — validateIconRefs (mendixlabs/mxcli#1008) and forEachWidget each grew the placeholder arm separately, both with a comment saying a missed walk is silent in both directions — so the roots are collected once in allPageWidgets rather than added to a fourth walker later. Control: with allPageWidgets reduced to `return s.Widgets`, the placeholder test fails with the reported symptom (`got []`) and the bare-body control still passes, so the test detects the bug rather than merely agreeing with the fix. A reference that resolves stays silent, which is the direction a widened walk is most likely to get wrong. Refs: mendixlabs/mxcli#1149 Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01ExDoCXFBfqq3R6j2qKx2Kf --- mdl/executor/validate.go | 44 +++++- .../validate_page_placeholder_refs_test.go | 126 ++++++++++++++++++ 2 files changed, 167 insertions(+), 3 deletions(-) create mode 100644 mdl/executor/validate_page_placeholder_refs_test.go diff --git a/mdl/executor/validate.go b/mdl/executor/validate.go index 0dcf5f50c..8e4f59693 100644 --- a/mdl/executor/validate.go +++ b/mdl/executor/validate.go @@ -609,19 +609,21 @@ func validateWithContext(ctx *ExecContext, stmt ast.Statement, sc *scriptContext return mdlerrors.NewNotFound("module", s.Name.Module) } } + // Every widget-bearing field, not just the bare body — see pageWidgets. + pageWidgets := allPageWidgets(s) // Validate widget references (DataSource, Action, Snippet) - if refErrors := validateWidgetReferences(ctx, s.Widgets, sc); len(refErrors) > 0 { + if refErrors := validateWidgetReferences(ctx, pageWidgets, sc); len(refErrors) > 0 { return mdlerrors.NewValidationf("page '%s' has reference errors:\n - %s", s.Name.String(), strings.Join(refErrors, "\n - ")) } // Validate page context tree (parameter/selection/attribute bindings) - if ctxErrors := validatePageContextTree(ctx, s.Parameters, s.Widgets); len(ctxErrors) > 0 { + if ctxErrors := validatePageContextTree(ctx, s.Parameters, pageWidgets); len(ctxErrors) > 0 { return mdlerrors.NewValidationf("page '%s' has context errors:\n - %s", s.Name.String(), strings.Join(ctxErrors, "\n - ")) } // CE1571: a microflow call must be given an argument per parameter — // as a data source and as an action alike (mendixlabs/mxcli#1082). - if argErrors := validateFlowArguments(ctx, s.Parameters, s.Widgets, sc); len(argErrors) > 0 { + if argErrors := validateFlowArguments(ctx, s.Parameters, pageWidgets, sc); len(argErrors) > 0 { return mdlerrors.NewValidationf("page '%s' has argument errors:\n - %s", s.Name.String(), strings.Join(argErrors, "\n - ")) } @@ -1329,3 +1331,39 @@ func validateMicroflowRules(stmt *ast.CreateMicroflowStmt) error { func (sc *scriptContext) recordFlowParams(qualifiedName string, params []ast.MicroflowParam, ret *ast.MicroflowReturnType) { sc.flowParams[strings.ToLower(qualifiedName)] = astFlowSignature(params, ret) } + +// allPageWidgets returns every widget a CREATE PAGE statement carries: the bare +// body AND the content of each `placeholder { … }` block. +// +// A page's widgets live in two fields. `Widgets` is the bare body, which binds +// to Main; content addressed to a named layout placeholder is held apart in +// `Placeholders` (issue #532). The three page validators below were wired to +// `Widgets` alone, so a widget inside a placeholder block was validated by +// nothing — and `placeholder Main { … }` is the shape mxcli's own skills, +// examples and DESCRIBE output all use, so this was the common case rather than +// an edge one. Measured on a blank Mendix 11.14.0 project, the same button +// twice: +// +// placeholder Main { actionbutton btn (Action: MICROFLOW Mod.NoSuchMicroflow) } +// -> ✓ All references valid +// actionbutton btn (Action: MICROFLOW Mod.NoSuchMicroflow) +// -> microflow not found: Mod.NoSuchMicroflow +// +// This is the third time the same walk has been missed in this file's +// neighbourhood: validateIconRefs (mendixlabs/mxcli#1008) and forEachWidget +// both had to grow the placeholder arm separately. Collecting the roots once, +// here, is what stops the fourth. +func allPageWidgets(s *ast.CreatePageStmtV3) []*ast.WidgetV3 { + if len(s.Placeholders) == 0 { + return s.Widgets + } + out := make([]*ast.WidgetV3, 0, len(s.Widgets)) + out = append(out, s.Widgets...) + for _, ph := range s.Placeholders { + if ph == nil { + continue + } + out = append(out, ph.Widgets...) + } + return out +} diff --git a/mdl/executor/validate_page_placeholder_refs_test.go b/mdl/executor/validate_page_placeholder_refs_test.go new file mode 100644 index 000000000..1eab099cd --- /dev/null +++ b/mdl/executor/validate_page_placeholder_refs_test.go @@ -0,0 +1,126 @@ +// SPDX-License-Identifier: Apache-2.0 + +package executor + +import ( + "strings" + "testing" + + "github.com/mendixlabs/mxcli/mdl/ast" + "github.com/mendixlabs/mxcli/mdl/visitor" +) + +// mendixlabs/mxcli#1149, the half underneath the reported one. A page's widgets +// live in TWO fields — `Widgets` is the bare body, and content addressed to a +// named layout placeholder is held apart in `Placeholders` (#532) — and the +// three page validators in validate.go were wired to the first alone. So every +// reference inside a `placeholder X { … }` block was validated by nothing. +// +// That is not an edge shape: `placeholder Main { … }` is what mxcli's own +// skills, its bug-test examples and #1057's own repro all write. Measured on a +// blank Mendix 11.14.0 project, the same button in the two positions: +// +// create or replace page MyFirstModule.PhMf (…) { +// placeholder Main { +// actionbutton btn (Action: MICROFLOW MyFirstModule.NoSuchMicroflow) +// } +// }; -> ✓ All references valid +// +// create or replace page MyFirstModule.BareMf (…) { +// actionbutton btn (Action: MICROFLOW MyFirstModule.NoSuchMicroflow) +// }; -> microflow not found: …NoSuchMicroflow +// +// validateIconRefs (#1008) and forEachWidget each had to grow the same +// placeholder arm on their own; this is the third copy of one walk, which is +// why allPageWidgets collects the roots once. + +// parsePageStmt parses one CREATE PAGE statement through the real parser, so the +// test is about how the AST actually comes out rather than how it is imagined. +func parsePageStmt(t *testing.T, src string) *ast.CreatePageStmtV3 { + t.Helper() + prog, errs := visitor.Build(src) + if len(errs) > 0 { + t.Fatalf("source does not parse: %v", errs) + } + stmt, ok := prog.Statements[0].(*ast.CreatePageStmtV3) + if !ok { + t.Fatalf("parsed %T, want *ast.CreatePageStmtV3", prog.Statements[0]) + } + return stmt +} + +const placeholderButtonPage = `create or replace page Mod.P (Title: 'T', Layout: Atlas_Core.Atlas_Default) { + placeholder Main { + actionbutton btn (Caption: 'Go', Action: MICROFLOW Mod.NoSuchMicroflow) + } +}` + +const bareButtonPage = `create or replace page Mod.P (Title: 'T', Layout: Atlas_Core.Atlas_Default) { + actionbutton btn (Caption: 'Go', Action: MICROFLOW Mod.NoSuchMicroflow) +}` + +// The reported escape: a reference inside a placeholder block must be resolved. +func TestPageRefs_PlaceholderContentIsValidated(t *testing.T) { + ctx, _ := newMockCtx(t) + stmt := parsePageStmt(t, placeholderButtonPage) + + // The parser really does hold this content apart — if it ever stops doing + // so, this test would pass for the wrong reason. + if len(stmt.Widgets) != 0 || len(stmt.Placeholders) == 0 { + t.Fatalf("expected the button in Placeholders and nothing in Widgets; got %d bare, %d placeholders", + len(stmt.Widgets), len(stmt.Placeholders)) + } + + errs := validateWidgetReferences(ctx, allPageWidgets(stmt), newScriptContext()) + if len(errs) != 1 || !strings.Contains(errs[0], "Mod.NoSuchMicroflow") { + t.Fatalf("a microflow reference inside `placeholder Main { … }` was not resolved; got %v", errs) + } +} + +// CONTROL: the same button in the bare body, which has always been reported. +// It is what makes the test above a statement about the placeholder walk rather +// than about whether the check works at all. +func TestPageRefs_BareBodyStillValidated(t *testing.T) { + ctx, _ := newMockCtx(t) + stmt := parsePageStmt(t, bareButtonPage) + + errs := validateWidgetReferences(ctx, allPageWidgets(stmt), newScriptContext()) + if len(errs) != 1 || !strings.Contains(errs[0], "Mod.NoSuchMicroflow") { + t.Fatalf("the bare-body control stopped working; got %v", errs) + } +} + +// CONTROL: a page whose references all resolve stays clean through the merged +// walk — otherwise the fix is indistinguishable from reporting everything. +func TestPageRefs_PlaceholderContentThatResolvesStaysSilent(t *testing.T) { + ctx, _ := newMockCtx(t) + sc := newScriptContext() + sc.microflows["Mod.NoSuchMicroflow"] = true // created earlier in the same script + + stmt := parsePageStmt(t, placeholderButtonPage) + if errs := validateWidgetReferences(ctx, allPageWidgets(stmt), sc); len(errs) != 0 { + t.Errorf("a resolvable reference inside a placeholder was reported: %v", errs) + } +} + +// allPageWidgets must merge, not replace: a page with both a bare body and a +// placeholder block has to have BOTH walked. Returning only one set would make +// the test above pass while re-opening the hole from the other side. +func TestAllPageWidgets_MergesBothFields(t *testing.T) { + stmt := parsePageStmt(t, `create or replace page Mod.P (Title: 'T', Layout: Atlas_Core.Atlas_Default) { + container bare { dynamictext d1 (Content: 'x') } + placeholder Sidebar { container inPh { dynamictext d2 (Content: 'y') } } +}`) + + var names []string + for _, w := range allPageWidgets(stmt) { + names = append(names, w.Name) + } + if len(names) != 2 { + t.Fatalf("got roots %v, want the bare container and the placeholder's", names) + } + joined := strings.Join(names, ",") + if !strings.Contains(joined, "bare") || !strings.Contains(joined, "inPh") { + t.Errorf("got roots %v, want both `bare` and `inPh`", names) + } +} From f5961faace17ccbf5c88ed9ca061645c6b4b8356 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 21 Sep 2026 08:13:02 +0000 Subject: [PATCH 02/15] fix(pages): resolve the image a static or dynamic image widget names MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit mendixlabs/mxcli#1149 reports that a Selection helper's custom states cannot hold a StaticImageViewer. The authoring half landed with #1057 — on main the reporter's page describes back with three `staticimage` lines and their `Image:` clauses, and re-running the description reports `Unchanged page`. What was still missing is the rest of the request, "a reference to an image from an image collection": the name could be written and nothing resolved it. widgetRefCollector keyed the image reference on the widget TYPE, so the pluggable `image` widget was collected and the two widgets #1057 had just given the same property were not — `staticimage`'s `Image` and `dynamicimage`'s `DefaultImage`. Measured on a blank Mendix 11.14.0 project: mxcli check sh.mdl -p app.mpr --references -> Check passed! mxcli exec sh.mdl -p app.mpr -> Created page … mx check -> [error] [CE1613] "The selected image 'Atlas_UI_Resources.Atlas_Icons.checkbox_checked' no longer exists." at Static image 'imgAll' (x3, one per state) Fixed, the same script reports all three at check time. With references that resolve, `check` passes and mxbuild reports no CE1613 — only CE0582, Mendix's own React-client deprecation of static images, which #1057's bug test already records as not an mxcli defect. It is a table now (imageRefProps), not a condition on one type name: the set grew and the condition did not, and `validate_widgets.go` already accepted these keys while DESCRIBE already emitted them, so only the resolver disagreed. Adding a widget that names an image means adding a row. Control: with the table reduced to the pluggable widget alone, the three symptom tests fail with the reported symptom (the reference accepted, `got 0 errors`) while both controls — a resolvable image under every spelling, and an unrelated `Image` property on a widget that is not in the table — still pass. Closes mendixlabs/mxcli#1149 Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01ExDoCXFBfqq3R6j2qKx2Kf --- .../fix-issue/findings/mdl-executor.jsonl | 1 + CHANGELOG.md | 4 + ...idgets-1149-image-reference-resolution.mdl | 101 +++++++++++ mdl/executor/helpers.go | 33 +++- .../validate_widget_image_ref_collect_test.go | 167 ++++++++++++++++++ 5 files changed, 302 insertions(+), 4 deletions(-) create mode 100644 mdl-examples/bug-tests/widgets-1149-image-reference-resolution.mdl create mode 100644 mdl/executor/validate_widget_image_ref_collect_test.go diff --git a/.claude/skills/fix-issue/findings/mdl-executor.jsonl b/.claude/skills/fix-issue/findings/mdl-executor.jsonl index 0c67963f0..057547703 100644 --- a/.claude/skills/fix-issue/findings/mdl-executor.jsonl +++ b/.claude/skills/fix-issue/findings/mdl-executor.jsonl @@ -662,3 +662,4 @@ {"area": "mdl/backend/pagemutator", "date": "2026-09-20", "symptom": "`alter page … { set DataSource = DATABASE Mod.Entity on dvCust; }` passes `check`, prints `Altered page …` with exit 0, and leaves the DataView with no datasource: `describe page` renders `dataview dvCust {` with the property gone, and the only other signal is CE7007 at `mx check`", "cause": "`serializeDataSourceBson` mapped every `*pages.DatabaseSource` to a `Forms$DataViewSource` — the *context* source — with the entity in `EntityRef` and `SourceVariable` left null. A DATABASE source has no single stored shape: the widget holding it decides (`Forms$ListViewXPathSource` on a list view, `CustomWidgets$CustomWidgetXPathSource` on a pluggable widget, `Forms$GridXPathSource` on a grid), and a DATA VIEW has no database form at all — which is why CREATE PAGE's `dataViewSourceToGen` already refused that pairing while SET wrote it silently", "file": "`mdl/backend/pagemutator/mutator.go` (`SetWidgetDataSource`, new `databaseSourceRefusal`, `serializeDataSourceBson`)", "insight": "**The \"gone entirely\" in the report was DESCRIBE, not the document.** The DataSource was present and well-formed BSON; `parseContextSource` returns nil for a `Forms$DataViewSource` with no `SourceVariable`, so the reader rendered nothing. Chasing a deleted property would have been the wrong hunt — diff the stored BSON before believing a describe-shaped symptom. **The refusal belongs in the mutator, not the validator**: `validateAlterSetProperties` dry-runs the real setter against a `pagemutator.Probe()` copy, so one refusal makes `check -p --references` and `exec` agree by construction; a second copy of the rule in the validator is the duplicate-resolver drift CLAUDE.md warns about. **Refusing beat rebuilding the shapes here** — writing ListViewXPathSource in raw BSON would duplicate `listViewSourceToGen` in a second currency, and REPLACE already reaches the real builder. **Two remedies, not one**: on a data view `use REPLACE` is a dead end (CREATE PAGE refuses it too), so the message names the sources a data view can take; on a list view REPLACE genuinely works, so it names REPLACE. Getting that backwards sends the author in a circle. **Same generalisable shape as #855/#1101**: when SET and REPLACE express different vocabularies for one property, SET is a whitelist extended one bug report at a time. **`make check-mdl` runs `mxcli check` WITHOUT `-p`**, so a document-dependent refusal cannot be a `.fail.mdl` — it would be reported as a negative test that unexpectedly passed. Write the passing shape and comment the refused statements, as #1063 does. Measured on two copies of a real 11.13.0 app: faulty → `Check passed!`, exit 0, `mx check` 1 error CE7007 at Data view 'dvCust'; fixed → both refuse with exit 1, datasource unchanged, `mx check` 0 errors. Tests `mdl/backend/pagemutator/mutator_datasource_test.go`, `mdl/executor/validate_alter_set_test.go`; example `mdl-examples/bug-tests/1032-alter-page-set-database-datasource.mdl`. upstream #1032", "refs": ["#855", "#1032"], "ce": ["CE7007"]} {"area": "mdl/executor", "date": "2026-09-20", "symptom": "`describe page` → `exec` over a **Studio Pro-authored** page silently drops six things, `mx check` 0 errors throughout. The one that matters: `IsPasswordBox True → False` — a **password field round-trips into a plaintext text box**, and describe → rename → exec is mxcli's copy operation. Also `Validation.Expression` blanked, a DataView's `ReadOnlyStyle Text → Control`, `PopupCloseAction` wiped, and two typed-array markers", "cause": "Four different causes behind one symptom, which is why triage came first: (1) IsPasswordBox — model and writer carried it, nothing parsed it, nothing emitted it; (2) Validation — `widgetValidationToGen()` wrote a DEFAULT EMPTY Forms$WidgetValidation over whatever was stored, on five widget types; (3) ReadOnlyStyle — wired for CheckBox only, and a DataView's draws no MDL-WIDGET07 warning because `staticWidgetKnownProps` is deliberately a union across widget types; (4) PopupCloseAction — `pageToGen` wrote \"\" unconditionally. Plus ParameterMappings/OutputMappings markers", "file": "`mdl/executor/cmd_pages_describe_parse.go` + `_output.go` (extract/emit), `cmd_pages_builder_v3_widgets.go` (consume), `cmd_pages_builder_v3.go`, `mdl/visitor/visitor_page_v3.go`, `mdl/ast/ast_page_v3.go`, `sdk/pages/*`, `mdl/backend/modelsdk/widget_write.go` + `page_write.go`, `mdl/executor/validate_widgets.go` (describe vocabulary)", "insight": "**Triage the layer before writing anything** — describer / grammar / builder have different fixes and this one issue had all three. The quickest probe is to run the property through `mxcli check`: MDL-WIDGET07 names an unrecognised one, and *silence is not acceptance* — the known-props list is a union across widget types, so a DataView's ReadOnlyStyle passed check and was dropped anyway. **Emit an expression QUOTED, not bracketed**: `[...]` is the XPath-constraint spelling and `propertyValueV3` parses it as an ARRAY, so `GetStringProp` yields \"\" — the emitter's own unit test was green while the real round trip still lost the value (storage form is not input form). **Measure the default before keeping it**: a DataView's ReadOnlyStyle is Control on 47 of 56, never Inherit, so the 'obvious' Inherit that every other input widget uses would have been wrong. Markers likewise measured, not assumed: ParameterMappings is marker 2 on 220 of 220 lists in every parent type, OutputMappings present on 91 of 91 — and an EMPTY list needs `MandatoryListMarkers` since `RegisterListMarker` keys on a child that is not there. Result 17 → 9 differences, the 9 being ako/mxcli#549", "refs": ["#550", "#541", "#549", "#490"]} {"area": "mdl/executor", "date": "2026-09-20", "symptom": "MDL-PAGEARG01 refused a list widget's OWN row action: `datagrid dg (DataSource: DATABASE M.E, onClick: SHOW_PAGE M.Edit(E: $currentObject))` was rejected at `check` with \"widget `dg` is not inside a data view, list view or grid row\" \u2014 and since exec refuses a script whose check errors, the slice could not be applied at all. On a `listview` the message contradicted itself. mxbuild 11.14.0 accepts the stored pages at 0 errors.", "cause": "The #1029 guard judged EVERY widget's own action in the context its PARENT supplies: `argContextForSubtreeOf` returns the parent context for a childless widget and `validate_widgets.go` passed the inherited `argCtx` to `validateShowPageArguments`. Right for a button, wrong for the widget that ESTABLISHES the context \u2014 a list widget's onClick is row-scoped, so the row it renders is the context object. Added `argContextForOwnAction`: a widget that binds a source of its own supplies the context for its own action; a source in a shape the pass cannot read (the bare-entity shorthand) degrades to UNKNOWN so the guard stands down rather than refusing what it cannot prove is discarded.", "file": "`mdl/executor/cmd_pages_showpage_args.go` (argContextForOwnAction, argContextForSubtreeOf), `mdl/executor/validate_widgets.go`", "insight": "**A false refusal costs more than a missing rule now that exec refuses on a check error** \u2014 the blast radius is 'this project cannot be built with this mxcli', not 'a warning is noisy'. Two things would have caught it before release: judging the rule against the widget kinds it NAMES in its own message (the listview refusal reads 'lvA is not inside a \u2026 list view'), and running it against mxbuild rather than against intuition. The mxbuild run paid for itself twice: it also showed that `DataSource: M.E` (bare-entity shorthand) on a datagrid is silently dropped, so that case is CE0488 + a REAL CE1571 \u2014 the stand-down is still correct, but the shorthand case must not be written into a bug test as mxbuild-clean (#576). Control the fix with the widget kinds STILL refused (a foreign variable, a sibling button beside the grid), or it is indistinguishable from deleting the rule.", "refs": ["#552", "#576", "mendixlabs/mxcli#1029", "#939"]} +{"area": "mdl/executor", "date": "2026-09-21", "symptom": "A page's image-collection reference passed `mxcli check --references` and failed the build. Reported as \"no MDL syntax for a StaticImageViewer inside a Selection helper custom state\" — the authoring half was already closed by #1057; what was left is that nothing RESOLVED the name it made writable. Measured on a blank Mendix 11.14.0 project: `staticimage imgAll (Image: 'Atlas_UI_Resources.Atlas_Icons.checkbox_checked')` in a custom state -> check passed, exec created the page, `mx check` -> 3x CE1613 \"The selected image … no longer exists.\"", "cause": "TWO independent holes, and either alone leaves the reported script unchecked. (1) widgetRefCollector keyed the image reference on the widget TYPE — `if w.Type == \"image\"` — so the pluggable widget was collected and `staticimage` (which #1057 had just given the SAME `Image:` property) and `dynamicimage`'s `DefaultImage` were not; replaced with an imageRefProps table. (2) A page's widgets live in two AST fields: `Widgets` is the bare body, `Placeholders` holds `placeholder X { … }` content (#532). validate.go passed `s.Widgets` alone to validateWidgetReferences, validatePageContextTree AND validateFlowArguments, so EVERY reference inside a placeholder block — microflow, nanoflow, page, snippet, entity, image — was validated by nothing; added allPageWidgets to collect both roots once.", "file": "`mdl/executor/helpers.go` (widgetRefCollector.collectFromWidget, imageRefProps), `mdl/executor/validate.go` (allPageWidgets)", "insight": "**When a capability gets a new spelling, grep for who RESOLVES the old one.** #1057 added `Image:` to a second and third widget and moved on; the resolver keyed on the type name, so the new spellings were unchecked from the day they shipped. A property list and a resolver list that describe the same property are two copies — `validate_widgets.go` already accepted `Image`/`DefaultImage` for these widgets and DESCRIBE already emitted them, and only the resolver disagreed. **The placeholder hole is the more useful lesson: it was the THIRD copy of one walk.** validateIconRefs (#1008) and forEachWidget had each grown the `Placeholders` arm separately, with a comment saying a missed walk is silent both ways — and the three validators next door still had not. When a fix is 'add the missing arm to this walker', the question is how many walkers there are; collect the roots once instead. **Do not reason about a bug report from the issue text alone when the version is older than the fix** — the reported symptom did not reproduce on main at all, and running the reporter's own script end to end is what turned 'already fixed, close it' into two real defects. **Control both directions**: a reference that resolves must stay silent, because a walker that can suddenly see a whole new region of the tree is as likely to report correct scripts as broken ones.", "refs": ["mendixlabs/mxcli#1149", "mendixlabs/mxcli#1057", "mendixlabs/mxcli#1008", "#532"]} diff --git a/CHANGELOG.md b/CHANGELOG.md index a63e97ace..42dc23e81 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -38,6 +38,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). ### Fixed +- **A page's image-collection reference passed `mxcli check --references` and failed the build** (mendixlabs/mxcli#1149) — `staticimage imgAll (Image: 'Atlas_UI_Resources.Atlas_Icons.checkbox_checked')` in a Selection helper's custom state checked clean, exec'd cleanly and then came back as `[error] [CE1613] "The selected image … no longer exists."`, once per state. The report asks for syntax, but the syntax landed with #1057 — describe emits the three `staticimage` lines and re-running the description reports `Unchanged page`, measured on a blank 11.14.0 project. What was missing is that nothing resolved the name #1057 had made writable. + + Two holes, and fixing either alone leaves the reported script unchecked. The image reference was collected by **widget type** (`if w.Type == "image"`), so the pluggable widget was resolved and the two widgets #1057 gave the same property — `staticimage`'s `Image` and `dynamicimage`'s `DefaultImage` — were not; it is a table now, so adding a widget that names an image means adding a row. And a page's widgets live in **two** AST fields: `Widgets` is the bare body, while `placeholder { … }` content is held apart in `Placeholders` (#532). All three page validators walked the first alone, so **every** reference inside a placeholder block — microflow, nanoflow, page, snippet, entity, image — was validated by nothing. Measured, the same button in the two positions: inside `placeholder Main` → `✓ All references valid`; in the bare body → `microflow not found`. That is the shape mxcli's own skills, examples and DESCRIBE output write, so it was the common case rather than an edge one, and it is the third copy of one walk — `validateIconRefs` (#1008) and `forEachWidget` had each grown the placeholder arm separately — so the roots are now collected once. + - **`raise error;` on a microflow's main flow passed check and exec, then failed the build** (mendixlabs/mxcli#1030) — with `[error] [CE0710] "The main flow cannot join an error flow or end in an error event."`, one per microflow. Mendix's error event *re-raises the error being handled*, so it is legal only where an error is in scope: inside an `on error { … }` handler. Studio Pro will not draw the connection from the normal flow to an error event; mxcli could, and did. It is now **MDL084**, at error severity, so `exec`'s pre-flight refuses the script with nothing written (`--no-check` still applies it, for reproducing the build failure). The report's own diagnosis — a trailing End event appended because `RaiseErrorStmt` never set the builder's "ends with return" flag — is not the cause: `isTerminalStmt` has treated it as a terminator all along, and the graph mxcli builds for `raise error;` is exactly the one the report asks for (start event, error event, one sequence flow, no trailing End event, no outgoing flow), pinned by a test. No wiring makes a main-flow error event legal, which is why the fix is a refusal rather than a builder change. diff --git a/mdl-examples/bug-tests/widgets-1149-image-reference-resolution.mdl b/mdl-examples/bug-tests/widgets-1149-image-reference-resolution.mdl new file mode 100644 index 000000000..493761320 --- /dev/null +++ b/mdl-examples/bug-tests/widgets-1149-image-reference-resolution.mdl @@ -0,0 +1,101 @@ +-- mendixlabs/mxcli#1149 — the images a Selection helper's custom states show +-- could be written, and nothing resolved their names. +-- +-- The report opens on the symptom #1057 had already closed: DESCRIBE PAGE +-- printing the widget as a note instead of MDL. On main that round-trips — +-- measured on a blank Mendix 11.14.0 project, the page below describes back with +-- the three `staticimage` lines and their `Image:` clauses, and re-running the +-- description reports `Unchanged page`. What the report asks for underneath is +-- the part that was still missing: "a reference to an image from an image +-- collection". You could write the reference. Nothing checked it. +-- +-- TWO defects, and fixing either alone leaves the reported script unchecked. +-- +-- 1. imageRefErrors was wired for the pluggable `image` widget and keyed on the +-- widget TYPE, so the widget #1057 had just given the same reference — +-- `staticimage`, which is what Studio Pro puts in these slots — was never +-- collected. Nor was a dynamic image's `DefaultImage` fallback. +-- +-- 2. A page's widgets live in two AST fields: `Widgets` is the bare body, and +-- content addressed to a named layout placeholder is held apart in +-- `Placeholders` (#532). The three page validators walked the first alone, so +-- EVERY reference inside a `placeholder X { … }` block — microflow, nanoflow, +-- page, snippet, entity, image — escaped `mxcli check --references`. That is +-- the shape mxcli's own skills, its examples and its DESCRIBE output write. +-- +-- Measured on a blank Mendix 11.14.0 project (mxbuild 11.14.0): +-- +-- pre-fix check --references -> Check passed! +-- exec -> Created page … +-- mx check -> [error] [CE1613] "The selected image +-- 'Atlas_UI_Resources.Atlas_Icons.checkbox_checked' no longer +-- exists." at Static image 'imgAll' (x3, one per state) +-- +-- fixed check --references -> image collection not found: +-- Atlas_UI_Resources.Atlas_Icons (referenced as …) x3 +-- +-- control (same button, the two positions, pre-fix): +-- placeholder Main { actionbutton … MICROFLOW Mod.NoSuch } +-- -> ✓ All references valid +-- actionbutton … MICROFLOW Mod.NoSuch (bare body) +-- -> microflow not found: Mod.NoSuch +-- +-- Atlas_UI_Resources.Atlas_Icons is not invented for the test: it is the kind of +-- name a reader guesses from the icon they want. A blank app has seven image +-- collections and none of them is it. + +-- The reported page, with references that DO resolve. `mxcli check +-- --references` passes and mxbuild reports no CE1613. +-- +-- mxbuild 11.14.0 does report CE0582 ("Widget static image is not supported in +-- React client") for a static image ANYWHERE, these slots included. That is +-- Mendix's own deprecation advice, not an mxcli defect — the same note +-- widgets-1057-staticimage-slot-round-trip.mdl carries. A new page should use +-- the pluggable `image` widget. + +create or replace page MyFirstModule.ImageRefsResolve ( + Title: 'Image references resolve', Layout: Atlas_Core.Atlas_Default +) { + placeholder Main { + selectionhelper sh1 (renderStyle: 'custom') { + customallselected slot1 { + staticimage imgAll (Image: 'MyFirstModule.Images.gallery') + } + customsomeselected slot2 { + staticimage imgSome (Image: 'MyFirstModule.Images.combobox') + } + customnoneselected slot3 { + staticimage imgNone (Image: 'MyFirstModule.Images.dg2') + } + } + } +}; + +-- The same three references outside a placeholder block, plus a dynamic image's +-- fallback. Defect 1 on its own: before the fix this passed `check --references` +-- too, with the widgets in the position the walk DOES reach — which is what +-- separates the two halves. + +create or replace page MyFirstModule.ImageRefsBareBody ( + Title: 'Image references, bare body', Layout: Atlas_Core.Atlas_Default +) { + staticimage imgBare (Image: 'MyFirstModule.Images.microflow') + image imgPluggable (Image: 'MyFirstModule.Images.nanoflow') +}; + +-- CONTROL for defect 2, in the direction that matters: a reference that resolves +-- must stay silent now that the placeholder arm is walked. A validator that +-- reports everything it can suddenly see is not a fix — it is a fix and a +-- regression, and the escape was silent for long enough that plenty of correct +-- scripts have only ever been checked in the bare-body position. + +create or replace page MyFirstModule.PlaceholderRefsAreFine ( + Title: 'Placeholder references resolve', Layout: Atlas_Core.Atlas_Default +) { + placeholder Main { + actionbutton btnHome ( + Caption: 'Home', + Action: SHOW_PAGE MyFirstModule.ImageRefsResolve + ) + } +}; diff --git a/mdl/executor/helpers.go b/mdl/executor/helpers.go index d0945f761..fb1d46909 100644 --- a/mdl/executor/helpers.go +++ b/mdl/executor/helpers.go @@ -353,10 +353,12 @@ func (c *widgetRefCollector) collectFromWidget(w *ast.WidgetV3) { c.snippets = append(c.snippets, snippet) } - // An image widget's Image collection entry. Only the image widget spells it - // this way; `Image` on any other widget type is not a collection reference. - if strings.EqualFold(w.Type, "image") { - if img := strings.TrimSpace(w.GetStringProp("Image")); img != "" { + // An image-collection entry, Module.Collection.Image. Which property holds + // one is a property of the WIDGET, so the table decides — `Image` on a + // widget that is not in it is not a collection reference and must not be + // resolved as one. + for _, key := range imageRefProps[strings.ToLower(w.Type)] { + if img := strings.TrimSpace(w.GetStringProp(key)); img != "" { c.images = append(c.images, img) } } @@ -830,3 +832,26 @@ func buildConstantQualifiedNames(ctx *ExecContext) (map[string]bool, bool) { // in it; treat that as "cannot tell" rather than "the project has none". return result, len(consts) > 0 } + +// imageRefProps maps a widget type to the properties that hold an +// image-collection entry (Module.Collection.Image). +// +// It is a table rather than a condition on the widget type because the set grew +// and the condition did not. `Image:` was wired for the pluggable `image` +// widget, and mendixlabs/mxcli#1057 then gave the SAME reference to +// `staticimage` — the widget Studio Pro puts in a Selection helper's custom +// states, which is the whole reason #1057 exists — and to `dynamicimage`'s +// fallback. Neither was collected, so a name that does not resolve passed +// `mxcli check --references` and failed the build (mendixlabs/mxcli#1149): +// +// [error] [CE1613] "The selected image 'Atlas_UI_Resources.Atlas_Icons. +// checkbox_checked' no longer exists." at Static image 'imgAll' +// +// Adding a widget that names an image means adding a row here. The keys are the +// ones `validate_widgets.go` already accepts for these widgets and DESCRIBE +// already emits, so the two lists say the same thing about the same property. +var imageRefProps = map[string][]string{ + "image": {"Image"}, + "staticimage": {"Image"}, + "dynamicimage": {"DefaultImage"}, +} diff --git a/mdl/executor/validate_widget_image_ref_collect_test.go b/mdl/executor/validate_widget_image_ref_collect_test.go new file mode 100644 index 000000000..3839b68c3 --- /dev/null +++ b/mdl/executor/validate_widget_image_ref_collect_test.go @@ -0,0 +1,167 @@ +// SPDX-License-Identifier: Apache-2.0 + +package executor + +import ( + "strings" + "testing" + + "github.com/mendixlabs/mxcli/mdl/ast" + "github.com/mendixlabs/mxcli/mdl/backend/mock" + "github.com/mendixlabs/mxcli/mdl/types" + "github.com/mendixlabs/mxcli/model" +) + +// mendixlabs/mxcli#1149. The reporter wants to put the images a Selection +// helper's custom states show into MDL — #1057 made `staticimage` authorable and +// gave it `Image: 'Module.Collection.Image'`, and this is the half that came +// with it: nothing resolves that name. +// +// imageRefErrors was wired for the pluggable `image` widget and keyed on the +// widget TYPE, so the widget #1057 had just given the same reference was not +// collected — nor was a dynamic image's `DefaultImage` fallback. Measured on a +// blank Mendix 11.14.0 project, the reporter's own shape: +// +// selectionhelper sh1 (renderStyle: 'custom') { +// customallselected slot1 { +// staticimage imgAll (Image: 'Atlas_UI_Resources.Atlas_Icons.checkbox_checked') +// } +// … +// } +// +// mxcli check sh.mdl -p SH1149.mpr --references -> Check passed! +// mxcli exec sh.mdl -p SH1149.mpr -> Created page … +// mx check -> [error] [CE1613] "The selected +// image 'Atlas_UI_Resources.Atlas_Icons.checkbox_checked' no longer exists." +// at Static image 'imgAll' (x3, one per state) +// +// There is no such collection in a blank app (`show image collections` lists +// seven, none of them Atlas_UI_Resources) — the name is what a reader guesses +// from the icon they want, and mxcli had nothing to say about it until the +// build. Same defect class as #1057 itself: check clean, exec clean, build red. + +// imageCollectionCtx is a project holding exactly one image collection with one +// image in it, so a reference can be right or wrong in a measurable way. +func imageCollectionCtx(t *testing.T) *ExecContext { + t.Helper() + mod := mkModule("MyFirstModule") + ic := &types.ImageCollection{ + BaseElement: model.BaseElement{ID: nextID("ic")}, + ContainerID: mod.ID, + Name: "Images", + Images: []types.Image{{Name: "gallery"}}, + } + h := mkHierarchy(mod) + withContainer(h, ic.ContainerID, mod.ID) + + mb := &mock.MockBackend{ + IsConnectedFunc: func() bool { return true }, + ListImageCollectionsFunc: func() ([]*types.ImageCollection, error) { return []*types.ImageCollection{ic}, nil }, + } + ctx, _ := newMockCtx(t, withBackend(mb), withHierarchy(h)) + return ctx +} + +// imageRefWidget is one widget of the given type carrying one image reference +// under the given property key. +func imageRefWidget(widgetType, propKey, ref string) []*ast.WidgetV3 { + return []*ast.WidgetV3{{ + Name: "img1", + Type: widgetType, + Properties: map[string]any{propKey: ref}, + }} +} + +// The reported symptom, at the layer it lives in. +func TestWidgetRefs_StaticImageReferenceIsResolved(t *testing.T) { + ctx := imageCollectionCtx(t) + + errs := validateWidgetReferences(ctx, + imageRefWidget("staticimage", "Image", "Atlas_UI_Resources.Atlas_Icons.checkbox_checked"), + newScriptContext()) + if len(errs) != 1 { + t.Fatalf("a staticimage naming an image collection the project does not have was accepted — "+ + "that is CE1613 at build time; got %d errors: %v", len(errs), errs) + } + if !strings.Contains(errs[0], "Atlas_UI_Resources.Atlas_Icons") { + t.Errorf("the message should name the missing collection: %s", errs[0]) + } +} + +// The same reference on a dynamic image's fallback, which #1057's sibling commit +// made authorable on the same day and left unresolved for the same reason. +func TestWidgetRefs_DynamicImageDefaultImageIsResolved(t *testing.T) { + ctx := imageCollectionCtx(t) + + errs := validateWidgetReferences(ctx, + imageRefWidget("dynamicimage", "DefaultImage", "MyFirstModule.Images.NoSuchImage"), + newScriptContext()) + if len(errs) != 1 { + t.Fatalf("a dynamicimage fallback naming an image the collection does not hold was "+ + "accepted; got %d errors: %v", len(errs), errs) + } + if !strings.Contains(errs[0], "NoSuchImage") { + t.Errorf("the message should name the missing image: %s", errs[0]) + } +} + +// The reporter's actual shape: the image sits inside a pluggable widget's child +// slot, not at the top of the page. Collection recurses through Children, so +// this passes for free once the type is recognised — and it is the case the +// issue is about, so it is asserted rather than assumed. +func TestWidgetRefs_StaticImageInsideACustomStateIsResolved(t *testing.T) { + ctx := imageCollectionCtx(t) + + widgets := []*ast.WidgetV3{{ + Name: "sh1", + Type: "selectionhelper", + Properties: map[string]any{"renderStyle": "custom"}, + Children: []*ast.WidgetV3{{ + Name: "customallselected1", + Type: "customallselected", + Properties: map[string]any{}, + Children: imageRefWidget("staticimage", "Image", "MyFirstModule.Images.NoSuchIcon"), + }}, + }} + + errs := validateWidgetReferences(ctx, widgets, newScriptContext()) + if len(errs) != 1 || !strings.Contains(errs[0], "NoSuchIcon") { + t.Fatalf("an unresolvable image inside a Selection helper custom state was accepted; got %v", errs) + } +} + +// CONTROL. An image that DOES resolve must stay silent under every spelling — +// otherwise the fix is indistinguishable from forbidding the feature, which is +// what #1057 made possible in the first place. +func TestWidgetRefs_ResolvableImagesStaySilent(t *testing.T) { + cases := []struct{ widgetType, propKey string }{ + {"staticimage", "Image"}, + {"dynamicimage", "DefaultImage"}, + {"image", "Image"}, + } + for _, tc := range cases { + t.Run(tc.widgetType+"."+tc.propKey, func(t *testing.T) { + ctx := imageCollectionCtx(t) + errs := validateWidgetReferences(ctx, + imageRefWidget(tc.widgetType, tc.propKey, "MyFirstModule.Images.gallery"), + newScriptContext()) + if len(errs) != 0 { + t.Errorf("an image that exists was reported: %v", errs) + } + }) + } +} + +// CONTROL. `Image` is only an image-collection reference on the widgets that +// spell it that way. A widget carrying an unrelated `Image` property must not be +// dragged into the check — reporting it would break scripts that are correct. +func TestWidgetRefs_ImagePropOnAnotherWidgetIsNotAnImageReference(t *testing.T) { + ctx := imageCollectionCtx(t) + + errs := validateWidgetReferences(ctx, + imageRefWidget("container", "Image", "not.a.reference"), + newScriptContext()) + if len(errs) != 0 { + t.Errorf("a non-image widget's Image property was resolved as an image collection entry: %v", errs) + } +} From 832e9c8023b86cc12231552e702df1cda54cb2e4 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 21 Sep 2026 12:42:11 +0000 Subject: [PATCH 03/15] feat(settings): manage workflow groups via MDL MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Workflow Groups (App Settings > Workflows > Groups) could not be created, listed or modified through mxcli, so a scripted project setup had to stop and open Studio Pro to define the buckets a user task's group targeting selects from. They are project settings — peers of UserEntity and DefaultTaskParallelism — and were missing from ALTER SETTINGS WORKFLOWS and DESCRIBE SETTINGS. Closes mendixlabs/mxcli#272. alter settings workflows add [or modify] group 'Approvers' (Description: '…'); alter settings workflows modify group 'Approvers' (Description: '…'); alter settings workflows remove group 'Approvers'; show workflow groups; The four verbs match the LANGUAGE forms rather than the issue's proposed ADD/ALTER/DROP, because the two clauses share a grammar rule and the same reasoning: a group is identified by its name alone, so it is addressed the way a language's code is. ADD OR MODIFY is the upsert DESCRIBE SETTINGS emits, so a described project replays onto itself quietly. Pinned against real Mendix documents rather than a guess: - Settings$WorkflowGroup stores Name and Description and nothing else (modelsdk/gen, generated/metamodel and mendixmodelsdk 4.115.0 all agree), so Description is the only option and an unknown key is refused rather than written into a property the type does not declare. - The Groups typed-array marker is 2, not the 3 every other settings child list uses — measured on a blank 11.13.0 project, whose empty list is `Groups: [2]`. - The version floor is 11.2.0, not the 11.6 the issue quotes: both the type and the groups property read `introduced: "11.2.0"` in the Model SDK's StructureVersionInfo, and an 11.1.0 project's workflows settings part has no Groups key at all while an 11.13.0 one has it. Release notes date the feature's GA; the metamodel decides whether the document loads. Two things the implementation turns on: - The write is an overlay on the preserved raw part, so adding Groups to the model and the read path is not enough. Without the overlay call the executor reports "Added workflow group: Auditors (3 group(s))" and writes nothing, with mx check clean either way — the shape the enabled-language list already went through. A stored element that did not decode is refused rather than dropped (ADR-0005). - A group's element $ID is its identity in the runtime database. Booting the app materialises one system$workflowgroup row per entry whose modelguid is that $ID read as a .NET GUID (measured on 11.13.0: stored bytes 7c5fc4cf05c3394fa6718a3d4603e9a5, row cfc45f7c-c305-4f39-a671-8a3d4603e9a5), so a description edit updates the row in place instead of orphaning every group membership under a perfectly valid model. Verified end to end on Mendix 11.13.0: exec writes the document, mx check is 0 errors, the booted runtime materialises both groups, a replay is elided (no unit mtime moves) and describe -> exec round-trips including quote escaping. Controls recorded: deleting the one overlay call reproduces the silent no-write verbatim; reverting the option key from identifierOrKeyword to IDENTIFIER fails the visitor test with "mismatched input 'Description' expecting IDENTIFIER" — Description is an MDL keyword, so the feature's only option was a parse error. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01TDVJRtXZmaXQGENThpcmDV --- .../fix-issue/findings/mdl-backend.jsonl | 1 + .../fix-issue/findings/mdl-grammar.jsonl | 1 + .../skills/fix-issue/findings/mdl-other.jsonl | 1 + .../skills/mendix/project-settings/SKILL.md | 43 ++- cmd/mxcli/syntax/features_misc.go | 27 ++ .../src/reference/settings/alter-settings.md | 40 ++- .../src/reference/settings/show-settings.md | 11 + docs/01-project/MDL_QUICK_REFERENCE.md | 7 +- .../14-project-settings-examples.mdl | 44 +++ mdl/ast/ast_query.go | 3 + mdl/ast/ast_settings.go | 12 + mdl/backend/modelsdk/settings_read.go | 26 ++ mdl/backend/modelsdk/settings_write.go | 25 +- .../settings_write_workflow_groups_test.go | 197 +++++++++++++ mdl/executor/cmd_settings.go | 24 ++ mdl/executor/cmd_settings_workflow_groups.go | 247 ++++++++++++++++ .../cmd_settings_workflow_groups_test.go | 278 ++++++++++++++++++ mdl/executor/executor_query.go | 2 + mdl/grammar/domains/MDLCatalog.g4 | 3 + mdl/grammar/domains/MDLSettings.g4 | 40 ++- mdl/settingsoverlay/settingsoverlay.go | 50 ++++ mdl/settingsoverlay/settingsoverlay_test.go | 131 +++++++++ mdl/visitor/visitor_query.go | 3 + mdl/visitor/visitor_settings.go | 53 +++- mdl/visitor/visitor_settings_test.go | 139 +++++++++ model/types.go | 20 ++ sdk/versions/mendix-11.yaml | 4 + sdk/versions/registry_test.go | 30 ++ 28 files changed, 1433 insertions(+), 29 deletions(-) create mode 100644 mdl/backend/modelsdk/settings_write_workflow_groups_test.go create mode 100644 mdl/executor/cmd_settings_workflow_groups.go create mode 100644 mdl/executor/cmd_settings_workflow_groups_test.go diff --git a/.claude/skills/fix-issue/findings/mdl-backend.jsonl b/.claude/skills/fix-issue/findings/mdl-backend.jsonl index d75355329..5f26332fd 100644 --- a/.claude/skills/fix-issue/findings/mdl-backend.jsonl +++ b/.claude/skills/fix-issue/findings/mdl-backend.jsonl @@ -121,3 +121,4 @@ {"area": "mdl/backend", "date": "2026-09-20", "symptom": "`describe page` → `exec` over a **Studio Pro-authored** page reports `Replaced page`, not `Unchanged` — the rebuild is not semantically equal to what was stored, so ADR-0008's elision cannot fire and the unit churns in version control on every re-run. `mx check` is 0 errors either way. Measured on ako/TestApp Rules.RuleAction_NewEdit at 11.14.0: fourteen differences", "cause": "Four independent classes, all 'the rebuild writes a constant where Studio Pro stores a value': (1) `pageToGen` hardcoded Autofocus/CanvasWidth/CanvasHeight; (2) save_changes/cancel_changes/close_page/delete_object never wrote `DisabledDuringExecution`, and save_changes wrote `SyncAutomatically` true; (3) `AttributeRef.EntityRef` emitted only on the navigated branch; (4) `Forms$PageVariable` had only the one name field set, so the other five keys were never marked dirty", "file": "`mdl/backend/modelsdk/page_write.go` (`carryStoredPageHeader`, `bsonInt`), `widget_write.go` (four action cases + two `RegisterTypeDefaults`), `modelsdk/codec/defaults.go` + `encoder.go` (new `FalseFields`)", "insight": "**mxcli round-tripping its own output proves NOTHING about this class** — measured: the MDL bug-test reports `Unchanged` on the unfixed build too, because mxcli writes the page and mxcli describes it, so its constants agree with themselves. The reference must be a Studio Pro document; a committed CI fixture only works if it is one. **A population selected by name can confirm whatever it excluded**: the first sweep filtered `$Type` on `endswith(\"ClientAction\")`, got a tidy 'True on 82 of 82', and so missed `Forms$NoAction` (False on 83 of ~7,300) and `Forms$MicroflowAction` (5 of 81) — scan by the PROPERTY, not by a name pattern. **Hardcoded looked safe and was not**: CanvasWidth takes seven distinct values across 67 pages and the hardcoded 1200 matched 4, so a round trip moved the canvas of 63. **The int width bit this fix once**: Studio Pro stores both canvas dimensions as int64 while the gen setter takes int32, so the natural `.(int32)` assertion matched nothing — and the first unit test passed anyway because its own fixture wrote int32, i.e. the test encoded the assumption under test (bson-numeric-width). Prefer `TypeDefaults` over patching each construction site: `Forms$PageVariable` is built in three places. Remaining after the fix: 1 of 14, a pluggable-widget Object property — CE0463 territory, deliberately out of scope", "refs": ["#541", "#529"]} {"area": "mdl/backend", "date": "2026-09-20", "symptom": "An access rule on an entity carrying `AutoOwner` (or `AutoChangedBy`) made mxbuild report the whole module as **CE0066** \"Entity access is out of date\" \u2014 and `UPDATE SECURITY`, the documented repair for exactly that error, printed `Reconciled 1 access rule(s) in module Mod` and left the error standing. Four lines reproduce it on a clean production-security app: `alter entity M.Fab add attribute Owner: AutoOwner;` + `update security;`. The original reporter bisected 9 entities and 36 rules one grant at a time behind a ~40s `mx check` to find it, because CE0066 names only the module.", "cause": "mxcli wrote a MemberAccess for the implicit `System.owner` / `System.changedBy` association, in TWO places that had to agree: the GRANT handler (`cmd_security_write.go`) and `ReconcileMemberAccesses`. Mendix maintains those members itself and treats a rule naming one as out of date. The audit DATE members were already known to work this way (issuetracker #20) \u2014 the owner/changedBy pair was assumed to be the opposite case because they are associations rather than attributes, and Mendix really does add them implicitly. Fixed by writing no entry for any of the four, and by REMOVING a stored one in the reconcile (an explicit case before the foreign-module branch, which otherwise preserves `System.*` forever on the grounds that System is not loaded).", "file": "`mdl/backend/modelsdk/domainmodel_security_write.go` (isAuditMemberRef, ReconcileMemberAccesses), `mdl/executor/cmd_security_write.go`", "insight": "**The decisive probe was removing the entry, not adding anything.** CE0066 says 'out of date', which reads as 'something is missing' and sends you looking for a member to add; the model had one too many. A build flag (`MXCLI_PROBE_NO_SYSOWNER`) that dropped the entry took the module from CE0066 to 0 errors in one mxbuild run and settled it. The same repo's earlier finding had already written the rule down \u2014 *'Ask mxbuild what it wants instead of inferring symmetry'* \u2014 and this defect is that exact inference, made in the same file for the sibling members. **A fix here is not done when the new writes are correct**: `update security` exists to repair a project an older mxcli damaged, so the reconcile has to remove the entry, not merely stop adding it. Measured separately: a stale `System.owner` entry survived even after the flag was turned off, because `!assocRefBelongsTo` preserved it as an unverifiable foreign-module reference.", "refs": ["#554", "#524", "issuetracker #20"]} {"area": "mdl/backend", "date": "2026-09-20", "symptom": "`DROP ENTITY` left every CROSS-MODULE association pointing at the deleted entity in place. Dropping the local BY-ID (FROM) end made mxbuild 11.14.0 unable to LOAD the project: `System.AggregateException \u2026 (The given key '' was not present in the dictionary.)` at `StreamingBsonUnitReader.ResolvePostponedProperties()` \u2014 no CE code, no document named, so the obvious reading is 'the project is corrupt, restore from git'. Dropping the BY-NAME (TO) end is milder and still wrong: CE1613 at the cross-module association. `show associations` shows a raw GUID where the parent entity should be.", "cause": "`removeAssocsReferencing` swept `dm.AssociationsItems()` and asserted `*genDm.Association` per item, so the SEPARATE `CrossAssociations` collection was never looked at. Fixed with `removeCrossAssocsReferencing`, matching BOTH ends because a cross-module association addresses them differently \u2014 FROM by element id (local), TO by qualified name (another module) \u2014 called in DeleteEntity locally and in its cascade over the other domain models.", "file": "`mdl/backend/modelsdk/domainmodel_alter.go` (removeCrossAssocsReferencing, DeleteEntity)", "insight": "**Reported against a view entity; nothing about it was view-entity specific.** The reporter met it dropping view entities (whose associations are DERIVED from OQL, so there is no CREATE ASSOCIATION to undo) and filed it that way. The first probe \u2014 a view entity and its source entity in the SAME module \u2014 did not reproduce at all, and that negative is the useful one: it says the variable is cross-module, not view-ness. A plain `create association A.X from A.X to B.Y` plus `drop entity A.X` reproduces the identical crash. Two lessons: when a repro fails, vary the dimension the report did not mention before doubting the report, and treat a collection-typed `.(*T)` assertion in a cascade as a place where a sibling type hides. mxbuild's diagnostic distinguishes the two ends for free \u2014 a dangling 16-byte pointer is a LOAD crash, a dangling qualified name is CE1613 \u2014 so testing only one end proves half the fix.", "refs": ["#553", "#556"]} +{"area": "mdl/backend", "date": "2026-09-21", "symptom": "`alter settings workflows add group 'Auditors'` reports \"Added workflow group: Auditors (3 group(s))\" and writes nothing \u2014 `show workflow groups` still lists 2, and `mx check` is 0 errors either way", "cause": "`UpdateProjectSettings` overlays the workflows part field by field onto the PRESERVED raw part, so a child LIST that nothing rebuilds is carried through from disk unchanged. Adding `Groups` to the semantic model and to the read path is not enough; the write needs `settingsoverlay.WorkflowGroups(ws, rawPart)`. Identical shape to the enabled-language list the same function already documents", "file": "`mdl/backend/modelsdk/settings_write.go` + `mdl/settingsoverlay/settingsoverlay.go` (`WorkflowGroups`)", "insight": "For anything under Settings$ProjectSettings, the executor's success message proves NOTHING \u2014 it reports the in-memory model, and the overlay is where a list quietly fails to land. Assert on the re-read document, not the handler's output. Two more things a reference project settles in one dump and a guess gets wrong: the `Groups` typed-array marker is 2, not the 3 every other settings child list uses (`ArrayMarker` preserves a stored one, but the fallback matters on a fresh list), and the element's `$ID` is the RUNTIME's identity \u2014 a booted 11.13.0 app keys `system$workflowgroup.modelguid` on it, byte-identical once the .NET GUID field order is undone, so re-minting it on a description edit would orphan every group membership with a perfectly valid model. Control: deleting the one overlay call reproduces the symptom verbatim. mendixlabs/mxcli#272", "refs": ["mendixlabs/mxcli#272"]} diff --git a/.claude/skills/fix-issue/findings/mdl-grammar.jsonl b/.claude/skills/fix-issue/findings/mdl-grammar.jsonl index fd5e9c69d..4c4b92ad0 100644 --- a/.claude/skills/fix-issue/findings/mdl-grammar.jsonl +++ b/.claude/skills/fix-issue/findings/mdl-grammar.jsonl @@ -57,3 +57,4 @@ {"area": "mdl/grammar", "date": "2026-09-08", "symptom": "Adding lexer tokens for a new feature broke an unrelated, previously-passing MDL example: `editable: never` on a list view stopped parsing after NEVER became a keyword", "cause": "A new lexer token steals every existing use of that word as an identifier or property value unless it is also added to the `keyword` rule in MDLSettings.g4. NEVER, ONLINE, SYNC and PRESERVE were added for offline sync; NEVER was already a real page property value", "file": "`mdl/grammar/domains/MDLSettings.g4` (keyword rule)", "insight": "Before adding a lexer token, grep the examples for that word as a value or name — the collision is with EXISTING scripts, so nothing in the new feature's own tests can find it. TestKeywordRuleCoverage catches the omission but only asserts the rule LISTS the token; add a test that the word still parses as an identifier, which is the property that actually matters. Here the two failures had one cause: the coverage test named the tokens and check-mdl named the victim file, and the file name (maint2-editable-never-create-page.mdl) said which word", "refs": ["PROPOSAL_offline_sync_configuration.md"]} {"area": "mdl/grammar", "date": "2026-09-15", "symptom": "Re-executing `describe workflow` output failed with `mismatched input 'boundary' expecting ';'` for any user task, call microflow or wait for notification that has two or more boundary events.", "cause": "formatBoundaryEvents emits `boundary event timer '…' { … }` per event (boundaryEventKeyword includes the prefix), and the syntax topic documents that per-clause form, but MDLWorkflow.g4 had `(BOUNDARY EVENT workflowBoundaryEventClause+)?` — one keyword, then clauses.", "fix": "All four sites accept `(BOUNDARY EVENT workflowBoundaryEventClause ((BOUNDARY EVENT)? workflowBoundaryEventClause)*)?`, so both the per-clause and the shared form parse; the visitor is unchanged.", "file": "mdl/grammar/domains/MDLWorkflow.g4", "insight": "A round trip that ends in `diff describe-1 describe-2` is vacuous when the exec in between fails: the second describe reads the unchanged document and matches. It reported IDENTICAL here while the exec had died on a parse error that a grep filter hid. Assert the exec itself — zero parse errors and a rewrite verb — before diffing. The integration round-trip tests had the same blind spot: they compare describe output but never feed it back to the parser, so a grammar/describer disagreement on a construct with more than one instance could not be seen. A test that re-parses describe output (TestWorkflowDescribe_TwoBoundaryEventsReparse) is the cheap guard."} {"area": "mdl/grammar", "date": "2026-09-18", "symptom": "Lint rule SEC005 reports \"strict mode is disabled\" and MDL has no statement that turns it on — the rule's own suggestion said \"not settable via MDL\". A lint rule with no remedy, recorded on the reporting project as the one finding left Open", "cause": "StrictMode was read everywhere and written nowhere: `security_read.go` reads it, `show security` prints it, the Starlark rule lints it, and `ProjectSecurity.SetStrictMode` existed in gen and was never called. `alterProjectSecurityStatement` had three variants (LEVEL, DEMO USERS, GUEST ACCESS) and no fourth", "file": "`mdl/grammar/MDLLexer.g4` + `domains/MDLSecurity.g4` + `domains/MDLSettings.g4` (keyword rule), `mdl/ast/ast_security.go`, `mdl/visitor/visitor_security.go`, `mdl/executor/cmd_security_write.go`, `mdl/backend/security.go`, `mdl/backend/modelsdk/security_write.go`, `mdl/backend/mock/mock_security.go`, `.claude/lint-rules/sec_strict_mode.star`", "insight": "**Writing a property gen merely offers is the trap; this is not one.** StrictMode is declared by BOTH generated sources and mxcli already reads it from real projects, which is the evidence that separates it from the Layout placeholder properties that make a document Studio Pro cannot open. **The AST field must be a POINTER** — a bare bool would disable strict mode on every DEMO USERS toggle, since \"said nothing\" and \"asked for off\" would be the same value (a test pins this). New tokens STRICT and MODE both go in the parser's `keyword` rule: `mode` is an entirely plausible attribute name and a keyword left out of that rule silently breaks every model already using the word (`TestKeywordRuleCoverage` catches it; a parse test pins it too). **Update the lint rule's suggestion in the same change** — a remedy that still says \"Studio Pro only\" leaves the finding exactly as unhelpful as before. No level-dependent refusal was added: the model stores StrictMode independently of SecurityLevel, and the rule already scopes its own advice to Production", "refs": ["#526"], "rules": ["SEC005"]} +{"area": "mdl/grammar", "date": "2026-09-21", "symptom": "A new settings option list keyed on `IDENTIFIER` makes the feature's ONLY option a parse error: `alter settings workflows add group 'Approvers' (Description: '\u2026')` \u2192 \"mismatched input 'Description' expecting IDENTIFIER\"", "cause": "`Description` is an MDL lexer keyword (DESCRIPTION, from the security statements), so it never matches IDENTIFIER. The rule was copied from `languageOption`, whose keys (CheckCompleteness, CustomDateFormat\u2026) all happen to be plain identifiers \u2014 so the pattern looked safe and was not", "file": "`mdl/grammar/domains/MDLSettings.g4` (`settingsItemOption`) + `mdl/visitor/visitor_settings.go` (`collectSettingsItemOptions`)", "insight": "Any `( key: value )` option list must key on `identifierOrKeyword`, not IDENTIFIER, and the visitor must read it with `unquoteIdentifier(ctx.IdentifierOrKeyword().GetText())`. Before writing one, grep MDLLexer.g4 for each key you intend to accept \u2014 the check costs seconds and the failure lands on the single statement the feature exists for. Copying an existing option rule proves nothing about your key set. Control: reverting the rule to IDENTIFIER fails TestAlterSettings_WorkflowGroup with exactly that message. mendixlabs/mxcli#272", "refs": ["mendixlabs/mxcli#272"]} diff --git a/.claude/skills/fix-issue/findings/mdl-other.jsonl b/.claude/skills/fix-issue/findings/mdl-other.jsonl index c7ea61b16..2bc072fa2 100644 --- a/.claude/skills/fix-issue/findings/mdl-other.jsonl +++ b/.claude/skills/fix-issue/findings/mdl-other.jsonl @@ -68,3 +68,4 @@ {"area": "mdl/catalog", "date": "2026-09-17", "symptom": "A microflow that runs only as an **entity event handler** is reported as unused from three directions at once: `show callers of Mod.ACT_Order_Validate` says `(no callers found)`, `CATALOG.GRAPH_DEAD_ASSETS` lists it, and `mxcli lint` emits `[QUAL004] ... is not called from anywhere.` with the suggestion **\"Remove if unused\"** \u2014 on a microflow that runs on every commit. Reported as 32 dead of 36 handlers across 24 entities", "cause": "`mdl/catalog` touched `Entity.EventHandlers` in exactly one place and threw the list away: `hasEventHandlers = 1` in `builder_modules.go`. No `refs` row was ever emitted, and no table held the handlers, so the reference graph had no ENTITY -> MICROFLOW edge for them. The `calculate` edge two lines below in `buildReferences` is the same shape and was already there, which is why the infrastructure looked complete", "file": "`mdl/catalog/builder_entity_events.go` (new), `mdl/catalog/builder_references.go` (`RefKindEvent` + `extractEventHandlerRefs`), `mdl/catalog/tables.go` (`entity_event_handlers_data` + view, `CatalogSchemaVersion` 11->12), `mdl/catalog/catalog.go` (`Tables()`), `mdl/catalog/builder.go` (field + build step), `mdl/catalog/builder_graph.go` (`graphRefKinds`), `mdl/executor/cmd_search.go` (`callerRefKinds`), `.claude/lint-rules/orphaned_elements.star`", "insight": "**The third consumer of a new RefKind is a schema version, not a list.** Beyond the three kind lists the scheduled-event fix named (`callerRefKinds`, `graphRefKinds`, the QUAL004 rule), a new edge needs `CatalogSchemaVersion` bumped: refs are only written by REFRESH CATALOG FULL and `NewFromFile` applies the schema with CREATE TABLE IF NOT EXISTS, so without the bump an existing `.mxcli/catalog.db` gains the empty table and keeps serving the pre-fix edge set \u2014 the wrong answer, from a cache, after the fix shipped. `migrateIfSchemaMismatch` drops and rebuilds on a mismatch (verified by hand-editing catalog_meta back to '11'). **A flag is a missing table wearing a value**: `HasEventHandlers` and `NavigationProfile.OfflineEntityCount` are the same defect, and the fix is the same pair \u2014 rows for what it does, an edge for whether it is reachable. Do not encode the detail in the kind: eight kinds (`before_commit`, `after_delete`, ...) would enter every consumer's list to say one thing, so the moment/event go in the table and the edge stays one `event`. **The control has to be the binary, not the test**: stubbing `extractEventHandlerRefs` to emit nothing and rebuilding reproduced `(no callers found)` + 2 dead microflows on the same project, which is what proves the assertion detects something. mendixlabs/mxcli#1127; repro `mdl-examples/bug-tests/catalog-1127-entity-event-handler-refs.mdl`", "refs": ["mendixlabs/mxcli#1127"]} {"area":"mdl/catalog","date":"2026-09-18","symptom":"A Starlark lint rule written from the bundled write-lint-rules skill matches zero rows and reports a clean pass — or, with an allowlist, inverts into flagging everything (138 of 282 ACT_ microflows on one real project, 49% false positives)","cause":"The skill's example tables are the only documentation of the lint API and nothing tied them to the values the catalog emits. action_type listed Mendix BSON *storage* names (CreateChangeAction, CommitAction, ShowFormAction, CloseFormAction, ShowHomeFormAction) against a catalog that labels an action with its SDK name via getMicroflowActionType; source_type/target_type/element_type/access_type were lower-cased against an upper-case vocabulary; data_type was lower-cased against TitleCase from AttributeType.GetTypeName","file":"`.claude/skills/mendix/write-lint-rules/SKILL.md` (six rows), `mdl/catalog/builder_references.go` (RefObject* constants + RefSourceObjectTypes/RefTargetObjectTypes), `mdl/catalog/builder_permissions.go` (PermissionElement*/AccessType* constants), test `mdl/catalog/lint_rule_doc_vocabulary_test.go`","insight":"**Fix the documentation the rule author reads, not just the rule that was reported.** The identical defect was found and fixed in CONV010's allowlist a month earlier (finding 2026-08-17, pinned by lint_rule_vocabulary_test.go) — and recurred, because the *source* the author copied from was never corrected. A rule pinned to the labeller and a doc that is not is one fix, not two. **Check the sibling rows before believing the report's scope**: element_type, access_type and data_type had the same lower-casing and nobody had reported them; data_type ('string' vs 'String') is the most-used filter in a lint rule, so it was the most expensive one. **A doc value is only pinnable against a named vocabulary**, so the fix is half refactor: the emitters' scattered ALL-CAPS literals became RefObject*/PermissionElement*/AccessType* constants with published lists, mirroring the SourceObjectTypes precedent already in this package. action_type needs no list — the label IS the Go type name (%T), so the test reads the isMicroflowAction marker methods out of sdk/microflows/microflows_actions.go with go/ast. **Three legs, not one, when proving a filter fix**: old values -> 7/7 flagged, corrected -> 0, corrected-minus-one -> exactly that one. Leg C is the control; without it a silent rule and a correct rule both report zero, which is the bug itself","refs":["mendixlabs/mxcli#1027"],"rules":["CONV010"]} {"area":"mdl/executor","date":"2026-09-18","symptom":"A page parameter passed as an argument to a nanoflow/microflow BUTTON action is not wired — Studio Pro reports CE1571 \"No argument has been selected for parameter 'X' and no default is available\" on opening the page, while `mx check`, `mxcli check --references` and `mxcli lint` are all clean. Reported as an asymmetry: of two arguments, the one matching the enclosing dataview's DataSource 'works' and the other does not","cause":"Mendix stores a flow argument in one of TWO slots of Forms$MicroflowParameterMapping / Forms$NanoflowParameterMapping: a reference to a page parameter, snippet parameter or page variable goes in `Variable` as a Forms$PageVariable; a literal or expression goes in `Expression`. mxcli only ever wrote `Expression: \"$Name\"`, which binds nothing. The read side was wrong in the mirror image — the three action describers and flowSourceArgs looked for a `Name` key on that sub-document, which Forms$PageVariable does not have","file":"`sdk/pages/pages_widgets_action.go` (VariableKind on both mapping types), `mdl/executor/cmd_pages_flow_args.go` (new: classifyFlowArgValue + pageVariableArgValue), `mdl/executor/cmd_pages_builder_v3.go` (3 of the 4 copies of the $-rule), `mdl/backend/modelsdk/widget_write.go` (bindParameterMappingValue), `mdl/executor/cmd_pages_describe_output.go` + `cmd_pages_describe_datasource.go` (read)","insight":"**The reported asymmetry is a red herring — both arguments were written identically and NEITHER was bound.** Studio Pro supplies a default for the one that is the dataview's object and reports the other; 'and no default is available' in CE1571 says exactly that. Time spent on why $Dto worked is wasted. **mxbuild is not a detector here**: `mx check` on the reported project is 0 errors before AND after the fix, so the usual two-copies-of-a-real-project run proves nothing and the reporter is right that it only shows in Studio Pro. **Get the reference from a Marketplace .mpk — it contains a whole Studio Pro-authored `project.mpr`**: `mxcli marketplace download --output x.mpk && unzip -o x.mpk project.mpr`, then `mxcli bson dump` it. A blank app is useless for this (every mapping list in it is empty); Workflow Commons 4.11.0 gave 101 flow parameter mappings, of which 95 bind through Variable and 6 through Expression — and all 6 of those are Boolean literals, so the $-prefixed Expression mxcli wrote occurs ZERO times. `marketplace install` refuses that package (javasource path guard), so extract rather than install. **The PageVariable slot follows what the name refers to** (PageParameter 20, SnippetParameter 58, Widget 17) — a snippet is the COMMON case, not the corner, and `paramScope` is the right oracle because it holds only entity-typed parameters, which is the same set Mendix binds this way. **Leave $currentObject alone**: no reference for the bare form was measured and show_page already depends on the context object being inferred (MDL-PAGEARG01), so changing it on a guess risks the case that works. **The read bug hid the write bug**: describe printed `Action: microflow M.F` with no arguments for Studio Pro content, so a round-trip looked lossless and the missing binding never showed up as a diff","refs":["mendixlabs/mxcli#1140","mendixlabs/mxcli#835"],"ce":["CE1571"]} +{"area": "mdl/versions", "date": "2026-09-21", "symptom": "A version gate copied from the issue text (\"Workflow Groups are GA from Mendix 11.6\") is wrong by four minors", "cause": "Mendix's release notes date the FEATURE's general availability; the metamodel floor is when the type and its property were introduced, and that is what decides whether the document loads. `Settings$WorkflowGroup` and `WorkflowsProjectSettingsPart.groups` are both `introduced: \"11.2.0\"`", "file": "`sdk/versions/mendix-11.yaml` (`workflows.groups`)", "insight": "The arbiter for a metamodel floor is the Model SDK's own StructureVersionInfo: `npm pack mendixmodelsdk && tar xzf \u2026 && grep -n '' package/src/gen/.js`, then read BOTH the class's `versionInfo.introduced` and its `properties..introduced` \u2014 a property can arrive later than its type. Release notes, proposal text and a number already written down in this repo are all downstream of it (same trap as mendixlabs/mxcli#1121). Corroborate it against two real projects rather than trusting one source: `mxcli new` at a version either side of the floor and diff the document's keys \u2014 an 11.1.0 workflows settings part has no `Groups` key at all, an 11.13.0 one carries `Groups: [2]`, which also proves the refusal is right rather than over-cautious (writing the property below the floor would be inventing a key). mendixlabs/mxcli#272", "refs": ["mendixlabs/mxcli#272"]} diff --git a/.claude/skills/mendix/project-settings/SKILL.md b/.claude/skills/mendix/project-settings/SKILL.md index ce8f6e6fd..cd32d8e89 100644 --- a/.claude/skills/mendix/project-settings/SKILL.md +++ b/.claude/skills/mendix/project-settings/SKILL.md @@ -13,7 +13,7 @@ Use this skill when the user wants to: - Change the after-startup or before-shutdown microflows - Modify hash algorithms, Java versions, or rounding modes - View or modify language settings -- Configure workflow settings (user entity, parallelism) +- Configure workflow settings (user entity, parallelism, groups) ## Commands @@ -230,6 +230,47 @@ alter settings workflows DefaultTaskParallelism = 3; ``` +### Workflow Groups (Mendix 11.2+) + +The named buckets under App Settings ▸ Workflows ▸ Groups that a user task's +group targeting selects from. + +```sql +alter settings workflows add group 'Approvers' (Description: 'Primary approval group'); +alter settings workflows add group 'Reviewers'; + +-- the upsert, and what `describe settings` emits +alter settings workflows add or modify group 'Approvers' (Description: 'Approves budget requests'); + +-- changes only the options it names +alter settings workflows modify group 'Reviewers' (Description: 'Second-line review'); + +alter settings workflows remove group 'Reviewers'; + +show workflow groups; +``` + +Four things worth knowing: + +- **`Description` is the only option.** A `Settings$WorkflowGroup` stores `Name` + and `Description` and nothing else, so there is no identifier to set and the + **name is the group's identity** — which is what MODIFY and REMOVE address, and + why a second group differing only in case is refused rather than created. +- **The version floor is 11.2, not 11.6.** `Settings$WorkflowGroup` and the + `groups` property were both introduced in the 11.2 metamodel; Mendix's release + notes call the feature GA in 11.6, which is a different question from whether + the document loads. On an earlier version the statement is refused with the + floor. +- **An edit keeps the group's runtime identity.** Booting the app materialises one + `System.WorkflowGroup` row per entry, keyed on the group's element id — so + changing a description updates that row in place rather than replacing it, and + the group memberships and assigned user tasks survive. +- **Nothing in the model points at a group.** A user task targets groups through a + microflow or an XPath returning `System.WorkflowGroup` objects, never by a + reference to the settings entry, so REMOVE has no dangling reference to check + for. What it does mean is that the removed group's runtime row stops being + maintained while the user tasks already assigned to it keep their association. + ## Common Patterns ### PostgreSQL Configuration diff --git a/cmd/mxcli/syntax/features_misc.go b/cmd/mxcli/syntax/features_misc.go index 6ee7ed8e7..ce5f2bf9d 100644 --- a/cmd/mxcli/syntax/features_misc.go +++ b/cmd/mxcli/syntax/features_misc.go @@ -374,6 +374,7 @@ create or modify translations in Administration for nl_NL ( "database type", "constant override", "language", "add language", "remove language", "enable language", "translations", "optimistic locking", "concurrency", "lost update", + "workflow group", "workflow groups", "add group", "task assignment", }, Syntax: `ALTER SETTINGS MODEL = ; ALTER SETTINGS CONFIGURATION '' = , ...; @@ -385,6 +386,9 @@ ALTER SETTINGS LANGUAGE ADD OR MODIFY '' [(...)]; ALTER SETTINGS LANGUAGE MODIFY '' (CheckCompleteness: true, ...); ALTER SETTINGS LANGUAGE REMOVE ''; ALTER SETTINGS WORKFLOWS UserEntity = ''; +ALTER SETTINGS WORKFLOWS ADD [OR MODIFY] GROUP '' [(Description: '')]; +ALTER SETTINGS WORKFLOWS MODIFY GROUP '' (Description: ''); +ALTER SETTINGS WORKFLOWS REMOVE GROUP ''; CREATE [OR MODIFY] CONFIGURATION '' [ = , ...]; DROP CONFIGURATION '';`, Example: `ALTER SETTINGS MODEL AfterStartupMicroflow = 'Module.MF_Startup'; @@ -436,6 +440,29 @@ ALTER SETTINGS LANGUAGE REMOVE 'de_DE'; -- removing it would strip work the statement does not name. Say it on purpose -- with: create or replace translations for ( ); +-- WORKFLOW GROUPS are the named buckets under App Settings > Workflows > Groups +-- that a user task's group targeting selects from. Mendix 11.2+ (the metamodel +-- floor for Settings$WorkflowGroup — the release notes' "GA in 11.6" is a +-- different question from whether the document loads). +ALTER SETTINGS WORKFLOWS ADD GROUP 'Approvers' (Description: 'Primary approval group'); +ALTER SETTINGS WORKFLOWS ADD GROUP 'Reviewers'; +ALTER SETTINGS WORKFLOWS MODIFY GROUP 'Reviewers' (Description: 'Second-line review'); +ALTER SETTINGS WORKFLOWS REMOVE GROUP 'Reviewers'; +SHOW WORKFLOW GROUPS; + +-- Description is the ONLY option: a Settings$WorkflowGroup stores Name and +-- Description and nothing else, so there is no identifier to set and the NAME is +-- the group's identity — which is what MODIFY and REMOVE address, and why adding +-- a second group differing only in case is refused. ADD OR MODIFY is the upsert +-- and what DESCRIBE SETTINGS emits. +-- +-- Nothing in the model references a group: a user task targets groups through a +-- microflow or an XPath returning System.WorkflowGroup objects. The coupling is +-- at RUNTIME, where Mendix materialises one System.WorkflowGroup row per entry, +-- keyed on the group's element id — so MODIFY edits the row in place and REMOVE +-- stops it being maintained, while user tasks already assigned to it keep their +-- association. + -- DatabaseType must be a Mendix database type: -- Db2, Hsqldb, MySql, Oracle, PostgreSql, SapHana, SqlServer -- (matched case-insensitively and stored in the spelling above). diff --git a/docs-site/src/reference/settings/alter-settings.md b/docs-site/src/reference/settings/alter-settings.md index 2445c3286..7add6e09d 100644 --- a/docs-site/src/reference/settings/alter-settings.md +++ b/docs-site/src/reference/settings/alter-settings.md @@ -20,6 +20,9 @@ ALTER SETTINGS LANGUAGE REMOVE 'code' ALTER SETTINGS WORKFLOWS key = value + ALTER SETTINGS WORKFLOWS ADD [OR MODIFY] GROUP 'name' [( Description: 'text' )] + ALTER SETTINGS WORKFLOWS MODIFY GROUP 'name' ( Description: 'text' ) + ALTER SETTINGS WORKFLOWS REMOVE GROUP 'name' ## Description @@ -34,7 +37,10 @@ Modifies project settings by category. Each category has its own syntax and avai **LANGUAGE** settings control localization: the default language code, and the list of **enabled** languages — the only ones a build emits translations for. -**WORKFLOWS** settings control the workflow engine, including the user entity used for workflow tasks and default task parallelism. +**WORKFLOWS** settings control the workflow engine, including the user entity +used for workflow tasks and default task parallelism, and the **workflow +groups** — the named buckets under App Settings ▸ Workflows ▸ Groups that a user +task's group targeting selects from. Groups need Mendix **11.2** or later. ## Parameters @@ -129,6 +135,38 @@ the run reports how many source strings are affected. ALTER SETTINGS WORKFLOWS UserEntity = 'Administration.Account'; ``` +### Manage workflow groups + +```sql +ALTER SETTINGS WORKFLOWS ADD GROUP 'Approvers' (Description: 'Primary approval group'); +ALTER SETTINGS WORKFLOWS ADD GROUP 'Reviewers'; + +-- the upsert, and what DESCRIBE SETTINGS emits +ALTER SETTINGS WORKFLOWS ADD OR MODIFY GROUP 'Approvers' (Description: 'Approves budget requests'); + +-- changes only the options it names +ALTER SETTINGS WORKFLOWS MODIFY GROUP 'Reviewers' (Description: 'Second-line review'); + +ALTER SETTINGS WORKFLOWS REMOVE GROUP 'Reviewers'; +``` + +`Description` is the only option, because a workflow group stores a name and a +description and nothing else. The **name is the group's identity**, which is what +`MODIFY` and `REMOVE` address; adding a second group whose name differs only in +case is refused rather than creating one no statement could address +unambiguously. + +Changing a description keeps the group's stored identity, so the +`System.WorkflowGroup` row the runtime maintains for it is updated in place — +group memberships and already-assigned user tasks survive the edit. + +`REMOVE` has no dangling reference to check for: nothing in the model points at a +settings group, since a user task targets groups through a microflow or an XPath +returning `System.WorkflowGroup` objects. The coupling is at runtime, where the +removed group's row simply stops being maintained. + +List the groups with [`SHOW WORKFLOW GROUPS`](show-settings.md). + ### Set Java version ```sql diff --git a/docs-site/src/reference/settings/show-settings.md b/docs-site/src/reference/settings/show-settings.md index dbc929ffa..0b2725514 100644 --- a/docs-site/src/reference/settings/show-settings.md +++ b/docs-site/src/reference/settings/show-settings.md @@ -53,6 +53,17 @@ DESCRIBE SETTINGS MODEL; DESCRIBE SETTINGS CONFIGURATION; ``` +### List workflow groups + +```sql +SHOW WORKFLOW GROUPS; +``` + +Lists the workflow groups from App Settings ▸ Workflows ▸ Groups with their +descriptions. It reads the project settings directly, so no catalog refresh is +needed. Groups are created with +[`ALTER SETTINGS WORKFLOWS ... GROUP`](alter-settings.md). + ### Describe workflow settings ```sql diff --git a/docs/01-project/MDL_QUICK_REFERENCE.md b/docs/01-project/MDL_QUICK_REFERENCE.md index d59ef8a2d..a5222bb16 100644 --- a/docs/01-project/MDL_QUICK_REFERENCE.md +++ b/docs/01-project/MDL_QUICK_REFERENCE.md @@ -948,7 +948,12 @@ still flagged rather than guessed at. | Enable or modify (upsert) | `alter settings LANGUAGE add or modify 'de_DE' (CheckCompleteness: true);` | What `describe settings` emits, so a described project replays onto itself or onto one that already has the language | | Modify a language | `alter settings LANGUAGE modify 'de_DE' (CheckCompleteness: true);` | Changes only the options it names. `CheckCompleteness` turns on error reporting for texts with no translation in that language (the default language is always checked regardless) | | Disable a language | `alter settings LANGUAGE remove 'de_DE';` | The **default** language is refused (every missing translation falls back on it). Translations are NOT deleted — they stay in the model and stop being built; the run reports how many | -| Alter workflows | `alter settings workflows key = value;` | UserEntity, DefaultTaskParallelism | +| Alter workflows | `alter settings workflows key = value;` | UserEntity, DefaultTaskParallelism, WorkflowEngineParallelism | +| Add a workflow group | `alter settings workflows add group 'Approvers' [(Description: 'Primary approval group')];` | The buckets under App Settings > Workflows > Groups that a user task's group targeting selects from. Mendix **11.2+**. `Description` is the only option — a `Settings$WorkflowGroup` stores Name and Description and nothing else, so the **name is the identity** and a second group differing only in case is refused | +| Add or modify (upsert) | `alter settings workflows add or modify group 'Approvers' (Description: '...');` | What `describe settings` emits, so a described project replays onto itself | +| Modify a workflow group | `alter settings workflows modify group 'Approvers' (Description: '...');` | Changes only the options it names, and keeps the group's element id — which is the **runtime's identity** for it (Mendix materialises one `System.WorkflowGroup` row per entry, keyed on that id), so an edit updates the row instead of replacing it | +| Remove a workflow group | `alter settings workflows remove group 'Approvers';` | Nothing in the model references a group (a user task targets groups through a microflow or an XPath returning `System.WorkflowGroup` objects), so there is nothing to dangle — the coupling is at runtime | +| List workflow groups | `show workflow groups;` | Reads the settings directly; no catalog refresh needed | | List languages | `show languages;` | ⚠️ languages that have TRANSLATIONS, not enabled ones (a stock app reports 8 while 1 is enabled). For the enabled list use `describe settings`. Requires `refresh catalog full` | ## Business Events diff --git a/mdl-examples/doctype-tests/14-project-settings-examples.mdl b/mdl-examples/doctype-tests/14-project-settings-examples.mdl index fb2fc7aca..577670d68 100644 --- a/mdl-examples/doctype-tests/14-project-settings-examples.mdl +++ b/mdl-examples/doctype-tests/14-project-settings-examples.mdl @@ -206,6 +206,48 @@ alter settings language DefaultLanguageCode = 'en_US'; */ alter settings workflows UserEntity = 'System.User'; +/** + * Example 4.3: Add a workflow group (App Settings > Workflows > Groups) + * + * The named buckets a user task's group targeting selects from. Description is + * the only option — a Settings$WorkflowGroup stores Name and Description and + * nothing else — and the NAME is the group's identity, which is how MODIFY and + * REMOVE address one. Needs Mendix 11.2+. + */ +alter settings workflows add group 'Approvers' (Description: 'Primary approval group'); +alter settings workflows add group 'Reviewers'; + +/** + * Example 4.4: Change a group's description + * + * Only the options the statement names are touched, so this cannot blank a + * description somebody wrote in Studio Pro. + */ +alter settings workflows modify group 'Reviewers' (Description: 'Second-line review'); + +/** + * Example 4.5: Add or modify — what DESCRIBE SETTINGS emits + * + * Adds the group when it is absent and changes it when it is there, so a + * described project re-executes against itself without an error. + */ +alter settings workflows add or modify group 'Approvers' (Description: 'Approves budget requests'); + +/** + * Example 4.6: List the groups + */ +show workflow groups; + +/** + * Example 4.7: Remove a group + * + * The model has no reference to a settings group — a user task targets groups + * through a microflow or an XPath returning System.WorkflowGroup objects — so + * there is nothing to dangle. The coupling is at runtime, where Mendix + * materialises one System.WorkflowGroup row per entry. + */ +alter settings workflows remove group 'Reviewers'; + -- ============================================================================ -- SUMMARY -- ============================================================================ @@ -219,6 +261,8 @@ alter settings workflows UserEntity = 'System.User'; -- DROP CONFIGURATION -- ALTER SETTINGS LANGUAGE -- ALTER SETTINGS WORKFLOWS +-- ALTER SETTINGS WORKFLOWS ADD [OR MODIFY] / MODIFY / REMOVE GROUP (11.2+) +-- SHOW WORKFLOW GROUPS -- SHOW SETTINGS / DESCRIBE SETTINGS (read-only) -- SHOW LANGUAGES (lists language codes with string counts; requires refresh catalog full) -- diff --git a/mdl/ast/ast_query.go b/mdl/ast/ast_query.go index 3cdfaae33..50974ccfb 100644 --- a/mdl/ast/ast_query.go +++ b/mdl/ast/ast_query.go @@ -96,6 +96,7 @@ const ( ShowContractChannels // SHOW CONTRACT CHANNELS FROM Module.Service (AsyncAPI) ShowContractMessages // SHOW CONTRACT MESSAGES FROM Module.Service (AsyncAPI) ShowLanguages // SHOW LANGUAGES + ShowWorkflowGroups // SHOW WORKFLOW GROUPS (project settings, not a module listing) ShowJsonStructures // SHOW JSON STRUCTURES [IN module] ShowMessageDefinitionCollections // SHOW MESSAGE DEFINITION COLLECTIONS [IN module] ShowImportMappings // SHOW IMPORT MAPPINGS [IN module] @@ -240,6 +241,8 @@ func (t ShowObjectType) String() string { return "CONTRACT MESSAGES" case ShowLanguages: return "LANGUAGES" + case ShowWorkflowGroups: + return "WORKFLOW GROUPS" case ShowJsonStructures: return "JSON STRUCTURES" case ShowMessageDefinitionCollections: diff --git a/mdl/ast/ast_settings.go b/mdl/ast/ast_settings.go index 5155c4ae2..8728c8fe7 100644 --- a/mdl/ast/ast_settings.go +++ b/mdl/ast/ast_settings.go @@ -23,6 +23,18 @@ type AlterSettingsStmt struct { // its languages. UpsertLanguage bool RemoveLanguage bool + // For WORKFLOWS ADD/MODIFY/REMOVE GROUP: the group's name. A + // Settings$WorkflowGroup declares Name and Description and no identifier, so + // the name is the group's identity — the same reason a language is addressed + // by its code. + GroupName string + AddGroup bool + ModifyGroup bool + // UpsertGroup is ADD OR MODIFY: add the group when it is not there, change + // its description when it is. It is what DESCRIBE emits, so a described + // project re-executes against a project that already has some of its groups. + UpsertGroup bool + RemoveGroup bool } func (s *AlterSettingsStmt) isStatement() {} diff --git a/mdl/backend/modelsdk/settings_read.go b/mdl/backend/modelsdk/settings_read.go index 3a8f18efa..16c45d58d 100644 --- a/mdl/backend/modelsdk/settings_read.go +++ b/mdl/backend/modelsdk/settings_read.go @@ -109,6 +109,7 @@ func projectSettingsFromGen(g *genSet.ProjectSettings) *model.ProjectSettings { DefaultTaskParallelism: int(p.DefaultTaskParallelism()), WorkflowEngineParallelism: int(p.WorkflowEngineParallelism()), } + ps.Workflows.Groups, ps.Workflows.GroupsIncomplete = workflowGroupsFromGen(p) setBase(&ps.Workflows.BaseElement, p, "Settings$WorkflowsProjectSettingsPart") case *genSet.DistributionSettings: ps.Distribution = &model.DistributionSettings{ @@ -288,3 +289,28 @@ func setBase(b *model.BaseElement, el interface{ ID() element.ID }, typeName str b.ID = model.ID(el.ID()) b.TypeName = typeName } + +// workflowGroupsFromGen converts the Groups part-list of a workflows settings +// part to the semantic model. The second return reports that some element of the +// stored list did not decode as a Settings$WorkflowGroup: the write path rebuilds +// the list from the returned slice, so such an element would be dropped by any +// rewrite, and UpdateProjectSettings refuses rather than lose it. +func workflowGroupsFromGen(p *genSet.WorkflowsProjectSettingsPart) ([]model.WorkflowGroup, bool) { + items := p.GroupsItems() + if len(items) == 0 { + return nil, false + } + groups := make([]model.WorkflowGroup, 0, len(items)) + incomplete := false + for _, it := range items { + g, ok := it.(*genSet.WorkflowGroup) + if !ok { + incomplete = true + continue + } + wg := model.WorkflowGroup{Name: g.Name(), Description: g.Description()} + setBase(&wg.BaseElement, g, "Settings$WorkflowGroup") + groups = append(groups, wg) + } + return groups, incomplete +} diff --git a/mdl/backend/modelsdk/settings_write.go b/mdl/backend/modelsdk/settings_write.go index b7f4f0fc0..2003ef6cf 100644 --- a/mdl/backend/modelsdk/settings_write.go +++ b/mdl/backend/modelsdk/settings_write.go @@ -65,7 +65,15 @@ func (b *Backend) UpdateProjectSettings(ps *model.ProjectSettings) error { rawPart["UserEntity"] = ps.Workflows.UserEntity rawPart["DefaultTaskParallelism"] = settingsoverlay.SafeInt64(ps.Workflows.DefaultTaskParallelism) rawPart["WorkflowEngineParallelism"] = settingsoverlay.SafeInt64(ps.Workflows.WorkflowEngineParallelism) - settings = append(settings, rawPart) + // The workflow groups, not just the scalars. Without this an + // ALTER SETTINGS WORKFLOWS ADD GROUP would report success and + // write nothing — the stored list was carried through from the + // preserved part and the new group dropped on the floor, the same + // shape as the enabled-language list above. + if err := guardWorkflowGroups(ps.Workflows); err != nil { + return err + } + settings = append(settings, settingsoverlay.WorkflowGroups(ps.Workflows, rawPart)) } else { settings = append(settings, rawPart) } @@ -92,3 +100,18 @@ func (b *Backend) UpdateProjectSettings(ps *model.ProjectSettings) error { func overlayModelSettings(ms *model.ModelSettings, raw map[string]any) map[string]any { return settingsoverlay.SetModelSettings(ms, raw) } + +// guardWorkflowGroups refuses a rewrite that would drop a stored group the read +// path could not represent (ADR-0005 guard-don't-drop). The Groups list is +// rebuilt from ws.Groups, so a stored element that did not decode as a +// Settings$WorkflowGroup — a future variant of the part, say — would be silently +// deleted by a statement that only meant to rename one group. Counting cannot +// detect that, because REMOVE GROUP legitimately shortens the list; the read +// flags it instead. +func guardWorkflowGroups(ws *model.WorkflowsSettings) error { + if ws.GroupsIncomplete { + return fmt.Errorf("UpdateProjectSettings: the stored workflow group list holds an element mxcli " + + "could not read; refusing to rewrite Groups, which would drop it") + } + return nil +} diff --git a/mdl/backend/modelsdk/settings_write_workflow_groups_test.go b/mdl/backend/modelsdk/settings_write_workflow_groups_test.go new file mode 100644 index 000000000..617ce0082 --- /dev/null +++ b/mdl/backend/modelsdk/settings_write_workflow_groups_test.go @@ -0,0 +1,197 @@ +// SPDX-License-Identifier: Apache-2.0 + +package modelsdkbackend + +import ( + "testing" + + "go.mongodb.org/mongo-driver/bson" + + "github.com/mendixlabs/mxcli/model" +) + +// workflowsPart returns the Settings$WorkflowsProjectSettingsPart of a settings +// document's raw parts. The fixture carries one, as every Mendix 11 project does. +func workflowsPart(t *testing.T, ps *model.ProjectSettings) map[string]any { + t.Helper() + for _, p := range ps.RawParts { + if p["$Type"] == "Settings$WorkflowsProjectSettingsPart" { + return p + } + } + t.Fatalf("no Settings$WorkflowsProjectSettingsPart in the settings document") + return nil +} + +// TestUpdateProjectSettings_WorkflowGroupsReachStorage is the write-path proof. +// +// The failure it exists to catch is silent: without the Groups overlay in +// UpdateProjectSettings the executor reports "Added workflow group: Approvers" +// and the stored Groups list is carried through from the preserved part +// unchanged, so nothing is written. That is the shape the enabled-language list +// already went through, and no `mx check` reports it — the document stays valid, +// it just does not have the group in it. +func TestUpdateProjectSettings_WorkflowGroupsReachStorage(t *testing.T) { + proj := copyFixture(t) + b := New() + if err := b.Connect(proj); err != nil { + t.Fatalf("connect: %v", err) + } + t.Cleanup(func() { _ = b.Disconnect() }) + + ps, err := b.GetProjectSettings() + if err != nil { + t.Fatalf("GetProjectSettings: %v", err) + } + if ps.Workflows == nil { + t.Fatalf("workflows settings not read") + } + if len(ps.Workflows.Groups) != 0 { + t.Fatalf("fixture already has groups: %+v", ps.Workflows.Groups) + } + partCount := len(ps.RawParts) + + ps.Workflows.Groups = []model.WorkflowGroup{ + {Name: "Approvers", Description: "Primary approval group"}, + {Name: "Reviewers"}, + } + if err := b.UpdateProjectSettings(ps); err != nil { + t.Fatalf("UpdateProjectSettings: %v", err) + } + + b2 := New() + if err := b2.Connect(proj); err != nil { + t.Fatalf("reconnect: %v", err) + } + t.Cleanup(func() { _ = b2.Disconnect() }) + ps2, err := b2.GetProjectSettings() + if err != nil { + t.Fatalf("GetProjectSettings(2): %v", err) + } + got := ps2.Workflows.Groups + if len(got) != 2 { + t.Fatalf("Groups = %+v, want the two written groups", got) + } + if got[0].Name != "Approvers" || got[0].Description != "Primary approval group" { + t.Errorf("group[0] = %+v", got[0]) + } + if got[1].Name != "Reviewers" || got[1].Description != "" { + t.Errorf("group[1] = %+v, want Reviewers with no description", got[1]) + } + if len(ps2.RawParts) != partCount { + t.Errorf("part count changed: was %d, now %d (the overlay dropped a part)", partCount, len(ps2.RawParts)) + } + + // The stored list keeps the marker the project already carried. A blank + // 11.13.0 project stores `Groups: [2]`, not the 3 the other settings child + // lists use, and rebuilding one with the wrong marker silently downgrades it. + raw := workflowsPart(t, ps2) + arr, ok := raw["Groups"].(bson.A) + if !ok || len(arr) != 3 { + t.Fatalf("stored Groups = %#v, want marker + two groups", raw["Groups"]) + } + if m, _ := arr[0].(int32); m != 2 { + t.Errorf("stored Groups marker = %#v, want 2 (the marker the fixture carried)", arr[0]) + } + // Exactly the four keys the type declares. mxbuild tolerates a fifth; Studio + // Pro throws at MprProperty.cs and will not open the project. + // The re-read document decodes into bson.M, which is a map[string]any under a + // different name, so both spellings have to be accepted here. + var g map[string]any + switch v := arr[1].(type) { + case bson.M: + g = v + case map[string]any: + g = v + default: + t.Fatalf("stored group = %#v, want a document", arr[1]) + } + for k := range g { + switch k { + case "$ID", "$Type", "Name", "Description": + default: + t.Errorf("stored group carries undeclared key %q = %#v", k, g[k]) + } + } + if g["$Type"] != "Settings$WorkflowGroup" { + t.Errorf("stored group $Type = %v", g["$Type"]) + } +} + +// TestUpdateProjectSettings_WorkflowGroupKeepsItsStoredID is the data-safety half. +// +// A group's element $ID is its identity in the runtime database: booting the app +// materialises one system$workflowgroup row per entry whose `modelguid` is that +// $ID read as a .NET GUID (measured on Mendix 11.13.0 — stored $ID bytes +// 7c5fc4cf05c3394fa6718a3d4603e9a5, row modelguid +// cfc45f7c-c305-4f39-a671-8a3d4603e9a5). Minting a fresh one on a description +// edit would make the runtime treat it as a different group, orphaning the +// system$workflowgroup_user memberships and the user tasks already targeting it — +// with a perfectly valid model, so no build and no check would report it. Same +// class as the entity GUID case in CLAUDE.md. +func TestUpdateProjectSettings_WorkflowGroupKeepsItsStoredID(t *testing.T) { + proj := copyFixture(t) + b := New() + if err := b.Connect(proj); err != nil { + t.Fatalf("connect: %v", err) + } + t.Cleanup(func() { _ = b.Disconnect() }) + + ps, _ := b.GetProjectSettings() + ps.Workflows.Groups = []model.WorkflowGroup{{Name: "Approvers", Description: "old"}} + if err := b.UpdateProjectSettings(ps); err != nil { + t.Fatalf("UpdateProjectSettings: %v", err) + } + + ps2, _ := b.GetProjectSettings() + if len(ps2.Workflows.Groups) != 1 { + t.Fatalf("Groups = %+v, want the written group back", ps2.Workflows.Groups) + } + before := ps2.Workflows.Groups[0].ID + if before == "" { + t.Fatalf("stored group has no $ID") + } + + ps2.Workflows.Groups[0].Description = "new" + if err := b.UpdateProjectSettings(ps2); err != nil { + t.Fatalf("UpdateProjectSettings(2): %v", err) + } + + ps3, _ := b.GetProjectSettings() + if len(ps3.Workflows.Groups) != 1 { + t.Fatalf("Groups = %+v, want the group back after the edit", ps3.Workflows.Groups) + } + if got := ps3.Workflows.Groups[0]; got.ID != before { + t.Errorf("group $ID changed on a description edit: %v -> %v", before, got.ID) + } else if got.Description != "new" { + t.Errorf("Description = %q, want the edit to have landed", got.Description) + } +} + +// TestUpdateProjectSettings_RefusesUnreadableGroupList is guard-don't-drop +// (ADR-0005): the Groups list is rebuilt from the semantic slice, so an element +// the read could not convert would be deleted by any rewrite — including one that +// only meant to change a description. Refuse instead of losing it. +func TestUpdateProjectSettings_RefusesUnreadableGroupList(t *testing.T) { + proj := copyFixture(t) + b := New() + if err := b.Connect(proj); err != nil { + t.Fatalf("connect: %v", err) + } + t.Cleanup(func() { _ = b.Disconnect() }) + + ps, _ := b.GetProjectSettings() + ps.Workflows.GroupsIncomplete = true + ps.Workflows.Groups = []model.WorkflowGroup{{Name: "Approvers"}} + + err := b.UpdateProjectSettings(ps) + if err == nil { + t.Fatal("a rewrite over an unreadable group list was accepted") + } + // The control: with the flag clear the very same write goes through, so the + // refusal is the flag's doing and not something else about this document. + ps.Workflows.GroupsIncomplete = false + if err := b.UpdateProjectSettings(ps); err != nil { + t.Fatalf("the same write was refused with the flag clear: %v", err) + } +} diff --git a/mdl/executor/cmd_settings.go b/mdl/executor/cmd_settings.go index 4d8ea5f8f..3289d215b 100644 --- a/mdl/executor/cmd_settings.go +++ b/mdl/executor/cmd_settings.go @@ -85,6 +85,9 @@ func listSettings(ctx *ExecContext) error { if ws.DefaultTaskParallelism > 0 { values = append(values, fmt.Sprintf("TaskParallelism: %d", ws.DefaultTaskParallelism)) } + if len(ws.Groups) > 0 { + values = append(values, fmt.Sprintf("%d group(s)", len(ws.Groups))) + } tr.Rows = append(tr.Rows, []any{"Workflow Settings", strings.Join(values, ", ")}) } @@ -225,6 +228,16 @@ func describeSettings(ctx *ExecContext, configName string) error { if len(parts) > 0 { fmt.Fprintf(ctx.Output, "alter settings workflows\n%s;\n\n", strings.Join(parts, ",\n")) } + // The groups, in stored order. `add or modify` so a described project + // re-executes against a project that already has some of them. + for _, g := range ws.Groups { + fmt.Fprintf(ctx.Output, + "alter settings workflows add or modify group '%s' (\n Description: '%s'\n);\n", + escapeMDLString(g.Name), escapeMDLString(g.Description)) + } + if len(ws.Groups) > 0 { + fmt.Fprintln(ctx.Output) + } } return nil @@ -397,6 +410,17 @@ func alterSettings(ctx *ExecContext, stmt *ast.AlterSettingsStmt) error { } } + // ADD/REMOVE GROUP change the workflows part's Groups list rather than a key + // on it, so they are dispatched before the key/value sections — as the + // language forms below are, and for the same reason. + if stmt.AddGroup || stmt.ModifyGroup || stmt.UpsertGroup || stmt.RemoveGroup { + if section != "workflows" { + return mdlerrors.NewUnsupported(fmt.Sprintf( + "GROUP is only defined for the WORKFLOWS section, not %s", stmt.Section)) + } + return alterSettingsWorkflowGroup(ctx, ps, stmt) + } + // ADD/REMOVE change the list of ENABLED languages rather than a key on the // settings part, so they are dispatched before the key/value sections. if stmt.AddLanguage || stmt.ModifyLanguage || stmt.UpsertLanguage || stmt.RemoveLanguage { diff --git a/mdl/executor/cmd_settings_workflow_groups.go b/mdl/executor/cmd_settings_workflow_groups.go new file mode 100644 index 000000000..a6eb75d91 --- /dev/null +++ b/mdl/executor/cmd_settings_workflow_groups.go @@ -0,0 +1,247 @@ +// SPDX-License-Identifier: Apache-2.0 + +package executor + +import ( + "fmt" + "sort" + "strings" + + "github.com/mendixlabs/mxcli/mdl/ast" + mdlerrors "github.com/mendixlabs/mxcli/mdl/errors" + "github.com/mendixlabs/mxcli/model" +) + +// Workflow groups are the entries under App Settings ▸ Workflows ▸ Groups: named +// buckets a user task's group targeting selects from. They live on the +// Settings$WorkflowsProjectSettingsPart as a Groups part-list, which is why they +// are ALTER SETTINGS WORKFLOWS rather than a document of their own. +// +// A Settings$WorkflowGroup stores exactly Name and Description and declares no +// identifier property — agreed by modelsdk/gen, generated/metamodel and the +// Mendix Model SDK's own gen (mendixmodelsdk 4.115.0), and confirmed by a blank +// 11.13.0 project, whose empty list is stored as `Groups: [2]`. So the NAME is +// the group's identity, the same way a language's code is, and that is what +// these statements address a group by. +// +// Nothing in the metamodel points at a group: a user task targets groups through +// a microflow or an XPath returning System.WorkflowGroup objects, never by a +// reference to the settings entry. There is therefore no dangling reference to +// check on REMOVE — the coupling is at runtime, where Mendix materialises one +// System.WorkflowGroup per entry, keyed by name. + +// workflowGroupOptionKeys names every option the ADD/MODIFY forms accept, for the +// error that lists what would have worked. +const workflowGroupOptionKeys = "Description" + +// alterSettingsWorkflowGroup dispatches the ADD / ADD OR MODIFY / MODIFY / +// REMOVE GROUP forms of ALTER SETTINGS WORKFLOWS. +func alterSettingsWorkflowGroup(ctx *ExecContext, ps *model.ProjectSettings, stmt *ast.AlterSettingsStmt) error { + if ps.Workflows == nil { + return mdlerrors.NewNotFound("settings section", "workflows") + } + if err := checkFeature(ctx, "workflows", "groups", + "alter settings workflows add group ''", + "Workflow groups (Settings$WorkflowGroup) were introduced in the 11.2 metamodel. "+ + "On an earlier version, model the buckets as your own entity and target user tasks with a microflow."); err != nil { + return err + } + if strings.TrimSpace(stmt.GroupName) == "" { + return mdlerrors.NewValidation( + "a workflow group needs a name — write e.g. `alter settings workflows add group 'Approvers';`") + } + switch { + case stmt.UpsertGroup: + return alterSettingsWorkflowGroupUpsert(ctx, ps, stmt) + case stmt.AddGroup: + return alterSettingsWorkflowGroupAdd(ctx, ps, stmt) + case stmt.ModifyGroup: + return alterSettingsWorkflowGroupModify(ctx, ps, stmt) + } + return alterSettingsWorkflowGroupRemove(ctx, ps, stmt) +} + +// alterSettingsWorkflowGroupAdd appends a group. Studio Pro appends rather than +// sorting, so the stored order is the order groups were added; the overlay +// rebuilds the list from this slice in the same order. +func alterSettingsWorkflowGroupAdd(ctx *ExecContext, ps *model.ProjectSettings, stmt *ast.AlterSettingsStmt) error { + ws := ps.Workflows + if i := indexOfWorkflowGroup(ws.Groups, stmt.GroupName); i >= 0 { + return mdlerrors.NewValidationf( + "workflow group %q already exists — change it with "+ + "`alter settings workflows modify group '%s' (Description: '…')`, or use `add or modify` to do either", + ws.Groups[i].Name, ws.Groups[i].Name) + } + g := model.WorkflowGroup{Name: stmt.GroupName} + if err := applyWorkflowGroupOptions(&g, stmt.Properties); err != nil { + return err + } + ws.Groups = append(ws.Groups, g) + if err := ctx.Backend.UpdateProjectSettings(ps); err != nil { + return mdlerrors.NewBackend("update workflow settings", err) + } + fmt.Fprintf(ctx.Output, "Added workflow group: %s (%d group(s))\n", g.Name, len(ws.Groups)) + return nil +} + +// alterSettingsWorkflowGroupUpsert is ADD OR MODIFY: add the group when it is not +// there, change the named options when it is. It exists for the same reason the +// language form does — DESCRIBE has to emit something that re-executes against a +// project that already has some of its groups. +func alterSettingsWorkflowGroupUpsert(ctx *ExecContext, ps *model.ProjectSettings, stmt *ast.AlterSettingsStmt) error { + if i := indexOfWorkflowGroup(ps.Workflows.Groups, stmt.GroupName); i >= 0 { + if len(stmt.Properties) == 0 { + // Nothing to change and nothing to add: report the state rather than + // an error, so a replay of a described project is quiet. + fmt.Fprintf(ctx.Output, "Unchanged workflow group: %s\n", ps.Workflows.Groups[i].Name) + return nil + } + return alterSettingsWorkflowGroupModify(ctx, ps, stmt) + } + return alterSettingsWorkflowGroupAdd(ctx, ps, stmt) +} + +// alterSettingsWorkflowGroupModify changes an existing group's description. +// +// Only the options the statement NAMES are touched, which is what distinguishes +// MODIFY from re-adding: a statement that mentions no option cannot silently +// blank a description somebody wrote in Studio Pro. The group's stored document +// is preserved either way, so its $ID survives. +func alterSettingsWorkflowGroupModify(ctx *ExecContext, ps *model.ProjectSettings, stmt *ast.AlterSettingsStmt) error { + ws := ps.Workflows + idx := indexOfWorkflowGroup(ws.Groups, stmt.GroupName) + if idx < 0 { + return mdlerrors.NewValidationf( + "workflow group %q does not exist in this project (%s) — add it first with "+ + "`alter settings workflows add group '%s';`", + stmt.GroupName, workflowGroupsSummary(ws.Groups), stmt.GroupName) + } + if len(stmt.Properties) == 0 { + return mdlerrors.NewValidationf( + "no properties given — write e.g. `alter settings workflows modify group '%s' (Description: '…');`", + stmt.GroupName) + } + + g := ws.Groups[idx] + if err := applyWorkflowGroupOptions(&g, stmt.Properties); err != nil { + return err + } + changed := sortedSettingsOptionKeys(stmt.Properties) + unchanged := g == ws.Groups[idx] + ws.Groups[idx] = g + + if err := ctx.Backend.UpdateProjectSettings(ps); err != nil { + return mdlerrors.NewBackend("update workflow settings", err) + } + // Say which of the two happened: a replayed DESCRIBE names every option of + // every group, so reporting "Modified" for all of them would describe a + // rewrite that ADR-0008 elides. + if unchanged { + fmt.Fprintf(ctx.Output, "Unchanged workflow group: %s\n", g.Name) + } else { + fmt.Fprintf(ctx.Output, "Modified workflow group: %s (%s)\n", g.Name, strings.Join(changed, ", ")) + } + return nil +} + +// alterSettingsWorkflowGroupRemove drops a group from the settings list. +// +// This is not refused when a workflow is in flight, because the model cannot +// tell: nothing in it references a settings group. What removal does mean is +// said instead — the runtime materialises System.WorkflowGroup rows from this +// list, so a removed group's row stops being maintained while the user tasks +// already assigned to it keep their association. +func alterSettingsWorkflowGroupRemove(ctx *ExecContext, ps *model.ProjectSettings, stmt *ast.AlterSettingsStmt) error { + ws := ps.Workflows + idx := indexOfWorkflowGroup(ws.Groups, stmt.GroupName) + if idx < 0 { + return mdlerrors.NewValidationf( + "workflow group %q does not exist in this project (%s)", + stmt.GroupName, workflowGroupsSummary(ws.Groups)) + } + name := ws.Groups[idx].Name + ws.Groups = append(ws.Groups[:idx], ws.Groups[idx+1:]...) + if err := ctx.Backend.UpdateProjectSettings(ps); err != nil { + return mdlerrors.NewBackend("update workflow settings", err) + } + fmt.Fprintf(ctx.Output, "Removed workflow group: %s (%d group(s))\n", name, len(ws.Groups)) + return nil +} + +// applyWorkflowGroupOptions writes the named ( key: value ) options onto a group. +// An unknown key is refused with the list of what would have worked, rather than +// stored under a property Settings$WorkflowGroup does not declare. +func applyWorkflowGroupOptions(g *model.WorkflowGroup, props map[string]any) error { + for _, key := range sortedSettingsOptionKeys(props) { + valStr := settingsValueToString(props[key]) + switch key { + case "Description": + g.Description = valStr + default: + return mdlerrors.NewUnsupported(fmt.Sprintf( + "unknown workflow group option: %s\n valid keys: %s", key, workflowGroupOptionKeys)) + } + } + return nil +} + +// indexOfWorkflowGroup finds a group by name, case-insensitively — Studio Pro +// will not let two groups differ only in case, so treating 'Approvers' and +// 'approvers' as the same group is what keeps ADD from creating a second one a +// script can no longer address unambiguously. +func indexOfWorkflowGroup(groups []model.WorkflowGroup, name string) int { + for i, g := range groups { + if strings.EqualFold(g.Name, name) { + return i + } + } + return -1 +} + +// workflowGroupsSummary renders the existing group names for an error message. +func workflowGroupsSummary(groups []model.WorkflowGroup) string { + if len(groups) == 0 { + return "this project has no workflow groups" + } + names := make([]string, 0, len(groups)) + for _, g := range groups { + names = append(names, g.Name) + } + sort.Strings(names) + return "existing: " + strings.Join(names, ", ") +} + +// sortedSettingsOptionKeys returns a settings-option map's keys in a stable order: iterating +// the map directly would make the reported option list — and any error naming the +// first bad key — differ between runs. +func sortedSettingsOptionKeys(m map[string]any) []string { + keys := make([]string, 0, len(m)) + for k := range m { + keys = append(keys, k) + } + sort.Strings(keys) + return keys +} + +// listWorkflowGroups backs SHOW WORKFLOW GROUPS. +func listWorkflowGroups(ctx *ExecContext) error { + if !ctx.Connected() { + return mdlerrors.NewNotConnected() + } + ps, err := ctx.Backend.GetProjectSettings() + if err != nil { + return mdlerrors.NewBackend("read project settings", err) + } + var groups []model.WorkflowGroup + if ps.Workflows != nil { + groups = ps.Workflows.Groups + } + tr := &TableResult{ + Columns: []string{"Name", "Description"}, + Summary: fmt.Sprintf("(%d workflow group(s))", len(groups)), + } + for _, g := range groups { + tr.Rows = append(tr.Rows, []any{g.Name, g.Description}) + } + return writeResult(ctx, tr) +} diff --git a/mdl/executor/cmd_settings_workflow_groups_test.go b/mdl/executor/cmd_settings_workflow_groups_test.go new file mode 100644 index 000000000..dbdb3758e --- /dev/null +++ b/mdl/executor/cmd_settings_workflow_groups_test.go @@ -0,0 +1,278 @@ +// SPDX-License-Identifier: Apache-2.0 + +// ALTER SETTINGS WORKFLOWS ADD/MODIFY/REMOVE GROUP and SHOW WORKFLOW GROUPS — +// the workflow groups under App Settings ▸ Workflows ▸ Groups. +// +// Pinned against a real Mendix 11.13.0 project: the workflows settings part +// stores `Groups: [2]` (a typed array whose marker is 2, not the 3 the other +// settings child lists use), and each entry is +// +// {$ID:…, $Type:"Settings$WorkflowGroup", Name:"Approvers", Description:"…"} +// +// and nothing else. Verified end to end by booting the app: the runtime +// materialises one system$workflowgroup row per entry, whose `modelguid` is the +// element's own $ID read as a .NET GUID — so the group's $ID is its database +// identity, and preserving it across a MODIFY is data safety, not tidiness. +package executor + +import ( + "bytes" + "strings" + "testing" + + "github.com/mendixlabs/mxcli/mdl/ast" + "github.com/mendixlabs/mxcli/mdl/backend/mock" + "github.com/mendixlabs/mxcli/model" +) + +func wfGroupCtx(t *testing.T, ps *model.ProjectSettings) (*ExecContext, *bytes.Buffer, **model.ProjectSettings) { + t.Helper() + var written *model.ProjectSettings + out := &bytes.Buffer{} + b := &mock.MockBackend{ + IsConnectedFunc: func() bool { return true }, + GetProjectSettingsFunc: func() (*model.ProjectSettings, error) { return ps, nil }, + UpdateProjectSettingsFunc: func(p *model.ProjectSettings) error { written = p; return nil }, + } + return &ExecContext{Backend: b, Output: out}, out, &written +} + +func settingsWithGroups(groups ...model.WorkflowGroup) *model.ProjectSettings { + return &model.ProjectSettings{Workflows: &model.WorkflowsSettings{ + UserEntity: "System.User", DefaultTaskParallelism: 3, WorkflowEngineParallelism: 5, + Groups: groups, + }} +} + +func addGroupStmt(name string, props map[string]any) *ast.AlterSettingsStmt { + if props == nil { + props = map[string]any{} + } + return &ast.AlterSettingsStmt{Section: "workflows", AddGroup: true, GroupName: name, Properties: props} +} + +func TestWorkflowGroupAdd_AppendsWithDescription(t *testing.T) { + ps := settingsWithGroups() + ctx, out, written := wfGroupCtx(t, ps) + + err := alterSettingsWorkflowGroup(ctx, ps, + addGroupStmt("Approvers", map[string]any{"Description": "Primary approval group"})) + if err != nil { + t.Fatal(err) + } + got := (*written).Workflows.Groups + if len(got) != 1 || got[0].Name != "Approvers" || got[0].Description != "Primary approval group" { + t.Fatalf("Groups = %+v, want one Approvers group with its description", got) + } + if !strings.Contains(out.String(), "Added workflow group: Approvers") { + t.Errorf("output = %q", out.String()) + } +} + +// Studio Pro appends rather than sorting, so the stored order is the order groups +// were added. The overlay rebuilds the list from this slice in the same order. +func TestWorkflowGroupAdd_AppendsInStatementOrder(t *testing.T) { + ps := settingsWithGroups(model.WorkflowGroup{Name: "Approvers"}) + ctx, _, written := wfGroupCtx(t, ps) + + if err := alterSettingsWorkflowGroup(ctx, ps, addGroupStmt("Reviewers", nil)); err != nil { + t.Fatal(err) + } + got := (*written).Workflows.Groups + if len(got) != 2 || got[0].Name != "Approvers" || got[1].Name != "Reviewers" { + t.Errorf("Groups = %+v, want [Approvers Reviewers]", got) + } +} + +func TestWorkflowGroupAdd_RefusesDuplicate(t *testing.T) { + // Case-insensitively: Studio Pro will not let two groups differ only in case, + // and a second 'approvers' would be a group no statement could address. + ps := settingsWithGroups(model.WorkflowGroup{Name: "Approvers"}) + ctx, _, written := wfGroupCtx(t, ps) + + err := alterSettingsWorkflowGroup(ctx, ps, addGroupStmt("approvers", nil)) + if err == nil { + t.Fatal("adding an existing group was accepted") + } + if !strings.Contains(err.Error(), "already exists") { + t.Errorf("error = %v", err) + } + if *written != nil { + t.Error("a refused ADD still wrote the settings") + } +} + +func TestWorkflowGroupAdd_RefusesUnknownOption(t *testing.T) { + // Description is the only property Settings$WorkflowGroup declares. Storing + // anything else would put a key in the document that the type does not have, + // which mxbuild tolerates and Studio Pro refuses to open. + ps := settingsWithGroups() + ctx, _, written := wfGroupCtx(t, ps) + + err := alterSettingsWorkflowGroup(ctx, ps, + addGroupStmt("Approvers", map[string]any{"Descriptio": "typo"})) + if err == nil { + t.Fatal("an unknown group option was accepted") + } + if !strings.Contains(err.Error(), "unknown workflow group option") || + !strings.Contains(err.Error(), "Description") { + t.Errorf("error = %v, want it to name the valid keys", err) + } + if *written != nil { + t.Error("a refused ADD still wrote the settings") + } +} + +func TestWorkflowGroupAdd_RefusesBlankName(t *testing.T) { + ps := settingsWithGroups() + ctx, _, _ := wfGroupCtx(t, ps) + if err := alterSettingsWorkflowGroup(ctx, ps, addGroupStmt(" ", nil)); err == nil { + t.Fatal("a blank group name was accepted") + } +} + +func TestWorkflowGroupModify_TouchesOnlyNamedOptions(t *testing.T) { + ps := settingsWithGroups( + model.WorkflowGroup{Name: "Approvers", Description: "old"}, + model.WorkflowGroup{Name: "Reviewers", Description: "keep me"}, + ) + ctx, out, written := wfGroupCtx(t, ps) + + err := alterSettingsWorkflowGroup(ctx, ps, &ast.AlterSettingsStmt{ + Section: "workflows", ModifyGroup: true, GroupName: "Approvers", + Properties: map[string]any{"Description": "new"}, + }) + if err != nil { + t.Fatal(err) + } + got := (*written).Workflows.Groups + if got[0].Description != "new" { + t.Errorf("Approvers description = %q, want new", got[0].Description) + } + if got[1].Description != "keep me" { + t.Errorf("MODIFY disturbed a sibling group: %+v", got[1]) + } + if !strings.Contains(out.String(), "Modified workflow group: Approvers") { + t.Errorf("output = %q", out.String()) + } +} + +func TestWorkflowGroupModify_RefusesMissingGroup(t *testing.T) { + ps := settingsWithGroups(model.WorkflowGroup{Name: "Approvers"}) + ctx, _, _ := wfGroupCtx(t, ps) + + err := alterSettingsWorkflowGroup(ctx, ps, &ast.AlterSettingsStmt{ + Section: "workflows", ModifyGroup: true, GroupName: "Nope", + Properties: map[string]any{"Description": "x"}, + }) + if err == nil { + t.Fatal("modifying a group that does not exist was accepted") + } + // The error names what does exist: a script that mistypes a group should not + // have to go and run `show workflow groups` to find out. + if !strings.Contains(err.Error(), "Approvers") { + t.Errorf("error = %v, want it to list the existing groups", err) + } +} + +// ADD OR MODIFY is what DESCRIBE emits, so replaying a described project must be +// quiet rather than an error — and must report "Unchanged" rather than claiming a +// write that ADR-0008 elides. +func TestWorkflowGroupUpsert_IsQuietOnReplay(t *testing.T) { + ps := settingsWithGroups(model.WorkflowGroup{Name: "Approvers", Description: "d"}) + ctx, out, _ := wfGroupCtx(t, ps) + + err := alterSettingsWorkflowGroup(ctx, ps, &ast.AlterSettingsStmt{ + Section: "workflows", UpsertGroup: true, GroupName: "Approvers", + Properties: map[string]any{"Description": "d"}, + }) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(out.String(), "Unchanged workflow group: Approvers") { + t.Errorf("output = %q, want the unchanged verb", out.String()) + } +} + +func TestWorkflowGroupUpsert_AddsWhenAbsent(t *testing.T) { + ps := settingsWithGroups() + ctx, _, written := wfGroupCtx(t, ps) + + err := alterSettingsWorkflowGroup(ctx, ps, &ast.AlterSettingsStmt{ + Section: "workflows", UpsertGroup: true, GroupName: "Approvers", + Properties: map[string]any{"Description": "d"}, + }) + if err != nil { + t.Fatal(err) + } + if got := (*written).Workflows.Groups; len(got) != 1 || got[0].Name != "Approvers" { + t.Errorf("Groups = %+v, want the group added", got) + } +} + +func TestWorkflowGroupRemove(t *testing.T) { + ps := settingsWithGroups( + model.WorkflowGroup{Name: "Approvers"}, + model.WorkflowGroup{Name: "Reviewers"}, + ) + ctx, out, written := wfGroupCtx(t, ps) + + err := alterSettingsWorkflowGroup(ctx, ps, + &ast.AlterSettingsStmt{Section: "workflows", RemoveGroup: true, GroupName: "Reviewers"}) + if err != nil { + t.Fatal(err) + } + got := (*written).Workflows.Groups + if len(got) != 1 || got[0].Name != "Approvers" { + t.Errorf("Groups = %+v, want only Approvers", got) + } + if !strings.Contains(out.String(), "Removed workflow group: Reviewers") { + t.Errorf("output = %q", out.String()) + } +} + +func TestWorkflowGroupRemove_RefusesMissingGroup(t *testing.T) { + ps := settingsWithGroups(model.WorkflowGroup{Name: "Approvers"}) + ctx, _, written := wfGroupCtx(t, ps) + + err := alterSettingsWorkflowGroup(ctx, ps, + &ast.AlterSettingsStmt{Section: "workflows", RemoveGroup: true, GroupName: "Nope"}) + if err == nil { + t.Fatal("removing a group that does not exist was accepted") + } + if *written != nil { + t.Error("a refused REMOVE still wrote the settings") + } +} + +// GROUP outside the WORKFLOWS section is a mistake worth naming: the grammar +// shares the clause with the LANGUAGE forms, so `alter settings language add +// group 'X'` parses and would otherwise reach the language handler. +func TestAlterSettings_GroupOutsideWorkflowsIsRefused(t *testing.T) { + ps := &model.ProjectSettings{ + Language: &model.LanguageSettings{DefaultLanguageCode: "en_US"}, + RawParts: []map[string]any{{"$Type": "Settings$LanguageSettings"}}, + } + ctx, _, _ := wfGroupCtx(t, ps) + err := alterSettings(ctx, &ast.AlterSettingsStmt{ + Section: "language", AddGroup: true, GroupName: "X", Properties: map[string]any{}, + }) + if err == nil || !strings.Contains(err.Error(), "only defined for the WORKFLOWS section") { + t.Fatalf("error = %v, want a refusal naming the section", err) + } +} + +func TestShowWorkflowGroups(t *testing.T) { + ps := settingsWithGroups( + model.WorkflowGroup{Name: "Approvers", Description: "Primary approval group"}, + ) + ctx, out, _ := wfGroupCtx(t, ps) + + if err := listWorkflowGroups(ctx); err != nil { + t.Fatal(err) + } + s := out.String() + if !strings.Contains(s, "Approvers") || !strings.Contains(s, "Primary approval group") { + t.Errorf("output = %q", s) + } +} diff --git a/mdl/executor/executor_query.go b/mdl/executor/executor_query.go index d5f2d5b27..417981622 100644 --- a/mdl/executor/executor_query.go +++ b/mdl/executor/executor_query.go @@ -125,6 +125,8 @@ func execShow(ctx *ExecContext, s *ast.ShowStmt) error { return listSettings(ctx) case ast.ShowLanguages: return listLanguages(ctx) + case ast.ShowWorkflowGroups: + return listWorkflowGroups(ctx) case ast.ShowFragments: return listFragments(ctx) case ast.ShowDatabaseConnections: diff --git a/mdl/grammar/domains/MDLCatalog.g4 b/mdl/grammar/domains/MDLCatalog.g4 index a7a1d2a0a..90667202f 100644 --- a/mdl/grammar/domains/MDLCatalog.g4 +++ b/mdl/grammar/domains/MDLCatalog.g4 @@ -24,6 +24,9 @@ showStatement | showOrList NANOFLOWS (IN (qualifiedName | IDENTIFIER))? | showOrList RULES (IN (qualifiedName | IDENTIFIER))? | showOrList WORKFLOWS (IN (qualifiedName | IDENTIFIER))? + // The project-level workflow groups (App Settings > Workflows > Groups), + // not a per-module listing: they are settings, so there is no IN . + | showOrList WORKFLOW GROUPS | showOrList PAGES (IN (qualifiedName | IDENTIFIER))? | showOrList SNIPPETS (IN (qualifiedName | IDENTIFIER))? | showOrList BUILDING BLOCKS (IN (qualifiedName | IDENTIFIER))? diff --git a/mdl/grammar/domains/MDLSettings.g4 b/mdl/grammar/domains/MDLSettings.g4 index bc84de598..cfb31a232 100644 --- a/mdl/grammar/domains/MDLSettings.g4 +++ b/mdl/grammar/domains/MDLSettings.g4 @@ -19,17 +19,31 @@ options { tokenVocab = MDLLexer; } * ALTER SETTINGS LANGUAGE MODIFY 'ar_SD' (Key: Value, ...); * ALTER SETTINGS LANGUAGE REMOVE 'ar_SD'; * ALTER SETTINGS WORKFLOWS Key = Value, ...; + * ALTER SETTINGS WORKFLOWS ADD [OR MODIFY] GROUP 'Approvers' [(Description: '...')]; + * ALTER SETTINGS WORKFLOWS MODIFY GROUP 'Approvers' (Description: '...'); + * ALTER SETTINGS WORKFLOWS REMOVE GROUP 'Approvers'; * * ADD/REMOVE name the ENABLED languages — the list Studio Pro shows under * App Settings > Languages, and the only languages a build emits anything for. * A language is identified by its code alone: Studio Pro's "Arabic, Sudan" is * derived from `ar_SD` for display and is not stored (verified against a * Studio Pro-authored reference on 11.13.0). + * + * The GROUP forms name the workflow groups under App Settings > Workflows > + * Groups. They take the same four verbs, and for the same reason: a group is + * identified by its name alone (Settings$WorkflowGroup declares Name and + * Description and no identifier), so it is addressed the way a language is. + * The GROUP keyword is what separates the two — without it the clause is about + * languages, which is the only other thing ALTER SETTINGS adds and removes. */ alterSettingsClause - : settingsSection ADD OR MODIFY STRING_LITERAL languageOptions? - | settingsSection ADD STRING_LITERAL languageOptions? - | settingsSection MODIFY STRING_LITERAL languageOptions + : settingsSection ADD OR MODIFY GROUP STRING_LITERAL settingsItemOptions? + | settingsSection ADD GROUP STRING_LITERAL settingsItemOptions? + | settingsSection MODIFY GROUP STRING_LITERAL settingsItemOptions + | settingsSection REMOVE GROUP STRING_LITERAL + | settingsSection ADD OR MODIFY STRING_LITERAL settingsItemOptions? + | settingsSection ADD STRING_LITERAL settingsItemOptions? + | settingsSection MODIFY STRING_LITERAL settingsItemOptions | settingsSection REMOVE STRING_LITERAL | settingsSection settingsAssignment (COMMA settingsAssignment)* | CONSTANT STRING_LITERAL (VALUE settingsValue | DROP) (IN CONFIGURATION STRING_LITERAL)? @@ -47,16 +61,22 @@ settingsAssignment : IDENTIFIER EQUALS settingsValue ; -// The optional properties of an added language, in the ( key: value ) form every -// other MDL statement uses. All five are what Texts$Language stores; omitting -// them reproduces what Studio Pro's Add Language dialog writes. +// The optional properties of an added language or workflow group, in the +// ( key: value ) form every other MDL statement uses. For a language all five +// are what Texts$Language stores; omitting them reproduces what Studio Pro's Add +// Language dialog writes. For a workflow group the only option is Description. // ( CheckCompleteness: true, CustomDateFormat: 'yyyy-MM-dd' ) -languageOptions - : LPAREN languageOption (COMMA languageOption)* RPAREN +// ( Description: 'Primary approval group' ) +settingsItemOptions + : LPAREN settingsItemOption (COMMA settingsItemOption)* RPAREN ; -languageOption - : IDENTIFIER COLON settingsValue +// The key is identifierOrKeyword, not IDENTIFIER: `Description` is an MDL +// keyword (DESCRIPTION, from the security statements), so a group's only option +// would otherwise be a parse error — "mismatched input 'Description' expecting +// IDENTIFIER" — on the one statement the feature exists for. +settingsItemOption + : identifierOrKeyword COLON settingsValue ; settingsValue diff --git a/mdl/settingsoverlay/settingsoverlay.go b/mdl/settingsoverlay/settingsoverlay.go index e38f6e6ce..12bbde55b 100644 --- a/mdl/settingsoverlay/settingsoverlay.go +++ b/mdl/settingsoverlay/settingsoverlay.go @@ -494,3 +494,53 @@ func newLanguage(siblings []map[string]any) map[string]any { "CustomDateTimeFormat": "", } } + +// WorkflowGroupListMarker is the typed-array marker Studio Pro writes for the +// Groups list of a Settings$WorkflowsProjectSettingsPart. It is 2, not the 3 the +// other settings child lists use — measured on a blank Mendix 11.13.0 project, +// whose empty list is stored as `Groups: [2]`. It is only a fallback: a stored +// list keeps whatever marker it already carries. +const WorkflowGroupListMarker = int32(2) + +// WorkflowGroups rebuilds the Groups list of a raw +// Settings$WorkflowsProjectSettingsPart, overlaying each modelled group onto the +// raw group it was read from. Groups are matched by name, which is also how the +// executor addresses one (ALTER SETTINGS WORKFLOWS ... GROUP '') — a +// Settings$WorkflowGroup declares no identifier property, so the name is the +// identity. +// +// raw is mutated and returned. +func WorkflowGroups(ws *model.WorkflowsSettings, raw map[string]any) map[string]any { + rawGroups := ArrayElements(raw["Groups"]) + byName := make(map[string]map[string]any, len(rawGroups)) + for _, rg := range rawGroups { + name, _ := rg["Name"].(string) + byName[strings.ToLower(name)] = rg + } + + groups := bson.A{ArrayMarker(raw["Groups"], WorkflowGroupListMarker)} + for _, g := range ws.Groups { + groups = append(groups, WorkflowGroupDoc(g, byName[strings.ToLower(g.Name)])) + } + raw["Groups"] = groups + return raw +} + +// WorkflowGroupDoc overlays one group onto its preserved document. A group with +// no counterpart on disk is newly added, and unlike a language or a server +// configuration it needs no sibling to copy a shape from: Settings$WorkflowGroup +// declares exactly Name and Description and nothing else, in every source that +// describes the type (modelsdk/gen, generated/metamodel, mendixmodelsdk 4.115.0), +// so there is no version-specific spelling to guess at. +func WorkflowGroupDoc(g model.WorkflowGroup, raw map[string]any) map[string]any { + if raw == nil { + raw = map[string]any{} + } + raw["$Type"] = "Settings$WorkflowGroup" + if raw["$ID"] == nil { + raw["$ID"] = elementID(g.ID) + } + raw["Name"] = g.Name + raw["Description"] = g.Description + return raw +} diff --git a/mdl/settingsoverlay/settingsoverlay_test.go b/mdl/settingsoverlay/settingsoverlay_test.go index 989ab1498..ad18c5a16 100644 --- a/mdl/settingsoverlay/settingsoverlay_test.go +++ b/mdl/settingsoverlay/settingsoverlay_test.go @@ -412,3 +412,134 @@ func TestSetJavaVersion_ConvertsToStoredDialect(t *testing.T) { t.Errorf("%s = %#v, want %q", JavaVersionEnumKey, got, "Java17") } } + +// TestWorkflowGroups_ShapePinnedToStudioPro pins the document mxcli writes for a +// workflow group against what a real Mendix project stores. +// +// The reference is a blank 11.13.0 project created with `mxcli new`: its +// workflows settings part stores `Groups: [2]` — an empty typed array whose +// marker is 2, not the 3 the other settings child lists use. Settings$WorkflowGroup +// declares exactly Name and Description (modelsdk/gen, generated/metamodel and +// mendixmodelsdk 4.115.0 all agree, and nothing else appears in the type), so a +// group document is those two keys plus $ID and $Type and nothing more. Writing a +// key the type does not declare is the mendixlabs/mxcli#759 failure shape: mxbuild +// accepts it and Studio Pro cannot open the project. +func TestWorkflowGroups_ShapePinnedToStudioPro(t *testing.T) { + raw := map[string]any{ + "$Type": "Settings$WorkflowsProjectSettingsPart", + "DefaultTaskParallelism": int64(3), + "Groups": bson.A{int32(2)}, + "OnWorkflowEvent": bson.A{int32(2)}, + "UserEntity": "System.User", + "WorkflowEngineParallelism": int64(5), + } + ws := &model.WorkflowsSettings{ + Groups: []model.WorkflowGroup{{Name: "Approvers", Description: "Primary approval group"}}, + } + + out := WorkflowGroups(ws, raw) + + groups, ok := out["Groups"].(bson.A) + if !ok || len(groups) != 2 { + t.Fatalf("Groups = %#v, want a 2-element array (marker + one group)", out["Groups"]) + } + if groups[0] != int32(2) { + t.Errorf("Groups marker = %#v, want int32(2) — the marker the stored list carried", groups[0]) + } + g, ok := groups[1].(map[string]any) + if !ok { + t.Fatalf("group = %#v, want a document", groups[1]) + } + if g["$Type"] != "Settings$WorkflowGroup" { + t.Errorf("$Type = %v, want Settings$WorkflowGroup", g["$Type"]) + } + if g["Name"] != "Approvers" || g["Description"] != "Primary approval group" { + t.Errorf("group = %#v, want Name/Description from the model", g) + } + if g["$ID"] == nil { + t.Error("a new group got no $ID") + } + // Exactly the four keys and no fifth: a property Settings$WorkflowGroup does + // not declare makes a document Studio Pro refuses to open, and mxbuild — which + // tolerates unknown properties — would never report it. + for k := range g { + switch k { + case "$ID", "$Type", "Name", "Description": + default: + t.Errorf("group carries undeclared key %q = %#v", k, g[k]) + } + } + // Every other key of the settings part passes through untouched. + if out["UserEntity"] != "System.User" || out["OnWorkflowEvent"] == nil { + t.Errorf("overlay disturbed a sibling key: %#v", out) + } +} + +// TestWorkflowGroups_PreservesStoredIdentityAndUnknownKeys covers the overlay half +// of guard-don't-drop: a group that is already on disk keeps its $ID and any key +// mxcli does not model, so MODIFY rewrites a description rather than replacing the +// element. A fresh $ID here would make Studio Pro paint the group as new. +func TestWorkflowGroups_PreservesStoredIdentityAndUnknownKeys(t *testing.T) { + raw := map[string]any{ + "$Type": "Settings$WorkflowsProjectSettingsPart", + "Groups": bson.A{int32(2), map[string]any{ + "$ID": "stored-id-sentinel", + "$Type": "Settings$WorkflowGroup", + "Name": "Approvers", + "Description": "old", + "SomeFutureKey": "keep me", + }}, + } + ws := &model.WorkflowsSettings{ + Groups: []model.WorkflowGroup{{Name: "Approvers", Description: "new"}}, + } + + groups := WorkflowGroups(ws, raw)["Groups"].(bson.A) + g := groups[1].(map[string]any) + if g["$ID"] != "stored-id-sentinel" { + t.Errorf("$ID = %v, want the stored one preserved", g["$ID"]) + } + if g["Description"] != "new" { + t.Errorf("Description = %v, want the modelled value", g["Description"]) + } + if g["SomeFutureKey"] != "keep me" { + t.Errorf("a key mxcli does not model was dropped: %#v", g) + } +} + +// TestWorkflowGroups_MatchesStoredGroupCaseInsensitively mirrors how the executor +// addresses a group. Two groups differing only in case would be one group as far +// as every statement is concerned, so the overlay must not treat them as two +// documents and mint a second $ID. +func TestWorkflowGroups_MatchesStoredGroupCaseInsensitively(t *testing.T) { + raw := map[string]any{ + "Groups": bson.A{int32(2), map[string]any{ + "$ID": "stored-id-sentinel", "$Type": "Settings$WorkflowGroup", "Name": "Approvers", + }}, + } + ws := &model.WorkflowsSettings{Groups: []model.WorkflowGroup{{Name: "APPROVERS"}}} + g := WorkflowGroups(ws, raw)["Groups"].(bson.A)[1].(map[string]any) + if g["$ID"] != "stored-id-sentinel" { + t.Errorf("$ID = %v, want the stored group matched case-insensitively", g["$ID"]) + } +} + +// TestWorkflowGroups_RemovedGroupLeavesTheList is the REMOVE half: the list is +// rebuilt from the model, so a group the model no longer carries must not survive +// in the preserved document. +func TestWorkflowGroups_RemovedGroupLeavesTheList(t *testing.T) { + raw := map[string]any{ + "Groups": bson.A{int32(2), + map[string]any{"$Type": "Settings$WorkflowGroup", "Name": "Approvers"}, + map[string]any{"$Type": "Settings$WorkflowGroup", "Name": "Reviewers"}, + }, + } + ws := &model.WorkflowsSettings{Groups: []model.WorkflowGroup{{Name: "Approvers"}}} + groups := WorkflowGroups(ws, raw)["Groups"].(bson.A) + if len(groups) != 2 { + t.Fatalf("Groups = %#v, want marker + one surviving group", groups) + } + if groups[1].(map[string]any)["Name"] != "Approvers" { + t.Errorf("wrong group survived: %#v", groups[1]) + } +} diff --git a/mdl/visitor/visitor_query.go b/mdl/visitor/visitor_query.go index 48e770b47..e8bbf273a 100644 --- a/mdl/visitor/visitor_query.go +++ b/mdl/visitor/visitor_query.go @@ -688,6 +688,9 @@ func (b *Builder) ExitShowStatement(ctx *parser.ShowStatementContext) { } } b.statements = append(b.statements, stmt) + } else if ctx.WORKFLOW() != nil && ctx.GROUPS() != nil { + // SHOW WORKFLOW GROUPS + b.statements = append(b.statements, &ast.ShowStmt{ObjectType: ast.ShowWorkflowGroups}) } else if ctx.LANGUAGES() != nil { // SHOW LANGUAGES b.statements = append(b.statements, &ast.ShowStmt{ObjectType: ast.ShowLanguages}) diff --git a/mdl/visitor/visitor_settings.go b/mdl/visitor/visitor_settings.go index bf61c373f..a9463908b 100644 --- a/mdl/visitor/visitor_settings.go +++ b/mdl/visitor/visitor_settings.go @@ -65,6 +65,19 @@ func (b *Builder) ExitAlterSettingsClause(ctx *parser.AlterSettingsClauseContext val := settingsValueText(svCtx) stmt.Properties[key] = val } + } else if ctx.SettingsSection() != nil && ctx.GROUP() != nil { + // ALTER SETTINGS WORKFLOWS ADD [OR MODIFY] GROUP 'Approvers' [( Description: '…' )] + // ALTER SETTINGS WORKFLOWS MODIFY GROUP 'Approvers' ( Description: '…' ) + // ALTER SETTINGS WORKFLOWS REMOVE GROUP 'Approvers' + stmt.Section = ctx.SettingsSection().GetText() + stmt.UpsertGroup = ctx.ADD() != nil && ctx.OR() != nil && ctx.MODIFY() != nil + stmt.AddGroup = ctx.ADD() != nil && !stmt.UpsertGroup + stmt.ModifyGroup = ctx.MODIFY() != nil && !stmt.UpsertGroup + stmt.RemoveGroup = ctx.REMOVE() != nil + if all := ctx.AllSTRING_LITERAL(); len(all) > 0 { + stmt.GroupName = unquoteString(all[0].GetText()) + } + collectSettingsItemOptions(ctx.SettingsItemOptions(), stmt.Properties) } else if ctx.SettingsSection() != nil && (ctx.ADD() != nil || ctx.MODIFY() != nil || ctx.REMOVE() != nil) { // ALTER SETTINGS LANGUAGE ADD 'ar_SD' [( key: value, … )] // ALTER SETTINGS LANGUAGE MODIFY 'ar_SD' ( key: value, … ) @@ -77,21 +90,7 @@ func (b *Builder) ExitAlterSettingsClause(ctx *parser.AlterSettingsClauseContext if all := ctx.AllSTRING_LITERAL(); len(all) > 0 { stmt.LanguageCode = unquoteString(all[0].GetText()) } - if opts := ctx.LanguageOptions(); opts != nil { - if oc, ok := opts.(*parser.LanguageOptionsContext); ok && oc != nil { - for _, o := range oc.AllLanguageOption() { - lo, ok := o.(*parser.LanguageOptionContext) - if !ok || lo == nil || lo.IDENTIFIER() == nil || lo.SettingsValue() == nil { - continue - } - sv, ok := lo.SettingsValue().(*parser.SettingsValueContext) - if !ok || sv == nil { - continue - } - stmt.Properties[lo.IDENTIFIER().GetText()] = settingsValueText(sv) - } - } - } + collectSettingsItemOptions(ctx.SettingsItemOptions(), stmt.Properties) } else if ctx.SettingsSection() != nil { // ALTER SETTINGS MODEL|LANGUAGE|WORKFLOWS Key = Value, ... stmt.Section = ctx.SettingsSection().GetText() @@ -152,6 +151,30 @@ func (b *Builder) ExitCreateConfigurationStatement(ctx *parser.CreateConfigurati b.statements = append(b.statements, stmt) } +// collectSettingsItemOptions reads a ( key: value, … ) option list — the shared +// form behind ALTER SETTINGS LANGUAGE's language options and WORKFLOWS' group +// options — into the statement's property map. +func collectSettingsItemOptions(opts parser.ISettingsItemOptionsContext, into map[string]any) { + if opts == nil { + return + } + oc, ok := opts.(*parser.SettingsItemOptionsContext) + if !ok || oc == nil { + return + } + for _, o := range oc.AllSettingsItemOption() { + so, ok := o.(*parser.SettingsItemOptionContext) + if !ok || so == nil || so.IdentifierOrKeyword() == nil || so.SettingsValue() == nil { + continue + } + sv, ok := so.SettingsValue().(*parser.SettingsValueContext) + if !ok || sv == nil { + continue + } + into[unquoteIdentifier(so.IdentifierOrKeyword().GetText())] = settingsValueText(sv) + } +} + // settingsValueText extracts the string value from a SettingsValue context. func settingsValueText(ctx *parser.SettingsValueContext) string { if sl := ctx.STRING_LITERAL(); sl != nil { diff --git a/mdl/visitor/visitor_settings_test.go b/mdl/visitor/visitor_settings_test.go index 5d3b4bf7c..071c4f3dd 100644 --- a/mdl/visitor/visitor_settings_test.go +++ b/mdl/visitor/visitor_settings_test.go @@ -3,11 +3,28 @@ package visitor import ( + "strings" "testing" "github.com/mendixlabs/mxcli/mdl/ast" ) +// mustParseSettings builds a program and fails the test on any parse error. +func mustParseSettings(t *testing.T, src string) *ast.Program { + t.Helper() + prog, errs := Build(src) + if len(errs) > 0 { + for _, e := range errs { + t.Errorf("Parse error: %v", e) + } + t.FailNow() + } + if len(prog.Statements) == 0 { + t.Fatalf("no statements parsed from %q", src) + } + return prog +} + func TestAlterSettings_Model(t *testing.T) { input := `ALTER SETTINGS MODEL DefaultLanguage = 'en_US';` prog, errs := Build(input) @@ -88,3 +105,125 @@ func TestCreateConfiguration(t *testing.T) { t.Errorf("Got %v", stmt.Properties["DatabaseHost"]) } } + +// TestAlterSettings_WorkflowGroup covers the four GROUP verbs and, with them, the +// one thing that made the first draft of this grammar unusable: `Description` is +// an MDL keyword (DESCRIPTION, from the security statements), so an option key +// typed as IDENTIFIER made the feature's only option a parse error — "mismatched +// input 'Description' expecting IDENTIFIER" — on the statement the feature exists +// for. The option key is identifierOrKeyword for that reason. +func TestAlterSettings_WorkflowGroup(t *testing.T) { + tests := []struct { + name string + src string + check func(*testing.T, *ast.AlterSettingsStmt) + }{ + { + "add with description", + "alter settings workflows add group 'Approvers' (Description: 'Primary approval group');", + func(t *testing.T, s *ast.AlterSettingsStmt) { + if !s.AddGroup || s.UpsertGroup || s.ModifyGroup || s.RemoveGroup { + t.Errorf("verbs = add:%t upsert:%t modify:%t remove:%t", s.AddGroup, s.UpsertGroup, s.ModifyGroup, s.RemoveGroup) + } + if s.Properties["Description"] != "Primary approval group" { + t.Errorf("Properties = %#v", s.Properties) + } + }, + }, + { + "add without options", + "alter settings workflows add group 'Reviewers';", + func(t *testing.T, s *ast.AlterSettingsStmt) { + if !s.AddGroup || len(s.Properties) != 0 { + t.Errorf("stmt = %+v", s) + } + }, + }, + { + "add or modify", + "alter settings workflows add or modify group 'Approvers' (Description: 'd');", + func(t *testing.T, s *ast.AlterSettingsStmt) { + if !s.UpsertGroup || s.AddGroup || s.ModifyGroup { + t.Errorf("verbs = add:%t upsert:%t modify:%t", s.AddGroup, s.UpsertGroup, s.ModifyGroup) + } + }, + }, + { + "modify", + "alter settings workflows modify group 'Approvers' (Description: 'd');", + func(t *testing.T, s *ast.AlterSettingsStmt) { + if !s.ModifyGroup || s.UpsertGroup { + t.Errorf("verbs = upsert:%t modify:%t", s.UpsertGroup, s.ModifyGroup) + } + }, + }, + { + "remove", + "alter settings workflows remove group 'Approvers';", + func(t *testing.T, s *ast.AlterSettingsStmt) { + if !s.RemoveGroup { + t.Errorf("stmt = %+v", s) + } + }, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + prog := mustParseSettings(t, tc.src) + stmt, ok := prog.Statements[0].(*ast.AlterSettingsStmt) + if !ok { + t.Fatalf("Expected AlterSettingsStmt, got %T", prog.Statements[0]) + } + if !strings.EqualFold(stmt.Section, "workflows") { + t.Errorf("Section = %q, want workflows", stmt.Section) + } + if stmt.GroupName != "Approvers" && stmt.GroupName != "Reviewers" { + t.Errorf("GroupName = %q", stmt.GroupName) + } + // The GROUP forms must not be mistaken for the LANGUAGE ones: they + // share the clause, and a stray AddLanguage would send the statement + // to the language handler. + if stmt.AddLanguage || stmt.ModifyLanguage || stmt.UpsertLanguage || stmt.RemoveLanguage { + t.Errorf("a GROUP statement set a LANGUAGE verb: %+v", stmt) + } + tc.check(t, stmt) + }) + } +} + +// The LANGUAGE forms share the clause and must keep working unchanged — the +// options rule they use was renamed and generalised for the GROUP forms. +func TestAlterSettings_LanguageStillParsesAfterGroupForms(t *testing.T) { + prog := mustParseSettings(t, "alter settings LANGUAGE add or modify 'ar_SD' (CheckCompleteness: true);") + stmt := prog.Statements[0].(*ast.AlterSettingsStmt) + if !stmt.UpsertLanguage || stmt.LanguageCode != "ar_SD" { + t.Fatalf("stmt = %+v", stmt) + } + if stmt.Properties["CheckCompleteness"] != "true" { + t.Errorf("Properties = %#v", stmt.Properties) + } + if stmt.AddGroup || stmt.UpsertGroup || stmt.GroupName != "" { + t.Errorf("a LANGUAGE statement set a GROUP field: %+v", stmt) + } +} + +func TestShowWorkflowGroups_Parses(t *testing.T) { + prog := mustParseSettings(t, "show workflow groups;") + stmt, ok := prog.Statements[0].(*ast.ShowStmt) + if !ok { + t.Fatalf("Expected ShowStmt, got %T", prog.Statements[0]) + } + if stmt.ObjectType != ast.ShowWorkflowGroups { + t.Errorf("ObjectType = %v, want ShowWorkflowGroups", stmt.ObjectType) + } +} + +// `show workflows` and `show workflow groups` are different statements sharing a +// prefix; the listing one must not be captured by the new alternative. +func TestShowWorkflows_StillListsWorkflows(t *testing.T) { + prog := mustParseSettings(t, "show workflows;") + stmt := prog.Statements[0].(*ast.ShowStmt) + if stmt.ObjectType != ast.ShowWorkflows { + t.Errorf("ObjectType = %v, want ShowWorkflows", stmt.ObjectType) + } +} diff --git a/model/types.go b/model/types.go index 1395509e3..6e7d30868 100644 --- a/model/types.go +++ b/model/types.go @@ -1051,6 +1051,26 @@ type WorkflowsSettings struct { UserEntity string `json:"userEntity,omitempty"` DefaultTaskParallelism int `json:"defaultTaskParallelism,omitempty"` WorkflowEngineParallelism int `json:"workflowEngineParallelism,omitempty"` + // Groups are the workflow groups from App > Settings > Workflows > Groups. + // The runtime materialises one System.WorkflowGroup per entry, which is what + // a user task's group targeting selects from. + Groups []WorkflowGroup `json:"groups,omitempty"` + // GroupsIncomplete records that the stored Groups list held an element the + // read could not convert. The write path rebuilds the list from Groups, so + // rewriting it would drop that element; UpdateProjectSettings refuses + // instead (ADR-0005 guard-don't-drop). + GroupsIncomplete bool `json:"-"` +} + +// WorkflowGroup represents a Settings$WorkflowGroup: one entry of the workflows +// settings part's Groups list. Name and Description are the only two properties +// the type declares (modelsdk/gen, generated/metamodel and mendixmodelsdk 4.115.0 +// all agree), and there is no separate identifier — Name is the group's identity, +// which is how ALTER SETTINGS WORKFLOWS ... GROUP addresses one. +type WorkflowGroup struct { + BaseElement + Name string `json:"name"` + Description string `json:"description,omitempty"` } // JarDeploymentSettings represents Settings$JarDeploymentSettings. diff --git a/sdk/versions/mendix-11.yaml b/sdk/versions/mendix-11.yaml index 0e21cf097..afa8f6e16 100644 --- a/sdk/versions/mendix-11.yaml +++ b/sdk/versions/mendix-11.yaml @@ -180,6 +180,10 @@ features: basic: min_version: "9.0.0" mdl: "CREATE WORKFLOW Module.Name ..." + groups: + min_version: "11.2.0" + mdl: "ALTER SETTINGS WORKFLOWS ADD [OR MODIFY] GROUP 'Approvers' (Description: '...')" + notes: "Settings$WorkflowGroup and WorkflowsProjectSettingsPart.groups were both introduced in 11.2.0 (mendixmodelsdk 4.115.0 StructureVersionInfo). Mendix's release notes call the feature GA in 11.6; the metamodel floor is what decides whether the document loads. Corroborated against real projects: a blank 11.13.0 app's workflows settings part carries `Groups: [2]`, an 11.1.0 one has no Groups key at all — so writing the property below the floor would be inventing a key, the mendixlabs/mxcli#759 failure shape." parallel_split: min_version: "9.0.0" user_task: diff --git a/sdk/versions/registry_test.go b/sdk/versions/registry_test.go index c6f693b3a..3d782700f 100644 --- a/sdk/versions/registry_test.go +++ b/sdk/versions/registry_test.go @@ -274,3 +274,33 @@ func TestAgentDocumentsAreGated(t *testing.T) { }) } } + +// TestWorkflowGroupsFloorIs11_2 pins the workflow-group gate to the metamodel, +// not to the release notes. +// +// mendixlabs/mxcli#272 states "Workflow Groups are GA from Mendix 11.6". The +// arbiter for whether the document loads is the Model SDK's own +// StructureVersionInfo, and there both Settings$WorkflowGroup and +// WorkflowsProjectSettingsPart.groups read `introduced: "11.2.0"` +// (mendixmodelsdk 4.115.0, package/src/gen/settings.js). Gating on 11.6 would +// refuse four minors of projects that store the property perfectly well. +func TestWorkflowGroupsFloorIs11_2(t *testing.T) { + reg, err := Load() + if err != nil { + t.Fatalf("Load: %v", err) + } + cases := []struct { + v SemVer + want bool + }{ + {SemVer{Major: 10, Minor: 24, Patch: 0}, false}, + {SemVer{Major: 11, Minor: 1, Patch: 0}, false}, + {SemVer{Major: 11, Minor: 2, Patch: 0}, true}, + {SemVer{Major: 11, Minor: 13, Patch: 0}, true}, + } + for _, tc := range cases { + if got := reg.IsAvailable("workflows", "groups", tc.v); got != tc.want { + t.Errorf("IsAvailable(workflows, groups, %v) = %v, want %v", tc.v, got, tc.want) + } + } +} From d614dbfffc8666545517481d4a81734ab179877b Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 21 Sep 2026 16:29:51 +0000 Subject: [PATCH 04/15] fix(executor): resolve a retrieve's sort hop across ancestors and modules MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `retrieve $AccountList from Administration.Account sort by System.Language.Code asc;` — MDL that `mxcli describe` had just emitted — passed `mxcli check` and was refused by `mxcli exec`: sort by attribute 'System.Language.Code' does not belong to entity 'Administration.Account' Mendix stores a sort over an association as an AttributeRef naming the far entity plus an EntityRef of EntityRefSteps. DESCRIBE emits only the attribute's qualified name — MDL has no spelling for the hop — so replaying a described retrieve has to re-derive the step. inferSortEntityRefSteps made three assumptions that hold only when the hop starts on the retrieved entity in its own module: it searched one domain model (the retrieved entity's), matched only associations whose parent was the retrieved entity itself, and qualified the association it found with the retrieved entity's module. Administration.Account reaches System.Language through System.User_Language, which is declared on the ancestor System.User and stored in the System module — all three are wrong there. It is now a generalization-chain walk that looks each ancestor up in its own module and qualifies the association with the module storing it. The destination end is matched with entityIsSubtypeOf rather than by equality, since an association may point at a specialization of the entity that declares the attribute. Measured on a blank 11.12.3 app: fixed exec ✓, describe → exec is a fixed point, mx check 0 errors, stored hop reads System.User_Language → System.Language fix reverted check ✓, exec fails with the reported message hop dropped exec ✓, mx check CE7247 "Cannot sort on attribute 'System.Language.Code'. Attribute 'System.Language.Code' is not an attribute of entity 'Administration.Account'." An attribute of an entity that is neither in the chain nor one hop away is still refused, in the unit tests and in the real project. Known residue, recorded in the finding and in the code: where several associations reach one entity, DESCRIBE cannot say which was stored and the nearest ancestor's first hop wins. Spelling the hop needs grammar. Fixes mendixlabs/mxcli#1152 Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01NuVi6vytSqoqAtZK8YzvD2 --- .../fix-issue/findings/mdl-executor.jsonl | 1 + docs/01-project/MDL_QUICK_REFERENCE.md | 3 +- .../microflow-1152-sort-over-association.mdl | 30 +++ .../modelsdk/microflow_retrievesort_test.go | 95 +++++++++ .../cmd_microflows_builder_actions.go | 111 +++++++--- .../cmd_microflows_sort_association_test.go | 193 ++++++++++++++++++ 6 files changed, 405 insertions(+), 28 deletions(-) create mode 100644 mdl-examples/bug-tests/microflow-1152-sort-over-association.mdl create mode 100644 mdl/executor/cmd_microflows_sort_association_test.go diff --git a/.claude/skills/fix-issue/findings/mdl-executor.jsonl b/.claude/skills/fix-issue/findings/mdl-executor.jsonl index 057547703..8b0a99e0e 100644 --- a/.claude/skills/fix-issue/findings/mdl-executor.jsonl +++ b/.claude/skills/fix-issue/findings/mdl-executor.jsonl @@ -663,3 +663,4 @@ {"area": "mdl/executor", "date": "2026-09-20", "symptom": "`describe page` → `exec` over a **Studio Pro-authored** page silently drops six things, `mx check` 0 errors throughout. The one that matters: `IsPasswordBox True → False` — a **password field round-trips into a plaintext text box**, and describe → rename → exec is mxcli's copy operation. Also `Validation.Expression` blanked, a DataView's `ReadOnlyStyle Text → Control`, `PopupCloseAction` wiped, and two typed-array markers", "cause": "Four different causes behind one symptom, which is why triage came first: (1) IsPasswordBox — model and writer carried it, nothing parsed it, nothing emitted it; (2) Validation — `widgetValidationToGen()` wrote a DEFAULT EMPTY Forms$WidgetValidation over whatever was stored, on five widget types; (3) ReadOnlyStyle — wired for CheckBox only, and a DataView's draws no MDL-WIDGET07 warning because `staticWidgetKnownProps` is deliberately a union across widget types; (4) PopupCloseAction — `pageToGen` wrote \"\" unconditionally. Plus ParameterMappings/OutputMappings markers", "file": "`mdl/executor/cmd_pages_describe_parse.go` + `_output.go` (extract/emit), `cmd_pages_builder_v3_widgets.go` (consume), `cmd_pages_builder_v3.go`, `mdl/visitor/visitor_page_v3.go`, `mdl/ast/ast_page_v3.go`, `sdk/pages/*`, `mdl/backend/modelsdk/widget_write.go` + `page_write.go`, `mdl/executor/validate_widgets.go` (describe vocabulary)", "insight": "**Triage the layer before writing anything** — describer / grammar / builder have different fixes and this one issue had all three. The quickest probe is to run the property through `mxcli check`: MDL-WIDGET07 names an unrecognised one, and *silence is not acceptance* — the known-props list is a union across widget types, so a DataView's ReadOnlyStyle passed check and was dropped anyway. **Emit an expression QUOTED, not bracketed**: `[...]` is the XPath-constraint spelling and `propertyValueV3` parses it as an ARRAY, so `GetStringProp` yields \"\" — the emitter's own unit test was green while the real round trip still lost the value (storage form is not input form). **Measure the default before keeping it**: a DataView's ReadOnlyStyle is Control on 47 of 56, never Inherit, so the 'obvious' Inherit that every other input widget uses would have been wrong. Markers likewise measured, not assumed: ParameterMappings is marker 2 on 220 of 220 lists in every parent type, OutputMappings present on 91 of 91 — and an EMPTY list needs `MandatoryListMarkers` since `RegisterListMarker` keys on a child that is not there. Result 17 → 9 differences, the 9 being ako/mxcli#549", "refs": ["#550", "#541", "#549", "#490"]} {"area": "mdl/executor", "date": "2026-09-20", "symptom": "MDL-PAGEARG01 refused a list widget's OWN row action: `datagrid dg (DataSource: DATABASE M.E, onClick: SHOW_PAGE M.Edit(E: $currentObject))` was rejected at `check` with \"widget `dg` is not inside a data view, list view or grid row\" \u2014 and since exec refuses a script whose check errors, the slice could not be applied at all. On a `listview` the message contradicted itself. mxbuild 11.14.0 accepts the stored pages at 0 errors.", "cause": "The #1029 guard judged EVERY widget's own action in the context its PARENT supplies: `argContextForSubtreeOf` returns the parent context for a childless widget and `validate_widgets.go` passed the inherited `argCtx` to `validateShowPageArguments`. Right for a button, wrong for the widget that ESTABLISHES the context \u2014 a list widget's onClick is row-scoped, so the row it renders is the context object. Added `argContextForOwnAction`: a widget that binds a source of its own supplies the context for its own action; a source in a shape the pass cannot read (the bare-entity shorthand) degrades to UNKNOWN so the guard stands down rather than refusing what it cannot prove is discarded.", "file": "`mdl/executor/cmd_pages_showpage_args.go` (argContextForOwnAction, argContextForSubtreeOf), `mdl/executor/validate_widgets.go`", "insight": "**A false refusal costs more than a missing rule now that exec refuses on a check error** \u2014 the blast radius is 'this project cannot be built with this mxcli', not 'a warning is noisy'. Two things would have caught it before release: judging the rule against the widget kinds it NAMES in its own message (the listview refusal reads 'lvA is not inside a \u2026 list view'), and running it against mxbuild rather than against intuition. The mxbuild run paid for itself twice: it also showed that `DataSource: M.E` (bare-entity shorthand) on a datagrid is silently dropped, so that case is CE0488 + a REAL CE1571 \u2014 the stand-down is still correct, but the shorthand case must not be written into a bug test as mxbuild-clean (#576). Control the fix with the widget kinds STILL refused (a foreign variable, a sibling button beside the grid), or it is indistinguishable from deleting the rule.", "refs": ["#552", "#576", "mendixlabs/mxcli#1029", "#939"]} {"area": "mdl/executor", "date": "2026-09-21", "symptom": "A page's image-collection reference passed `mxcli check --references` and failed the build. Reported as \"no MDL syntax for a StaticImageViewer inside a Selection helper custom state\" — the authoring half was already closed by #1057; what was left is that nothing RESOLVED the name it made writable. Measured on a blank Mendix 11.14.0 project: `staticimage imgAll (Image: 'Atlas_UI_Resources.Atlas_Icons.checkbox_checked')` in a custom state -> check passed, exec created the page, `mx check` -> 3x CE1613 \"The selected image … no longer exists.\"", "cause": "TWO independent holes, and either alone leaves the reported script unchecked. (1) widgetRefCollector keyed the image reference on the widget TYPE — `if w.Type == \"image\"` — so the pluggable widget was collected and `staticimage` (which #1057 had just given the SAME `Image:` property) and `dynamicimage`'s `DefaultImage` were not; replaced with an imageRefProps table. (2) A page's widgets live in two AST fields: `Widgets` is the bare body, `Placeholders` holds `placeholder X { … }` content (#532). validate.go passed `s.Widgets` alone to validateWidgetReferences, validatePageContextTree AND validateFlowArguments, so EVERY reference inside a placeholder block — microflow, nanoflow, page, snippet, entity, image — was validated by nothing; added allPageWidgets to collect both roots once.", "file": "`mdl/executor/helpers.go` (widgetRefCollector.collectFromWidget, imageRefProps), `mdl/executor/validate.go` (allPageWidgets)", "insight": "**When a capability gets a new spelling, grep for who RESOLVES the old one.** #1057 added `Image:` to a second and third widget and moved on; the resolver keyed on the type name, so the new spellings were unchecked from the day they shipped. A property list and a resolver list that describe the same property are two copies — `validate_widgets.go` already accepted `Image`/`DefaultImage` for these widgets and DESCRIBE already emitted them, and only the resolver disagreed. **The placeholder hole is the more useful lesson: it was the THIRD copy of one walk.** validateIconRefs (#1008) and forEachWidget had each grown the `Placeholders` arm separately, with a comment saying a missed walk is silent both ways — and the three validators next door still had not. When a fix is 'add the missing arm to this walker', the question is how many walkers there are; collect the roots once instead. **Do not reason about a bug report from the issue text alone when the version is older than the fix** — the reported symptom did not reproduce on main at all, and running the reporter's own script end to end is what turned 'already fixed, close it' into two real defects. **Control both directions**: a reference that resolves must stay silent, because a walker that can suddenly see a whole new region of the tree is as likely to report correct scripts as broken ones.", "refs": ["mendixlabs/mxcli#1149", "mendixlabs/mxcli#1057", "mendixlabs/mxcli#1008", "#532"]} +{"area": "mdl/executor", "date": "2026-09-21", "symptom": "`retrieve $AccountList from Administration.Account sort by System.Language.Code asc;` — MDL that `mxcli describe` had just emitted — passed `mxcli check` and was refused by `mxcli exec`: \"sort by attribute 'System.Language.Code' does not belong to entity 'Administration.Account'\". Reported as a check/exec inconsistency (mendixlabs/mxcli#1152); the real defect is that the round trip cannot replay its own output for any sort over an association reached from an ANCESTOR.", "cause": "inferSortEntityRefSteps searched ONE domain model — the retrieved entity's own module — for associations whose parent was the retrieved entity ITSELF, and qualified the association it found with the retrieved entity's module. All three assumptions hold only when the hop starts on the retrieved entity in its own module. Administration.Account reaches System.Language through System.User_Language, declared on System.User and stored in the System module: parent is an ancestor, the domain model is another module's, and the qualified name carries THAT module. Rewritten as a generalization-chain walk that looks each ancestor up in its own module and qualifies the association with the module storing it; the destination end is matched with entityIsSubtypeOf rather than by equality, since an association may point at a specialization of the entity that declares the attribute.", "file": "`mdl/executor/cmd_microflows_builder_actions.go` (inferSortEntityRefSteps); tests `mdl/executor/cmd_microflows_sort_association_test.go`, `mdl/backend/modelsdk/microflow_retrievesort_test.go`; example `mdl-examples/bug-tests/microflow-1152-sort-over-association.mdl`", "insight": "**The second control is the one that pays.** Reverting the fix reproduces the refusal, which only proves the test fires. The control that taught something was building a binary that DERIVES the hop and does not WRITE it — exec succeeds and mxbuild 11.12.3 answers CE7247 \"Cannot sort on attribute 'System.Language.Code'. Attribute 'System.Language.Code' is not an attribute of entity 'Administration.Account'\" — the executor's refusal message almost word for word, from the other end of the pipeline. That is what fixes the qualified name as load-bearing: the stored EntityRefStep must read System.User_Language, and the pre-existing code would have written Administration.User_Language had it found anything at all. **Skip the theory that check is missing a rule**: check has no sort-attribute rule at all and resolves no hops, so it was never going to disagree with exec here — the inconsistency in the report is a symptom of the false refusal, not a second defect. **Known residue, stated because the round trip rests on it**: DESCRIBE emits only the attribute's qualified name, so where several associations reach one entity the replay picks the nearest ancestor's first and can silently land on the other hop. Spelling the hop needs grammar (sortColumn is qualifiedName|IDENTIFIER, no `/` path) and is a language change, not a fix."} diff --git a/docs/01-project/MDL_QUICK_REFERENCE.md b/docs/01-project/MDL_QUICK_REFERENCE.md index d59ef8a2d..e6cf8a659 100644 --- a/docs/01-project/MDL_QUICK_REFERENCE.md +++ b/docs/01-project/MDL_QUICK_REFERENCE.md @@ -510,7 +510,8 @@ it is for pages. | Commit | `commit $entity [without events] [refresh];` | **Omitted = with events**, matching Studio Pro's default. `without events` is the deviation and the only form that changes the stored value; `with events` still parses and means the default | | Delete | `delete $entity [refresh];` | | | Rollback | `rollback $entity [refresh];` | Reverts uncommitted changes | -| Retrieve (DB) | `retrieve $Var from Module.Entity [where condition];` | Database XPath retrieve | +| Retrieve (DB) | `retrieve $Var from Module.Entity [where condition] [sort by Attr asc\|desc, ...] [limit n [offset n]];` | Database XPath retrieve. `limit 1` with no `offset` binds a single **object**, not a one-element list (MDL-RETRIEVE01) | +| Retrieve (DB), sorted | `sort by Attr asc` / `sort by Module.Other.Attr asc` | A bare name is qualified with the entity **declaring** it, which may be an ancestor. A qualified name may also be an attribute of an entity reached by **one association hop** — mxcli derives the hop (walking the generalization chain, across modules: `Administration.Account` reaches `System.Language.Code` through `System.User_Language`) and stores it as the `EntityRef` Mendix needs; without it the build is CE7247. Where several associations reach the same entity MDL cannot say which, and the nearest one wins (mendixlabs/mxcli#1152) | | Retrieve (Assoc) | `retrieve $list from $Parent/Module.AssocName;` | Retrieve by association | | Add to list | `add expression to $list;` | Also accepts existing `add $item to $list;` form | | Aggregate a list | `$Total = sum($list.Attr);` / `$Total = sum($list, expression);` | `count` (list only), `sum`, `average`, `minimum`, `maximum` — attribute or expression over `$currentObject` | diff --git a/mdl-examples/bug-tests/microflow-1152-sort-over-association.mdl b/mdl-examples/bug-tests/microflow-1152-sort-over-association.mdl new file mode 100644 index 000000000..0b87be201 --- /dev/null +++ b/mdl-examples/bug-tests/microflow-1152-sort-over-association.mdl @@ -0,0 +1,30 @@ +-- mendixlabs/mxcli#1152 — "roundtrip for retrieve DB with sorting over +-- association does not work". +-- +-- `mxcli describe` emitted this retrieve, `mxcli check` reported no errors, and +-- `mxcli exec` refused it: +-- +-- Error: microflow 'ExamplesModule.ACT_sort_test ' has validation errors: +-- sort by attribute 'System.Language.Code' does not belong to entity +-- 'Administration.Account' +-- +-- The hop that reaches System.Language is System.User_Language, declared on the +-- ANCESTOR System.User and stored in the SYSTEM module's domain model. The +-- executor searched only the retrieved entity's own module, for associations +-- whose parent was the retrieved entity itself, so it found nothing. +-- +-- Measured on 11.12.3 in a blank app: with the hop written, `mx check` is +-- 0 errors. With the hop dropped but the attribute path kept — the shape a +-- half-fix produces — mxbuild reports CE7247 "Cannot sort on attribute +-- 'System.Language.Code'. Attribute 'System.Language.Code' is not an attribute +-- of entity 'Administration.Account'." + +create or modify microflow MyFirstModule.ACT_sort_test () +begin + @start(96, 200) + @position(236, 200) + retrieve $AccountList from Administration.Account + sort by System.Language.Code asc; + @position(700, 200) + return; +end; diff --git a/mdl/backend/modelsdk/microflow_retrievesort_test.go b/mdl/backend/modelsdk/microflow_retrievesort_test.go index 9aaed6d6e..6b28da94c 100644 --- a/mdl/backend/modelsdk/microflow_retrievesort_test.go +++ b/mdl/backend/modelsdk/microflow_retrievesort_test.go @@ -92,3 +92,98 @@ func TestRetrieveSourceToGen_SortByRoundTrip(t *testing.T) { t.Errorf("sort[1] = {%q, %q}, want {SortBug.Ticket.Name, Ascending}", out.Sorting[1].AttributeQualifiedName, out.Sorting[1].Direction) } } + +// TestRetrieveSourceToGen_SortOverAssociationWritesEntityRefSteps guards the +// BSON a sort over an association produces. The attribute's qualified name names +// the FAR entity, so on its own it is a reference Mendix cannot resolve against +// the retrieved entity — the hop has to be stored beside it, as an +// AttributeRef.EntityRef (DomainModels$IndirectEntityRef) of EntityRefSteps. +// +// This is the write half of mendixlabs/mxcli#1152, where the executor could not derive +// the hop and refused the statement outright; a fix that derived it but did not +// persist it would leave the same dangling reference in the document. +func TestRetrieveSourceToGen_SortOverAssociationWritesEntityRefSteps(t *testing.T) { + in := µflows.DatabaseRetrieveSource{ + EntityQualifiedName: "Administration.Account", + Sorting: []*microflows.SortItem{{ + AttributeQualifiedName: "System.Language.Code", + Direction: microflows.SortDirectionAscending, + EntityRefSteps: []microflows.EntityRefStep{{ + Association: "System.User_Language", + DestinationEntity: "System.Language", + }}, + }}, + } + + el := retrieveSourceToGen(in) + if el == nil { + t.Fatal("retrieveSourceToGen returned nil") + } + raw, err := (&codec.Encoder{}).Encode(el) + if err != nil { + t.Fatalf("encode: %v", err) + } + + sortings, ok := bson.Raw(raw).Lookup("NewSortings").DocumentOK() + if !ok { + t.Fatal("no NewSortings envelope") + } + arr, ok := sortings.Lookup("Sortings").ArrayOK() + if !ok { + t.Fatal("no Sortings array") + } + vals, err := arr.Values() + if err != nil { + t.Fatalf("Sortings values: %v", err) + } + var item bson.Raw + for _, v := range vals { + // The first entry is the typed-array marker, not a document. + if d, ok := v.DocumentOK(); ok { + item = d + break + } + } + if item == nil { + t.Fatal("Sortings array carries no sort item") + } + ref, ok := item.Lookup("AttributeRef").DocumentOK() + if !ok { + t.Fatal("sort item has no AttributeRef") + } + if got := ref.Lookup("Attribute").StringValue(); got != "System.Language.Code" { + t.Errorf("Attribute = %q, want System.Language.Code", got) + } + entityRef, ok := ref.Lookup("EntityRef").DocumentOK() + if !ok { + t.Fatal("AttributeRef has no EntityRef — the association hop was dropped, " + + "leaving a sort attribute that does not resolve against the retrieved entity") + } + if got := entityRef.Lookup("$Type").StringValue(); got != "DomainModels$IndirectEntityRef" { + t.Errorf("EntityRef $Type = %q, want DomainModels$IndirectEntityRef", got) + } + steps, ok := entityRef.Lookup("Steps").ArrayOK() + if !ok { + t.Fatal("EntityRef has no Steps array") + } + stepVals, err := steps.Values() + if err != nil { + t.Fatalf("Steps values: %v", err) + } + var step bson.Raw + for _, v := range stepVals { + if d, ok := v.DocumentOK(); ok { + step = d + break + } + } + if step == nil { + t.Fatal("Steps array carries no step") + } + if got := step.Lookup("Association").StringValue(); got != "System.User_Language" { + t.Errorf("Association = %q, want System.User_Language", got) + } + if got := step.Lookup("DestinationEntity").StringValue(); got != "System.Language" { + t.Errorf("DestinationEntity = %q, want System.Language", got) + } +} diff --git a/mdl/executor/cmd_microflows_builder_actions.go b/mdl/executor/cmd_microflows_builder_actions.go index 75a16ac59..ffbb170c3 100644 --- a/mdl/executor/cmd_microflows_builder_actions.go +++ b/mdl/executor/cmd_microflows_builder_actions.go @@ -1247,42 +1247,99 @@ func retrieveXPathConstraint(expr ast.Expression) string { return visitor.FormatXPathConstraint("[" + xpath + "]") } +// inferSortEntityRefSteps finds the one association hop that reaches the entity +// DECLARING the sort attribute, for a `sort by Module.Entity.Attribute` whose +// entity is neither the retrieved one nor an ancestor of it. +// +// Mendix stores such a sort as an AttributeRef whose AttributeQualifiedName +// names the far entity plus an EntityRef carrying one EntityRefStep per hop. +// DESCRIBE emits only the attribute's qualified name — MDL has no spelling for +// the hop — so replaying a described retrieve has to re-derive the step, and +// the whole round trip rests on that derivation. +// +// The association is not necessarily declared on the retrieved entity, nor in +// its module. `Administration.Account` reaches `System.Language` through +// `System.User_Language`, which is declared on the ANCESTOR `System.User` and +// stored in the SYSTEM module's domain model. Searching only the retrieved +// entity's own module for associations whose parent is the retrieved entity +// itself found nothing, so a retrieve `mxcli describe` had just emitted was +// refused by `exec` with "does not belong to entity" while `mxcli check` passed +// — mendixlabs/mxcli#1152. Same shape as the inherited-attribute defect +// (CapTrackV2 §13): the resolver that walks the generalization chain existed, +// and this path did not call it. +// +// So the walk is over the generalization chain, and each ancestor is looked up +// in ITS OWN module — which is also where the association's qualified name +// comes from. Qualifying with the retrieved entity's module is what the +// same-module case made look right, and it is wrong exactly in the case that +// was broken. +// +// The destination end is matched with entityIsSubtypeOf rather than by equality, +// because an association may point at a SPECIALIZATION of the entity that +// declares the attribute; Mendix stores the declaring entity in the path either +// way. +// +// Limitation, stated because the round trip depends on it: where several +// associations reach the same entity, MDL cannot say which one was stored, and +// the nearest entity's first association wins. The order is deterministic (both +// the chain walk and dm.Associations are ordered), so a replay is stable — but +// a model with two hops to one entity can still round-trip to the other one. +// Spelling the hop would need grammar, and is a language change, not a fix. func (fb *flowBuilder) inferSortEntityRefSteps(sourceEntityQN, attrPath string) []microflows.EntityRefStep { attrEntityQN := entityQualifiedNameFromAttribute(attrPath) if attrEntityQN == "" || attrEntityQN == sourceEntityQN { return nil } - parts := strings.SplitN(sourceEntityQN, ".", 2) - if len(parts) != 2 || parts[0] == "" { - return nil - } - if fb.backend == nil { - return nil - } - mod, err := fb.backend.GetModuleByName(parts[0]) - if err != nil || mod == nil { - return nil - } - dm, err := fb.backend.GetDomainModel(mod.ID) - if err != nil || dm == nil { + if fb == nil || fb.backend == nil { return nil } - entityNames := make(map[model.ID]string, len(dm.Entities)) - for _, e := range dm.Entities { - entityNames[e.ID] = parts[0] + "." + e.Name - } - for _, assoc := range dm.Associations { - parentQN := entityNames[assoc.ParentID] - childQN := entityNames[assoc.ChildID] - if parentQN == sourceEntityQN && childQN == attrEntityQN { - return []microflows.EntityRefStep{{Association: parts[0] + "." + assoc.Name, DestinationEntity: childQN}} + seen := make(map[string]bool) + for currentQN := sourceEntityQN; currentQN != ""; { + if seen[currentQN] { + return nil } - } - for _, assoc := range dm.CrossAssociations { - parentQN := entityNames[assoc.ParentID] - if parentQN == sourceEntityQN && assoc.ChildRef == attrEntityQN { - return []microflows.EntityRefStep{{Association: parts[0] + "." + assoc.Name, DestinationEntity: assoc.ChildRef}} + seen[currentQN] = true + + parts := strings.SplitN(currentQN, ".", 2) + if len(parts) != 2 || parts[0] == "" { + return nil + } + moduleName := parts[0] + mod, err := fb.backend.GetModuleByName(moduleName) + if err != nil || mod == nil { + return nil + } + dm, err := fb.backend.GetDomainModel(mod.ID) + if err != nil || dm == nil { + return nil + } + entityNames := make(map[model.ID]string, len(dm.Entities)) + for _, e := range dm.Entities { + entityNames[e.ID] = moduleName + "." + e.Name + } + for _, assoc := range dm.Associations { + if entityNames[assoc.ParentID] != currentQN { + continue + } + childQN := entityNames[assoc.ChildID] + if childQN != "" && fb.entityIsSubtypeOf(childQN, attrEntityQN) { + return []microflows.EntityRefStep{{Association: moduleName + "." + assoc.Name, DestinationEntity: childQN}} + } + } + for _, assoc := range dm.CrossAssociations { + if entityNames[assoc.ParentID] != currentQN { + continue + } + if assoc.ChildRef != "" && fb.entityIsSubtypeOf(assoc.ChildRef, attrEntityQN) { + return []microflows.EntityRefStep{{Association: moduleName + "." + assoc.Name, DestinationEntity: assoc.ChildRef}} + } } + + entity := dm.FindEntityByName(parts[1]) + if entity == nil { + return nil + } + currentQN = entity.GeneralizationRef } return nil } diff --git a/mdl/executor/cmd_microflows_sort_association_test.go b/mdl/executor/cmd_microflows_sort_association_test.go new file mode 100644 index 000000000..c05647330 --- /dev/null +++ b/mdl/executor/cmd_microflows_sort_association_test.go @@ -0,0 +1,193 @@ +// SPDX-License-Identifier: Apache-2.0 + +package executor + +import ( + "strings" + "testing" + + "github.com/mendixlabs/mxcli/mdl/ast" + "github.com/mendixlabs/mxcli/mdl/backend/mock" + "github.com/mendixlabs/mxcli/model" + "github.com/mendixlabs/mxcli/sdk/domainmodel" + "github.com/mendixlabs/mxcli/sdk/microflows" +) + +// mendixlabs/mxcli#1152 — "roundtrip for retrieve DB with sorting over association does +// not work". `mxcli describe` emitted +// +// retrieve $AccountList from Administration.Account +// sort by System.Language.Code asc; +// +// `mxcli check` reported no errors and `mxcli exec` refused it: +// +// Error: microflow 'ExamplesModule.ACT_sort_test ' has validation errors: +// sort by attribute 'System.Language.Code' does not belong to entity +// 'Administration.Account' +// +// The hop that reaches System.Language is System.User_Language, which is +// declared on the ANCESTOR System.User and stored in the SYSTEM module's domain +// model. inferSortEntityRefSteps looked only in the retrieved entity's own +// module (Administration) and only at associations whose parent was the +// retrieved entity itself, so it found nothing and the sort was refused. +// +// This backend is that shape with synthetic names: SyntheticApp.AppUser +// generalizes SyntheticBase.User, and the association to SyntheticBase.Language +// lives on the base, in the base's module. +func associationSortBackend() *mock.MockBackend { + appModuleID := model.ID("synthetic-app-module") + baseModuleID := model.ID("synthetic-base-module") + return &mock.MockBackend{ + GetModuleByNameFunc: func(name string) (*model.Module, error) { + switch name { + case "SyntheticApp": + return &model.Module{BaseElement: model.BaseElement{ID: appModuleID}, Name: name}, nil + case "SyntheticBase": + return &model.Module{BaseElement: model.BaseElement{ID: baseModuleID}, Name: name}, nil + } + return nil, nil + }, + GetDomainModelFunc: func(id model.ID) (*domainmodel.DomainModel, error) { + switch id { + case appModuleID: + profile := &domainmodel.Entity{Name: "Profile"} + profile.ID = model.ID("SyntheticApp.Profile") + appUser := &domainmodel.Entity{ + Name: "AppUser", + GeneralizationRef: "SyntheticBase.User", + Attributes: []*domainmodel.Attribute{ + {Name: "FullName", Type: &domainmodel.StringAttributeType{}}, + }, + } + appUser.ID = model.ID("SyntheticApp.AppUser") + own := &domainmodel.Association{ + Name: "AppUser_Profile", + ParentID: appUser.ID, + ChildID: profile.ID, + Type: domainmodel.AssociationTypeReference, + } + own.ID = model.ID("SyntheticApp.AppUser_Profile") + return &domainmodel.DomainModel{ + ContainerID: appModuleID, + Entities: []*domainmodel.Entity{appUser, profile}, + Associations: []*domainmodel.Association{own}, + }, nil + case baseModuleID: + user := &domainmodel.Entity{ + Name: "User", + Attributes: []*domainmodel.Attribute{ + {Name: "Name", Type: &domainmodel.StringAttributeType{}}, + }, + } + user.ID = model.ID("SyntheticBase.User") + language := &domainmodel.Entity{ + Name: "Language", + Attributes: []*domainmodel.Attribute{ + {Name: "Code", Type: &domainmodel.StringAttributeType{}}, + }, + } + language.ID = model.ID("SyntheticBase.Language") + other := &domainmodel.Entity{Name: "Other"} + other.ID = model.ID("SyntheticBase.Other") + assoc := &domainmodel.Association{ + Name: "User_Language", + ParentID: user.ID, + ChildID: language.ID, + Type: domainmodel.AssociationTypeReference, + } + assoc.ID = model.ID("SyntheticBase.User_Language") + return &domainmodel.DomainModel{ + ContainerID: baseModuleID, + Entities: []*domainmodel.Entity{user, language, other}, + Associations: []*domainmodel.Association{assoc}, + }, nil + } + return nil, nil + }, + } +} + +// assocSortOf builds the retrieve against associationSortBackend and returns the +// stored sort attribute path, its EntityRefSteps, and any build errors. +func assocSortOf(t *testing.T, attr string) (path string, steps []microflows.EntityRefStep, errs []string) { + t.Helper() + fb := &flowBuilder{backend: associationSortBackend(), spacing: 100} + fb.addRetrieveAction(&ast.RetrieveStmt{ + Variable: "Users", + Source: ast.QualifiedName{Module: "SyntheticApp", Name: "AppUser"}, + SortColumns: []ast.SortColumnDef{{Attribute: attr, Order: "ASC"}}, + }) + var items []*microflows.SortItem + for _, obj := range fb.objects { + act, ok := obj.(*microflows.ActionActivity) + if !ok { + continue + } + ra, ok := act.Action.(*microflows.RetrieveAction) + if !ok { + continue + } + ds, ok := ra.Source.(*microflows.DatabaseRetrieveSource) + if !ok { + continue + } + items = append(items, ds.Sorting...) + } + if len(items) > 1 { + t.Fatalf("got %d sort columns, want at most 1", len(items)) + } + if len(items) == 1 { + path, steps = items[0].AttributeQualifiedName, items[0].EntityRefSteps + } + return path, steps, fb.errors +} + +// The reported case: the association is on an ancestor, in the ancestor's +// module. Before the fix this produced "does not belong to entity". +func TestSortBy_AssociationOnAncestorInAnotherModule(t *testing.T) { + path, steps, errs := assocSortOf(t, "SyntheticBase.Language.Code") + if len(errs) > 0 { + t.Fatalf("describe emitted this sort and exec refused it (mendixlabs/mxcli#1152): %v", errs) + } + if path != "SyntheticBase.Language.Code" { + t.Errorf("stored attribute %q, want %q", path, "SyntheticBase.Language.Code") + } + if len(steps) != 1 { + t.Fatalf("got %d entity ref steps, want 1: %+v", len(steps), steps) + } + // The association is qualified with the module that STORES it, not with the + // retrieved entity's module — the two differ exactly in this case. + if steps[0].Association != "SyntheticBase.User_Language" { + t.Errorf("association %q, want %q", steps[0].Association, "SyntheticBase.User_Language") + } + if steps[0].DestinationEntity != "SyntheticBase.Language" { + t.Errorf("destination %q, want %q", steps[0].DestinationEntity, "SyntheticBase.Language") + } +} + +// CONTROL 1: an association declared on the retrieved entity itself, in its own +// module, still resolves. A fix that only looked at ancestors would break the +// case that already worked. +func TestSortBy_AssociationOnTheRetrievedEntityStillResolves(t *testing.T) { + _, steps, errs := assocSortOf(t, "SyntheticApp.Profile.Label") + if len(errs) > 0 { + t.Fatalf("unexpected errors: %v", errs) + } + if len(steps) != 1 || steps[0].Association != "SyntheticApp.AppUser_Profile" || + steps[0].DestinationEntity != "SyntheticApp.Profile" { + t.Errorf("got %+v, want one hop SyntheticApp.AppUser_Profile -> SyntheticApp.Profile", steps) + } +} + +// CONTROL 2: an entity that is neither in the generalization chain nor reachable +// by one association hop is still refused. Accepting anything qualified would +// turn a diagnosable mistake into a CE1613 at the far end of a build. +func TestSortBy_UnreachableEntityIsStillRefused(t *testing.T) { + _, _, errs := assocSortOf(t, "SyntheticBase.Other.Name") + if len(errs) == 0 { + t.Fatal("an attribute of an unreachable entity must still be refused") + } + if !strings.Contains(strings.Join(errs, " "), "does not belong to entity") { + t.Errorf("unexpected message: %v", errs) + } +} From fe5a7408ee5e5f9e3b3a9c8c117170cd44b9a536 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 21 Sep 2026 16:59:40 +0000 Subject: [PATCH 05/15] feat(mdl): let a sort column name the association it navigates MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The inference fix restored the reported script but left the round trip lossy. `describe` emitted only the sort attribute's qualified name, so replaying it had to re-derive the hop — and where two associations reach the same entity there is nothing to derive it from. Measured on 11.12.3, with Order_ShipTo and Order_BillTo both Order → Address (an ordinary shape, not a corner case). A microflow sorting by the BILLING address, run through describe → exec: stored hop before: MyFirstModule.Order_BillTo describe emits: sort by MyFirstModule.Address.City asc stored hop after: MyFirstModule.Order_ShipTo mx check is 0 errors on both sides. The app sorts by the wrong address and nothing in the toolchain says so. A sort column now takes an association path — one `/` per hop, the last segment names the attribute — in a microflow retrieve and in a page datasource alike: retrieve $Orders from Sales.Order sort by Sales.Order_BillTo/Sales.Address.City asc; listview lv (datasource: database from Sales.Order sort by Sales.Order_BillTo/City asc) `qualifiedName (SLASH qualifiedName)*` is the shape MDLCatalog.g4 already uses for Association/Entity, and `Assoc/Attr` is how DataGrid2 columns and dynamictext params navigate, so this adds no new idiom. Inference stays as the fallback: every script written before keeps working. Two reads were missing, not one. sortItemsFromRaw read AttributeRef.Attribute and skipped AttributeRef.EntityRef, and the page reader did the same — so the hop was written and never read back, and DESCRIBE could not have emitted it even with a spelling in hand. Fixing only the microflow half would have shipped a describer that emits the path for microflows and silently drops it for pages. A hop that does not resolve, or does not start at the entity in hand, is refused rather than written: a step with an empty DestinationEntity makes the project unopenable rather than merely wrong. Measured, both halves, with controls: fixed mx check 0 errors; describe → exec replays every sort to "Unchanged" (the write elided — the rebuild is identical) hops ignored the named Order_BillTo is stored as Order_ShipTo hop dropped, mx check CE7247 "Cannot sort on attribute microflow 'System.Language.Code' …" hop dropped, mx check CE7247 "… is not an attribute of entity page 'MyFirstModule.Order'." at Sort bar of list view 'lvOrders' Refs mendixlabs/mxcli#1152 Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01NuVi6vytSqoqAtZK8YzvD2 --- .../fix-issue/findings/mdl-executor.jsonl | 1 + .../mendix/master-detail-pages/SKILL.md | 3 +- .claude/skills/mendix/overview-pages/SKILL.md | 7 +- .../reference/data-operations.md | 25 +++ cmd/mxcli/syntax/features_microflow.go | 11 +- cmd/mxcli/syntax/features_page.go | 2 +- docs/01-project/MDL_QUICK_REFERENCE.md | 2 +- .../microflow-1152-sort-association-path.mdl | 73 +++++++ mdl/ast/ast_page.go | 12 +- mdl/ast/ast_page_v3.go | 7 +- .../modelsdk/microflow_read_actions.go | 40 ++++ .../modelsdk/microflow_retrievesort_test.go | 71 +++++++ mdl/backend/modelsdk/widget_write.go | 14 +- .../cmd_microflows_builder_actions.go | 148 +++++++++++++- mdl/executor/cmd_microflows_format_action.go | 11 ++ .../cmd_microflows_sort_assoc_path_test.go | 182 ++++++++++++++++++ mdl/executor/cmd_pages_builder_v3.go | 35 +++- mdl/executor/cmd_pages_describe.go | 5 +- mdl/executor/cmd_pages_describe_datasource.go | 35 +++- .../cmd_pages_sort_association_test.go | 108 +++++++++++ mdl/grammar/domains/MDLPage.g4 | 14 +- mdl/visitor/visitor_microflow_statements.go | 22 ++- mdl/visitor/visitor_page_v3.go | 7 +- .../visitor_sort_association_path_test.go | 97 ++++++++++ sdk/pages/pages_datasources.go | 9 +- 25 files changed, 917 insertions(+), 24 deletions(-) create mode 100644 mdl-examples/bug-tests/microflow-1152-sort-association-path.mdl create mode 100644 mdl/executor/cmd_microflows_sort_assoc_path_test.go create mode 100644 mdl/executor/cmd_pages_sort_association_test.go create mode 100644 mdl/visitor/visitor_sort_association_path_test.go diff --git a/.claude/skills/fix-issue/findings/mdl-executor.jsonl b/.claude/skills/fix-issue/findings/mdl-executor.jsonl index 8b0a99e0e..dc6cd1d80 100644 --- a/.claude/skills/fix-issue/findings/mdl-executor.jsonl +++ b/.claude/skills/fix-issue/findings/mdl-executor.jsonl @@ -664,3 +664,4 @@ {"area": "mdl/executor", "date": "2026-09-20", "symptom": "MDL-PAGEARG01 refused a list widget's OWN row action: `datagrid dg (DataSource: DATABASE M.E, onClick: SHOW_PAGE M.Edit(E: $currentObject))` was rejected at `check` with \"widget `dg` is not inside a data view, list view or grid row\" \u2014 and since exec refuses a script whose check errors, the slice could not be applied at all. On a `listview` the message contradicted itself. mxbuild 11.14.0 accepts the stored pages at 0 errors.", "cause": "The #1029 guard judged EVERY widget's own action in the context its PARENT supplies: `argContextForSubtreeOf` returns the parent context for a childless widget and `validate_widgets.go` passed the inherited `argCtx` to `validateShowPageArguments`. Right for a button, wrong for the widget that ESTABLISHES the context \u2014 a list widget's onClick is row-scoped, so the row it renders is the context object. Added `argContextForOwnAction`: a widget that binds a source of its own supplies the context for its own action; a source in a shape the pass cannot read (the bare-entity shorthand) degrades to UNKNOWN so the guard stands down rather than refusing what it cannot prove is discarded.", "file": "`mdl/executor/cmd_pages_showpage_args.go` (argContextForOwnAction, argContextForSubtreeOf), `mdl/executor/validate_widgets.go`", "insight": "**A false refusal costs more than a missing rule now that exec refuses on a check error** \u2014 the blast radius is 'this project cannot be built with this mxcli', not 'a warning is noisy'. Two things would have caught it before release: judging the rule against the widget kinds it NAMES in its own message (the listview refusal reads 'lvA is not inside a \u2026 list view'), and running it against mxbuild rather than against intuition. The mxbuild run paid for itself twice: it also showed that `DataSource: M.E` (bare-entity shorthand) on a datagrid is silently dropped, so that case is CE0488 + a REAL CE1571 \u2014 the stand-down is still correct, but the shorthand case must not be written into a bug test as mxbuild-clean (#576). Control the fix with the widget kinds STILL refused (a foreign variable, a sibling button beside the grid), or it is indistinguishable from deleting the rule.", "refs": ["#552", "#576", "mendixlabs/mxcli#1029", "#939"]} {"area": "mdl/executor", "date": "2026-09-21", "symptom": "A page's image-collection reference passed `mxcli check --references` and failed the build. Reported as \"no MDL syntax for a StaticImageViewer inside a Selection helper custom state\" — the authoring half was already closed by #1057; what was left is that nothing RESOLVED the name it made writable. Measured on a blank Mendix 11.14.0 project: `staticimage imgAll (Image: 'Atlas_UI_Resources.Atlas_Icons.checkbox_checked')` in a custom state -> check passed, exec created the page, `mx check` -> 3x CE1613 \"The selected image … no longer exists.\"", "cause": "TWO independent holes, and either alone leaves the reported script unchecked. (1) widgetRefCollector keyed the image reference on the widget TYPE — `if w.Type == \"image\"` — so the pluggable widget was collected and `staticimage` (which #1057 had just given the SAME `Image:` property) and `dynamicimage`'s `DefaultImage` were not; replaced with an imageRefProps table. (2) A page's widgets live in two AST fields: `Widgets` is the bare body, `Placeholders` holds `placeholder X { … }` content (#532). validate.go passed `s.Widgets` alone to validateWidgetReferences, validatePageContextTree AND validateFlowArguments, so EVERY reference inside a placeholder block — microflow, nanoflow, page, snippet, entity, image — was validated by nothing; added allPageWidgets to collect both roots once.", "file": "`mdl/executor/helpers.go` (widgetRefCollector.collectFromWidget, imageRefProps), `mdl/executor/validate.go` (allPageWidgets)", "insight": "**When a capability gets a new spelling, grep for who RESOLVES the old one.** #1057 added `Image:` to a second and third widget and moved on; the resolver keyed on the type name, so the new spellings were unchecked from the day they shipped. A property list and a resolver list that describe the same property are two copies — `validate_widgets.go` already accepted `Image`/`DefaultImage` for these widgets and DESCRIBE already emitted them, and only the resolver disagreed. **The placeholder hole is the more useful lesson: it was the THIRD copy of one walk.** validateIconRefs (#1008) and forEachWidget had each grown the `Placeholders` arm separately, with a comment saying a missed walk is silent both ways — and the three validators next door still had not. When a fix is 'add the missing arm to this walker', the question is how many walkers there are; collect the roots once instead. **Do not reason about a bug report from the issue text alone when the version is older than the fix** — the reported symptom did not reproduce on main at all, and running the reporter's own script end to end is what turned 'already fixed, close it' into two real defects. **Control both directions**: a reference that resolves must stay silent, because a walker that can suddenly see a whole new region of the tree is as likely to report correct scripts as broken ones.", "refs": ["mendixlabs/mxcli#1149", "mendixlabs/mxcli#1057", "mendixlabs/mxcli#1008", "#532"]} {"area": "mdl/executor", "date": "2026-09-21", "symptom": "`retrieve $AccountList from Administration.Account sort by System.Language.Code asc;` — MDL that `mxcli describe` had just emitted — passed `mxcli check` and was refused by `mxcli exec`: \"sort by attribute 'System.Language.Code' does not belong to entity 'Administration.Account'\". Reported as a check/exec inconsistency (mendixlabs/mxcli#1152); the real defect is that the round trip cannot replay its own output for any sort over an association reached from an ANCESTOR.", "cause": "inferSortEntityRefSteps searched ONE domain model — the retrieved entity's own module — for associations whose parent was the retrieved entity ITSELF, and qualified the association it found with the retrieved entity's module. All three assumptions hold only when the hop starts on the retrieved entity in its own module. Administration.Account reaches System.Language through System.User_Language, declared on System.User and stored in the System module: parent is an ancestor, the domain model is another module's, and the qualified name carries THAT module. Rewritten as a generalization-chain walk that looks each ancestor up in its own module and qualifies the association with the module storing it; the destination end is matched with entityIsSubtypeOf rather than by equality, since an association may point at a specialization of the entity that declares the attribute.", "file": "`mdl/executor/cmd_microflows_builder_actions.go` (inferSortEntityRefSteps); tests `mdl/executor/cmd_microflows_sort_association_test.go`, `mdl/backend/modelsdk/microflow_retrievesort_test.go`; example `mdl-examples/bug-tests/microflow-1152-sort-over-association.mdl`", "insight": "**The second control is the one that pays.** Reverting the fix reproduces the refusal, which only proves the test fires. The control that taught something was building a binary that DERIVES the hop and does not WRITE it — exec succeeds and mxbuild 11.12.3 answers CE7247 \"Cannot sort on attribute 'System.Language.Code'. Attribute 'System.Language.Code' is not an attribute of entity 'Administration.Account'\" — the executor's refusal message almost word for word, from the other end of the pipeline. That is what fixes the qualified name as load-bearing: the stored EntityRefStep must read System.User_Language, and the pre-existing code would have written Administration.User_Language had it found anything at all. **Skip the theory that check is missing a rule**: check has no sort-attribute rule at all and resolves no hops, so it was never going to disagree with exec here — the inconsistency in the report is a symptom of the false refusal, not a second defect. **Known residue, stated because the round trip rests on it**: DESCRIBE emits only the attribute's qualified name, so where several associations reach one entity the replay picks the nearest ancestor's first and can silently land on the other hop. Spelling the hop needs grammar (sortColumn is qualifiedName|IDENTIFIER, no `/` path) and is a language change, not a fix."} +{"area": "mdl/executor", "date": "2026-09-21", "symptom": "Follow-up to the sort-hop inference fix: with the hop derivable but not SAYABLE, `describe → exec` still silently changed the program wherever two associations reach the same entity. Measured on 11.12.3 with Order_ShipTo and Order_BillTo (both Order -> Address): a microflow sorting by the BILLING address came back sorting by the SHIPPING one, `mx check` 0 errors on both sides. Same for a page datasource's sort bar.", "cause": "DESCRIBE emitted only the sort attribute's qualified name and the reader never looked at the hop at all — `sortItemsFromRaw` read AttributeRef.Attribute and skipped AttributeRef.EntityRef, so the association was written and never read back. MDL had no spelling for it either (`sortColumn : (qualifiedName | IDENTIFIER)`). Closed end to end: sortColumn takes `qualifiedName (SLASH qualifiedName)*` (the shape MDLCatalog.g4 already uses for Association/Entity), SortColumnDef/OrderByItemV3 carry the hops, the executor resolves the NAMED association instead of inferring, both readers reconstruct EntityRef.Steps, both describers emit `Assoc/.../Attr`, and the page writers moved from attributeRefToGen to inputAttributeRefToGen. Inference stays as the fallback, so every script written before still works.", "file": "`mdl/grammar/domains/MDLPage.g4` (sortColumn) + `mdl/ast/ast_page.go`/`ast_page_v3.go` + `mdl/visitor/visitor_microflow_statements.go` (sortColumnHops) + `visitor_page_v3.go` + `mdl/executor/cmd_microflows_builder_actions.go` (resolveSortAssociationPath, lookupSortHop, entityChainModules) + `cmd_microflows_format_action.go` + `cmd_pages_builder_v3.go` (resolveAssociationAttributePathForEntity) + `cmd_pages_describe_datasource.go` (sortAttributeHops, sortColumnPath) + `mdl/backend/modelsdk/microflow_read_actions.go` (entityRefStepsFromRaw) + `widget_write.go` + `sdk/pages/pages_datasources.go` (GridSort.AttributeRefSteps)", "insight": "**The measurement that decides whether a lossy describer is worth a language change is a CONSTRUCTED one.** The corpus agrees with the inference rule by construction — every document mxcli itself wrote stores the association inference would have picked, so the round trip is a fixed point on everything to hand and looks faithful. The case that matters had to be built: two associations to one entity, then the stored hop edited to the one inference does NOT pick. Byte-patching the .mxunit is enough and takes a minute — `Order_ShipTo` and `Order_BillTo` are the same length, so a `sed` on the BSON needs no resize — and the replay flipped it back immediately. **Control on a binary that drops the hop, not just on one that reverts the fix**: reverting only proves the test fires, while dropping the hop gets mxbuild to say CE7247 \"Cannot sort on attribute … is not an attribute of entity …\" — the executor's own refusal message from the other end of the pipeline, which is what proves the EntityRef load-bearing rather than cosmetic. **Two reads were missing, not one**: the microflow reader and the page reader each drop the hop separately, and fixing only the half named in the report would have shipped a describer that emits the path for microflows and silently drops it for pages. **The strongest round-trip evidence is 'Unchanged'** — with identity preservation and write elision, replaying DESCRIBE output on a correct implementation elides the write entirely, so `Unchanged microflow: …` is a stronger result than any byte comparison."} diff --git a/.claude/skills/mendix/master-detail-pages/SKILL.md b/.claude/skills/mendix/master-detail-pages/SKILL.md index 2e958c1ae..d97d7ac17 100644 --- a/.claude/skills/mendix/master-detail-pages/SKILL.md +++ b/.claude/skills/mendix/master-detail-pages/SKILL.md @@ -65,7 +65,7 @@ gallery widgetName ( ``` **Properties:** -- `datasource: database from entity sort by attr asc|desc` - Entity data source with optional sorting +- `datasource: database from entity sort by attr asc|desc` - Entity data source with optional sorting; `sort by Assoc/Attr asc` sorts over an association - `selection: single` - Selection mode (Single for master-detail) - Template content inside TEMPLATE widget (requires name) @@ -168,6 +168,7 @@ template template1 { | Database source | `datasource: database from Module.Entity` | | Selection binding | `datasource: selection widgetName` | | Sort by | `datasource: database from entity sort by Name asc` | +| Sort over an association | `datasource: database from entity sort by Order_BillTo/City asc` — one `/` per hop, last segment is the attribute. Name the hop when two associations reach the same entity; inference cannot tell them apart (mendixlabs/mxcli#1152) | | Where filter | `datasource: database from entity where [IsActive = true]` | | Selection mode | `selection: single` | | Attribute binding | `attribute: attributename` | diff --git a/.claude/skills/mendix/overview-pages/SKILL.md b/.claude/skills/mendix/overview-pages/SKILL.md index 664d7bcb4..d3308ef81 100644 --- a/.claude/skills/mendix/overview-pages/SKILL.md +++ b/.claude/skills/mendix/overview-pages/SKILL.md @@ -155,7 +155,12 @@ datagrid GridName ( **Properties:** - `datasource: database from Module.Entity` - Entity data source (required) - `where [condition]` - Optional XPath filter (inline after entity in DataSource) -- `sort by attr asc|desc` - Optional sorting (inline after WHERE: `sort by Name asc, Price desc`) +- `sort by attr asc|desc` - Optional sorting (inline after WHERE: `sort by Name asc, Price desc`). + A sort may navigate associations, one `/` per hop, with the last segment the attribute: + `sort by Order_BillTo/City asc`. **Name the hop when more than one association reaches the + same entity** — a bare `Module.Address.City` is resolved by inference, which cannot tell + `Order_ShipTo` from `Order_BillTo`, and the wrong one builds cleanly and sorts by the wrong + thing (mendixlabs/mxcli#1152) - `selection: Multi` - Multi-selection (`Multi`, `Single`, or omit for none) - `PagingPosition: both` - Pagination bar position (`top`, `bottom`, `both`) - `designproperties: ['Compact': on, 'Hover': on, 'Striped': on]` - Atlas design tokens diff --git a/.claude/skills/mendix/write-microflows/reference/data-operations.md b/.claude/skills/mendix/write-microflows/reference/data-operations.md index f5793e4cb..6ed4590d2 100644 --- a/.claude/skills/mendix/write-microflows/reference/data-operations.md +++ b/.claude/skills/mendix/write-microflows/reference/data-operations.md @@ -270,6 +270,31 @@ retrieve $Recent from Sales.Order - The keyword is **`sort by`** (one or more `Module.Entity.Attr asc|desc`, comma-separated). `order by` is **not** valid on a microflow `retrieve` — it's reserved for `select ... from CATALOG.*` queries and will cause a parse error here. - `limit` and `offset` accept **a variable or expression**, not only a literal — `limit $PageSize`, `offset $Offset`, even `limit $Base + 5` all work. A bare literal (`limit 20`) is just the simplest case. +- A bare attribute name is qualified with the entity that **declares** it, which may be an ancestor — `sort by Name` on a specialization of `System.User` stores `System.User.Name`, which is what mxbuild resolves. + +**Sorting over an association** — one `/` per hop, the last segment is the attribute: + +```mdl +retrieve $Orders from Sales.Order + sort by Sales.Order_BillTo/Sales.Address.City asc; + +-- the hop may be unqualified, and the attribute bare +retrieve $Orders from Sales.Order + sort by Order_BillTo/City asc; +``` + +- **Name the hop when more than one association reaches the same entity.** Writing the + attribute alone (`sort by Sales.Address.City`) makes mxcli infer the association: it + walks the generalization chain and crosses modules — `Administration.Account` reaches + `System.Language.Code` through `System.User_Language`, declared on `System.User` — but it + cannot tell `Order_ShipTo` from `Order_BillTo` and takes the nearest one. Measured on + 11.12.3: a microflow sorting by the billing address came back from `describe → exec` + sorting by the shipping one, at 0 errors on both sides (mendixlabs/mxcli#1152). +- `describe microflow` emits the hop whenever one is stored, so a described sort replays + to the same model. An association that does not exist, or does not start at the entity + in hand, is **refused** rather than written — Mendix stores a sort over an association + as an `EntityRef` beside the attribute, and an attribute of a far entity without one is + **CE7247** "Cannot sort on attribute …". ### Retrieve by Association (in-memory, over an association path) diff --git a/cmd/mxcli/syntax/features_microflow.go b/cmd/mxcli/syntax/features_microflow.go index dfc1b61e0..b3de1d437 100644 --- a/cmd/mxcli/syntax/features_microflow.go +++ b/cmd/mxcli/syntax/features_microflow.go @@ -84,20 +84,27 @@ func init() { }, // Retrieve-by-association was missing here, so it read as unsupported // even though it works and the write-microflows skill documents it. - Syntax: "-- From the database\nRETRIEVE $Var FROM Module.Entity\n [WHERE condition]\n [SORT BY attr ASC|DESC]\n [LIMIT n] [OFFSET n];\n\n-- Over an association, from an object you already have\nRETRIEVE $Var FROM $Object/Module.Association;", + Syntax: "-- From the database\nRETRIEVE $Var FROM Module.Entity\n [WHERE condition]\n [SORT BY attr ASC|DESC]\n [LIMIT n] [OFFSET n];\n\n-- Sort over an association: one `/` per hop, the last segment is the attribute\nRETRIEVE $Var FROM Module.Entity\n SORT BY Module.Assoc/Module.Other.Attr ASC;\n\n-- Over an association, from an object you already have\nRETRIEVE $Var FROM $Object/Module.Association;", Example: "-- LIMIT 1 binds a single OBJECT (Mendix's \"First object\" range), not a\n" + "-- one-element list — hence the singular variable name here.\n" + "RETRIEVE $Customer FROM MyModule.Customer\n WHERE Code = $CustomerCode\n LIMIT 1;\n\n" + "-- Any other LIMIT is a bounded range, which is a list.\n" + "RETRIEVE $Orders FROM MyModule.Order\n WHERE Status = 'Pending'\n SORT BY CreateDate DESC\n LIMIT 10 OFFSET 0;\n\n" + "-- Follow an association rather than querying the database\nRETRIEVE $Orders FROM $Customer/MyModule.Order_Customer;\nRETRIEVE $Customer FROM $Order/MyModule.Order_Customer;\n\n" + + "-- Sort on an attribute of an associated entity. Name the association\n" + + "-- when two of them reach the same entity.\n" + + "RETRIEVE $Orders FROM MyModule.Order\n SORT BY MyModule.Order_BillTo/MyModule.Address.City ASC;\n\n" + "-- Notes:\n" + "-- * LIMIT 1 with no OFFSET is the one form that binds an object. HEAD(),\n" + "-- COUNT() or a LOOP over it is CE0097 at build time; mxcli reports it as\n" + "-- MDL-RETRIEVE01 at check time.\n" + "-- * LIMIT 1 OFFSET n is a bounded range, so that one IS a list.\n" + "-- * `import from mapping … limit 1` means the opposite — a one-element\n" + - "-- list — and `… first` is its object form.", + "-- list — and `… first` is its object form.\n" + + "-- * SORT BY may navigate associations. Name the hop when more than one\n" + + "-- reaches the same entity — mxcli infers a single hop, but it cannot\n" + + "-- tell Order_ShipTo from Order_BillTo, and the wrong one builds\n" + + "-- cleanly and sorts by the wrong thing.", SeeAlso: []string{"microflow.object-operations", "xpath"}, }) diff --git a/cmd/mxcli/syntax/features_page.go b/cmd/mxcli/syntax/features_page.go index 8cfd6b9b3..e42fa1d05 100644 --- a/cmd/mxcli/syntax/features_page.go +++ b/cmd/mxcli/syntax/features_page.go @@ -237,7 +237,7 @@ CREATE PAGE Sales.Detail (Title: 'Detail', Layout: Atlas_Core.Atlas_Default) { "datasource", "data source", "database", "microflow", "selection", "variable", "binding", "binds", "association", "data from context", }, - Syntax: "DataSource: $Variable -- Parameter/variable binding\nDataSource: DATABASE Module.Entity -- Database query\nDataSource: DATABASE Module.Entity WHERE [Attr != ''] SORT BY Attr ASC\n -- ...optionally constrained and sorted\nDataSource: DATABASE Module.Entity SEARCH BY Attr, Attr2\n -- LIST VIEW only: the attributes its\n -- search bar filters on. Mirrors SORT BY,\n -- but takes no direction.\nDataSource: MICROFLOW Module.MF -- Microflow datasource, no parameters\nDataSource: MICROFLOW Module.MF($P) -- ...one argument per PARAMETER, required:\n -- Mendix does NOT auto-map an object in\n -- scope, not even one of the exact type,\n -- so a missing argument is CE1571\nDataSource: SELECTION widgetName -- Selection from another widget\nDataSource: $currentObject/Module.Assoc -- Over an association (\"data from context\")\n -- list widget → to-many collection\n -- nested DATAVIEW → the to-one referenced object\nAttribute: AttributeName -- Attribute binding (inputs)", + Syntax: "DataSource: $Variable -- Parameter/variable binding\nDataSource: DATABASE Module.Entity -- Database query\nDataSource: DATABASE Module.Entity WHERE [Attr != ''] SORT BY Attr ASC\n -- ...optionally constrained and sorted\nDataSource: DATABASE Module.Entity SORT BY Module.Assoc/Attr ASC\n -- ...sorted over an association. Name the\n -- hop when two reach the same entity —\n -- the wrong one builds cleanly and sorts\n -- by the wrong thing.\nDataSource: DATABASE Module.Entity SEARCH BY Attr, Attr2\n -- LIST VIEW only: the attributes its\n -- search bar filters on. Mirrors SORT BY,\n -- but takes no direction.\nDataSource: MICROFLOW Module.MF -- Microflow datasource, no parameters\nDataSource: MICROFLOW Module.MF($P) -- ...one argument per PARAMETER, required:\n -- Mendix does NOT auto-map an object in\n -- scope, not even one of the exact type,\n -- so a missing argument is CE1571\nDataSource: SELECTION widgetName -- Selection from another widget\nDataSource: $currentObject/Module.Assoc -- Over an association (\"data from context\")\n -- list widget → to-many collection\n -- nested DATAVIEW → the to-one referenced object\nAttribute: AttributeName -- Attribute binding (inputs)", Example: "-- Database datasource with grid\nDATAGRID grid (DataSource: DATABASE Module.Customer) {\n COLUMN colName (Attribute: Name, Caption: 'Name')\n}\n\n-- Microflow datasource\nDATAVIEW dv (DataSource: MICROFLOW Module.GetData) {\n TEXTBOX txtName (Label: 'Name', Attribute: Name)\n}\n\n-- Over an association: a nested DataView shows the referenced (to-one) object\nDATAVIEW dvOrder (DataSource: $Order) {\n DATAVIEW dvCustomer (DataSource: $currentObject/Order_Customer) {\n TEXTBOX txtCustName (Label: 'Name', Attribute: Name)\n }\n}\n\n-- Over an association: a list widget shows the (to-many) collection\nLISTVIEW lvLines (DataSource: $currentObject/Order_OrderLine) {\n DYNAMICTEXT dtLine (Content: 'Line')\n}", SeeAlso: []string{"page.widgets", "page.create"}, }) diff --git a/docs/01-project/MDL_QUICK_REFERENCE.md b/docs/01-project/MDL_QUICK_REFERENCE.md index e6cf8a659..5056ef594 100644 --- a/docs/01-project/MDL_QUICK_REFERENCE.md +++ b/docs/01-project/MDL_QUICK_REFERENCE.md @@ -511,7 +511,7 @@ it is for pages. | Delete | `delete $entity [refresh];` | | | Rollback | `rollback $entity [refresh];` | Reverts uncommitted changes | | Retrieve (DB) | `retrieve $Var from Module.Entity [where condition] [sort by Attr asc\|desc, ...] [limit n [offset n]];` | Database XPath retrieve. `limit 1` with no `offset` binds a single **object**, not a one-element list (MDL-RETRIEVE01) | -| Retrieve (DB), sorted | `sort by Attr asc` / `sort by Module.Other.Attr asc` | A bare name is qualified with the entity **declaring** it, which may be an ancestor. A qualified name may also be an attribute of an entity reached by **one association hop** — mxcli derives the hop (walking the generalization chain, across modules: `Administration.Account` reaches `System.Language.Code` through `System.User_Language`) and stores it as the `EntityRef` Mendix needs; without it the build is CE7247. Where several associations reach the same entity MDL cannot say which, and the nearest one wins (mendixlabs/mxcli#1152) | +| Retrieve (DB), sorted | `sort by Attr asc` / `sort by Module.Other.Attr asc` / `sort by Module.Assoc/Module.Other.Attr asc` | A bare name is qualified with the entity **declaring** it, which may be an ancestor. A sort may also navigate associations — one `/` per hop, the last segment is the attribute — and mxcli stores the hops as the `EntityRef` Mendix needs; without them the build is **CE7247**. **Name the hop when more than one association reaches the same entity**: a bare `Module.Other.Attr` is resolved by inference, which walks the generalization chain across modules (`Administration.Account` reaches `System.Language.Code` through `System.User_Language`) but cannot tell `Order_ShipTo` from `Order_BillTo` — measured, a sort on the billing address round-tripped into one on the shipping address at 0 errors both sides (mendixlabs/mxcli#1152). The same spelling works in a page datasource's `sort by` | | Retrieve (Assoc) | `retrieve $list from $Parent/Module.AssocName;` | Retrieve by association | | Add to list | `add expression to $list;` | Also accepts existing `add $item to $list;` form | | Aggregate a list | `$Total = sum($list.Attr);` / `$Total = sum($list, expression);` | `count` (list only), `sum`, `average`, `minimum`, `maximum` — attribute or expression over `$currentObject` | diff --git a/mdl-examples/bug-tests/microflow-1152-sort-association-path.mdl b/mdl-examples/bug-tests/microflow-1152-sort-association-path.mdl new file mode 100644 index 000000000..1bd492530 --- /dev/null +++ b/mdl-examples/bug-tests/microflow-1152-sort-association-path.mdl @@ -0,0 +1,73 @@ +-- mendixlabs/mxcli#1152, second half — naming the association a sort navigates. +-- +-- Inference alone cannot make the round trip faithful. Two associations reaching +-- the same entity (Order_ShipTo and Order_BillTo, both Order -> Address) is an +-- ordinary shape, and the stored hop is not recoverable from the sort +-- attribute's name. Measured on 11.12.3 before this: a microflow sorting by the +-- BILLING address came back from `describe -> exec` sorting by the SHIPPING one, +-- at 0 errors on both sides — the round trip silently denoting a different +-- program. +-- +-- A sort column may now navigate associations, one `/` per hop, with the last +-- segment naming the attribute. DESCRIBE emits the hop whenever one is stored, +-- so a described sort replays to the same model — measured, the replay of the +-- first microflow below reports "Unchanged", the write being elided because the +-- rebuild is identical. +-- +-- Dropping the hop but keeping the attribute path — the shape a half-fix +-- produces — is CE7247 "Cannot sort on attribute ... is not an attribute of +-- entity ...", in a microflow and in a page's sort bar alike. + +create persistent entity Sort1152.Address ( City: String(200) ); +create persistent entity Sort1152.Order ( OrderNo: String(50) ); + +create association Sort1152.Order_ShipTo from Sort1152.Order to Sort1152.Address; +create association Sort1152.Order_BillTo from Sort1152.Order to Sort1152.Address; + +-- Named hop: Order_ShipTo is declared first and reaches the same entity, so +-- inference would pick it. The spelling is what makes the billing sort sayable. +create or modify microflow Sort1152.ACT_billing_sort () +begin + @start(96, 200) + @position(236, 200) + retrieve $Orders from Sort1152.Order + sort by Sort1152.Order_BillTo/Sort1152.Address.City asc; + @position(700, 200) + return; +end; + +-- The hop may be unqualified and the attribute bare. +create or modify microflow Sort1152.ACT_shipping_sort () +begin + @start(96, 200) + @position(236, 200) + retrieve $Orders from Sort1152.Order + sort by Order_ShipTo/City asc; + @position(700, 200) + return; +end; + +-- Inference still covers the unambiguous case, including an association declared +-- on an ANCESTOR in another module: System.User_Language is on System.User, and +-- Administration.Account generalizes it. +create or modify microflow Sort1152.ACT_language_sort () +begin + @start(96, 200) + @position(236, 200) + retrieve $AccountList from Administration.Account + sort by System.Language.Code asc; + @position(700, 200) + return; +end; + +-- A page datasource's sort takes the same path: it is stored as the same +-- DomainModels$AttributeRef, so it has the same EntityRef and the same CE7247. +create or replace page Sort1152.Orders_Overview ( + title: 'Orders', + Layout: Atlas_Core.Atlas_Default +) { + listview lvOrders (datasource: database from Sort1152.Order + sort by Sort1152.Order_BillTo/City asc) { + dynamictext txtNo (content: '{1}', contentparams: [{1} = OrderNo]) + } +} diff --git a/mdl/ast/ast_page.go b/mdl/ast/ast_page.go index 13a82abde..65bdd58cc 100644 --- a/mdl/ast/ast_page.go +++ b/mdl/ast/ast_page.go @@ -23,8 +23,16 @@ type PageVariable struct { // SortColumnDef represents a sort column: attribute ASC/DESC type SortColumnDef struct { - Attribute string // Qualified name or simple identifier - Order string // "ASC" or "DESC" + Attribute string // Qualified name or simple identifier — the FINAL segment + // Associations holds one qualified association name per `/` hop, in order, + // for a sort that navigates to another entity + // (`Sales.Order_BillTo/Sales.Address.City`). Empty for a sort on the + // retrieved entity's own (or inherited) attribute. Consumers must resolve + // these into the AttributeRef's EntityRef — dropping them stores a sort + // Mendix cannot resolve (CE7247), and guessing them picks the wrong + // association wherever two reach the same entity (mendixlabs/mxcli#1152). + Associations []string + Order string // "ASC" or "DESC" } // DataGridColumnDef represents a DataGrid2 column definition. diff --git a/mdl/ast/ast_page_v3.go b/mdl/ast/ast_page_v3.go index c5f2705e2..b8abe6c07 100644 --- a/mdl/ast/ast_page_v3.go +++ b/mdl/ast/ast_page_v3.go @@ -209,8 +209,11 @@ type FlowArgV3 struct { // OrderByItemV3 represents a sort column. type OrderByItemV3 struct { - Attribute string // Attribute path - Direction string // "ASC" or "DESC" + Attribute string // Attribute path — the FINAL segment + // Associations holds one qualified association name per `/` hop, in order. + // See ast.SortColumnDef.Associations. + Associations []string + Direction string // "ASC" or "DESC" } // ActionV3 represents a V3 action expression. diff --git a/mdl/backend/modelsdk/microflow_read_actions.go b/mdl/backend/modelsdk/microflow_read_actions.go index 33ec330bd..bc7d9193c 100644 --- a/mdl/backend/modelsdk/microflow_read_actions.go +++ b/mdl/backend/modelsdk/microflow_read_actions.go @@ -1253,12 +1253,52 @@ func sortItemsFromRaw(doc bson.Raw) []*microflows.SortItem { if ref, ok := sd.Lookup("AttributeRef").DocumentOK(); ok { // The AttributeRef stores its by-name reference under "Attribute". it.AttributeQualifiedName = rawStr(ref, "Attribute") + // …and the association hops, when the sort navigates to another + // entity, under EntityRef. Not reading them made the hop invisible to + // DESCRIBE, so a sort over an association could not be described and + // the replay had to guess which association was meant + // (mendixlabs/mxcli#1152). + it.EntityRefSteps = entityRefStepsFromRaw(ref) } out = append(out, it) } return out } +// entityRefStepsFromRaw reads the association hops of a DomainModels$AttributeRef +// — its EntityRef, a DomainModels$IndirectEntityRef whose Steps array holds one +// DomainModels$EntityRefStep per hop. A DirectEntityRef (or no EntityRef at all) +// is an own-entity attribute and yields no steps. +// +// The first Steps entry is the typed-array marker (an int, not a document) and is +// skipped by the DocumentOK guard. +func entityRefStepsFromRaw(ref bson.Raw) []microflows.EntityRefStep { + entityRef, ok := ref.Lookup("EntityRef").DocumentOK() + if !ok { + return nil + } + arr, ok := entityRef.Lookup("Steps").ArrayOK() + if !ok { + return nil + } + vals, err := arr.Values() + if err != nil { + return nil + } + var out []microflows.EntityRefStep + for _, v := range vals { + sd, ok := v.DocumentOK() + if !ok { + continue + } + out = append(out, microflows.EntityRefStep{ + Association: rawStr(sd, "Association"), + DestinationEntity: rawStr(sd, "DestinationEntity"), + }) + } + return out +} + // memberChangesFromGen reconstructs the attribute/association assignments of a // create/change-object action (the inverse of memberChangeToGen). func memberChangesFromGen(items []element.Element) []*microflows.MemberChange { diff --git a/mdl/backend/modelsdk/microflow_retrievesort_test.go b/mdl/backend/modelsdk/microflow_retrievesort_test.go index 6b28da94c..f693b28bb 100644 --- a/mdl/backend/modelsdk/microflow_retrievesort_test.go +++ b/mdl/backend/modelsdk/microflow_retrievesort_test.go @@ -187,3 +187,74 @@ func TestRetrieveSourceToGen_SortOverAssociationWritesEntityRefSteps(t *testing. t.Errorf("DestinationEntity = %q, want System.Language", got) } } + +// TestRetrieveSourceFromGen_SortOverAssociationReadsHops guards the READ half of +// mendixlabs/mxcli#1152. The hop was written and never read back, so DESCRIBE +// could not emit it even once MDL had a spelling for it — and a describer that +// drops what the writer stores is how a round trip silently changes a program. +func TestRetrieveSourceFromGen_SortOverAssociationReadsHops(t *testing.T) { + in := µflows.DatabaseRetrieveSource{ + EntityQualifiedName: "Sales.Order", + Sorting: []*microflows.SortItem{{ + AttributeQualifiedName: "Sales.Address.City", + Direction: microflows.SortDirectionAscending, + EntityRefSteps: []microflows.EntityRefStep{{ + Association: "Sales.Order_BillTo", + DestinationEntity: "Sales.Address", + }}, + }}, + } + + raw, err := (&codec.Encoder{}).Encode(retrieveSourceToGen(in)) + if err != nil { + t.Fatalf("encode: %v", err) + } + decoded, err := codec.NewDecoder(codec.DefaultRegistry).Decode(raw) + if err != nil { + t.Fatalf("decode: %v", err) + } + out, ok := retrieveSourceFromGen(decoded).(*microflows.DatabaseRetrieveSource) + if !ok { + t.Fatal("round-trip did not yield a DatabaseRetrieveSource") + } + if len(out.Sorting) != 1 { + t.Fatalf("Sorting = %d items, want 1", len(out.Sorting)) + } + got := out.Sorting[0] + if got.AttributeQualifiedName != "Sales.Address.City" { + t.Errorf("attribute = %q, want Sales.Address.City", got.AttributeQualifiedName) + } + if len(got.EntityRefSteps) != 1 { + t.Fatalf("EntityRefSteps = %+v, want one hop — the association the sort navigates "+ + "is invisible to DESCRIBE without it", got.EntityRefSteps) + } + if got.EntityRefSteps[0].Association != "Sales.Order_BillTo" || + got.EntityRefSteps[0].DestinationEntity != "Sales.Address" { + t.Errorf("hop = %+v, want Sales.Order_BillTo -> Sales.Address", got.EntityRefSteps[0]) + } +} + +// CONTROL: an own-entity sort carries a DirectEntityRef or no EntityRef at all, +// and must read back with no hops. A reader that manufactured an empty hop would +// make DESCRIBE emit a stray `/`. +func TestRetrieveSourceFromGen_PlainSortHasNoHops(t *testing.T) { + in := µflows.DatabaseRetrieveSource{ + EntityQualifiedName: "Sales.Order", + Sorting: []*microflows.SortItem{{ + AttributeQualifiedName: "Sales.Order.OrderNo", + Direction: microflows.SortDirectionAscending, + }}, + } + raw, err := (&codec.Encoder{}).Encode(retrieveSourceToGen(in)) + if err != nil { + t.Fatalf("encode: %v", err) + } + decoded, err := codec.NewDecoder(codec.DefaultRegistry).Decode(raw) + if err != nil { + t.Fatalf("decode: %v", err) + } + out := retrieveSourceFromGen(decoded).(*microflows.DatabaseRetrieveSource) + if len(out.Sorting) != 1 || len(out.Sorting[0].EntityRefSteps) != 0 { + t.Errorf("got %+v, want one sort item with no hops", out.Sorting) + } +} diff --git a/mdl/backend/modelsdk/widget_write.go b/mdl/backend/modelsdk/widget_write.go index 62ed823c3..15fa85c3e 100644 --- a/mdl/backend/modelsdk/widget_write.go +++ b/mdl/backend/modelsdk/widget_write.go @@ -1431,7 +1431,12 @@ func listViewSourceToGen(ds pages.DataSource) (element.Element, error) { item := genPg.NewGridSortItem() assignID(item) item.SetSortDirection(string(s.Direction)) - if ref := attributeRefToGen(s.AttributePath); ref != nil { + // inputAttributeRefToGen, not attributeRefToGen: a sort that navigates + // associations needs its hops stored as the AttributeRef's EntityRef, + // exactly as an input widget's binding does. Without them the far + // entity's attribute does not resolve and mxbuild answers CE7247 + // (mendixlabs/mxcli#1152). With no steps the two are identical. + if ref := inputAttributeRefToGen(s.AttributePath, s.AttributeRefSteps); ref != nil { item.SetAttributeRef(ref) } bar.AddSortItems(item) @@ -1504,7 +1509,12 @@ func customWidgetDataSourceToGen(ds pages.DataSource) (element.Element, error) { item := genPg.NewGridSortItem() assignID(item) item.SetSortDirection(string(s.Direction)) - if ref := attributeRefToGen(s.AttributePath); ref != nil { + // inputAttributeRefToGen, not attributeRefToGen: a sort that navigates + // associations needs its hops stored as the AttributeRef's EntityRef, + // exactly as an input widget's binding does. Without them the far + // entity's attribute does not resolve and mxbuild answers CE7247 + // (mendixlabs/mxcli#1152). With no steps the two are identical. + if ref := inputAttributeRefToGen(s.AttributePath, s.AttributeRefSteps); ref != nil { item.SetAttributeRef(ref) } bar.AddSortItems(item) diff --git a/mdl/executor/cmd_microflows_builder_actions.go b/mdl/executor/cmd_microflows_builder_actions.go index ffbb170c3..37eb86793 100644 --- a/mdl/executor/cmd_microflows_builder_actions.go +++ b/mdl/executor/cmd_microflows_builder_actions.go @@ -1100,7 +1100,19 @@ func (fb *flowBuilder) addRetrieveAction(s *ast.RetrieveStmt) model.ID { // Resolve attribute path - if just a simple name, prefix with entity attrPath := col.Attribute var entityRefSteps []microflows.EntityRefStep - if !strings.Contains(attrPath, ".") { + if len(col.Associations) > 0 { + // The script SPELLS the hops. Take them as written rather than + // inferring: inference cannot tell two associations reaching the + // same entity apart, and picking the wrong one is a model that + // builds cleanly and sorts by the wrong thing + // (mendixlabs/mxcli#1152). + resolved, finalQN, err := fb.resolveSortAssociationPath(entityQN, col.Associations, col.Attribute) + if err != nil { + fb.addError("sort by %s: %s", sortColumnText(col), err.Error()) + continue // Skip this sort column but continue processing others + } + entityRefSteps, attrPath = resolved, finalQN + } else if !strings.Contains(attrPath, ".") { // Qualify with the entity that DECLARES the attribute, which is // not always the one being retrieved. Mendix resolves a sort // reference against the declaring entity, so qualifying an @@ -1344,6 +1356,140 @@ func (fb *flowBuilder) inferSortEntityRefSteps(sourceEntityQN, attrPath string) return nil } +// sortColumnText renders a sort column the way it was authored, for messages. +func sortColumnText(col ast.SortColumnDef) string { + if len(col.Associations) == 0 { + return col.Attribute + } + return strings.Join(append(append([]string{}, col.Associations...), col.Attribute), "/") +} + +// resolveSortAssociationPath turns an authored `Assoc/…/Attribute` sort column +// into the EntityRefSteps Mendix stores alongside the attribute, plus the +// attribute's fully-qualified name. +// +// This is the spelling that exists because inference cannot be made correct: +// where two associations reach the same entity — `Order_ShipTo` and +// `Order_BillTo`, both `Order → Address`, an ordinary shape — the stored hop is +// not recoverable from the attribute name alone, and DESCRIBE emitted nothing +// else. Measured on 11.12.3: a microflow sorting by the billing address came +// back from `describe → exec` sorting by the shipping one, at 0 errors on both +// sides (mendixlabs/mxcli#1152). +// +// Everything here is refused rather than guessed. A hop that does not resolve, +// one that starts nowhere near the entity in hand, or a final attribute on an +// entity the last hop does not reach are each an error — a step written with an +// empty DestinationEntity is the one outcome worse than a refusal, since it +// makes the project unopenable (System.ArgumentNullException at +// EntityRefStep.set_DestinationEntityId) rather than merely wrong. +func (fb *flowBuilder) resolveSortAssociationPath(sourceEntityQN string, hops []string, attrName string) ([]microflows.EntityRefStep, string, error) { + if sourceEntityQN == "" { + return nil, "", fmt.Errorf("the retrieved entity is unknown, so the association path cannot be resolved") + } + if fb == nil || fb.backend == nil { + return nil, "", fmt.Errorf("no project is open, so the association path cannot be resolved") + } + + steps := make([]microflows.EntityRefStep, 0, len(hops)) + current := sourceEntityQN + for _, hop := range hops { + assocQN, info := fb.lookupSortHop(current, hop) + if info == nil { + return nil, "", fmt.Errorf("association '%s' was not found", hop) + } + var dest string + switch { + case fb.entityIsSubtypeOf(current, info.parentEntityQN): + dest = info.childEntityQN + case fb.entityIsSubtypeOf(current, info.childEntityQN): + dest = info.parentEntityQN + default: + return nil, "", fmt.Errorf("association '%s' connects %s and %s, neither of which is %s", + assocQN, info.parentEntityQN, info.childEntityQN, current) + } + if dest == "" { + return nil, "", fmt.Errorf("association '%s' has an unresolved end, so the entity it reaches is unknown", assocQN) + } + steps = append(steps, microflows.EntityRefStep{Association: assocQN, DestinationEntity: dest}) + current = dest + } + + // The final attribute is qualified with the entity that DECLARES it, which + // for an inherited attribute is an ancestor of the last hop's destination — + // the same rule (and the same CE1613 when broken) as a sort with no hops. + if strings.Count(attrName, ".") >= 2 { + owner := attrName[:strings.LastIndex(attrName, ".")] + if !fb.entityIsSubtypeOf(current, owner) { + return nil, "", fmt.Errorf("attribute '%s' does not belong to %s, which is where the association path ends", + attrName, current) + } + return steps, attrName, nil + } + if declared, ok := fb.resolveAttributeInEntityHierarchy(current, attrName); ok { + return steps, declared, nil + } + return nil, "", fmt.Errorf("entity %s has no attribute '%s'", current, attrName) +} + +// lookupSortHop resolves one segment of a sort column's association path. A +// qualified segment names its module outright; a bare one is looked for in the +// modules of the entity in hand and of its ancestors, because an association is +// stored in the module of the entity that DECLARES it — which for an inherited +// one is not the module of the entity being sorted (mendixlabs/mxcli#1152). +func (fb *flowBuilder) lookupSortHop(currentEntityQN, hop string) (string, *assocLookupResult) { + if i := strings.LastIndex(hop, "."); i > 0 { + if info := fb.lookupAssociation(hop[:i], hop[i+1:]); info != nil { + return hop, info + } + return hop, nil + } + for _, moduleName := range fb.entityChainModules(currentEntityQN) { + if info := fb.lookupAssociation(moduleName, hop); info != nil { + return moduleName + "." + hop, info + } + } + return hop, nil +} + +// entityChainModules lists the modules of an entity and of its ancestors, +// nearest first and without repeats. +func (fb *flowBuilder) entityChainModules(entityQN string) []string { + var out []string + seenModule := make(map[string]bool) + seenEntity := make(map[string]bool) + for currentQN := entityQN; currentQN != ""; { + if seenEntity[currentQN] { + break + } + seenEntity[currentQN] = true + parts := strings.SplitN(currentQN, ".", 2) + if len(parts) != 2 || parts[0] == "" { + break + } + if !seenModule[parts[0]] { + seenModule[parts[0]] = true + out = append(out, parts[0]) + } + if fb.backend == nil { + break + } + mod, err := fb.backend.GetModuleByName(parts[0]) + if err != nil || mod == nil { + break + } + dm, err := fb.backend.GetDomainModel(mod.ID) + if err != nil || dm == nil { + break + } + entity := dm.FindEntityByName(parts[1]) + if entity == nil { + break + } + currentQN = entity.GeneralizationRef + } + return out +} + func entityQualifiedNameFromAttribute(attrPath string) string { parts := strings.Split(attrPath, ".") if len(parts) < 3 { diff --git a/mdl/executor/cmd_microflows_format_action.go b/mdl/executor/cmd_microflows_format_action.go index 73604bb00..8ad527800 100644 --- a/mdl/executor/cmd_microflows_format_action.go +++ b/mdl/executor/cmd_microflows_format_action.go @@ -502,7 +502,18 @@ func formatAction( if len(dbSource.Sorting) > 0 { var sortParts []string for _, sortItem := range dbSource.Sorting { + // A sort that navigates associations is emitted with its hops, + // one `/` per step. Emitting the attribute alone is lossy in the + // way that hides longest: the replay has to guess which + // association was meant, and where two reach the same entity it + // can pick the other one — a model that builds cleanly and sorts + // by the wrong thing (mendixlabs/mxcli#1152). attrName := sortItem.AttributeQualifiedName + for i := len(sortItem.EntityRefSteps) - 1; i >= 0; i-- { + if assoc := sortItem.EntityRefSteps[i].Association; assoc != "" { + attrName = assoc + "/" + attrName + } + } order := "asc" if sortItem.Direction == microflows.SortDirectionDescending { order = "desc" diff --git a/mdl/executor/cmd_microflows_sort_assoc_path_test.go b/mdl/executor/cmd_microflows_sort_assoc_path_test.go new file mode 100644 index 000000000..b52976909 --- /dev/null +++ b/mdl/executor/cmd_microflows_sort_assoc_path_test.go @@ -0,0 +1,182 @@ +// SPDX-License-Identifier: Apache-2.0 + +package executor + +import ( + "strings" + "testing" + + "github.com/mendixlabs/mxcli/mdl/ast" + "github.com/mendixlabs/mxcli/mdl/backend/mock" + "github.com/mendixlabs/mxcli/model" + "github.com/mendixlabs/mxcli/sdk/domainmodel" + "github.com/mendixlabs/mxcli/sdk/microflows" +) + +// mendixlabs/mxcli#1152, second half — the residue inference leaves behind. +// +// Two associations reach the same entity (Order_ShipTo and Order_BillTo, both +// Order -> Address), which is an ordinary shape, not a corner case. The stored +// hop is then not recoverable from the sort attribute's name, so DESCRIBE +// dropping it means the replay has to guess. Measured on 11.12.3: a microflow +// sorting by the BILLING address came back from `describe -> exec` sorting by +// the SHIPPING one, at 0 errors on both sides. +// +// `sort by Mod.Order_BillTo/Mod.Address.City` is the spelling that removes the +// guess. These tests pin the resolution; the describe half is pinned in +// mdl/backend/modelsdk (the read) and by the formatter test below. +func twoHopBackend() *mock.MockBackend { + moduleID := model.ID("synthetic-sales-module") + order := &domainmodel.Entity{Name: "Order"} + order.ID = model.ID("Sales.Order") + address := &domainmodel.Entity{ + Name: "Address", + Attributes: []*domainmodel.Attribute{ + {Name: "City", Type: &domainmodel.StringAttributeType{}}, + }, + } + address.ID = model.ID("Sales.Address") + shipTo := &domainmodel.Association{Name: "Order_ShipTo", ParentID: order.ID, ChildID: address.ID, Type: domainmodel.AssociationTypeReference} + shipTo.ID = model.ID("Sales.Order_ShipTo") + billTo := &domainmodel.Association{Name: "Order_BillTo", ParentID: order.ID, ChildID: address.ID, Type: domainmodel.AssociationTypeReference} + billTo.ID = model.ID("Sales.Order_BillTo") + + return &mock.MockBackend{ + GetModuleByNameFunc: func(name string) (*model.Module, error) { + if name == "Sales" { + return &model.Module{BaseElement: model.BaseElement{ID: moduleID}, Name: name}, nil + } + return nil, nil + }, + GetDomainModelFunc: func(id model.ID) (*domainmodel.DomainModel, error) { + if id != moduleID { + return nil, nil + } + return &domainmodel.DomainModel{ + ContainerID: moduleID, + Entities: []*domainmodel.Entity{order, address}, + Associations: []*domainmodel.Association{shipTo, billTo}, + }, nil + }, + } +} + +// pathSortOf builds the retrieve for an authored sort column and returns the +// stored attribute path, its hops, and any build errors. +func pathSortOf(t *testing.T, col ast.SortColumnDef) (path string, steps []microflows.EntityRefStep, errs []string) { + t.Helper() + fb := &flowBuilder{backend: twoHopBackend(), spacing: 100} + fb.addRetrieveAction(&ast.RetrieveStmt{ + Variable: "Orders", + Source: ast.QualifiedName{Module: "Sales", Name: "Order"}, + SortColumns: []ast.SortColumnDef{col}, + }) + for _, obj := range fb.objects { + act, ok := obj.(*microflows.ActionActivity) + if !ok { + continue + } + ra, ok := act.Action.(*microflows.RetrieveAction) + if !ok { + continue + } + ds, ok := ra.Source.(*microflows.DatabaseRetrieveSource) + if !ok { + continue + } + for _, s := range ds.Sorting { + path, steps = s.AttributeQualifiedName, s.EntityRefSteps + } + } + return path, steps, fb.errors +} + +// The named association is the one stored — not the first one that happens to +// reach Address. This is the whole point of the spelling. +func TestSortPath_NamedAssociationIsTheOneStored(t *testing.T) { + path, steps, errs := pathSortOf(t, ast.SortColumnDef{ + Associations: []string{"Sales.Order_BillTo"}, + Attribute: "Sales.Address.City", + Order: "ASC", + }) + if len(errs) > 0 { + t.Fatalf("unexpected errors: %v", errs) + } + if path != "Sales.Address.City" { + t.Errorf("stored attribute %q, want Sales.Address.City", path) + } + if len(steps) != 1 || steps[0].Association != "Sales.Order_BillTo" || steps[0].DestinationEntity != "Sales.Address" { + t.Fatalf("stored hops %+v, want one Sales.Order_BillTo -> Sales.Address", steps) + } +} + +// CONTROL: the sibling association is equally reachable, so a resolver that +// ignored the name and searched the domain model would pass the test above by +// luck. Asking for the other one must store the other one. +func TestSortPath_TheOtherAssociationStoresTheOther(t *testing.T) { + _, steps, errs := pathSortOf(t, ast.SortColumnDef{ + Associations: []string{"Sales.Order_ShipTo"}, + Attribute: "Sales.Address.City", + Order: "ASC", + }) + if len(errs) > 0 { + t.Fatalf("unexpected errors: %v", errs) + } + if len(steps) != 1 || steps[0].Association != "Sales.Order_ShipTo" { + t.Fatalf("stored hops %+v, want one Sales.Order_ShipTo", steps) + } +} + +// An unqualified hop is looked up in the entity's own module. +func TestSortPath_UnqualifiedHopAndBareAttribute(t *testing.T) { + path, steps, errs := pathSortOf(t, ast.SortColumnDef{ + Associations: []string{"Order_BillTo"}, + Attribute: "City", + Order: "ASC", + }) + if len(errs) > 0 { + t.Fatalf("unexpected errors: %v", errs) + } + if path != "Sales.Address.City" { + t.Errorf("stored attribute %q, want Sales.Address.City — a bare attribute after a hop "+ + "is qualified with the entity the hop reaches", path) + } + if len(steps) != 1 || steps[0].Association != "Sales.Order_BillTo" { + t.Fatalf("stored hops %+v, want one Sales.Order_BillTo", steps) + } +} + +// A hop that does not exist is refused, not written with an empty destination — +// a step with no DestinationEntity makes the project unopenable rather than +// merely wrong. +func TestSortPath_UnknownHopIsRefused(t *testing.T) { + _, steps, errs := pathSortOf(t, ast.SortColumnDef{ + Associations: []string{"Sales.Order_Nowhere"}, + Attribute: "Sales.Address.City", + Order: "ASC", + }) + if len(errs) == 0 { + t.Fatal("an unknown association must be refused") + } + if len(steps) != 0 { + t.Errorf("a refused hop must store nothing, got %+v", steps) + } + if !strings.Contains(strings.Join(errs, " "), "was not found") { + t.Errorf("unexpected message: %v", errs) + } +} + +// An attribute that is not on the entity the path ends at is refused. +func TestSortPath_AttributeOffThePathIsRefused(t *testing.T) { + _, _, errs := pathSortOf(t, ast.SortColumnDef{ + Associations: []string{"Sales.Order_BillTo"}, + Attribute: "Sales.Order.OrderNo", + Order: "ASC", + }) + if len(errs) == 0 { + t.Fatal("an attribute off the end of the path must be refused") + } + if !strings.Contains(strings.Join(errs, " "), "does not belong to") { + t.Errorf("unexpected message: %v", errs) + } +} diff --git a/mdl/executor/cmd_pages_builder_v3.go b/mdl/executor/cmd_pages_builder_v3.go index 1b5934ab0..554f70f71 100644 --- a/mdl/executor/cmd_pages_builder_v3.go +++ b/mdl/executor/cmd_pages_builder_v3.go @@ -850,13 +850,32 @@ func (pb *pageBuilder) buildDataSourceV3(ds *ast.DataSourceV3) (pages.DataSource if strings.ToLower(ob.Direction) == "desc" { direction = pages.SortDirectionDescending } + attrPath := pb.resolveAttributePathForEntity(ob.Attribute, ds.Reference) + var steps []pages.AttributeRefStep + if len(ob.Associations) > 0 { + // A sort that navigates associations. Resolved through the same + // walker DataGrid2 columns and dynamictext params use, so the two + // cannot disagree about a path that means the same thing in both. + // Refused rather than flattened: an attribute of a far entity with + // no EntityRef beside it is CE7247 at build time + // (mendixlabs/mxcli#1152). + path := strings.Join(append(append([]string{}, ob.Associations...), ob.Attribute), "/") + finalQN, hops, ok := pb.resolveAssociationAttributePathForEntity(path, ds.Reference) + if !ok { + return nil, "", mdlerrors.NewValidation(fmt.Sprintf( + "sort by %s: the association path could not be resolved from %s", + path, ds.Reference)) + } + attrPath, steps = finalQN, hops + } sortItem := &pages.GridSort{ BaseElement: model.BaseElement{ ID: model.ID(types.GenerateID()), TypeName: "Forms$GridSort", }, - AttributePath: pb.resolveAttributePathForEntity(ob.Attribute, ds.Reference), - Direction: direction, + AttributePath: attrPath, + AttributeRefSteps: steps, + Direction: direction, } dbSource.Sorting = append(dbSource.Sorting, sortItem) } @@ -1840,6 +1859,18 @@ func (pb *pageBuilder) resolveAttributePathForEntity(attrName string, entityName return pb.resolveAttributePath(attrName) } +// resolveAssociationAttributePathForEntity resolves an `Assoc/.../Attr` path +// against an explicit root entity rather than the builder's current widget +// context — a datasource's sort is rooted in the datasource's own entity. +// Mirrors resolveAttributePathForEntity. +func (pb *pageBuilder) resolveAssociationAttributePathForEntity(path, entityName string) (string, []pages.AttributeRefStep, bool) { + oldContext := pb.entityContext + pb.entityContext = entityName + defer func() { pb.entityContext = oldContext }() + + return pb.resolveAssociationAttributePath(path) +} + // resolveTemplateAttributePath resolves template parameter values like $widgetName.Attribute // to fully qualified entity paths like Module.Entity.Attribute. // It handles patterns like: diff --git a/mdl/executor/cmd_pages_describe.go b/mdl/executor/cmd_pages_describe.go index 314679566..589fa0b3c 100644 --- a/mdl/executor/cmd_pages_describe.go +++ b/mdl/executor/cmd_pages_describe.go @@ -509,7 +509,10 @@ func resolveLayoutName(ctx *ExecContext, layoutID model.ID) string { // rawSortColumn represents a sort column for describe output. type rawSortColumn struct { Attribute string // Qualified name or simple identifier - Order string // "ASC" or "DESC" + // Associations holds one qualified association name per hop, when the sort + // navigates to another entity (mendixlabs/mxcli#1152). + Associations []string + Order string // "ASC" or "DESC" } // rawDataSource represents a data source for describe output. diff --git a/mdl/executor/cmd_pages_describe_datasource.go b/mdl/executor/cmd_pages_describe_datasource.go index c4c3d1088..00a5b52f8 100644 --- a/mdl/executor/cmd_pages_describe_datasource.go +++ b/mdl/executor/cmd_pages_describe_datasource.go @@ -215,6 +215,7 @@ func parseSortColumns(ds map[string]any) []rawSortColumn { col := rawSortColumn{Order: "asc"} if attrRef, ok := sortItem["AttributeRef"].(map[string]any); ok { col.Attribute = shortAttributeName(extractString(attrRef["Attribute"])) + col.Associations = sortAttributeHops(attrRef) } if gridSortDirection(sortItem) == "Descending" { col.Order = "desc" @@ -226,6 +227,38 @@ func parseSortColumns(ds map[string]any) []rawSortColumn { return cols } +// sortAttributeHops reads the association hops of a stored AttributeRef — its +// EntityRef.Steps, one DomainModels$EntityRefStep per hop. An own-entity +// attribute carries a DirectEntityRef or no EntityRef and yields none. +// +// Without this a sort over an association described as a bare attribute name, +// and the replay had to guess the hop back (mendixlabs/mxcli#1152). +func sortAttributeHops(attrRef map[string]any) []string { + entityRef, ok := attrRef["EntityRef"].(map[string]any) + if !ok || entityRef == nil { + return nil + } + var hops []string + for _, raw := range getBsonArrayElements(entityRef["Steps"]) { + step, ok := raw.(map[string]any) + if !ok { + continue + } + if assoc := extractString(step["Association"]); assoc != "" { + hops = append(hops, assoc) + } + } + return hops +} + +// sortColumnPath renders a sort column as `Assoc/.../Attribute`. +func sortColumnPath(col rawSortColumn) string { + if len(col.Associations) == 0 { + return col.Attribute + } + return strings.Join(append(append([]string{}, col.Associations...), col.Attribute), "/") +} + // dataSourceExpr renders a datasource as the MDL that reproduces it — the part // after `DataSource: `. Returns "" when there is nothing to emit. // @@ -251,7 +284,7 @@ func dataSourceExpr(ds *rawDataSource) string { if len(ds.SortColumns) > 0 { parts := make([]string, 0, len(ds.SortColumns)) for _, col := range ds.SortColumns { - parts = append(parts, col.Attribute+" "+col.Order) + parts = append(parts, sortColumnPath(col)+" "+col.Order) } expr += " sort by " + strings.Join(parts, ", ") } diff --git a/mdl/executor/cmd_pages_sort_association_test.go b/mdl/executor/cmd_pages_sort_association_test.go new file mode 100644 index 000000000..48877ea12 --- /dev/null +++ b/mdl/executor/cmd_pages_sort_association_test.go @@ -0,0 +1,108 @@ +// SPDX-License-Identifier: Apache-2.0 + +package executor + +import ( + "strings" + "testing" + + "github.com/mendixlabs/mxcli/sdk/pages" +) + +// mendixlabs/mxcli#1152, page half. A page datasource's `sort by` is stored as +// the same DomainModels$AttributeRef a microflow retrieve's sort is, so it has +// the same two halves: the hops must be READ (or DESCRIBE drops them) and +// EMITTED as `Assoc/Attr` (or the replay has to guess which association). + +// TestParseSortColumns_ReadsAssociationHops is the read half. +func TestParseSortColumns_ReadsAssociationHops(t *testing.T) { + ds := map[string]any{ + "SortBar": map[string]any{ + "SortItems": []any{ + map[string]any{ + "SortDirection": "Ascending", + "AttributeRef": map[string]any{ + "Attribute": "Sales.Address.City", + "EntityRef": map[string]any{ + "Steps": []any{ + map[string]any{ + "Association": "Sales.Order_BillTo", + "DestinationEntity": "Sales.Address", + }, + }, + }, + }, + }, + }, + }, + } + cols := parseSortColumns(ds) + if len(cols) != 1 { + t.Fatalf("got %d sort columns, want 1", len(cols)) + } + if got := strings.Join(cols[0].Associations, ","); got != "Sales.Order_BillTo" { + t.Errorf("Associations = %q, want Sales.Order_BillTo — the hop is invisible to "+ + "DESCRIBE without it", got) + } + if got := sortColumnPath(cols[0]); got != "Sales.Order_BillTo/City" { + t.Errorf("rendered %q, want Sales.Order_BillTo/City", got) + } +} + +// CONTROL: an own-entity sort reads back with no hops and renders as the bare +// attribute — a reader that manufactured an empty hop would emit a stray `/`. +func TestParseSortColumns_PlainSortHasNoHops(t *testing.T) { + ds := map[string]any{ + "SortBar": map[string]any{ + "SortItems": []any{ + map[string]any{ + "SortDirection": "Descending", + "AttributeRef": map[string]any{"Attribute": "Sales.Order.OrderNo"}, + }, + }, + }, + } + cols := parseSortColumns(ds) + if len(cols) != 1 { + t.Fatalf("got %d sort columns, want 1", len(cols)) + } + if len(cols[0].Associations) != 0 { + t.Errorf("Associations = %v, want none", cols[0].Associations) + } + if got := sortColumnPath(cols[0]); got != "OrderNo" { + t.Errorf("rendered %q, want OrderNo", got) + } +} + +// TestDataSourceExpr_EmitsSortAssociationPath is the emit half: the rendered +// datasource must carry the hop, because that text is what a person replays. +func TestDataSourceExpr_EmitsSortAssociationPath(t *testing.T) { + ds := &rawDataSource{ + Type: "database", + Reference: "Sales.Order", + SortColumns: []rawSortColumn{ + {Attribute: "City", Associations: []string{"Sales.Order_BillTo"}, Order: "asc"}, + {Attribute: "OrderNo", Order: "desc"}, + }, + } + got := dataSourceExpr(ds) + want := "database from Sales.Order sort by Sales.Order_BillTo/City asc, OrderNo desc" + if got != want { + t.Errorf("dataSourceExpr =\n %q\nwant\n %q", got, want) + } +} + +// A GridSort carrying hops must reach the writer with them attached — the field +// exists so the page writer can build the EntityRef, and a builder that resolved +// the path but dropped the steps would store an attribute of a far entity with +// nothing to reach it, which is CE7247. +func TestGridSortCarriesAttributeRefSteps(t *testing.T) { + s := &pages.GridSort{ + AttributePath: "Sales.Address.City", + AttributeRefSteps: []pages.AttributeRefStep{{Association: "Sales.Order_BillTo", DestinationEntity: "Sales.Address"}}, + Direction: pages.SortDirectionAscending, + } + if len(s.AttributeRefSteps) != 1 || s.AttributeRefSteps[0].Association != "Sales.Order_BillTo" { + t.Fatalf("GridSort hops = %+v", s.AttributeRefSteps) + } +} diff --git a/mdl/grammar/domains/MDLPage.g4 b/mdl/grammar/domains/MDLPage.g4 index bdb8a6e40..904136ccf 100644 --- a/mdl/grammar/domains/MDLPage.g4 +++ b/mdl/grammar/domains/MDLPage.g4 @@ -75,8 +75,20 @@ variableDeclaration : VARIABLE COLON dataType EQUALS STRING_LITERAL // $varName: Boolean = 'expression' ; +// A sort column. The name may navigate associations, one `/` per hop, with the +// final segment naming the attribute: +// +// sort by Name asc +// sort by Sales.Order.Name asc +// sort by Sales.Order_BillTo/Sales.Address.City asc +// +// Mendix stores the hops as the AttributeRef's EntityRef, and without a spelling +// for them `describe` had to drop them and `exec` had to guess — which silently +// picked the wrong association wherever two reach the same entity +// (mendixlabs/mxcli#1152). `qualifiedName SLASH qualifiedName` is the same shape +// MDLCatalog.g4 uses for `Association/Entity`. sortColumn - : (qualifiedName | IDENTIFIER) (ASC | DESC)? + : (qualifiedName (SLASH qualifiedName)* | IDENTIFIER) (ASC | DESC)? ; // One attribute of a List View's search bar. No direction — unlike a sort diff --git a/mdl/visitor/visitor_microflow_statements.go b/mdl/visitor/visitor_microflow_statements.go index db4eed1f9..34cca43af 100644 --- a/mdl/visitor/visitor_microflow_statements.go +++ b/mdl/visitor/visitor_microflow_statements.go @@ -1517,8 +1517,12 @@ func buildSortColumnMicroflow(ctx parser.ISortColumnContext) *ast.SortColumnDef // keeping the quotes produced a nonsense reference that only failed on write // ("attribute does not belong to entity"), unlike everywhere else where quoting // is safe (FINDINGS #13). - if qn := colCtx.QualifiedName(); qn != nil { - col.Attribute = unquoteQualifiedName(qn.GetText()) + // + // Several qualifiedNames mean an association path: every segment but the last + // is a hop, the last is the attribute (mendixlabs/mxcli#1152). + if qns := colCtx.AllQualifiedName(); len(qns) > 0 { + col.Associations = sortColumnHops(qns) + col.Attribute = unquoteQualifiedName(qns[len(qns)-1].GetText()) } else if id := colCtx.IDENTIFIER(); id != nil { col.Attribute = unquoteIdentifier(id.GetText()) } @@ -1531,6 +1535,20 @@ func buildSortColumnMicroflow(ctx parser.ISortColumnContext) *ast.SortColumnDef return col } +// sortColumnHops returns every segment of a sort column's association path but +// the last — the last is the attribute. Shared by the microflow and page sort +// column builders so the two cannot disagree about where the attribute is. +func sortColumnHops(qns []parser.IQualifiedNameContext) []string { + if len(qns) < 2 { + return nil + } + hops := make([]string, 0, len(qns)-1) + for _, qn := range qns[:len(qns)-1] { + hops = append(hops, getQualifiedNameText(qn)) + } + return hops +} + // buildIfStatement converts IF statement context to IfStmt. func buildIfStatement(ctx parser.IIfStatementContext) *ast.IfStmt { if ctx == nil { diff --git a/mdl/visitor/visitor_page_v3.go b/mdl/visitor/visitor_page_v3.go index 43231c3d3..52beda937 100644 --- a/mdl/visitor/visitor_page_v3.go +++ b/mdl/visitor/visitor_page_v3.go @@ -1418,8 +1418,11 @@ func buildSortColumnAsOrderBy(ctx parser.ISortColumnContext) ast.OrderByItemV3 { scCtx := ctx.(*parser.SortColumnContext) item := ast.OrderByItemV3{Direction: "ASC"} - if qn := scCtx.QualifiedName(); qn != nil { - item.Attribute = getQualifiedNameText(qn) + // Several qualifiedNames mean an association path: every segment but the last + // is a hop, the last is the attribute (mendixlabs/mxcli#1152). + if qns := scCtx.AllQualifiedName(); len(qns) > 0 { + item.Associations = sortColumnHops(qns) + item.Attribute = getQualifiedNameText(qns[len(qns)-1]) } else if id := scCtx.IDENTIFIER(); id != nil { item.Attribute = id.GetText() } diff --git a/mdl/visitor/visitor_sort_association_path_test.go b/mdl/visitor/visitor_sort_association_path_test.go new file mode 100644 index 000000000..150ececbc --- /dev/null +++ b/mdl/visitor/visitor_sort_association_path_test.go @@ -0,0 +1,97 @@ +// SPDX-License-Identifier: Apache-2.0 + +package visitor + +import ( + "strings" + "testing" + + "github.com/mendixlabs/mxcli/mdl/ast" +) + +// mendixlabs/mxcli#1152 — a sort column may navigate associations, one `/` per +// hop, with the final segment naming the attribute. Without the spelling, +// DESCRIBE had to drop the hop and the replay had to guess which association +// was meant. +func sortColumnsOf(t *testing.T, sortClause string) []ast.SortColumnDef { + t.Helper() + input := `CREATE MICROFLOW Sales.ListOrders () +BEGIN + RETRIEVE $Orders FROM Sales.Order ` + sortClause + `; + RETURN; +END;` + prog, errs := Build(input) + if len(errs) > 0 { + t.Fatalf("parse errors for %q: %v", sortClause, errs) + } + mf, ok := prog.Statements[0].(*ast.CreateMicroflowStmt) + if !ok { + t.Fatalf("statement = %T, want *ast.CreateMicroflowStmt", prog.Statements[0]) + } + retr, ok := mf.Body[0].(*ast.RetrieveStmt) + if !ok { + t.Fatalf("body[0] = %T, want *ast.RetrieveStmt", mf.Body[0]) + } + return retr.SortColumns +} + +func TestSortColumn_AssociationPathParses(t *testing.T) { + cols := sortColumnsOf(t, "SORT BY Sales.Order_BillTo/Sales.Address.City ASC") + if len(cols) != 1 { + t.Fatalf("got %d sort columns, want 1", len(cols)) + } + if got := strings.Join(cols[0].Associations, ","); got != "Sales.Order_BillTo" { + t.Errorf("Associations = %q, want Sales.Order_BillTo", got) + } + if cols[0].Attribute != "Sales.Address.City" { + t.Errorf("Attribute = %q, want Sales.Address.City — the LAST segment is the attribute", cols[0].Attribute) + } + if cols[0].Order != "ASC" { + t.Errorf("Order = %q, want ASC", cols[0].Order) + } +} + +func TestSortColumn_MultipleHopsParse(t *testing.T) { + cols := sortColumnsOf(t, "SORT BY Order_Customer/Customer_Country/Name DESC") + if len(cols) != 1 { + t.Fatalf("got %d sort columns, want 1", len(cols)) + } + if got := strings.Join(cols[0].Associations, ","); got != "Order_Customer,Customer_Country" { + t.Errorf("Associations = %q, want Order_Customer,Customer_Country", got) + } + if cols[0].Attribute != "Name" || cols[0].Order != "DESC" { + t.Errorf("got {%q, %q}, want {Name, DESC}", cols[0].Attribute, cols[0].Order) + } +} + +// CONTROL: a sort with no hop must still parse to no hops and the whole name as +// the attribute. A change that read every qualifiedName as a hop would leave +// ordinary sorts with an empty attribute. +func TestSortColumn_PlainSortHasNoHops(t *testing.T) { + for _, clause := range []string{"SORT BY Name ASC", "SORT BY Sales.Order.Name ASC"} { + cols := sortColumnsOf(t, clause) + if len(cols) != 1 { + t.Fatalf("%s: got %d sort columns, want 1", clause, len(cols)) + } + if len(cols[0].Associations) != 0 { + t.Errorf("%s: Associations = %v, want none", clause, cols[0].Associations) + } + if cols[0].Attribute == "" { + t.Errorf("%s: Attribute is empty", clause) + } + } +} + +// Multiple sort columns, one with hops and one without, keep their own paths. +func TestSortColumn_MixedColumns(t *testing.T) { + cols := sortColumnsOf(t, "SORT BY Sales.Order_BillTo/City ASC, OrderNo DESC") + if len(cols) != 2 { + t.Fatalf("got %d sort columns, want 2", len(cols)) + } + if len(cols[0].Associations) != 1 || cols[0].Attribute != "City" { + t.Errorf("col[0] = %+v, want one hop and City", cols[0]) + } + if len(cols[1].Associations) != 0 || cols[1].Attribute != "OrderNo" { + t.Errorf("col[1] = %+v, want no hops and OrderNo", cols[1]) + } +} diff --git a/sdk/pages/pages_datasources.go b/sdk/pages/pages_datasources.go index 4f4fe6e24..ce46bc16c 100644 --- a/sdk/pages/pages_datasources.go +++ b/sdk/pages/pages_datasources.go @@ -43,8 +43,13 @@ func (DatabaseSource) isDataSource() {} // GridSort represents sorting configuration. type GridSort struct { model.BaseElement - AttributePath string `json:"attributePath"` - Direction SortDirection `json:"direction"` + AttributePath string `json:"attributePath"` + // AttributeRefSteps carries the association hops when the sort navigates to + // another entity. Mendix stores them as the AttributeRef's EntityRef; an + // attribute path naming a far entity without them is CE7247 "Cannot sort on + // attribute …" (mendixlabs/mxcli#1152). + AttributeRefSteps []AttributeRefStep `json:"attributeRefSteps,omitempty"` + Direction SortDirection `json:"direction"` } // SortDirection represents the sort direction. From 10d6892b5cad72a82247ef242596e38f0f5fb9f1 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 21 Sep 2026 18:05:25 +0000 Subject: [PATCH 06/15] fix: session-auth OData needs X-Csrf-Token, and the vega pack said it did not MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The mendix-vega-charts skill pack stated that "same-origin requests carry the session cookie, so an endpoint authenticated by session is reachable from a chart on a page of the same app without any token handling". The cookie does go; it is not sufficient. A URL-fed Vega-Lite spec therefore drew axes and a full legend with zero data points and no error, because Vega reads the 401 body as an empty dataset. Measured on a freshly built 11.14.0 app, one URL, four requests: no credentials 401 session cookie only 401 session cookie + X-Csrf-Token 200, correct payload basic auth 200, correct payload Rows 2 and 3 differ in exactly one header, and the 401 body is byte-identical to the one reported. Through vega's own loader against that running app: 0 marks / 3 axes / no error without the token, 1 mark with it — the reported symptom, reproduced end to end. The docs fix is the reported ask, but a doc saying "the widget must supply the header" over a widget that does not is half a fix, so the pack's widget now installs a loader that attaches the token — to same-origin requests ONLY. The token authenticates this session against this app, and the same-origin decision resolves the URI against the page rather than testing it for a scheme: a protocol-relative //elsewhere.example/rows.json carries no scheme, so a scheme test reads it as relative and hands another host a working session credential. - widget/src/csrf.ts: isSameOrigin / readCsrfToken / withCsrfHeader, pure and non-mutating (vega reuses one options object across a spec's fetches, so writing a header into it would leak the token to later URLs) - widget/test/csrf.test.ts: 9 cases, node's own runner over .ts, no deps - make check-skill-pack-js + a CI step: nothing else in this repo compiles a pack's widget, so logic shipped there was ungated - two guards on the vendored pack in cmd/mxcli/skillpacks_test.go: a pack documenting a relative-URL data source must mention X-Csrf-Token, and a widget attaching the header must resolve the uri with new URL() Controls, each run with the fix reverted: - guard stubbed to pre-fix behaviour: "X-Csrf-Token missing: Mendix answers this request 401", 4 of 9 cases fail - the issue's own suggested regex in place of the origin resolution: 4 of 9 fail, including the protocol-relative leak - both Go guards reverted: both fail, naming the file Also verified: the modified widget typechecks clean under --strict against the real vega / vega-embed / react types. Fixes ako/mxcli#574 Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01BwJzSGaDx7vzTE2g4QN7kz --- .claude/skills/fix-issue/findings/other.jsonl | 1 + .../skills/packs/mendix-vega-charts/SKILL.md | 53 ++++++++- .../skills/packs/mendix-vega-charts/pack.yaml | 2 +- .../references/failure-modes.md | 40 +++++++ .../widget/src/VegaChart.tsx | 23 ++++ .../mendix-vega-charts/widget/src/csrf.ts | 88 +++++++++++++++ .../widget/test/csrf.test.ts | 106 ++++++++++++++++++ .github/workflows/push-test.yml | 6 + Makefile | 20 +++- cmd/mxcli/skillpacks_test.go | 89 +++++++++++++++ .../bug-tests/odata-574-session-auth-csrf.mdl | 98 ++++++++++++++++ 11 files changed, 520 insertions(+), 6 deletions(-) create mode 100644 .claude/skills/packs/mendix-vega-charts/widget/src/csrf.ts create mode 100644 .claude/skills/packs/mendix-vega-charts/widget/test/csrf.test.ts create mode 100644 mdl-examples/bug-tests/odata-574-session-auth-csrf.mdl diff --git a/.claude/skills/fix-issue/findings/other.jsonl b/.claude/skills/fix-issue/findings/other.jsonl index 3fa07fd25..51edfa988 100644 --- a/.claude/skills/fix-issue/findings/other.jsonl +++ b/.claude/skills/fix-issue/findings/other.jsonl @@ -15,3 +15,4 @@ {"area": "model", "date": "2026-08-25", "raw": "| A multi-segment member in an **inline REST** mapping (`\"Title\" = \"fields/Title\"` inside `Response:`/`Body: MAPPING`) passes every gate — `mxcli check`, `exec`, `mx check`, `describe` — and the column is **empty at runtime** | The inline REST serializer is a SEPARATE code path from the mapping documents, and it appended the member text verbatim: `jsonPath + \"\\|\" + m.ExposedName` stores `(Object)\\|fields/Title`, one member whose NAME contains a slash, where Mendix stores `(Object)\\|fields\\|Title`. Nothing converted `/` to `\\|` on this path. Confirmed against four Studio Pro-authored inline response mappings in the demo apps, which all store full pipe paths (`(Object)\\|results\\|bindings\\|(Object)\\|caseId\\|value`) | `model/mapping_paths.go` (`InlineMappingPath`, `InlineMappingExposedName`), `sdk/mpr/writer_rest.go` (`serializeInlineMappingElement`), `mdl/backend/modelsdk/consumed_rest_write.go` (`restInlineMappingElementToGen`), `mdl/executor/cmd_rest_clients.go` (`inlineMemberName`) | **The document fixes do not reach this.** The mapping census (`PROPOSAL_mapping_coverage.md`) classifies mapping DOCUMENTS only and does not mention the inline form, so none of #248/#262–#268 improved it as a side effect — a reader who sees multi-segment paths documented as working will reasonably try one here. Store the pipe path and put the **last segment** in `ExposedName` (Studio Pro's own derivation uniquifies against siblings — `caseId\\|value` becomes `CaseId_Value` but `graphData\\|value` in the same document becomes plain `Value` — so it is not reproducible from the path and does not need to be; ExposedName is a label, JsonPath is what binds). **Fixing the path breaks DESCRIBE in the same motion**: the printer emitted `ExposedName`, which now holds only the last segment, so it produced `\"Title\" = \"Title\"` — output that parses, re-executes and binds the wrong member. Derive the member from the stored JsonPath relative to the enclosing element and drop the generated `(Object)`/`(Array)`/`(Wrapper)` markers. Both engines have their own copy of this serializer; patch both or they drift. Repro `mdl-examples/bug-tests/rest-inline-mapping-paths.mdl`, tests `sdk/mpr/writer_rest_inline_mapping_test.go`. mxcli-rest FINDINGS #36 |", "refs": ["#248", "#262", "#268", "#36"]} {"area": "web/dist", "date": "2026-08-30", "raw": "| After `mxcli test … --local`, an app another `mxcli run --local` is serving goes blank while still answering HTTP 200 (~1.7 KB, the Mendix SPA shell); the runtime log shows `Connector: 404 - file not found for file: dist%2Findex.js` and `deployment/web/dist` is gone | `cmd/mxcli/testrunner/localapp_options.go`, `cmd/mxcli/testrunner/runner_local.go` (`localTestDeployDir`), `cmd/mxcli/testrunner/runner.go` (`checkScratchDeploymentExists`) | A local test run already used its own ports and its own `_test` database — the code comment says why, verbatim — but shared the **deployment directory**, which is the one the *browser* reads. A headless test boot does not bundle the web client, so its build left the running app serving the shell over a 404: tests pass, run keeps running, app is blank, nothing reported at either end. **Detection was not the fix**: the two processes use different ports by design, so no port check can see it, and a lock file would only turn a silent blanking into a refusal. The test boot now builds into `/.mxcli/deployment-test/` — gitignored, already where the test runtime log lives — which makes the collision impossible. Note booting a runtime against the shared directory damages it even **without** a rebuild (the packaging step removes the bundle — FINDINGS §35, `ReportLostWebClientBundle`), so \"reuse the dev loop's tree read-only\" is not an alternative. Consequence to wire: `--skip-build` used to mean \"reuse deployment/\" and now has nothing until tests have run once, so it is refused with the reason rather than failing inside the runtime boot against a path the user never chose. Reported as mxcli-formula1 FINDINGS §62 |"} {"area": ".claude/skills/mendix/record-narrated-demo", "date": "2026-09-13", "symptom": "In a narrated demo recorded with CSS `zoom` (take.js's fix for a fixed-width Mendix page), narrate.js's `point()` highlight ring is drawn around the wrong control or off the edge of the frame, and the caption plate is the wrong height and sits outside the film's caption band. Nothing in the take, the beat assertions or the contact sheet reports anything.", "cause": "Under `html{zoom:z}` Chromium reports `getBoundingClientRect()` in ZOOM-ADJUSTED (video) pixels but `getComputedStyle()` and `style.*` in CSS pixels. `point()` read a rect and assigned it straight to `style.left/top/width/height`, so the ring landed at position x z (measured at z=1.6842: a target at (168,202) ringed at (274,330)). The plate had the mirror-image problem: its geometry was declared in CSS pixels, so a 96px bar reached the file as 96 x z = 162 video px against a 184px caption band.", "file": ".claude/skills/mendix/record-narrated-demo/narrate.js (`point`, `css`, `checkOverlay`)", "insight": "The overlay lives in the page's coordinate space and the film is specified in the frame's, and `zoom` is the only conversion between them - so every overlay number is now stated in VIDEO pixels and divided by a zoom passed to `configure()`. The trap is that the conversion runs in opposite directions depending on which API you read it back with, which is why the fix came with `checkOverlay()`: it measures the installed plate against the band and refuses the take, with the control being one line (build the overlay without telling it the zoom -> 'caption plate is 310 video px tall, the band is 184'). A design rule that can be measured in the page should be a check that throws at record time, not a note in a skill - the same argument PRODUCTION.md sec 12 makes for compositions.", "refs": ["ako/mxcli-intro-video video-system/DESIGN-LANGUAGE.md"]} +{"area":".claude/skills/packs","date":"2026-09-21","symptom":"A URL-fed Vega-Lite chart in the mendix-vega-charts pack drew its axes and a FULL legend with zero data points, no console error and no Vega warning. The skill stated that \"same-origin requests carry the session cookie, so an endpoint authenticated by session is reachable ... without any token handling\".","cause":"Mendix refuses a session-authenticated request without the session's CSRF token on READS too, not just writes and not just /xas/. The cookie is sent; it is not sufficient. Vega's loader read the 401 body as an empty dataset, so the failure surfaced as a plausible-looking empty chart rather than as an error.","file":".claude/skills/packs/mendix-vega-charts/widget/src/csrf.ts, .../SKILL.md, cmd/mxcli/skillpacks_test.go","insight":"THE LEGEND IS THE TELL: it is built from the spec's scales, not from rows, so a chart with a complete legend and no marks has had its DATA refused, while a chart with a broken legend has a spec problem. That one distinction separates the two hypotheses before any measurement. Two things then send the diagnosis the wrong way and cost the time: document.cookie shows only originURI=/login.html (XASSESSIONID and xasid are httpOnly, so the browser IS sending them and JavaScript cannot see them) and basic auth on the same URL returns the data, which reads as proof the endpoint is fine and the chart is broken. Isolate on the HEADER, not the URL: two requests, one added header, everything else equal -- 401 vs 200, measured on a fresh 11.14.0 app. Skip the plausible wrong turn of adding the header unconditionally: the issue's own suggested loader tests the URI with /^[a-z][a-z0-9+.-]*:\\/\\//i, which passes //elsewhere.example/rows.json (no scheme, another host) and hands that host a working session credential. Resolve with new URL(uri, base) and compare origins instead -- it also gets the converse right, an absolute URL naming the app's own origin IS the app. End-to-end control through vega's real loader against the running app: 0 marks / 3 axes / no error without the token, 1 mark with it, which reproduces the reported symptom exactly.","refs":["ako/mxcli#574"],"ce":[],"mendix":"11.14.0"} diff --git a/.claude/skills/packs/mendix-vega-charts/SKILL.md b/.claude/skills/packs/mendix-vega-charts/SKILL.md index 26e31a3fa..7f1c356ec 100644 --- a/.claude/skills/packs/mendix-vega-charts/SKILL.md +++ b/.claude/skills/packs/mendix-vega-charts/SKILL.md @@ -64,14 +64,59 @@ is fetched; the payload arrives with the page. } ``` -The widget needs no change for this — with no data bound it passes the spec +The spec needs no change for this — with no data bound the widget passes it through untouched and Vega's own loader does the fetch. Verified end to end against an endpoint served by the app itself: one `200`, six marks, no error, and `format.property` unwrapping the `{"value": […]}` envelope OData returns. -Same-origin requests carry the session cookie, so an endpoint authenticated by -session is reachable from a chart on a page of the same app without any token -handling. +### A session-authenticated endpoint needs `X-Csrf-Token` + +Same-origin requests carry the session cookie, and **the cookie alone is not +enough.** Mendix refuses a session-authenticated request without the session's +CSRF token — on a *read*, not just a write, and on `/odata/` as well as `/xas/`. +Measured on 11.14.0 against one URL, four requests: + +| request | result | +|---|---| +| no credentials | `401` | +| session cookie only | `401` | +| session cookie **+ `X-Csrf-Token`** | `200`, correct payload | +| basic auth | `200`, correct payload | + +Vega reports the `401` body as an empty dataset, so the chart draws its axes and +a full legend with **no marks and no error** — measured through this widget's own +loader: 0 marks and 3 axes without the token, 1 mark with it. + +**The widget shipped here supplies the header** ([`widget/src/csrf.ts`](widget/src/csrf.ts)), +reading the token from `mx.session.getConfig("csrftoken")` — it is not a cookie +and cannot ride along by itself. A widget built from an earlier copy of this pack +does not, and needs that loader added: + +```ts +const instance = vegaLoader(); +const fetchHttp = instance.http.bind(instance); +instance.http = (uri, options) => fetchHttp(uri, withCsrfHeader(uri, options, location.href, token)); +``` + +**Same-origin only**, and the check is a resolution rather than a string test: +the token authenticates this session against this app, so sending it to a +third-party host hands that host a working credential. Resolving the URI against +the page settles the two cases a `/^[a-z][a-z0-9+.-]*:\/\//i` test on the raw +string gets wrong — a protocol-relative `//elsewhere.example/rows.json` carries +no scheme and goes to another host, and an absolute URL naming the app's own +origin *is* the app. + +Two things that send a diagnosis the wrong way: + +- **`document.cookie` shows only `originURI=/login.html`.** That reads like a + missing session cookie and starts a hunt for a cookie problem that does not + exist — `XASSESSIONID` and `xasid` are `httpOnly`, so the browser sends them + and JavaScript cannot see them. +- **Basic auth on the same URL returns the data,** which looks like proof the + endpoint is fine and the chart is broken. Both are fine; the header is missing. + +The wider rule, and what each authentication method costs, is in +[`.claude/skills/mendix/odata-data-sharing/reference/errors-and-auth.md`](../../mendix/odata-data-sharing/reference/errors-and-auth.md). ### Which to use diff --git a/.claude/skills/packs/mendix-vega-charts/pack.yaml b/.claude/skills/packs/mendix-vega-charts/pack.yaml index a9e856afe..6146cd66c 100644 --- a/.claude/skills/packs/mendix-vega-charts/pack.yaml +++ b/.claude/skills/packs/mendix-vega-charts/pack.yaml @@ -1,6 +1,6 @@ # Skill pack manifest. See docs/11-proposals/PROPOSAL_skill_packs.md. name: mendix-vega-charts -version: 1.0.0 +version: 1.1.0 description: >- Vega-Lite charting through a pluggable widget that takes the specification and the data as separate properties, so the model emits rows and never assembles a diff --git a/.claude/skills/packs/mendix-vega-charts/references/failure-modes.md b/.claude/skills/packs/mendix-vega-charts/references/failure-modes.md index b83b99789..cb722f9f9 100644 --- a/.claude/skills/packs/mendix-vega-charts/references/failure-modes.md +++ b/.claude/skills/packs/mendix-vega-charts/references/failure-modes.md @@ -19,6 +19,46 @@ A mismatch leaves the spec's own (absent) data in place. No error either. empty string parses as nothing and the widget renders an empty chart rather than an error. +**Cause D — the spec is URL-fed and the endpoint answered `401`.** See the next +entry; this is the one that looks most like a spec problem and is not one. + +--- + +## A URL-fed chart is empty — axes and a full legend, zero data points + +The fetch came back `401` and Vega read the error body as an empty dataset. No console +error, no Vega warning, and the legend is *complete* — it is built from the spec's +scales, not from rows — so the chart looks like one whose data happens to be missing +rather than one whose request was refused. + +**Cause: Mendix refuses a session-authenticated request without the session's CSRF +token, on reads too.** Not just writes, and not just `/xas/`. Measured on 11.14.0, +one URL, requests differing in one header: + +| request | result | +|---|---| +| no credentials | `401` | +| session cookie only | `401` | +| session cookie **+ `X-Csrf-Token`** | `200`, correct payload | +| basic auth | `200`, correct payload | + +and through the widget's own loader: **0 marks, 3 axes, no error** without the token; +1 mark with it. + +Fix: the widget attaches the token to same-origin fetches +([`widget/src/csrf.ts`](../widget/src/csrf.ts)). A widget built from a copy of this +pack from before ako/mxcli#574 does not — check for that file before suspecting the +spec. + +Two measurements that mislead here, both worth knowing before you start: + +- **`document.cookie` shows only `originURI=/login.html`.** It reads like the session + cookie is missing, and sends you after a cookie problem that does not exist: + `XASSESSIONID` and `xasid` are `httpOnly`. The browser is sending them. +- **Basic auth on the same URL returns the data.** That looks like proof the endpoint + is fine and the chart is broken. The endpoint *is* fine. So is the chart. Isolate on + the header, not on the URL: two requests, one added header, everything else equal. + --- ## Numbers in the chart are smaller than the numbers in the database diff --git a/.claude/skills/packs/mendix-vega-charts/widget/src/VegaChart.tsx b/.claude/skills/packs/mendix-vega-charts/widget/src/VegaChart.tsx index 822bb1fe7..f9b815d02 100644 --- a/.claude/skills/packs/mendix-vega-charts/widget/src/VegaChart.tsx +++ b/.claude/skills/packs/mendix-vega-charts/widget/src/VegaChart.tsx @@ -1,7 +1,9 @@ import { useEffect, useMemo, useRef, useState, ReactElement } from "react"; import embed, { Result as EmbedResult, VisualizationSpec } from "vega-embed"; +import { loader as vegaLoader } from "vega"; import { VegaChartContainerProps } from "../typings/VegaChartProps"; +import { readCsrfToken, withCsrfHeader } from "./csrf"; // Not decoration: a spec using `"width": "container"` measures this element, and // without the width rule below it measures zero and the chart draws nothing — @@ -50,6 +52,26 @@ function cleanDatum(datum: Record): Record { return out; } +/** + * A Vega loader that authenticates fetches going back to this app. + * + * Only a URL-fed spec reaches this — a spec fed from the `chartData` attribute + * fetches nothing. Mendix refuses a session-authenticated read without the + * session's CSRF token, and Vega reports the 401 body as an empty dataset, so + * without this the chart draws its axes and legend and no marks. The token goes + * to same-origin requests only; see `csrf.ts`. Built per embed rather than once + * at module load, because the token is not there until the session is. + */ +function createLoader(): ReturnType { + const instance = vegaLoader(); + const fetchHttp = instance.http.bind(instance); + const token = readCsrfToken(); + const origin = typeof location === "undefined" ? "" : location.href; + instance.http = (uri: string, options?: RequestInit) => + fetchHttp(uri, withCsrfHeader(uri, options, origin, token)); + return instance; +} + export function VegaChart(props: VegaChartContainerProps): ReactElement { const { spec, chartData, datasetName, chartHeight, renderer, showActions, selection, onClick } = props; const hostRef = useRef(null); @@ -108,6 +130,7 @@ export function VegaChart(props: VegaChartContainerProps): ReactElement { embed(hostRef.current, resolvedSpec, { actions: showActions, renderer, + loader: createLoader(), // The app supplies its own type scale and palette; letting Vega apply // a theme on top would fight it. config: { background: "transparent" } diff --git a/.claude/skills/packs/mendix-vega-charts/widget/src/csrf.ts b/.claude/skills/packs/mendix-vega-charts/widget/src/csrf.ts new file mode 100644 index 000000000..0b4fdf192 --- /dev/null +++ b/.claude/skills/packs/mendix-vega-charts/widget/src/csrf.ts @@ -0,0 +1,88 @@ +/** + * Authenticating a Vega fetch against the app's own endpoints. + * + * A URL-fed spec — `{"data": {"url": "/odata/…"}}` — is fetched by Vega's own + * loader, not by the Mendix client, so it carries whatever `fetch` carries by + * default. Against a *session-authenticated* Mendix endpoint that is not + * enough: Mendix refuses a session-authenticated request, **including a read**, + * unless it also presents the session's CSRF token. Measured on 11.14.0, two + * requests differing in one header: + * + * session cookie only 401 + * session cookie + X-Csrf-Token 200 + * + * The failure is silent in the worst way — Vega treats the 401 body as an empty + * dataset, so the chart draws its axes and a full legend with no marks and no + * error. See `references/failure-modes.md`. + * + * The token is not a cookie and cannot ride along by itself; the client keeps it + * at `mx.session.getConfig("csrftoken")`. + * + * **Same-origin only.** The token authenticates this session against this app, + * so attaching it to a request bound for another host hands that host a working + * credential. A spec's `url` is author-controlled, but an author reaching a + * public data set over `https://` must not thereby leak the session — so the + * header is added only where the request is going back to the app itself. + */ + +/** The header Mendix reads the session's CSRF token from. */ +export const CSRF_HEADER = "X-Csrf-Token"; + +/** + * Does `uri` resolve to the same origin as `base`? + * + * Resolution, not string matching, because the cases that matter do not look + * alike. A protocol-relative `//elsewhere.example/rows.json` carries no scheme + * and so passes any "does it start with https://" test, while going to another + * host — which is exactly the request that must not carry the token. Conversely + * an absolute URL naming the app's own origin is the app, and refusing it would + * break a spec that spells its endpoint out in full. + * + * Anything that will not parse, and anything opaque (`data:`, `blob:`), is not + * same-origin: no header, which is the safe answer for both. + */ +export function isSameOrigin(uri: string, base: string): boolean { + try { + return new URL(uri, base).origin === new URL(base).origin; + } catch { + return false; + } +} + +/** + * Read the session's CSRF token from the Mendix client. + * + * Defensive about every step: the widget also runs in the Studio Pro preview + * and in a unit test, where `mx` does not exist, and a chart that throws on a + * missing global is worse than one that fetches without a token. + */ +export function readCsrfToken(): string | undefined { + const client = (globalThis as { mx?: { session?: { getConfig?: (key: string) => unknown } } }).mx; + const token = client?.session?.getConfig?.("csrftoken"); + return typeof token === "string" && token !== "" ? token : undefined; +} + +/** + * The request options for one Vega fetch: the caller's, plus credentials and + * the CSRF token where the request is going back to this app. + * + * Pure, and it never mutates what it is handed — Vega reuses its options object + * across the fetches of one spec, so writing a header into it would send this + * app's token to every later URL, including the third-party ones this function + * exists to protect. + */ +export function withCsrfHeader( + uri: string, + options: RequestInit | undefined, + base: string, + token: string | undefined +): RequestInit { + if (!isSameOrigin(uri, base)) { + return { ...options }; + } + const next: RequestInit = { ...options, credentials: "same-origin" }; + if (token) { + next.headers = { ...(options?.headers as Record | undefined), [CSRF_HEADER]: token }; + } + return next; +} diff --git a/.claude/skills/packs/mendix-vega-charts/widget/test/csrf.test.ts b/.claude/skills/packs/mendix-vega-charts/widget/test/csrf.test.ts new file mode 100644 index 000000000..d987d1617 --- /dev/null +++ b/.claude/skills/packs/mendix-vega-charts/widget/test/csrf.test.ts @@ -0,0 +1,106 @@ +/** + * ako/mxcli#574 — a URL-fed chart drew axes and a full legend with zero data + * points, because the fetch Vega made carried the session cookie and nothing + * else, and Mendix answers a session-authenticated OData read `401` without + * `X-Csrf-Token`. + * + * Run with node's own test runner, no dependencies and no build: + * + * node --test .claude/skills/packs/mendix-vega-charts/widget/test/csrf.test.ts + * + * or `make check-skill-pack-js`. + */ +import { test } from "node:test"; +import assert from "node:assert/strict"; + +import { CSRF_HEADER, isSameOrigin, readCsrfToken, withCsrfHeader } from "../src/csrf.ts"; + +const APP = "http://localhost:8080/p/home"; +const TOKEN = "6e8a0d3c-1f4b-4a21-9c77-0c2f5b1e4d90"; + +function headerOf(options: RequestInit): string | undefined { + return (options.headers as Record | undefined)?.[CSRF_HEADER]; +} + +// The reported symptom, at the layer the request is assembled: the fetch the +// widget makes for a relative spec URL must present the session's CSRF token. +// Without it Mendix answers 401 and Vega renders the empty chart. +test("a relative URL — the spec form the skill documents — carries the token", () => { + const options = withCsrfHeader("/odata/chartapi/v1/Rows?$filter=Yr eq 2026", undefined, APP, TOKEN); + assert.equal(headerOf(options), TOKEN, `${CSRF_HEADER} missing: Mendix answers this request 401`); + assert.equal(options.credentials, "same-origin"); +}); + +test("an absolute URL naming the app's own origin is the app, and carries it too", () => { + const options = withCsrfHeader("http://localhost:8080/odata/chartapi/v1/Rows", undefined, APP, TOKEN); + assert.equal(headerOf(options), TOKEN); +}); + +// The other half of the fix, and the reason it is not an unconditional header: +// the token authenticates this session against this app. +test("a third-party host is never handed the token", () => { + const options = withCsrfHeader("https://data.example.org/rows.json", undefined, APP, TOKEN); + assert.equal(headerOf(options), undefined); + assert.equal(options.credentials, undefined); +}); + +// The case a `/^[a-z][a-z0-9+.-]*:\/\//i` test on the URI gets wrong: no scheme, +// so it reads as relative, while the request goes to another host. +test("a protocol-relative URL is a third-party host, not a relative path", () => { + const options = withCsrfHeader("//data.example.org/rows.json", undefined, APP, TOKEN); + assert.equal(headerOf(options), undefined); +}); + +test("an opaque or unparseable URI gets no token", () => { + assert.equal(headerOf(withCsrfHeader("data:application/json,[]", undefined, APP, TOKEN)), undefined); + assert.equal(headerOf(withCsrfHeader("http://[", undefined, APP, TOKEN)), undefined); +}); + +// Vega reuses one options object across the fetches of a spec. Writing into it +// would send the token to every later URL, third-party ones included. +test("the caller's options are not mutated", () => { + const shared: RequestInit = { headers: { Accept: "application/json" } }; + const options = withCsrfHeader("/odata/chartapi/v1/Rows", shared, APP, TOKEN); + assert.equal(headerOf(options), TOKEN); + assert.equal(headerOf(shared), undefined, "the token leaked into the shared options object"); + assert.equal((options.headers as Record).Accept, "application/json"); +}); + +test("no token available — still same-origin credentials, no empty header", () => { + const options = withCsrfHeader("/odata/chartapi/v1/Rows", undefined, APP, undefined); + assert.equal(options.credentials, "same-origin"); + assert.equal(headerOf(options), undefined); +}); + +test("isSameOrigin resolves rather than string-matches", () => { + assert.equal(isSameOrigin("rows.json", APP), true); + assert.equal(isSameOrigin("../odata/v1/Rows", APP), true); + assert.equal(isSameOrigin("HTTP://LOCALHOST:8080/odata/v1/Rows", APP), true); + assert.equal(isSameOrigin("http://localhost:8081/odata/v1/Rows", APP), false, "a different port is a different origin"); + assert.equal(isSameOrigin("https://localhost:8080/odata/v1/Rows", APP), false, "a different scheme is a different origin"); +}); + +// `mx` exists only in the Mendix client. The widget also runs in the Studio Pro +// preview and here, so reading the token must not throw. +test("readCsrfToken tolerates every shape of absent client", () => { + const g = globalThis as { mx?: unknown }; + const saved = g.mx; + try { + delete g.mx; + assert.equal(readCsrfToken(), undefined); + g.mx = {}; + assert.equal(readCsrfToken(), undefined); + g.mx = { session: {} }; + assert.equal(readCsrfToken(), undefined); + g.mx = { session: { getConfig: () => "" } }; + assert.equal(readCsrfToken(), undefined, "an empty token is no token"); + g.mx = { session: { getConfig: (k: string) => (k === "csrftoken" ? TOKEN : undefined) } }; + assert.equal(readCsrfToken(), TOKEN); + } finally { + if (saved === undefined) { + delete g.mx; + } else { + g.mx = saved; + } + } +}); diff --git a/.github/workflows/push-test.yml b/.github/workflows/push-test.yml index ac645b8a0..fa5734cd2 100644 --- a/.github/workflows/push-test.yml +++ b/.github/workflows/push-test.yml @@ -122,6 +122,12 @@ jobs: # honouring the pre-existing-failure SKIP list). Previously this step only # iterated doctype-tests/, so bug-test regression fixtures had no CI gate. run: make check-mdl + - name: Check skill pack JavaScript + # Nothing else in this repo compiles a pack's widget — it is built in + # the user's project — so logic shipped there is otherwise ungated. The + # #574 case: the widget must attach the session's CSRF token to + # same-origin fetches and never to any other host. + run: make check-skill-pack-js - name: Check bug findings # One JSON object per line, an area, a date, and either the four # structured fields or a raw row. DuckDB rejects a whole file on one bad diff --git a/Makefile b/Makefile index 533490c10..ba84eece8 100644 --- a/Makefile +++ b/Makefile @@ -35,7 +35,7 @@ GO_BUILD_FLAGS = -trimpath # Clean version for VS Code extension (must be valid semver: major.minor.patch) VSCE_VERSION = $(shell echo "$(VERSION)" | sed 's/^v//; s/-.*//' | grep -E '^[0-9]+\.[0-9]+\.[0-9]+$$' || echo "0.0.0") -.PHONY: build build-debug size release clean test test-mdl check-mdl check-skill-mdl check-findings check-wiki-pages digest-status check-tunnel-deps check-widget-versions grammar completions sync-skills sync-skill-packs sync-commands sync-lint-rules sync-changelog sync-all docs documentation docs-site docs-serve vscode-ext vscode-install source-tree sbom sbom-report lint lint-go lint-ts fmt fmt-check vet +.PHONY: build build-debug size release clean test test-mdl check-mdl check-skill-mdl check-skill-pack-js check-findings check-wiki-pages digest-status check-tunnel-deps check-widget-versions grammar completions sync-skills sync-skill-packs sync-commands sync-lint-rules sync-changelog sync-all docs documentation docs-site docs-serve vscode-ext vscode-install source-tree sbom sbom-report lint lint-go lint-ts fmt fmt-check vet # Helper: copy file only if content differs (avoids mtime updates that invalidate go build cache) # Usage: $(call copy-if-changed,src,dst) @@ -274,6 +274,24 @@ check-skill-mdl: build check-findings: @scripts/check-findings.sh +# Unit-test the JavaScript a skill pack ships. Nothing else in this repo +# compiles a pack's widget — it is installed and built in the user's project — +# so logic that lives there has no gate at all unless it has one here. +# +# node's own test runner over .ts, no dependencies and no build step: Node 22 +# strips the types. Keep pack test files to erasable syntax, and out of the +# widget's tsconfig `include` (they live in widget/test/, which it does not +# name), or the user's `npm run build` type-checks them without @types/node. +check-skill-pack-js: + @found=0; \ + for f in .claude/skills/packs/*/widget/test/*.test.ts; do \ + [ -e "$$f" ] || continue; \ + found=1; \ + node --test "$$f" >/dev/null || { echo "FAILED: $$f"; node --test "$$f"; exit 1; }; \ + echo " ok $$f"; \ + done; \ + [ $$found -eq 1 ] || echo " (no skill-pack JS tests)" + # How far docs-wiki/bug-patterns/ has fallen behind the findings it digests. # Advisory, always exits 0 — see the header in the script for why. digest-status: diff --git a/cmd/mxcli/skillpacks_test.go b/cmd/mxcli/skillpacks_test.go index 84e51a41d..664456363 100644 --- a/cmd/mxcli/skillpacks_test.go +++ b/cmd/mxcli/skillpacks_test.go @@ -152,6 +152,95 @@ func TestWidgetPacksShipALockfile(t *testing.T) { } } +// TestUrlFedPacksDocumentSessionAuth — a pack that tells the reader to fetch +// data from a relative URL must say how that request authenticates. +// +// ako/mxcli#574: mendix-vega-charts documented a `{"data": {"url": "/odata/…"}}` +// spec and stated that "same-origin requests carry the session cookie, so an +// endpoint authenticated by session is reachable … without any token handling". +// The cookie does go. It is not sufficient — Mendix answers a +// session-authenticated OData READ 401 without `X-Csrf-Token` (measured on +// 11.14.0: cookie only 401, cookie + header 200), and Vega reads the 401 body as +// an empty dataset, so the chart draws axes and a full legend with no marks and +// no error. +// +// The check is on the *vendored* copy, which is what installs, and it is keyed +// on the relative URL in the docs rather than on this one pack: any pack that +// documents fetching from the app's own endpoints has the same obligation. +func TestUrlFedPacksDocumentSessionAuth(t *testing.T) { + fsys, err := packsFS() + if err != nil { + t.Fatalf("packsFS: %v", err) + } + // A data spec whose url is a relative path — i.e. the app's own endpoint. + relativeURLSpec := regexp.MustCompile(`"url"\s*:\s*"/`) + + err = fs.WalkDir(fsys, ".", func(p string, d fs.DirEntry, err error) error { + if err != nil || d.IsDir() || !strings.HasSuffix(p, ".md") { + return err + } + body, err := fs.ReadFile(fsys, p) + if err != nil { + return err + } + if !relativeURLSpec.Match(body) { + return nil + } + if !strings.Contains(string(body), "X-Csrf-Token") { + t.Errorf("%s documents fetching from a relative URL but never mentions "+ + "X-Csrf-Token; a session-authenticated read is refused 401 without it, "+ + "and the chart renders empty with no error (ako/mxcli#574)", p) + } + return nil + }) + if err != nil { + t.Fatal(err) + } +} + +// TestCsrfTokenIsNeverSentCrossOrigin — the other half of #574's fix. +// +// The CSRF token authenticates this session against this app, so a widget that +// attaches it to whatever URL a spec names would hand a third-party host a +// working credential. A widget source that sets the header must therefore +// decide same-origin by RESOLVING the uri against the page, not by testing the +// string for a scheme: a protocol-relative "//elsewhere.example/rows.json" +// passes any `^[a-z][a-z0-9+.-]*://` test and goes to another host. +// +// The decision itself is unit-tested in the pack +// (widget/test/csrf.test.ts, `make check-skill-pack-js`). This guards the +// vendored copy against losing it. +func TestCsrfTokenIsNeverSentCrossOrigin(t *testing.T) { + fsys, err := packsFS() + if err != nil { + t.Fatalf("packsFS: %v", err) + } + err = fs.WalkDir(fsys, ".", func(p string, d fs.DirEntry, err error) error { + // The widget's own tests name the header while asserting it is absent, + // which is the opposite of the hazard. + if err != nil || d.IsDir() || !strings.Contains(p, "/widget/") || strings.Contains(p, ".test.") { + return err + } + body, err := fs.ReadFile(fsys, p) + if err != nil { + return err + } + src := string(body) + if !strings.Contains(src, "X-Csrf-Token") { + return nil + } + if !strings.Contains(src, "new URL(") { + t.Errorf("%s attaches X-Csrf-Token but does not resolve the uri with new URL(); "+ + "a scheme test on the raw string sends this app's session token to "+ + "//another.host/x (ako/mxcli#574)", p) + } + return nil + }) + if err != nil { + t.Fatal(err) + } +} + // TestLockfilesAreNotRewritten — a lockfile must never be listed under // rewrite.files. Substitution is what keeps a widget id unique, but a lock // records resolved integrity hashes: rewriting one silently invalidates them diff --git a/mdl-examples/bug-tests/odata-574-session-auth-csrf.mdl b/mdl-examples/bug-tests/odata-574-session-auth-csrf.mdl new file mode 100644 index 000000000..d8a533ec4 --- /dev/null +++ b/mdl-examples/bug-tests/odata-574-session-auth-csrf.mdl @@ -0,0 +1,98 @@ +-- ako/mxcli#574 — a session-authenticated OData READ needs X-Csrf-Token. +-- +-- The mendix-vega-charts skill pack claimed that "same-origin requests carry +-- the session cookie, so an endpoint authenticated by session is reachable … +-- without any token handling". The cookie does go; it is not sufficient. A +-- URL-fed Vega-Lite spec therefore drew axes and a full legend with ZERO data +-- points and no error, because Vega reads the 401 body as an empty dataset. +-- +-- This script is the reproduction harness for the runtime half of that claim: +-- it builds the smallest app that can be asked the question — one persistent +-- entity, seeded with one row, published over OData with BOTH `basic` and +-- `session` authentication, so a single URL can be fetched four ways and only +-- the header differs between the two that matter. +-- +-- mxcli new CsrfProbe --version 11.14.0 --theme none --layout none --skip-init +-- mxcli exec odata-574-session-auth-csrf.mdl -p CsrfProbe/CsrfProbe.mpr +-- mxcli docker check -p CsrfProbe/CsrfProbe.mpr # 0 errors +-- mxcli run --local -p CsrfProbe/CsrfProbe.mpr --ensure-db +-- +-- URL=http://127.0.0.1:8080/odata/chartapi/v1/Rows +-- curl -s -c jar.txt -X POST http://127.0.0.1:8080/xas/ \ +-- -H 'Content-Type: application/json' -H 'X-Requested-With: XMLHttpRequest' \ +-- -d '{"action":"login","params":{"username":"probe","password":"ProbePassw0rd1!"}}' +-- # -> {"csrftoken":"…"}; XASSESSIONID and xasid land in jar.txt, both httpOnly, +-- # which is why document.cookie shows neither and a cookie hunt finds nothing. +-- +-- curl -s -w '%{http_code}\n' "$URL" # 401 no credentials +-- curl -s -w '%{http_code}\n' -b jar.txt "$URL" # 401 cookie only +-- curl -s -w '%{http_code}\n' -b jar.txt -H "X-Csrf-Token: $TOKEN" "$URL" # 200 +-- curl -s -w '%{http_code}\n' -u 'probe:ProbePassw0rd1!' "$URL" # 200 +-- +-- Measured on Mendix 11.14.0. Rows 2 and 3 differ in exactly one header, and the +-- 401 body is the one in the issue: +-- {"error":{"code":"401","message":"You are not authorized to access this resource"}} +-- +-- The fix is in the skill pack's widget: it attaches the token to same-origin +-- fetches only (.claude/skills/packs/mendix-vega-charts/widget/src/csrf.ts). +-- +-- Written against a `mxcli new` app, whose starting module is MyFirstModule. +-- Re-runnable: every statement is CREATE OR MODIFY / OR REPLACE. + +alter project security level production; +alter project security demo users on; + +create or modify module role MyFirstModule.User; +create or modify user role ApiUser (MyFirstModule.User, System.User); + +create or modify entity MyFirstModule.Row ( + -- CE6624: an OData key attribute needs a unique validation rule. + Cat: String(100) unique error 'Cat must be unique', + V: Decimal +); + +grant MyFirstModule.User on MyFirstModule.Row (CREATE, DELETE, READ *, WRITE *); + +create demo user 'probe' password 'ProbePassw0rd1!' (ApiUser); + +-- Seed one row, so a 200 over data can be told apart from a 200 over nothing. +-- Returns Boolean because an after-startup microflow must (CE0142). +create or replace microflow MyFirstModule.Seed () + returns Boolean as $Ok +begin + declare $Ok Boolean = true; + retrieve $Existing from MyFirstModule.Row; + $N = count($Existing); + if $N = 0 then + $R = create MyFirstModule.Row (Cat = 'Groceries', V = 697.70) commit; + end if; + set $Ok = true; + return $Ok; +end; + +alter settings MODEL AfterStartupMicroflow = 'MyFirstModule.Seed'; + +-- Both methods on one service: `basic` is the control that proves the endpoint +-- and the data are fine when `session` answers 401. +create or modify odata service MyFirstModule.ChartApi ( + path: 'odata/chartapi/v1/', + version: '1.0.0', + ODataVersion: OData4, + namespace: 'MyFirstModule.ChartApi', + ServiceName: 'ChartApi' +) +authentication basic, session +{ + publish entity MyFirstModule.Row as 'Rows' ( + ReadMode: source, + InsertMode: not_supported, + UpdateMode: not_supported, + DeleteMode: not_supported + ) + expose ( + Cat as 'Cat' (KEY, Filterable, Sortable), + V as 'V' (Filterable) + ); +}; + +grant access on odata service MyFirstModule.ChartApi to MyFirstModule.User; From 71851f74c60b58b412260a31ec73b6a929134ef2 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 21 Sep 2026 18:15:43 +0000 Subject: [PATCH 07/15] fix(check): report the members a script removes from the project (MDL087) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ako/mxcli#562. `create or modify entity` rebuilds the entity from the statement, so an attribute a LATER script added is deleted when the earlier script is re-run on its own. The reporter hit it with a calculated attribute: the entity is created in 01-domain-core.mdl and the attribute added in 03-logic.mdl, because the microflow it calculates by does not exist until then. Re-running slice 01 alone removed it, and the loss surfaced two slices later as CE1613 on a page — an error naming the page, never the script. The issue asks for two things. exec's warning already shipped (findings #24) and does fire; measured on a real 11.6.6 project re-running the reporter's slice 01, it prints the attribute by name, so the report predates it. The second ask did not exist: `mxcli check … -p app.mpr --references` printed "Check passed!" and said nothing, which is the one command that runs BEFORE anything is written. CheckEntityMemberDrops joins the catalog-backed tier of `check` as MDL087, a warning ("modify to this shape" is a legitimate intent; the defect was the silence, and the run still exits 0). It is deliberately NOT the same computation as the exec-time warning: - NET over the whole script, not per-statement. A script that rebuilds an entity and then `alter entity … add attribute`s the members back loses nothing — and that is the idiomatic full-script order, so a per-statement port would warn on every correct script. - Intent is tracked, not inferred. `drop attribute`, `rename attribute` and `drop entity` produce the same before/after diff as the accident. droppedEntityMembers now delegates to the shared entityMemberSet comparison rather than keeping a second hand-written diff: the audit system fields and an omitted `extends` are covered once, and an audit pseudo-type is a flag rather than an attribute on both sides. Reporting order is the entity's own attribute order, so the exec warning's output is unchanged. Controls recorded, both directions: - guard stubbed → 4 of 10 tests fail with the reported symptom ("got 0 violation(s)"), the silence tests still pass - net/intent logic removed → TestMDL087_ExplicitRemovalIsSilent fails on 3 of 4 spellings while the positive test still passes, so the positive test alone proves nothing Verified end to end on a real project: silent before, MDL087 after, with the concatenated slices and an explicit `drop attribute` both silent. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_011WGX3LgaNzHcnpAoHRzRZx --- .../fix-issue/findings/mdl-executor.jsonl | 1 + .claude/skills/mendix/check-syntax/SKILL.md | 44 +++ cmd/mxcli/cmd_check.go | 37 +- docs/01-project/MDL_QUICK_REFERENCE.md | 2 +- .../entity-562-or-modify-member-drop.mdl | 103 +++++ mdl/executor/cmd_entities.go | 43 +- mdl/executor/validate_entity_member_drops.go | 354 +++++++++++++++++ .../validate_entity_member_drops_test.go | 371 ++++++++++++++++++ 8 files changed, 905 insertions(+), 50 deletions(-) create mode 100644 mdl-examples/bug-tests/entity-562-or-modify-member-drop.mdl create mode 100644 mdl/executor/validate_entity_member_drops.go create mode 100644 mdl/executor/validate_entity_member_drops_test.go diff --git a/.claude/skills/fix-issue/findings/mdl-executor.jsonl b/.claude/skills/fix-issue/findings/mdl-executor.jsonl index 057547703..96bf59cb3 100644 --- a/.claude/skills/fix-issue/findings/mdl-executor.jsonl +++ b/.claude/skills/fix-issue/findings/mdl-executor.jsonl @@ -663,3 +663,4 @@ {"area": "mdl/executor", "date": "2026-09-20", "symptom": "`describe page` → `exec` over a **Studio Pro-authored** page silently drops six things, `mx check` 0 errors throughout. The one that matters: `IsPasswordBox True → False` — a **password field round-trips into a plaintext text box**, and describe → rename → exec is mxcli's copy operation. Also `Validation.Expression` blanked, a DataView's `ReadOnlyStyle Text → Control`, `PopupCloseAction` wiped, and two typed-array markers", "cause": "Four different causes behind one symptom, which is why triage came first: (1) IsPasswordBox — model and writer carried it, nothing parsed it, nothing emitted it; (2) Validation — `widgetValidationToGen()` wrote a DEFAULT EMPTY Forms$WidgetValidation over whatever was stored, on five widget types; (3) ReadOnlyStyle — wired for CheckBox only, and a DataView's draws no MDL-WIDGET07 warning because `staticWidgetKnownProps` is deliberately a union across widget types; (4) PopupCloseAction — `pageToGen` wrote \"\" unconditionally. Plus ParameterMappings/OutputMappings markers", "file": "`mdl/executor/cmd_pages_describe_parse.go` + `_output.go` (extract/emit), `cmd_pages_builder_v3_widgets.go` (consume), `cmd_pages_builder_v3.go`, `mdl/visitor/visitor_page_v3.go`, `mdl/ast/ast_page_v3.go`, `sdk/pages/*`, `mdl/backend/modelsdk/widget_write.go` + `page_write.go`, `mdl/executor/validate_widgets.go` (describe vocabulary)", "insight": "**Triage the layer before writing anything** — describer / grammar / builder have different fixes and this one issue had all three. The quickest probe is to run the property through `mxcli check`: MDL-WIDGET07 names an unrecognised one, and *silence is not acceptance* — the known-props list is a union across widget types, so a DataView's ReadOnlyStyle passed check and was dropped anyway. **Emit an expression QUOTED, not bracketed**: `[...]` is the XPath-constraint spelling and `propertyValueV3` parses it as an ARRAY, so `GetStringProp` yields \"\" — the emitter's own unit test was green while the real round trip still lost the value (storage form is not input form). **Measure the default before keeping it**: a DataView's ReadOnlyStyle is Control on 47 of 56, never Inherit, so the 'obvious' Inherit that every other input widget uses would have been wrong. Markers likewise measured, not assumed: ParameterMappings is marker 2 on 220 of 220 lists in every parent type, OutputMappings present on 91 of 91 — and an EMPTY list needs `MandatoryListMarkers` since `RegisterListMarker` keys on a child that is not there. Result 17 → 9 differences, the 9 being ako/mxcli#549", "refs": ["#550", "#541", "#549", "#490"]} {"area": "mdl/executor", "date": "2026-09-20", "symptom": "MDL-PAGEARG01 refused a list widget's OWN row action: `datagrid dg (DataSource: DATABASE M.E, onClick: SHOW_PAGE M.Edit(E: $currentObject))` was rejected at `check` with \"widget `dg` is not inside a data view, list view or grid row\" \u2014 and since exec refuses a script whose check errors, the slice could not be applied at all. On a `listview` the message contradicted itself. mxbuild 11.14.0 accepts the stored pages at 0 errors.", "cause": "The #1029 guard judged EVERY widget's own action in the context its PARENT supplies: `argContextForSubtreeOf` returns the parent context for a childless widget and `validate_widgets.go` passed the inherited `argCtx` to `validateShowPageArguments`. Right for a button, wrong for the widget that ESTABLISHES the context \u2014 a list widget's onClick is row-scoped, so the row it renders is the context object. Added `argContextForOwnAction`: a widget that binds a source of its own supplies the context for its own action; a source in a shape the pass cannot read (the bare-entity shorthand) degrades to UNKNOWN so the guard stands down rather than refusing what it cannot prove is discarded.", "file": "`mdl/executor/cmd_pages_showpage_args.go` (argContextForOwnAction, argContextForSubtreeOf), `mdl/executor/validate_widgets.go`", "insight": "**A false refusal costs more than a missing rule now that exec refuses on a check error** \u2014 the blast radius is 'this project cannot be built with this mxcli', not 'a warning is noisy'. Two things would have caught it before release: judging the rule against the widget kinds it NAMES in its own message (the listview refusal reads 'lvA is not inside a \u2026 list view'), and running it against mxbuild rather than against intuition. The mxbuild run paid for itself twice: it also showed that `DataSource: M.E` (bare-entity shorthand) on a datagrid is silently dropped, so that case is CE0488 + a REAL CE1571 \u2014 the stand-down is still correct, but the shorthand case must not be written into a bug test as mxbuild-clean (#576). Control the fix with the widget kinds STILL refused (a foreign variable, a sibling button beside the grid), or it is indistinguishable from deleting the rule.", "refs": ["#552", "#576", "mendixlabs/mxcli#1029", "#939"]} {"area": "mdl/executor", "date": "2026-09-21", "symptom": "A page's image-collection reference passed `mxcli check --references` and failed the build. Reported as \"no MDL syntax for a StaticImageViewer inside a Selection helper custom state\" — the authoring half was already closed by #1057; what was left is that nothing RESOLVED the name it made writable. Measured on a blank Mendix 11.14.0 project: `staticimage imgAll (Image: 'Atlas_UI_Resources.Atlas_Icons.checkbox_checked')` in a custom state -> check passed, exec created the page, `mx check` -> 3x CE1613 \"The selected image … no longer exists.\"", "cause": "TWO independent holes, and either alone leaves the reported script unchecked. (1) widgetRefCollector keyed the image reference on the widget TYPE — `if w.Type == \"image\"` — so the pluggable widget was collected and `staticimage` (which #1057 had just given the SAME `Image:` property) and `dynamicimage`'s `DefaultImage` were not; replaced with an imageRefProps table. (2) A page's widgets live in two AST fields: `Widgets` is the bare body, `Placeholders` holds `placeholder X { … }` content (#532). validate.go passed `s.Widgets` alone to validateWidgetReferences, validatePageContextTree AND validateFlowArguments, so EVERY reference inside a placeholder block — microflow, nanoflow, page, snippet, entity, image — was validated by nothing; added allPageWidgets to collect both roots once.", "file": "`mdl/executor/helpers.go` (widgetRefCollector.collectFromWidget, imageRefProps), `mdl/executor/validate.go` (allPageWidgets)", "insight": "**When a capability gets a new spelling, grep for who RESOLVES the old one.** #1057 added `Image:` to a second and third widget and moved on; the resolver keyed on the type name, so the new spellings were unchecked from the day they shipped. A property list and a resolver list that describe the same property are two copies — `validate_widgets.go` already accepted `Image`/`DefaultImage` for these widgets and DESCRIBE already emitted them, and only the resolver disagreed. **The placeholder hole is the more useful lesson: it was the THIRD copy of one walk.** validateIconRefs (#1008) and forEachWidget had each grown the `Placeholders` arm separately, with a comment saying a missed walk is silent both ways — and the three validators next door still had not. When a fix is 'add the missing arm to this walker', the question is how many walkers there are; collect the roots once instead. **Do not reason about a bug report from the issue text alone when the version is older than the fix** — the reported symptom did not reproduce on main at all, and running the reporter's own script end to end is what turned 'already fixed, close it' into two real defects. **Control both directions**: a reference that resolves must stay silent, because a walker that can suddenly see a whole new region of the tree is as likely to report correct scripts as broken ones.", "refs": ["mendixlabs/mxcli#1149", "mendixlabs/mxcli#1057", "mendixlabs/mxcli#1008", "#532"]} +{"area": "mdl/executor", "date": "2026-09-21", "symptom": "`create or modify entity` drops an attribute a LATER script added, silently. Reported shape: entity created in 01-domain-core.mdl, a calculated attribute added in 03-logic.mdl (its microflow does not exist until then); re-running slice 01 ALONE rebuilt the entity from its own statement and removed the attribute, with `Modified entity: ServiceCore.LithoSystem` as the only output. It surfaced two slices later as `[CE1613] \"The selected attribute 'ServiceCore.LithoSystem.OpenRequestCount' no longer exists.\" at Text 'dtOpen'` — an error naming the PAGE, never the script that removed the attribute. `mxcli check … -p app.mpr --references` said \"Check passed!\".", "ce": "CE1613", "rules": ["MDL087"], "cause": "Half the ask was already shipped and half was not, and the report could not tell them apart. exec's warning (droppedEntityMembers, findings #24, landed 320a304 two weeks before the report) DOES fire — measured on a real 11.6.6 project re-running the reporter's slice 01, it prints the attribute by name — so the reporter was on an older binary. What genuinely did not exist was the issue's second ask: `check` had no project-aware pass for member loss at all, so the one command that runs BEFORE anything is written was the silent one. Added CheckEntityMemberDrops (MDL087, warning) to cmd_check.go's catalog-backed tier, and refactored droppedEntityMembers to share its comparison.", "file": "`mdl/executor/validate_entity_member_drops.go` (new: entityMemberSet, droppedMembers, CheckEntityMemberDrops), `mdl/executor/cmd_entities.go` (droppedEntityMembers now delegates), `cmd/mxcli/cmd_check.go` (projectViolations)", "insight": "**Reproduce before theorising when the report predates a fix in the same area** — exec already printed the exact line the issue asks for, so reading the issue text alone leads either to 'already fixed, close it' or to reimplementing the shipped half. Running the reporter's own sequence against a real project separated the two halves in one command each, and the isolated-slice check printing `Check passed!` is what identified the actual gap. **A check-time twin of an exec-time warning must NOT be the same computation.** exec is per-statement because it is applying statements; check sees the whole script, so it has to be the NET effect — a script that rebuilds an entity and then `alter entity … add attribute`s the members back loses nothing, and that is the IDIOMATIC full-script order, so a per-statement port would warn on every correct script and be switched off within a day. **Intent has to be tracked, not inferred from the outcome**: `drop attribute` / `rename attribute` / `drop entity` produce the same before/after diff as the accident, and a pure diff cannot separate them. Both of those are separate controls, and the naive implementation fails each one specifically (measured: stubbing the net/intent logic fails TestMDL087_ExplicitRemovalIsSilent on 3 of 4 spellings while the positive test still passes — so the positive test alone proves nothing). **One comparison, two layers**: the audit system fields and an omitted `extends` were reported by exec and would have been missed by a second hand-written diff, which is why droppedEntityMembers was refactored onto the shared entityMemberSet rather than copied. An audit pseudo-type (`AutoOwner`) is a FLAG, not an attribute — exec `continue`s past it — so counting it as one makes a faithful restatement read as a drop.", "refs": ["ako/mxcli#562", "findings #24", "findings #13"]} diff --git a/.claude/skills/mendix/check-syntax/SKILL.md b/.claude/skills/mendix/check-syntax/SKILL.md index 5a5909f65..661a2be13 100644 --- a/.claude/skills/mendix/check-syntax/SKILL.md +++ b/.claude/skills/mendix/check-syntax/SKILL.md @@ -85,6 +85,50 @@ action, workflow, and the integration/agent document types. If you find one that `exec` refuses and `check` does not, that is a bug of exactly the shape `TestEveryCreateDocTypeIsProjectChecked` exists to prevent. +### It reports what the script REMOVES from the project + +`create or modify entity` is on the list above — it is never a conflict, because +"fine if it already exists" is exactly what it says. What it does **not** say is +that it rebuilds the entity from the statement, so every member the statement +omits is deleted. Slice an app into ordered scripts and that becomes a real +hazard: an attribute added by a later `alter entity` — a calculated one whose +microflow does not exist until then is the usual reason — is gone the moment the +earlier script is re-run on its own. **Script order is load-bearing, even though +each script is individually idempotent.** + +`check` now says so before anything is written, as **MDL087**: + +``` +⚠ applying this script to the project removes 1 member(s) from entity + ServiceCore.LithoSystem that it does not restate: OpenRequestCount + — anything still bound to them (widgets, microflows) fails the build with CE1613 + at ServiceCore.LithoSystem + → … or add them incrementally with 'alter entity ServiceCore.LithoSystem + add attribute : ;' in this script; if they are meant to go, + say so with 'alter entity … drop attribute ;' +``` + +`exec` prints the same list — but as it applies the statement, by which point the +attribute is gone. Left unreported entirely, the loss surfaces slices later as +`CE1613` on whatever still binds it, naming the *page*, not the script that +removed the attribute (ako/mxcli#562). + +Two properties of the rule are worth knowing, because they are what keep it from +becoming noise you learn to scroll past: + +- **It is the NET effect of the whole script, not one statement's.** A script + that rebuilds an entity and then adds the members back with `alter entity … + add attribute` loses nothing and is silent. So the *combined* slices check + clean and slice 01 alone does not, which is precisely the difference that bit. +- **An explicit removal is not reported.** `drop attribute`, `rename attribute` + and `drop entity` say what they do. Only a member the project holds, that the + script neither restates nor asks to remove, is a warning. + +It is a **warning**: "modify to this shape" is a legitimate intent and `check` +still exits 0. The defect was the silence, not the behaviour. It also covers the +members that are not attributes — the four audit system fields and an omitted +`extends` — because those drop the same way. + ### It resolves MEMBER names too, where it can establish the entity Resolution does not stop at the entity. An attribute named in a **create** or diff --git a/cmd/mxcli/cmd_check.go b/cmd/mxcli/cmd_check.go index db2e19a84..4a3aed62f 100644 --- a/cmd/mxcli/cmd_check.go +++ b/cmd/mxcli/cmd_check.go @@ -246,32 +246,41 @@ Examples: fmt.Printf("✓ All references valid\n") } - // Expression type checking is the catalog-backed tier: the rules that - // need an attribute's type, an enumeration's cases or a microflow's - // return type. It runs here rather than in the unconditional pass - // because those answers only exist once a project is connected, and - // after the reference check because a script naming things that do - // not exist has a more basic problem than a mistyped operand — and - // because building the catalog for a run that already failed is - // wasted work. + // The catalog-backed tier: the checks whose answers only exist once a + // project is connected. It runs after the reference check because a + // script naming things that do not exist has a more basic problem + // than a mistyped operand — and because building the catalog for a + // run that already failed is wasted work. // // Like every other violation this command emits, only an error // severity fails the run. Warnings and hints are advice, and a // checker whose first outing turns advice into a broken build is a // checker people turn off. - typeViolations := exec.TypeCheckProgram(prog) - if len(typeViolations) > 0 { + // + // MDL087 is what this script REMOVES from the project, which nothing + // reported until now (ako/mxcli#562). `create or modify entity` + // rebuilds the entity from the statement, so a member the script does + // not restate is deleted — and the loss only becomes visible slices + // later, as a CE1613 on whatever still binds it. exec prints the same + // list, but only as it applies the statement; by then it is gone. + // + // Expression type checking is the other half: the rules that need an + // attribute's type, an enumeration's cases or a microflow's return + // type. The scope-local tier already ran in the unconditional pass. + projectViolations := exec.CheckEntityMemberDrops(prog) + projectViolations = append(projectViolations, exec.TypeCheckProgram(prog)...) + if len(projectViolations) > 0 { if isStructured { - formatter.Format(typeViolations, os.Stderr) + formatter.Format(projectViolations, os.Stderr) } else { fmt.Fprintln(os.Stderr) - formatter.Format(typeViolations, os.Stderr) + formatter.Format(projectViolations, os.Stderr) } - if linter.Summarize(typeViolations).Errors > 0 { + if linter.Summarize(projectViolations).Errors > 0 { os.Exit(1) } } else if !isStructured { - fmt.Printf("✓ Expression types OK\n") + fmt.Printf("✓ Expression types OK, no unstated member drops\n") } } diff --git a/docs/01-project/MDL_QUICK_REFERENCE.md b/docs/01-project/MDL_QUICK_REFERENCE.md index a5222bb16..7c08cb1ee 100644 --- a/docs/01-project/MDL_QUICK_REFERENCE.md +++ b/docs/01-project/MDL_QUICK_REFERENCE.md @@ -93,7 +93,7 @@ Modifies an existing entity without full replacement. | Rename attribute | `alter entity Module.Name rename attribute OldName to NewName;` | Also rewrites stored references (microflow members, page widgets, validation/access rules) and XPath constraints. Microflow expressions are free text and are **not** rewritten | | Add index | `alter entity Module.Name add index [if not exists] [name] [on] (Col1 [asc\|desc], ...);` | `on` is optional (SQL-like). **Without `if not exists`, re-running is an error** — a second identical index fails the build with CE0072 | | Document an association | `/** What it links. */`
`create association Mod.C_P from Mod.C to Mod.P;`
or `... to Mod.P comment 'What it links.';` | Both spellings work on create; the doc comment wins when both are present. `comment` survives here — and only here among the CREATE statements — because it is an association's **only inline** spelling | -| Create if absent | `create entity if not exists Module.Name (...);`
`create association if not exists Module.Assoc from ... to ...;` | Skips when it already exists, leaving the stored definition untouched. Unlike `create or modify`, which rebuilds the element from the statement and drops any attribute the statement omits | +| Create if absent | `create entity if not exists Module.Name (...);`
`create association if not exists Module.Assoc from ... to ...;` | Skips when it already exists, leaving the stored definition untouched. Unlike `create or modify`, which rebuilds the element from the statement and drops any attribute the statement omits — `mxcli check … -p app.mpr --references` warns about that as **MDL087**, naming the members the script removes without restating them | | Add index (SQL form) | `create index IdxName on Module.Name (Col1 [asc\|desc], ...);` | Same effect as `alter entity … add index`. The index name is accepted and discarded — a Mendix index is identified by its columns | | Drop index | `alter entity Module.Name drop index [if exists] (Col1 [asc\|desc], ...);` | Selected by its columns — a Mendix index stores no name, so the columns are its identity, and they are what `describe entity` prints. The legacy positional form `drop index idx1` still works but shifts when an earlier index is dropped | | Add event handler | `alter entity Module.Name add event handler on before commit call Mod.MF($currentObject) [raise error];` | `($currentObject)` or `()`, RAISE ERROR only on BEFORE | diff --git a/mdl-examples/bug-tests/entity-562-or-modify-member-drop.mdl b/mdl-examples/bug-tests/entity-562-or-modify-member-drop.mdl new file mode 100644 index 000000000..bf2e13891 --- /dev/null +++ b/mdl-examples/bug-tests/entity-562-or-modify-member-drop.mdl @@ -0,0 +1,103 @@ +-- Bug test for ako/mxcli#562: +-- CREATE OR MODIFY ENTITY drops attributes added by a later ALTER without +-- saying so. +-- +-- Reported shape: the entity is created in 01-domain-core.mdl and a calculated +-- attribute added in 03-logic.mdl, because the microflow it calculates by does +-- not exist until then. Re-running slice 01 ALONE rebuilt the entity from its +-- own statement and dropped the attribute; `exec` reported only +-- `Modified entity: …`, and it surfaced two slices later as +-- +-- [error] [CE1613] "The selected attribute +-- 'ServiceCore.LithoSystem.OpenRequestCount' no longer exists." at Text 'dtOpen' +-- +-- Two halves, and this script exercises both: +-- +-- exec prints `⚠ create or modify entity … drops 1 existing member(s) …` +-- as it applies the statement (shipped earlier, see findings #24). +-- check reports MDL087 BEFORE anything is written — run it separately: +-- +-- mxcli check mdl-examples/bug-tests/entity-562-or-modify-member-drop.mdl \ +-- -p app.mpr --references +-- +-- against a project that ALREADY holds the attribute. MDL087 is a +-- warning, so the run still exits 0. +-- +-- Note MDL087 is a NET computation over the whole script: this file adds the +-- attribute back (Step 4) and explicitly drops another (Step 5), so checking +-- THIS file reports nothing. That is the point — re-running slice 01 on its own +-- is what loses a member, not the full script. + +create module BugTest562; + +-- ============================================================================ +-- Step 1 — slice 01: the entity's own statement. It does NOT mention +-- OpenRequestCount, because CALC_OpenRequestCount does not exist yet. +-- ============================================================================ + +@position(100,100) +create or modify persistent entity BugTest562.LithoSystem ( + Name: String(200), + SerialNumber: String(50), + Retired: Boolean +); + +-- ============================================================================ +-- Step 2 — slice 03: the microflow, then the calculated attribute. +-- ============================================================================ + +create or modify microflow BugTest562.CALC_OpenRequestCount ($LithoSystem: BugTest562.LithoSystem) +returns Integer as $Result +begin + declare $Result Integer = 0; + return $Result; +end; +/ + +alter entity BugTest562.LithoSystem + add attribute if not exists OpenRequestCount: Integer + calculated by BugTest562.CALC_OpenRequestCount; + +-- ============================================================================ +-- Step 3 — the bug: re-running slice 01 alone. Identical statement, and the +-- attribute Step 2 added is not in it. +-- +-- EXPECTED exec output (this is the fix; before it there was only +-- `Modified entity:`): +-- +-- ⚠ create or modify entity BugTest562.LithoSystem drops 1 existing +-- member(s) not listed in this statement: OpenRequestCount +-- They are removed from the entity; anything still bound to them +-- (widgets, microflows) will fail the build with CE1613. +-- To add attributes without disturbing the rest, use: alter entity +-- BugTest562.LithoSystem add attribute : ; +-- ============================================================================ + +@position(100,100) +create or modify persistent entity BugTest562.LithoSystem ( + Name: String(200), + SerialNumber: String(50), + Retired: Boolean +); + +-- ============================================================================ +-- Step 4 — the incremental spelling the warning points at. Re-adding what +-- Step 3 dropped leaves the entity whole, and is why MDL087 is net rather than +-- per-statement: the two slices run in order lose nothing. +-- ============================================================================ + +alter entity BugTest562.LithoSystem + add attribute if not exists OpenRequestCount: Integer + calculated by BugTest562.CALC_OpenRequestCount; + +-- ============================================================================ +-- Step 5 — CONTROL: an explicit removal. `drop attribute` says what it does, +-- so MDL087 must stay silent about Retired even though the member is gone. +-- A pure before/after diff cannot tell this from the accident in Step 3. +-- ============================================================================ + +alter entity BugTest562.LithoSystem + drop attribute if exists Retired; + +describe entity BugTest562.LithoSystem; +-- Expected: Name, SerialNumber and OpenRequestCount; no Retired. diff --git a/mdl/executor/cmd_entities.go b/mdl/executor/cmd_entities.go index 74fca5ff2..52a450233 100644 --- a/mdl/executor/cmd_entities.go +++ b/mdl/executor/cmd_entities.go @@ -666,42 +666,15 @@ func isViewEntity(e *domainmodel.Entity) bool { // droppedEntityMembers reports the members present on existing but absent from // replacement — i.e. what a CREATE OR MODIFY replace would delete. Named -// attributes are compared case-insensitively; the four audit system fields are -// reported when their flag is on in existing but off in replacement. Used to -// surface accidental data loss (findings #24). +// attributes are compared case-insensitively; the four audit system fields and +// the generalization are reported when existing carries one and replacement does +// not. Used to surface accidental data loss (findings #24). +// +// The comparison itself lives in droppedMembers, shared with the check-time +// MDL087 pass (ako/mxcli#562). Two hand-written diffs at two layers is how the +// audit fields came to be covered by one and not the other. func droppedEntityMembers(existing, replacement *domainmodel.Entity) []string { - keep := make(map[string]bool, len(replacement.Attributes)) - for _, a := range replacement.Attributes { - keep[strings.ToLower(a.Name)] = true - } - var dropped []string - for _, a := range existing.Attributes { - if !keep[strings.ToLower(a.Name)] { - dropped = append(dropped, a.Name) - } - } - // Audit system fields that were enabled and are no longer requested are also - // removed by the replace. - if existing.HasOwner && !replacement.HasOwner { - dropped = append(dropped, "owner (system field)") - } - if existing.HasChangedBy && !replacement.HasChangedBy { - dropped = append(dropped, "changedBy (system field)") - } - if existing.HasCreatedDate && !replacement.HasCreatedDate { - dropped = append(dropped, "createdDate (system field)") - } - if existing.HasChangedDate && !replacement.HasChangedDate { - dropped = append(dropped, "changedDate (system field)") - } - // An omitted EXTENDS un-inherits the entity, which is a bigger change than a - // dropped attribute and was the only one of these that happened in silence. - // It is reported rather than preserved because there is no "extends nothing" - // spelling, so preserving it would make an inheritance impossible to remove. - if existing.GeneralizationRef != "" && replacement.GeneralizationRef == "" { - dropped = append(dropped, "extends "+existing.GeneralizationRef+" (generalization)") - } - return dropped + return droppedMembers(memberSetFromEntity(existing), memberSetFromEntity(replacement)) } // execCreateViewEntity handles CREATE VIEW ENTITY statements. diff --git a/mdl/executor/validate_entity_member_drops.go b/mdl/executor/validate_entity_member_drops.go new file mode 100644 index 000000000..0742ea000 --- /dev/null +++ b/mdl/executor/validate_entity_member_drops.go @@ -0,0 +1,354 @@ +// SPDX-License-Identifier: Apache-2.0 + +package executor + +import ( + "context" + "fmt" + "strings" + + "github.com/mendixlabs/mxcli/mdl/ast" + "github.com/mendixlabs/mxcli/mdl/linter" + "github.com/mendixlabs/mxcli/sdk/domainmodel" +) + +// entityMemberDropRule is the check-time counterpart of the exec-time warning in +// execCreateEntity: a CREATE OR MODIFY ENTITY rebuilds the entity from the +// statement alone, so every member the statement omits is deleted. +const entityMemberDropRule = "MDL087" + +// entityMemberSet is the name-level member inventory of an entity — the named +// attributes, the four audit system fields and the generalization. +// +// It exists so the exec-time warning and the check-time one share ONE answer to +// "what counts as a member". The two run at different layers and in different +// currencies (exec holds two *domainmodel.Entity, check holds a stored entity +// and an AST statement), and a second hand-written comparison is how the audit +// fields came to be reported by one and not the other. +type entityMemberSet struct { + // attrs maps the lower-cased attribute name to the name as written, because + // Mendix attribute names are matched case-insensitively but reported as + // spelled. + attrs map[string]string + // order is the lower-cased keys in insertion order. droppedMembers reports + // in it, so the list reads in the entity's own attribute order — the same + // order `describe entity` prints — rather than alphabetically, and does not + // depend on map iteration. + order []string + hasOwner bool + hasChangedBy bool + hasCreatedDate bool + hasChangedDate bool + generalization string +} + +func newEntityMemberSet() *entityMemberSet { + return &entityMemberSet{attrs: map[string]string{}} +} + +func (m *entityMemberSet) addAttr(name string) { + key := strings.ToLower(name) + if _, seen := m.attrs[key]; !seen { + m.order = append(m.order, key) + } + m.attrs[key] = name +} + +// removeAttr drops the attribute but LEAVES its key in order, so a name that is +// removed and re-added keeps its original position. order is only ever read +// through attrs, so a stale key is skipped. +func (m *entityMemberSet) removeAttr(name string) { + delete(m.attrs, strings.ToLower(name)) +} + +func (m *entityMemberSet) hasAttr(name string) bool { + _, ok := m.attrs[strings.ToLower(name)] + return ok +} + +// applyAuditPseudoType records an AutoOwner/AutoChangedBy/AutoCreatedDate/ +// AutoChangedDate pseudo-type as the entity flag it becomes, and reports whether +// the type was one. A pseudo-type is not an attribute: execCreateEntity `continue`s +// past it, so counting it as one would make a faithful restatement look like a drop. +func (m *entityMemberSet) applyAuditPseudoType(kind ast.DataTypeKind) bool { + switch kind { + case ast.TypeAutoOwner: + m.hasOwner = true + case ast.TypeAutoChangedBy: + m.hasChangedBy = true + case ast.TypeAutoCreatedDate: + m.hasCreatedDate = true + case ast.TypeAutoChangedDate: + m.hasChangedDate = true + default: + return false + } + return true +} + +// memberSetFromEntity reads the inventory off a stored (or rebuilt) entity. +func memberSetFromEntity(e *domainmodel.Entity) *entityMemberSet { + m := newEntityMemberSet() + if e == nil { + return m + } + for _, a := range e.Attributes { + m.addAttr(a.Name) + } + m.hasOwner = e.HasOwner + m.hasChangedBy = e.HasChangedBy + m.hasCreatedDate = e.HasCreatedDate + m.hasChangedDate = e.HasChangedDate + m.generalization = e.GeneralizationRef + return m +} + +// memberSetFromCreateStmt reads the inventory a CREATE [OR MODIFY] ENTITY +// statement declares — i.e. exactly what the rebuilt entity will hold. +func memberSetFromCreateStmt(s *ast.CreateEntityStmt) *entityMemberSet { + m := newEntityMemberSet() + if s == nil { + return m + } + for _, a := range s.Attributes { + if m.applyAuditPseudoType(a.Type.Kind) { + continue + } + m.addAttr(a.Name) + } + if s.Generalization != nil { + m.generalization = s.Generalization.String() + } + return m +} + +// droppedMembers reports the members present in before and absent from after — +// what a rebuild would delete. Attribute names are compared case-insensitively +// and reported as stored. +func droppedMembers(before, after *entityMemberSet) []string { + if before == nil { + return nil + } + if after == nil { + after = newEntityMemberSet() + } + var dropped []string + for _, lower := range before.order { + name, present := before.attrs[lower] + if !present { + continue + } + if _, kept := after.attrs[lower]; !kept { + dropped = append(dropped, name) + } + } + // Audit system fields that were enabled and are no longer requested are also + // removed by the replace. + if before.hasOwner && !after.hasOwner { + dropped = append(dropped, "owner (system field)") + } + if before.hasChangedBy && !after.hasChangedBy { + dropped = append(dropped, "changedBy (system field)") + } + if before.hasCreatedDate && !after.hasCreatedDate { + dropped = append(dropped, "createdDate (system field)") + } + if before.hasChangedDate && !after.hasChangedDate { + dropped = append(dropped, "changedDate (system field)") + } + // An omitted EXTENDS un-inherits the entity, which is a bigger change than a + // dropped attribute and was the only one of these that happened in silence. + // It is reported rather than preserved because there is no "extends nothing" + // spelling, so preserving it would make an inheritance impossible to remove. + if before.generalization != "" && after.generalization == "" { + dropped = append(dropped, "extends "+before.generalization+" (generalization)") + } + return dropped +} + +// CheckEntityMemberDrops reports, per entity, the members this script removes +// from the connected project without having been asked to — MDL087. +// +// It is the check-time half of ako/mxcli#562: exec already prints the same list, +// but only as it applies the statement, by which point the attribute is gone and +// the loss surfaces slices later as a CE1613 on whatever still binds it. +// +// Two things make this a NET computation over the whole script rather than a +// per-statement one, unlike the exec-time warning: +// +// 1. Re-adding is idiomatic. A domain script that rebuilds an entity and then +// `alter entity … add attribute` the members whose microflows did not exist +// yet loses nothing, and warning on it would train people to ignore the rule. +// 2. An explicit removal is not a surprise. `drop attribute`, `rename +// attribute` and `drop entity` say what they do, so they are excluded by +// intent rather than by outcome — a pure before/after diff cannot tell them +// from an accident. +// +// The warning is therefore about the one case the reporter hit: a member the +// project holds, that this script does not restate and never asks to remove. +// +// Severity is a warning, not an error. "Modify to this shape" is a legitimate +// statement of intent and the user may well mean it; the defect was that it +// happened in silence. +func (e *Executor) CheckEntityMemberDrops(prog *ast.Program) []linter.Violation { + if e == nil { + return nil + } + return CheckEntityMemberDrops(e.newExecContext(context.Background()), prog) +} + +// entityDropState is one tracked entity's simulated member set plus the members +// this script explicitly asked to remove. +type entityDropState struct { + stored *entityMemberSet + // current is the member set after applying the statements seen so far. + current *entityMemberSet + // intentional holds lower-cased attribute names an explicit DROP or RENAME + // removed, and the audit/generalization sentinels for the same. + intentional map[string]bool + // skip marks an entity this script drops outright; a DROP ENTITY is as + // explicit as a statement gets and its members are not a loss to report. + skip bool +} + +// CheckEntityMemberDrops is the ExecContext-level entry point. A disconnected +// context returns no violations rather than an error: this check is entirely +// about what the PROJECT holds, so without one there is nothing it can say. +func CheckEntityMemberDrops(ctx *ExecContext, prog *ast.Program) []linter.Violation { + if ctx == nil || prog == nil || !ctx.Connected() { + return nil + } + + states := map[string]*entityDropState{} + var order []string + + // track returns the state for an entity, loading the stored member set on + // first use. An entity the project does not hold is tracked with a nil + // stored set, so nothing it does can be reported as a loss. + track := func(qn string) *entityDropState { + if st, ok := states[qn]; ok { + return st + } + st := &entityDropState{intentional: map[string]bool{}} + if stored, ok := findEntityByQN(ctx.Backend, qn); ok && stored != nil { + st.stored = memberSetFromEntity(stored) + st.current = memberSetFromEntity(stored) + } + states[qn] = st + order = append(order, qn) + return st + } + + for _, stmt := range prog.Statements { + switch s := stmt.(type) { + case *ast.CreateEntityStmt: + st := track(s.Name.String()) + if st.stored == nil { + // Created by this script: a rebuild of something the project does + // not hold removes nothing from it. + continue + } + // CREATE ENTITY IF NOT EXISTS never touches an existing definition, + // and a plain CREATE over an existing entity is refused by + // CheckProjectConflicts — neither rebuilds, so neither drops. + if s.IfNotExists || !s.CreateOrModify { + continue + } + st.current = memberSetFromCreateStmt(s) + case *ast.DropEntityStmt: + track(s.Name.String()).skip = true + case *ast.AlterEntityStmt: + applyAlterToDropState(track(s.Name.String()), s) + case *ast.AlterEntitiesStmt: + // The bulk form runs each action through the single-entity path, so + // its removals are just as explicit. Apply them to every entity this + // pass is already tracking within the statement's module scope; that + // cannot invent a drop, only cancel one out. + for _, qn := range order { + st := states[qn] + if s.Module != "" && !strings.EqualFold(splitQualifiedName(qn).Module, s.Module) { + continue + } + for _, action := range s.Actions { + applyAlterToDropState(st, action) + } + } + } + } + + var out []linter.Violation + for _, qn := range order { + st := states[qn] + if st.skip || st.stored == nil || st.current == nil { + continue + } + var dropped []string + for _, name := range droppedMembers(st.stored, st.current) { + if st.intentional[strings.ToLower(memberKey(name))] { + continue + } + dropped = append(dropped, name) + } + if len(dropped) == 0 { + continue + } + target := splitQualifiedName(qn) + out = append(out, linter.Violation{ + RuleID: entityMemberDropRule, + Severity: linter.SeverityWarning, + Message: fmt.Sprintf( + "applying this script to the project removes %d member(s) from entity %s that it does not restate: %s — anything still bound to them (widgets, microflows) fails the build with CE1613", + len(dropped), qn, strings.Join(dropped, ", ")), + Suggestion: fmt.Sprintf( + "if these members are meant to survive, add them to the create or modify statement, or add them incrementally with 'alter entity %s add attribute : ;' in this script; if they are meant to go, say so with 'alter entity %s drop attribute ;'", + qn, qn), + Location: linter.Location{Module: target.Module, DocumentType: "entity", DocumentName: target.Name}, + }) + } + return out +} + +// applyAlterToDropState folds one ALTER ENTITY action into the simulated member +// set. Only the four operations that change MEMBERSHIP are handled; MODIFY +// ATTRIBUTE, indexes, documentation and the rest cannot add or remove a member. +func applyAlterToDropState(st *entityDropState, s *ast.AlterEntityStmt) { + if st == nil || st.current == nil || st.stored == nil || s == nil { + return + } + switch s.Operation { + case ast.AlterEntityAddAttribute: + if s.Attribute == nil { + return + } + if st.current.applyAuditPseudoType(s.Attribute.Type.Kind) { + return + } + st.current.addAttr(s.Attribute.Name) + // An ADD after a rebuild dropped the same name restores it, so the + // earlier removal is no longer a loss. + delete(st.intentional, strings.ToLower(s.Attribute.Name)) + case ast.AlterEntityDropAttribute: + st.current.removeAttr(s.AttributeName) + st.intentional[strings.ToLower(s.AttributeName)] = true + case ast.AlterEntityRenameAttribute: + if !st.current.hasAttr(s.AttributeName) && !st.stored.hasAttr(s.AttributeName) { + return + } + st.current.removeAttr(s.AttributeName) + st.intentional[strings.ToLower(s.AttributeName)] = true + if s.NewName != "" { + st.current.addAttr(s.NewName) + delete(st.intentional, strings.ToLower(s.NewName)) + } + } +} + +// memberKey strips the parenthesised annotation droppedMembers adds to the +// non-attribute members ("owner (system field)"), so an intent lookup keyed on a +// bare attribute name cannot accidentally match one. +func memberKey(reported string) string { + if i := strings.Index(reported, " ("); i > 0 { + return reported[:i] + } + return reported +} diff --git a/mdl/executor/validate_entity_member_drops_test.go b/mdl/executor/validate_entity_member_drops_test.go new file mode 100644 index 000000000..c9da24b36 --- /dev/null +++ b/mdl/executor/validate_entity_member_drops_test.go @@ -0,0 +1,371 @@ +// SPDX-License-Identifier: Apache-2.0 + +package executor + +import ( + "strings" + "testing" + + "github.com/mendixlabs/mxcli/mdl/ast" + "github.com/mendixlabs/mxcli/mdl/backend/mock" + "github.com/mendixlabs/mxcli/mdl/linter" + "github.com/mendixlabs/mxcli/model" + "github.com/mendixlabs/mxcli/sdk/domainmodel" +) + +// ako/mxcli#562, reported verbatim: +// +// Re-running **slice 01 alone** rebuilt the entity from its own statement and +// dropped the attribute. `exec` reported only `Modified entity: +// ServiceCore.LithoSystem`; nothing said an attribute had been removed, and +// `mxcli lint` did not flag it. It surfaced two slices later as a page error: +// +// [error] [CE1613] "The selected attribute +// 'ServiceCore.LithoSystem.OpenRequestCount' no longer exists." +// at Text 'dtOpen' +// +// The exec half of the ask shipped in 320a304 and is covered by +// TestCreateOrModifyEntity_WarnsOnDrop. MEASURED on this branch against a real +// 11.6.6 project, re-running the reporter's slice 01: exec now prints +// +// ⚠ create or modify entity MyFirstModule.LithoSystem drops 1 existing +// member(s) not listed in this statement: OpenRequestCount +// +// while `mxcli check 01-domain-core.mdl -p app.mpr --references` printed +// "Check passed!" and said nothing. That is what these tests are about: the +// issue's second ask, "a warning at check time would be better still where the +// project is available" — before the attribute is gone rather than as it goes. + +// storedLithoSystem is the entity as slice 01 plus slice 03 leave it: two +// attributes from the CREATE, and the calculated one a later ALTER added. +func storedLithoSystem() *domainmodel.Entity { + return &domainmodel.Entity{ + BaseElement: model.BaseElement{ID: "ent-litho", TypeName: "DomainModels$Entity"}, + Name: "LithoSystem", + Persistable: true, + Attributes: []*domainmodel.Attribute{ + {BaseElement: model.BaseElement{ID: "a-name"}, Name: "Name"}, + {BaseElement: model.BaseElement{ID: "a-serial"}, Name: "SerialNumber"}, + {BaseElement: model.BaseElement{ID: "a-open"}, Name: "OpenRequestCount"}, + }, + } +} + +// dropCheckCtx wires a context whose project holds exactly the given entities in +// module ServiceCore, reachable through findEntityByQN's module-then-DM path. +func dropCheckCtx(t *testing.T, entities ...*domainmodel.Entity) *ExecContext { + t.Helper() + mod := mkModule("ServiceCore") + dm := &domainmodel.DomainModel{ + BaseElement: model.BaseElement{ID: nextID("dm")}, + ContainerID: mod.ID, + Entities: entities, + } + mb := &mock.MockBackend{ + IsConnectedFunc: func() bool { return true }, + GetModuleByNameFunc: func(name string) (*model.Module, error) { + if name == "ServiceCore" { + return mod, nil + } + return nil, nil + }, + GetDomainModelFunc: func(model.ID) (*domainmodel.DomainModel, error) { return dm, nil }, + ListDomainModelsFunc: func() ([]*domainmodel.DomainModel, error) { return []*domainmodel.DomainModel{dm}, nil }, + ListModulesFunc: func() ([]*model.Module, error) { return []*model.Module{mod}, nil }, + } + ctx, _ := newMockCtx(t, withBackend(mb)) + return ctx +} + +func litho(name string) ast.QualifiedName { + return ast.QualifiedName{Module: "ServiceCore", Name: name} +} + +func dropStrAttr(name string) ast.Attribute { + return ast.Attribute{Name: name, Type: ast.DataType{Kind: ast.TypeString, Length: 200}} +} + +// slice01 is the reporter's 01-domain-core.mdl: the entity's own statement, +// which never mentions OpenRequestCount because the microflow it calculates by +// does not exist until slice 03. +func slice01() *ast.CreateEntityStmt { + return &ast.CreateEntityStmt{ + Name: litho("LithoSystem"), + Kind: ast.EntityPersistent, + CreateOrModify: true, + Attributes: []ast.Attribute{dropStrAttr("Name"), dropStrAttr("SerialNumber")}, + } +} + +func findDropRule(vs []linter.Violation, id string) *linter.Violation { + for i := range vs { + if vs[i].RuleID == id { + return &vs[i] + } + } + return nil +} + +// TestMDL087_ReRunningOneSliceAloneIsReported is the issue's case: check the +// reporter's slice 01 against a project that already holds the attribute slice +// 03 added, and the member loss must be named before anything is written. +func TestMDL087_ReRunningOneSliceAloneIsReported(t *testing.T) { + ctx := dropCheckCtx(t, storedLithoSystem()) + prog := &ast.Program{Statements: []ast.Statement{slice01()}} + + vs := CheckEntityMemberDrops(ctx, prog) + v := findDropRule(vs, "MDL087") + if v == nil { + t.Fatalf("expected MDL087 for the attribute slice 01 does not restate, got %d violation(s): %+v", len(vs), vs) + } + if v.Severity != linter.SeverityWarning { + t.Errorf("MDL087 severity = %v, want warning — 'modify to this shape' is a legitimate intent; the defect was the silence", v.Severity) + } + // The reported symptom is the attribute's name arriving too late, in a + // CE1613 about a page. Both belong in the message. + for _, want := range []string{"OpenRequestCount", "ServiceCore.LithoSystem", "CE1613"} { + if !strings.Contains(v.Message, want) { + t.Errorf("MDL087 message does not mention %q: %s", want, v.Message) + } + } + if !strings.Contains(v.Suggestion, "alter entity ServiceCore.LithoSystem add attribute") { + t.Errorf("MDL087 should point at the incremental spelling, got: %s", v.Suggestion) + } + if v.Location.DocumentName != "LithoSystem" || v.Location.Module != "ServiceCore" { + t.Errorf("MDL087 location = %+v, want ServiceCore/LithoSystem", v.Location) + } +} + +// TestMDL087_FaithfulRestatementIsSilent is the CONTROL for the test above: the +// same pass over a script that restates every member must say nothing. Without +// it, a checker that flagged every `create or modify entity` would pass. +func TestMDL087_FaithfulRestatementIsSilent(t *testing.T) { + ctx := dropCheckCtx(t, storedLithoSystem()) + full := slice01() + full.Attributes = append(full.Attributes, + ast.Attribute{Name: "OpenRequestCount", Type: ast.DataType{Kind: ast.TypeInteger}}) + + if vs := CheckEntityMemberDrops(ctx, &ast.Program{Statements: []ast.Statement{full}}); len(vs) != 0 { + t.Fatalf("a statement restating every member must be silent, got %+v", vs) + } +} + +// TestMDL087_ReAddedLaterInTheSameScriptIsSilent is the whole reason this pass +// is a NET computation over the script rather than a per-statement one. The +// reporter's two slices concatenated — rebuild, then add the attribute back once +// its microflow exists — lose nothing, and warning on that is how a rule gets +// ignored. Note the ADD carries IF NOT EXISTS, which is how the slice is written. +func TestMDL087_ReAddedLaterInTheSameScriptIsSilent(t *testing.T) { + ctx := dropCheckCtx(t, storedLithoSystem()) + readd := &ast.AlterEntityStmt{ + Name: litho("LithoSystem"), + Operation: ast.AlterEntityAddAttribute, + IfNotExists: true, + Attribute: &ast.Attribute{ + Name: "OpenRequestCount", + Type: ast.DataType{Kind: ast.TypeInteger}, + Calculated: true, + CalculatedMicroflow: &ast.QualifiedName{Module: "ServiceCore", Name: "CALC_OpenRequestCount"}, + }, + } + prog := &ast.Program{Statements: []ast.Statement{slice01(), readd}} + if vs := CheckEntityMemberDrops(ctx, prog); len(vs) != 0 { + t.Fatalf("an attribute the script adds back is not a loss, got %+v", vs) + } + + // And the other order is still a loss: adding it and THEN rebuilding + // without it removes it just the same. + reversed := &ast.Program{Statements: []ast.Statement{readd, slice01()}} + if findDropRule(CheckEntityMemberDrops(ctx, reversed), "MDL087") == nil { + t.Error("a rebuild AFTER the add still drops the attribute — expected MDL087") + } +} + +// TestMDL087_ExplicitRemovalIsSilent covers the three spellings that say what +// they do. A pure before/after diff cannot tell these from an accident, which is +// why intent is tracked rather than inferred from the outcome. +func TestMDL087_ExplicitRemovalIsSilent(t *testing.T) { + cases := []struct { + name string + prog []ast.Statement + }{ + {"drop attribute", []ast.Statement{ + slice01(), + &ast.AlterEntityStmt{Name: litho("LithoSystem"), Operation: ast.AlterEntityDropAttribute, AttributeName: "OpenRequestCount"}, + }}, + {"drop attribute before the rebuild", []ast.Statement{ + &ast.AlterEntityStmt{Name: litho("LithoSystem"), Operation: ast.AlterEntityDropAttribute, AttributeName: "OpenRequestCount"}, + slice01(), + }}, + {"drop entity", []ast.Statement{ + &ast.DropEntityStmt{Name: litho("LithoSystem")}, + slice01(), + }}, + {"rename attribute", []ast.Statement{ + &ast.AlterEntityStmt{ + Name: litho("LithoSystem"), Operation: ast.AlterEntityRenameAttribute, + AttributeName: "OpenRequestCount", NewName: "OpenCount", + }, + func() ast.Statement { + s := slice01() + s.Attributes = append(s.Attributes, ast.Attribute{Name: "OpenCount", Type: ast.DataType{Kind: ast.TypeInteger}}) + return s + }(), + }}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + ctx := dropCheckCtx(t, storedLithoSystem()) + if vs := CheckEntityMemberDrops(ctx, &ast.Program{Statements: tc.prog}); len(vs) != 0 { + t.Errorf("an explicit removal must not warn, got %+v", vs) + } + }) + } +} + +// TestMDL087_EntityCreatedByThisScriptIsSilent — a rebuild of something the +// project does not hold removes nothing from it. +func TestMDL087_EntityCreatedByThisScriptIsSilent(t *testing.T) { + ctx := dropCheckCtx(t) // empty project + prog := &ast.Program{Statements: []ast.Statement{slice01(), slice01()}} + if vs := CheckEntityMemberDrops(ctx, prog); len(vs) != 0 { + t.Fatalf("expected silence for an entity this script creates, got %+v", vs) + } +} + +// TestMDL087_NonRebuildingFormsAreSilent — neither CREATE ENTITY IF NOT EXISTS +// (which never touches an existing definition) nor a plain CREATE (refused by +// CheckProjectConflicts) rebuilds, so neither drops. +func TestMDL087_NonRebuildingFormsAreSilent(t *testing.T) { + for _, tc := range []struct { + name string + shape func(*ast.CreateEntityStmt) + }{ + {"if not exists", func(s *ast.CreateEntityStmt) { s.CreateOrModify = false; s.IfNotExists = true }}, + {"plain create", func(s *ast.CreateEntityStmt) { s.CreateOrModify = false }}, + } { + t.Run(tc.name, func(t *testing.T) { + ctx := dropCheckCtx(t, storedLithoSystem()) + s := slice01() + tc.shape(s) + if vs := CheckEntityMemberDrops(ctx, &ast.Program{Statements: []ast.Statement{s}}); len(vs) != 0 { + t.Errorf("expected silence, got %+v", vs) + } + }) + } +} + +// TestMDL087_AuditFieldsAndGeneralization — the members that are not attributes +// drop the same way, and are reported by the same shared comparison the exec +// warning uses. An audit pseudo-type restated in the statement is NOT a drop, +// which is the control that separates "reported the flag" from "reported every +// entity that has one". +func TestMDL087_AuditFieldsAndGeneralization(t *testing.T) { + stored := storedLithoSystem() + stored.HasOwner = true + stored.HasCreatedDate = true + stored.GeneralizationRef = "ServiceCore.Asset" + + ctx := dropCheckCtx(t, stored) + s := slice01() + s.Attributes = append(s.Attributes, + ast.Attribute{Name: "OpenRequestCount", Type: ast.DataType{Kind: ast.TypeInteger}}, + // Owner is restated as its pseudo-type — kept, and not counted as an + // attribute either. + ast.Attribute{Name: "Owner", Type: ast.DataType{Kind: ast.TypeAutoOwner}}, + ) + + v := findDropRule(CheckEntityMemberDrops(ctx, &ast.Program{Statements: []ast.Statement{s}}), "MDL087") + if v == nil { + t.Fatal("expected MDL087 for the dropped createdDate and generalization") + } + for _, want := range []string{"createdDate (system field)", "extends ServiceCore.Asset (generalization)"} { + if !strings.Contains(v.Message, want) { + t.Errorf("message missing %q: %s", want, v.Message) + } + } + if strings.Contains(v.Message, "owner (system field)") { + t.Errorf("owner was restated as AutoOwner and must not be reported: %s", v.Message) + } + if strings.Contains(v.Message, "Owner,") || strings.Contains(v.Message, ": Owner") { + t.Errorf("an audit pseudo-type is not an attribute and must not be reported as one: %s", v.Message) + } +} + +// TestMDL087_SharedComparisonWithExecWarning pins the refactor that made the +// exec-time warning and this pass one answer. Two hand-written diffs at two +// layers is how the audit fields came to be covered by one and not the other, so +// the two entry points are asserted to agree on the same pair of entities. +func TestMDL087_SharedComparisonWithExecWarning(t *testing.T) { + stored := storedLithoSystem() + stored.HasChangedDate = true + stored.GeneralizationRef = "ServiceCore.Asset" + + rebuilt := &domainmodel.Entity{ + Name: "LithoSystem", + Attributes: []*domainmodel.Attribute{ + {Name: "Name"}, {Name: "SerialNumber"}, + }, + } + execSide := strings.Join(droppedEntityMembers(stored, rebuilt), ", ") + + ctx := dropCheckCtx(t, stored) + v := findDropRule(CheckEntityMemberDrops(ctx, &ast.Program{Statements: []ast.Statement{slice01()}}), "MDL087") + if v == nil { + t.Fatal("expected MDL087") + } + if !strings.Contains(v.Message, execSide) { + t.Errorf("check-time list and exec-time list disagree:\n exec: %s\n check: %s", execSide, v.Message) + } +} + +// TestMDL087_NoProjectIsSilent — the answer is entirely "what does the project +// hold that this script does not mention", so without a project the pass must +// report nothing rather than guess. +func TestMDL087_NoProjectIsSilent(t *testing.T) { + ctx, _ := newMockCtx(t, withBackend(&mock.MockBackend{IsConnectedFunc: func() bool { return false }})) + if vs := CheckEntityMemberDrops(ctx, &ast.Program{Statements: []ast.Statement{slice01()}}); len(vs) != 0 { + t.Fatalf("expected silence without a project, got %+v", vs) + } +} + +// TestMDL087_ReportsInStoredAttributeOrder pins the reporting order. It is the +// entity's own attribute order — what `describe entity` prints — not +// alphabetical and not map iteration order, so the list reads against the +// document the user is looking at. This is also what keeps the exec-time +// warning's output unchanged by the refactor onto the shared comparison. +func TestMDL087_ReportsInStoredAttributeOrder(t *testing.T) { + stored := &domainmodel.Entity{ + Name: "LithoSystem", + Persistable: true, + Attributes: []*domainmodel.Attribute{ + {Name: "Zeta"}, {Name: "Name"}, {Name: "Alpha"}, {Name: "Mid"}, + }, + } + // Only Name is restated, so three drop — and Zeta comes first because the + // entity stores it first. + keepName := &ast.CreateEntityStmt{ + Name: litho("LithoSystem"), + Kind: ast.EntityPersistent, + CreateOrModify: true, + Attributes: []ast.Attribute{dropStrAttr("Name")}, + } + + ctx := dropCheckCtx(t, stored) + v := findDropRule(CheckEntityMemberDrops(ctx, &ast.Program{Statements: []ast.Statement{keepName}}), "MDL087") + if v == nil { + t.Fatal("expected MDL087") + } + if !strings.Contains(v.Message, "Zeta, Alpha, Mid") { + t.Errorf("expected stored order 'Zeta, Alpha, Mid', got: %s", v.Message) + } + + // The exec-time helper reports the same list in the same order. + got := strings.Join(droppedEntityMembers(stored, &domainmodel.Entity{ + Name: "LithoSystem", + Attributes: []*domainmodel.Attribute{{Name: "Name"}}, + }), ", ") + if got != "Zeta, Alpha, Mid" { + t.Errorf("droppedEntityMembers order = %q, want \"Zeta, Alpha, Mid\"", got) + } +} From 95c1841f64f94e1362ab8e4614619e94160ff122 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 21 Sep 2026 18:15:53 +0000 Subject: [PATCH 08/15] Bind a pluggable widget's textTemplate to an attribute (#575) A textTemplate property took literal text only, so a caption written as treenode tn (headerType: 'text', headerCaption: 'Name', ...) passed check, exec and `mx check` and rendered the word "Name" on every node. The companion the report reached for did not exist: x widget `tn` (treenode) has no property `headerCaptionParams` -- did you mean `headerCaption`? [MDL-WIDGET01] `Params` was already the convention an object-list ITEM used for its text templates (#956); it stopped at the item boundary. At the widget level the engine read parameters from one place, the widget-wide `contentparams:` -- a single list for every template on the widget, which cannot say "this caption binds Name and that one binds Remarks". Three gaps: 1. resolveMapping's "TextTemplate" case consulted only GetContentParams(). 2. A texttemplate mapping addressed by its def.json SOURCE name (an Image's `ImageUrl:`, schema key `imageUrl`) fell through to the default branch, which set no parameters at all -- not even contentparams, so #928's fix only ever covered the schema-key spelling. 3. allowedWidgetProperties did not know the companion, so writing it was an MDL-WIDGET01 error and exec refused the whole script. Each text template now takes its own `Params`, under whichever spelling the template itself was written (schema key, alias, or source name); `contentparams:` stays the fallback and `'{AttrName}'` is untouched. An orphaned companion -- parameters beside a literal caption -- is reported as MDL-WIDGET21 instead of being dropped in silence, and DESCRIBE emits the companion so describe -> exec keeps the binding (it round-tripped to a bare `{1}`, which re-executes into CE0720). Measured on a blank Mendix 11.14.0 project, mxbuild 11.14.0, with the built-in pluggable Image standing in for the TreeNode -- it has the same shape (two text templates) and needs no download: pre-fix check -> MDL-WIDGET01 x4 on imageUrlParams / alternativeTextParams and their source-name spellings; exec refuses, nothing written. silent control: companions deleted, one shared `contentparams: [{1} = PictureUrl]` left -- check, exec and `mx check` (0 errors) all clean, and BOTH stored ClientTemplates bind PictureUrl, so the alt text renders the URL. Valid, buildable, wrong. fixed check -> Check passed! exec -> Created page mx check -> 0 errors stored BSON -> imageUrl={1}[PictureUrl], alternativeText={1}[Name] describe -> exec -> "Unchanged page" (round trip stable) Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01DXYNJwiutu5AjxLmG7Fgsu --- .../fix-issue/findings/mdl-executor.jsonl | 1 + .../mendix/create-page/reference/widgets.md | 19 +- .claude/skills/mendix/custom-widgets/SKILL.md | 13 +- cmd/mxcli/lsp_completion.go | 12 + cmd/mxcli/syntax/features_page.go | 9 +- docs-site/src/language/widget-types.md | 34 +++ docs-site/src/reference/capabilities.md | 1 - docs/01-project/MDL_QUICK_REFERENCE.md | 20 ++ ...gets-575-texttemplate-params-companion.mdl | 97 +++++++ mdl/executor/cmd_pages_describe.go | 20 +- mdl/executor/cmd_pages_describe_output.go | 8 + mdl/executor/cmd_pages_describe_pluggable.go | 32 +-- mdl/executor/validate_widget_contentparams.go | 52 ++++ mdl/executor/validate_widgets.go | 28 ++ mdl/executor/widget_engine.go | 94 ++++++- ...dget_texttemplate_named_params_575_test.go | 250 ++++++++++++++++++ 16 files changed, 651 insertions(+), 39 deletions(-) create mode 100644 mdl-examples/bug-tests/widgets-575-texttemplate-params-companion.mdl create mode 100644 mdl/executor/widget_texttemplate_named_params_575_test.go diff --git a/.claude/skills/fix-issue/findings/mdl-executor.jsonl b/.claude/skills/fix-issue/findings/mdl-executor.jsonl index 057547703..8a88d70c3 100644 --- a/.claude/skills/fix-issue/findings/mdl-executor.jsonl +++ b/.claude/skills/fix-issue/findings/mdl-executor.jsonl @@ -663,3 +663,4 @@ {"area": "mdl/executor", "date": "2026-09-20", "symptom": "`describe page` → `exec` over a **Studio Pro-authored** page silently drops six things, `mx check` 0 errors throughout. The one that matters: `IsPasswordBox True → False` — a **password field round-trips into a plaintext text box**, and describe → rename → exec is mxcli's copy operation. Also `Validation.Expression` blanked, a DataView's `ReadOnlyStyle Text → Control`, `PopupCloseAction` wiped, and two typed-array markers", "cause": "Four different causes behind one symptom, which is why triage came first: (1) IsPasswordBox — model and writer carried it, nothing parsed it, nothing emitted it; (2) Validation — `widgetValidationToGen()` wrote a DEFAULT EMPTY Forms$WidgetValidation over whatever was stored, on five widget types; (3) ReadOnlyStyle — wired for CheckBox only, and a DataView's draws no MDL-WIDGET07 warning because `staticWidgetKnownProps` is deliberately a union across widget types; (4) PopupCloseAction — `pageToGen` wrote \"\" unconditionally. Plus ParameterMappings/OutputMappings markers", "file": "`mdl/executor/cmd_pages_describe_parse.go` + `_output.go` (extract/emit), `cmd_pages_builder_v3_widgets.go` (consume), `cmd_pages_builder_v3.go`, `mdl/visitor/visitor_page_v3.go`, `mdl/ast/ast_page_v3.go`, `sdk/pages/*`, `mdl/backend/modelsdk/widget_write.go` + `page_write.go`, `mdl/executor/validate_widgets.go` (describe vocabulary)", "insight": "**Triage the layer before writing anything** — describer / grammar / builder have different fixes and this one issue had all three. The quickest probe is to run the property through `mxcli check`: MDL-WIDGET07 names an unrecognised one, and *silence is not acceptance* — the known-props list is a union across widget types, so a DataView's ReadOnlyStyle passed check and was dropped anyway. **Emit an expression QUOTED, not bracketed**: `[...]` is the XPath-constraint spelling and `propertyValueV3` parses it as an ARRAY, so `GetStringProp` yields \"\" — the emitter's own unit test was green while the real round trip still lost the value (storage form is not input form). **Measure the default before keeping it**: a DataView's ReadOnlyStyle is Control on 47 of 56, never Inherit, so the 'obvious' Inherit that every other input widget uses would have been wrong. Markers likewise measured, not assumed: ParameterMappings is marker 2 on 220 of 220 lists in every parent type, OutputMappings present on 91 of 91 — and an EMPTY list needs `MandatoryListMarkers` since `RegisterListMarker` keys on a child that is not there. Result 17 → 9 differences, the 9 being ako/mxcli#549", "refs": ["#550", "#541", "#549", "#490"]} {"area": "mdl/executor", "date": "2026-09-20", "symptom": "MDL-PAGEARG01 refused a list widget's OWN row action: `datagrid dg (DataSource: DATABASE M.E, onClick: SHOW_PAGE M.Edit(E: $currentObject))` was rejected at `check` with \"widget `dg` is not inside a data view, list view or grid row\" \u2014 and since exec refuses a script whose check errors, the slice could not be applied at all. On a `listview` the message contradicted itself. mxbuild 11.14.0 accepts the stored pages at 0 errors.", "cause": "The #1029 guard judged EVERY widget's own action in the context its PARENT supplies: `argContextForSubtreeOf` returns the parent context for a childless widget and `validate_widgets.go` passed the inherited `argCtx` to `validateShowPageArguments`. Right for a button, wrong for the widget that ESTABLISHES the context \u2014 a list widget's onClick is row-scoped, so the row it renders is the context object. Added `argContextForOwnAction`: a widget that binds a source of its own supplies the context for its own action; a source in a shape the pass cannot read (the bare-entity shorthand) degrades to UNKNOWN so the guard stands down rather than refusing what it cannot prove is discarded.", "file": "`mdl/executor/cmd_pages_showpage_args.go` (argContextForOwnAction, argContextForSubtreeOf), `mdl/executor/validate_widgets.go`", "insight": "**A false refusal costs more than a missing rule now that exec refuses on a check error** \u2014 the blast radius is 'this project cannot be built with this mxcli', not 'a warning is noisy'. Two things would have caught it before release: judging the rule against the widget kinds it NAMES in its own message (the listview refusal reads 'lvA is not inside a \u2026 list view'), and running it against mxbuild rather than against intuition. The mxbuild run paid for itself twice: it also showed that `DataSource: M.E` (bare-entity shorthand) on a datagrid is silently dropped, so that case is CE0488 + a REAL CE1571 \u2014 the stand-down is still correct, but the shorthand case must not be written into a bug test as mxbuild-clean (#576). Control the fix with the widget kinds STILL refused (a foreign variable, a sibling button beside the grid), or it is indistinguishable from deleting the rule.", "refs": ["#552", "#576", "mendixlabs/mxcli#1029", "#939"]} {"area": "mdl/executor", "date": "2026-09-21", "symptom": "A page's image-collection reference passed `mxcli check --references` and failed the build. Reported as \"no MDL syntax for a StaticImageViewer inside a Selection helper custom state\" — the authoring half was already closed by #1057; what was left is that nothing RESOLVED the name it made writable. Measured on a blank Mendix 11.14.0 project: `staticimage imgAll (Image: 'Atlas_UI_Resources.Atlas_Icons.checkbox_checked')` in a custom state -> check passed, exec created the page, `mx check` -> 3x CE1613 \"The selected image … no longer exists.\"", "cause": "TWO independent holes, and either alone leaves the reported script unchecked. (1) widgetRefCollector keyed the image reference on the widget TYPE — `if w.Type == \"image\"` — so the pluggable widget was collected and `staticimage` (which #1057 had just given the SAME `Image:` property) and `dynamicimage`'s `DefaultImage` were not; replaced with an imageRefProps table. (2) A page's widgets live in two AST fields: `Widgets` is the bare body, `Placeholders` holds `placeholder X { … }` content (#532). validate.go passed `s.Widgets` alone to validateWidgetReferences, validatePageContextTree AND validateFlowArguments, so EVERY reference inside a placeholder block — microflow, nanoflow, page, snippet, entity, image — was validated by nothing; added allPageWidgets to collect both roots once.", "file": "`mdl/executor/helpers.go` (widgetRefCollector.collectFromWidget, imageRefProps), `mdl/executor/validate.go` (allPageWidgets)", "insight": "**When a capability gets a new spelling, grep for who RESOLVES the old one.** #1057 added `Image:` to a second and third widget and moved on; the resolver keyed on the type name, so the new spellings were unchecked from the day they shipped. A property list and a resolver list that describe the same property are two copies — `validate_widgets.go` already accepted `Image`/`DefaultImage` for these widgets and DESCRIBE already emitted them, and only the resolver disagreed. **The placeholder hole is the more useful lesson: it was the THIRD copy of one walk.** validateIconRefs (#1008) and forEachWidget had each grown the `Placeholders` arm separately, with a comment saying a missed walk is silent both ways — and the three validators next door still had not. When a fix is 'add the missing arm to this walker', the question is how many walkers there are; collect the roots once instead. **Do not reason about a bug report from the issue text alone when the version is older than the fix** — the reported symptom did not reproduce on main at all, and running the reporter's own script end to end is what turned 'already fixed, close it' into two real defects. **Control both directions**: a reference that resolves must stay silent, because a walker that can suddenly see a whole new region of the tree is as likely to report correct scripts as broken ones.", "refs": ["mendixlabs/mxcli#1149", "mendixlabs/mxcli#1057", "mendixlabs/mxcli#1008", "#532"]} +{"area": "mdl/executor", "date": "2026-09-21", "symptom": "A pluggable widget's textTemplate property took only literal text, so it rendered the same string on every row. `treenode tn (headerType: 'text', headerCaption: 'Name')` passed check, exec and `mx check` and printed the word \"Name\" on every node; the companion the report reached for did not exist — \"widget `tn` (treenode) has no property `headerCaptionParams` — did you mean `headerCaption`? [MDL-WIDGET01]\".", "cause": "The `Params` convention was already how an object-list ITEM bound its text templates (#956, buildObjectListItem looks up matchedAlias+\"Params\"), and it simply stopped at the item boundary. At the WIDGET level the engine read parameters from ONE place: the widget-wide `contentparams:`. Three separate holes: (1) `resolveMapping` case \"TextTemplate\" consulted only `w.GetContentParams()`; (2) a texttemplate mapping addressed by its def.json SOURCE name (an Image's `ImageUrl:`, schema key `imageUrl`) fell through to the `default:` branch, which set no ClientParams at all — not even contentparams, so #928's fix covered only the schema-key spelling; (3) `allowedWidgetProperties` did not know the companion, so writing it was MDL-WIDGET01 and exec refused the script. Added `textTemplateParams` (own companion, then contentparams as fallback), `namedPropValueWithKey`/`templateParamNames` so the companion is found under whichever spelling the template was written in, `addTemplateParamsNames` in the validator, `validatePluggableTemplateParams` (MDL-WIDGET21) for an orphaned companion, and the DESCRIBE side for the Image's two templates.", "file": "`mdl/executor/widget_engine.go` (textTemplateParams, namedPropValueWithKey, templateParamNames, resolveMapping TextTemplate + default branches), `mdl/executor/validate_widgets.go` (addTemplateParamsNames), `mdl/executor/validate_widget_contentparams.go` (validatePluggableTemplateParams), `mdl/executor/cmd_pages_describe_pluggable.go` + `cmd_pages_describe_output.go`", "insight": "**The built-in pluggable Image is the cheap stand-in for any Marketplace widget with this shape** — it has TWO texttemplate properties (`imageUrl`, `alternativeText`), which is exactly the case one widget-wide `contentparams:` cannot express, and it needs no download. The silent control is what makes the bug visible: with the companions deleted and one shared `contentparams: [{1} = PictureUrl]` left, `check`, `exec` and `mx check` are all clean and BOTH stored ClientTemplates bind PictureUrl — measured in the BSON, on a blank 11.14.0 project. Do not reach for the MDL-WIDGET01 message as the whole bug: the reported spelling being rejected is the loud half, and fixing only that leaves the shared-list aliasing intact. Also worth knowing before theorising: a texttemplate mapping's def.json `source` is sometimes a source KIND (\"TextTemplate\") and sometimes a real MDL property name (\"ImageUrl\"), and the two take different branches of resolveMapping — a fix applied to one branch looks complete and covers half the widgets.", "refs": ["ako/mxcli#575", "mendixlabs/mxcli#928", "mendixlabs/mxcli#956"], "rules": ["MDL-WIDGET01", "MDL-WIDGET21"], "ce": ["CE0720"]} diff --git a/.claude/skills/mendix/create-page/reference/widgets.md b/.claude/skills/mendix/create-page/reference/widgets.md index 843c959d2..85c584134 100644 --- a/.claude/skills/mendix/create-page/reference/widgets.md +++ b/.claude/skills/mendix/create-page/reference/widgets.md @@ -768,8 +768,8 @@ alter page Mod.Home { For theme images, use paths relative to `theme/web/` (e.g., `img/logo.svg` → `theme/web/img/logo.svg`). -**A per-row image URL comes from the entity, two ways.** `imageUrl` is a text -template, so it takes either spelling: +**A per-row image URL comes from the entity, three ways.** `imageUrl` is a text +template, so it takes any of these spellings: ```sql -- named placeholder: shortest form for a single attribute @@ -782,8 +782,23 @@ pluggablewidget 'com.mendix.widget.web.image.Image' cardImage ( datasource: imageUrl, imageUrl: '{1}/{2}', contentparams: [{1} = BaseUrl, {2} = PictureUrl] ) + +-- `Params`: the property's OWN parameters. `contentparams` is one list +-- shared by every template on the widget, so it cannot bind `imageUrl` and +-- `alternativeText` to different attributes; this can (ako/mxcli#575). +pluggablewidget 'com.mendix.widget.web.image.Image' cardImage ( + datasource: imageUrl, + imageUrl: '{1}', imageUrlParams: [{1} = PictureUrl], + alternativeText: '{1}', alternativeTextParams: [{1} = Name] +) ``` +The same companion works on any pluggable widget's text-template property — a +TreeNode's `headerCaption`, a Timeline's `title` / `description` / +`timeIndication` — under the property's own name + `Params`. Without it a +text-template property took literal text only, so it rendered the same string +on every row with `check`, `exec` and `mx check` all clean. + Every `{N}` must have a matching parameter — Mendix rejects a shortfall with `CE0720` ("place holder index N is greater than …, the number of parameter(s)"). Parameters with no `{N}` to fill are reported by MDL-WIDGET21 rather than diff --git a/.claude/skills/mendix/custom-widgets/SKILL.md b/.claude/skills/mendix/custom-widgets/SKILL.md index bccb1e578..bd95ce76c 100644 --- a/.claude/skills/mendix/custom-widgets/SKILL.md +++ b/.claude/skills/mendix/custom-widgets/SKILL.md @@ -475,7 +475,8 @@ widgets take a different, simpler path than the MPR writer: whitelist. - **Supported property operations**: attribute, association, primitive, selection, datasource, widgets (child slots), object lists, expression, - texttemplate (including `{AttrName}` placeholders -> template parameters), + texttemplate (including `{AttrName}` placeholders and `Params` / + `contentparams` bindings -> template parameters), and action (`microflow Module.Flow`, `show_page Module.Page`, or none). - **Rejected loudly** (widget refused, nothing sent): actions *with argument mappings*, other action kinds (save/cancel/close/delete/create/open-link/ @@ -553,6 +554,16 @@ otherwise select a datasource mode. | `selection` | Sets `Value.Selection` (mode string) | `selection` | | `widgets` | Replaces `Value.Widgets` array with child widget BSON | child slot | | `texttemplate` | Sets text in `Value.TextTemplate` (Forms$ClientTemplate) | property name (resolved as string) | + +A `texttemplate` takes **text**, so a bare value renders the same string on every +row. Bind it with the property's own `Params` companion, named for +whichever spelling the template used (`ImageUrl:` pairs with `ImageUrlParams:`) +and taking the same `format (...)` block a `dynamictext` does — e.g. +`headerCaption: '{1}', headerCaptionParams: [{1} = Name]`, or a Timeline's +`title` / `description` bound separately. `contentparams:` is ONE list shared by +every template on the widget, so it only disambiguates a widget with a single +one; `'{AttrName}'` is the short form for one attribute. A companion whose +template has no `{N}` is **MDL-WIDGET21**, not a silent drop (ako/mxcli#575). | `action` | Sets `Value.Action` with serialized client action BSON | `onclick` (resolved from AST Action) | ### Mapping Order Constraints diff --git a/cmd/mxcli/lsp_completion.go b/cmd/mxcli/lsp_completion.go index d06f9f1fa..4e60c22fe 100644 --- a/cmd/mxcli/lsp_completion.go +++ b/cmd/mxcli/lsp_completion.go @@ -681,8 +681,19 @@ func (s *mdlServer) widgetPropertyCompletionItems(text, linePrefix string, curso }) } + // A text-template property's `Params` companion is offered beside it: + // the template takes literal text, so without the companion the obvious + // completion is also the one that renders the same string on every row + // (ako/mxcli#575). + addTemplateParams := func(m executor.PropertyMapping, detail string) { + if m.Operation != "texttemplate" || m.PropertyKey == "" { + return + } + addProp(m.PropertyKey+"Params", detail+" — `{N}` bindings", protocol.CompletionItemKindProperty) + } for _, m := range def.PropertyMappings { addProp(m.PropertyKey, "Property ("+m.Operation+")", protocol.CompletionItemKindProperty) + addTemplateParams(m, "Parameters for `"+m.PropertyKey+"`") } for _, mode := range def.Modes { for _, m := range mode.PropertyMappings { @@ -691,6 +702,7 @@ func (s *mdlServer) widgetPropertyCompletionItems(text, linePrefix string, curso detail += " [mode: " + mode.Name + "]" } addProp(m.PropertyKey, detail, protocol.CompletionItemKindProperty) + addTemplateParams(m, "Parameters for `"+m.PropertyKey+"`") } } for _, m := range def.ChildSlots { diff --git a/cmd/mxcli/syntax/features_page.go b/cmd/mxcli/syntax/features_page.go index 8cfd6b9b3..0bd739723 100644 --- a/cmd/mxcli/syntax/features_page.go +++ b/cmd/mxcli/syntax/features_page.go @@ -153,7 +153,14 @@ CREATE PAGE Sales.Detail (Title: 'Detail', Layout: Atlas_Core.Atlas_Default) { "-- entry writes a model mxbuild refuses (\"No image selected.\"); MDL-WIDGET22\n" + "-- reports that at check time, and a name that does not resolve is reported by\n" + "-- `check --references` rather than failing the build with CE1613.\n" + - "-- The alternatives are the URL form above, or `ImageType: icon`.\n\n" + + "-- The alternatives are the URL form above, or `ImageType: icon`.\n" + + "-- A text-template property (ImageUrl, AlternativeText, a pluggable widget's\n" + + "-- headerCaption/title/…) takes TEXT, so a bare value renders the same string\n" + + "-- on every row. Bind it with the property's own `Params` companion:\n" + + "IMAGE name (ImageType: imageUrl, ImageUrl: '{1}', ImageUrlParams: [{1} = PictureUrl],\n" + + " AlternativeText: '{1}', AlternativeTextParams: [{1} = Name])\n" + + "-- The widget-wide `contentparams:` is one list shared by every template on the\n" + + "-- widget; `'{AttrName}'` is the shortest form for a single attribute.\n\n" + "-- Any pluggable widget by its id (id FIRST, then the name)\nPLUGGABLEWIDGET 'com.mendix.widget.web.badge.Badge' name (value: 'x')\nCUSTOMWIDGET 'com.mendix.widget.custom.x.X' name (prop: 'x') -- legacy spelling\n\n" + "-- DYNAMICIMAGE shows the image held by an OBJECT, so it needs the entity that\n" + "-- object belongs to — reachable from the widget's context. Without it mxbuild\n" + diff --git a/docs-site/src/language/widget-types.md b/docs-site/src/language/widget-types.md index c4b94783e..9fadab07b 100644 --- a/docs-site/src/language/widget-types.md +++ b/docs-site/src/language/widget-types.md @@ -521,6 +521,40 @@ extracted yet: mxcli widget init -p app.mpr # extract definitions for every widget in widgets/ ``` +### Binding a text-template property + +Many pluggable widgets expose **text-template** properties — an Image's +`ImageUrl` and `AlternativeText`, a TreeNode's `headerCaption`, a Timeline's +`title` / `description` / `timeIndication`. They take *text*, so a bare value is +stored as a literal and renders the same string on every row, with `check`, +`exec` and `mx check` all clean. + +Bind one with the property's own `Params` companion: + +```sql +image cardImage ( + ImageType: imageUrl, + ImageUrl: '{1}', ImageUrlParams: [{1} = PictureUrl], + AlternativeText: '{1}', AlternativeTextParams: [{1} = Name] +) +``` + +The companion is the property's own name + `Params`, in whichever spelling the +template itself was written. It takes the same per-parameter `format (...)` +block a `dynamictext` does, and `DESCRIBE PAGE` emits it, so describe → exec +keeps the binding. + +Two shorter spellings remain: + +| Spelling | Use it for | +|----------|-----------| +| `'{AttrName}'` | one attribute, no formatting block | +| `contentparams: [...]` | a widget with a **single** text template — it is one list shared by every template on the widget | + +Every `{N}` needs a matching parameter (Mendix rejects a shortfall with +`CE0720`), and parameters with no `{N}` to fill are reported as MDL-WIDGET21 +rather than dropped in silence. + ## Common Widget Properties These properties are shared across many widget types: diff --git a/docs-site/src/reference/capabilities.md b/docs-site/src/reference/capabilities.md index c6e837414..5bf56c230 100644 --- a/docs-site/src/reference/capabilities.md +++ b/docs-site/src/reference/capabilities.md @@ -142,7 +142,6 @@ Everything mxcli can do, organized by use case. |---|---|---| | Design properties (Atlas v3) | Requires Mendix 11.0+ | Use CSS classes on 10.x | | REST query parameters | Requires Mendix 11.0+ | Build query string manually on 10.x | -| Pluggable widget ImageUrl mode | Cannot set imageUrl from MDL | Configure in Studio Pro | | Concurrent editing | Not supported | Close Studio Pro before mxcli writes | | Widget template drift | CE0463 on version mismatch | MPK augmentation handles most cases | | Marketplace module update | Existing modules are reported, not updated in place | Update via Studio Pro (preserves local edits and entity IDs) | diff --git a/docs/01-project/MDL_QUICK_REFERENCE.md b/docs/01-project/MDL_QUICK_REFERENCE.md index a5222bb16..dbecf32b0 100644 --- a/docs/01-project/MDL_QUICK_REFERENCE.md +++ b/docs/01-project/MDL_QUICK_REFERENCE.md @@ -1067,6 +1067,26 @@ resolve is reported by `mxcli check --references` rather than by the build (CE1613). The other two sources are `ImageType: imageUrl, ImageUrl: '…'` and `ImageType: icon`. +### Binding a pluggable widget's text-template property + +A text-template property (`ImageUrl`, a TreeNode's `headerCaption`, a Timeline's +`title` / `description`) takes **text**, so a bare value renders the same string +on every row — with `check`, `exec` and `mx check` all clean. Bind it with the +property's own `Params` companion: + +```sql +image cardImage ( + ImageType: imageUrl, + ImageUrl: '{1}', ImageUrlParams: [{1} = PictureUrl], + AlternativeText: '{1}', AlternativeTextParams: [{1} = Name] +); +``` + +The widget-wide `contentparams:` is one list shared by every template on the +widget, so it remains the convenience form for a widget with a single template; +`'{AttrName}'` is the shortest form for one attribute with no formatting block. +Parameters with no `{N}` to fill are reported as MDL-WIDGET21. + ## Icon Collections (read-only) Icon collections (`CustomIcons$CustomIconCollection`, e.g. `Atlas_Core.Atlas_Filled`) ship with the theme/Atlas. Their icons are referenced from a widget as `Module.Collection.IconName` (a button's `icon:`). Use these to discover valid icon names — icons have non-obvious names (it's `add`, not `plus`). diff --git a/mdl-examples/bug-tests/widgets-575-texttemplate-params-companion.mdl b/mdl-examples/bug-tests/widgets-575-texttemplate-params-companion.mdl new file mode 100644 index 000000000..abd7047a5 --- /dev/null +++ b/mdl-examples/bug-tests/widgets-575-texttemplate-params-companion.mdl @@ -0,0 +1,97 @@ +-- ============================================================================ +-- ako/mxcli#575: a pluggable widget's textTemplate property took only literal +-- text, so it rendered the same string on every row. +-- ============================================================================ +-- +-- Reported against a TreeNode, whose `headerCaption` is a textTemplate: +-- +-- treenode tnCustomer (headerType: 'text', headerCaption: 'Name', ...) +-- +-- passes `check`, `exec` and `mx check` -- and renders the literal word "Name" +-- on every node. The companion the report reached for did not exist: +-- +-- x widget `tn` (treenode) has no property `headerCaptionParams` +-- -- did you mean `headerCaption`? [MDL-WIDGET01] +-- +-- The `Params` convention was already the one an object-list ITEM used +-- (a File Uploader custom button's `ButtonCaptionParams`, #956); it simply +-- stopped at the item boundary. At the WIDGET level the only parameters the +-- engine read were the single, widget-wide `contentparams:` -- one list for +-- every template on the widget, which cannot say "this caption binds Name and +-- that one binds Remarks". +-- +-- This file uses the built-in pluggable Image rather than a Marketplace +-- TreeNode, because it has the same shape and needs no download: TWO +-- text-template properties, `imageUrl` and `alternativeText`, which is exactly +-- what one shared contentparams cannot address. +-- +-- Measured on a blank Mendix 11.14.0 project (mxbuild 11.14.0): +-- +-- pre-fix check -> widget `cardImage` (IMAGE) has no property +-- `imageUrlParams` -- did you mean `imageUrl`? +-- (x4: imageUrlParams, alternativeTextParams, and the +-- same two in the source-name spelling) [MDL-WIDGET01] +-- -> exec refuses; nothing is written. +-- +-- with the companions deleted and one shared `contentparams:` +-- left instead, check and exec pass and mx check is clean -- and +-- alternativeText renders PictureUrl, because both templates read +-- parameter {1} out of the same list. That is the reported class: +-- valid, buildable, and wrong. +-- +-- fixed check -> Check passed! +-- exec -> Created page Bug575.Catalogue +-- mx check -> 0 errors +-- describe -> each template round-trips with its own binding +-- +-- Usage: +-- mxcli exec mdl-examples/bug-tests/widgets-575-texttemplate-params-companion.mdl -p app.mpr +-- mx check app.mpr # 0 errors +-- ============================================================================ + +create module Bug575; + +create persistent entity Bug575.Product ( + Name: string(200), + PictureUrl: string(500) +); +/ + +create or modify page Bug575.Catalogue ( + Title: 'Catalogue', + Layout: Atlas_Core.Atlas_Default +) +{ + LISTVIEW lv (DataSource: DATABASE Bug575.Product) { + + -- The control: a built-in widget whose parameters always bound. It has one + -- template, so the widget-wide contentparams was never ambiguous for it. + DYNAMICTEXT dt ( Content: '{1}', ContentParams: [{1} = Name] ) + + -- The fix, in the schema-key spelling: each template names its own + -- parameters, so the two bind DIFFERENT attributes. + PLUGGABLEWIDGET 'com.mendix.widget.web.image.Image' cardImage ( + datasource: imageUrl, + imageUrl: '{1}', imageUrlParams: [{1} = PictureUrl], + alternativeText: '{1}', alternativeTextParams: [{1} = Name] + ) + + -- The same, in the source-name spelling the def.json also accepts. This + -- one carried no parameters at all before -- not even contentparams, which + -- #928 wired only into the schema-key path. + PLUGGABLEWIDGET 'com.mendix.widget.web.image.Image' cardImage2 ( + datasource: imageUrl, + ImageUrl: '{1}', ImageUrlParams: [{1} = PictureUrl], + AlternativeText: '{1}', AlternativeTextParams: [{1} = Name] + ) + + -- Unchanged: the widget-wide contentparams remains the fallback for a + -- widget with one template, and the `{AttrName}` convenience spelling is + -- untouched. + PLUGGABLEWIDGET 'com.mendix.widget.web.image.Image' cardImage3 ( + datasource: imageUrl, + imageUrl: '{1}', contentparams: [{1} = PictureUrl] + ) + } +} +/ diff --git a/mdl/executor/cmd_pages_describe.go b/mdl/executor/cmd_pages_describe.go index 314679566..1248b2ada 100644 --- a/mdl/executor/cmd_pages_describe.go +++ b/mdl/executor/cmd_pages_describe.go @@ -723,13 +723,19 @@ type rawWidget struct { // Pluggable Image widget properties ImageUrl string // Image URL (from textTemplate) AlternativeText string // Alt text (from textTemplate) - ImageWidth string // Width in pixels/percentage - ImageHeight string // Height in pixels/percentage - WidthUnit string // "auto", "pixels", "percentage" - HeightUnit string // "auto", "pixels", "percentage", "viewport" - DisplayAs string // "fullImage", "thumbnail" - Responsive string // "true", "false" - ImageType string // "image", "imageUrl", "icon" + // The `{N}` bindings of the two templates above, each under its own + // `Params` companion (#575). Without them a bound image described back + // as a bare `ImageUrl: '{1}'`, which re-executes into CE0720 — the round trip + // silently unbinding what it was asked to copy. + ImageUrlParams []string + AlternativeTextParams []string + ImageWidth string // Width in pixels/percentage + ImageHeight string // Height in pixels/percentage + WidthUnit string // "auto", "pixels", "percentage" + HeightUnit string // "auto", "pixels", "percentage", "viewport" + DisplayAs string // "fullImage", "thumbnail" + Responsive string // "true", "false" + ImageType string // "image", "imageUrl", "icon" // ImageObject is the image collection entry the widget shows, as the // three-part qualified name Module.Collection.Image. Empty when the source // is not an image collection, or when none is selected. Without it a diff --git a/mdl/executor/cmd_pages_describe_output.go b/mdl/executor/cmd_pages_describe_output.go index 7ae115b46..905836818 100644 --- a/mdl/executor/cmd_pages_describe_output.go +++ b/mdl/executor/cmd_pages_describe_output.go @@ -1923,9 +1923,17 @@ func describeImageWidgetProps(w rawWidget) []string { } if w.ImageUrl != "" { props = append(props, fmt.Sprintf("ImageUrl: %s", mdlQuote(w.ImageUrl))) + if len(w.ImageUrlParams) > 0 { + props = append(props, fmt.Sprintf("ImageUrlParams: [%s]", + strings.Join(formatParametersV3(w.ImageUrlParams), ", "))) + } } if w.AlternativeText != "" { props = append(props, fmt.Sprintf("AlternativeText: %s", mdlQuote(w.AlternativeText))) + if len(w.AlternativeTextParams) > 0 { + props = append(props, fmt.Sprintf("AlternativeTextParams: [%s]", + strings.Join(formatParametersV3(w.AlternativeTextParams), ", "))) + } } if w.WidthUnit != "" && w.WidthUnit != "auto" { props = append(props, fmt.Sprintf("WidthUnit: %s", w.WidthUnit)) diff --git a/mdl/executor/cmd_pages_describe_pluggable.go b/mdl/executor/cmd_pages_describe_pluggable.go index 681477883..d5ddb2683 100644 --- a/mdl/executor/cmd_pages_describe_pluggable.go +++ b/mdl/executor/cmd_pages_describe_pluggable.go @@ -1167,8 +1167,8 @@ func extractExplicitProperties(ctx *ExecContext, w map[string]any) []rawExplicit func extractImageProperties(ctx *ExecContext, w map[string]any, widget *rawWidget) { widget.ImageType = extractCustomWidgetPropertyString(ctx, w, "datasource") widget.ImageObject = extractCustomWidgetPropertyImage(ctx, w, "imageObject") - widget.ImageUrl = extractCustomWidgetPropertyTextTemplate(ctx, w, "imageUrl") - widget.AlternativeText = extractCustomWidgetPropertyTextTemplate(ctx, w, "alternativeText") + widget.ImageUrl, widget.ImageUrlParams = extractCustomWidgetPropertyTextTemplate(ctx, w, "imageUrl") + widget.AlternativeText, widget.AlternativeTextParams = extractCustomWidgetPropertyTextTemplate(ctx, w, "alternativeText") widget.ImageWidth = extractCustomWidgetPropertyString(ctx, w, "width") widget.ImageHeight = extractCustomWidgetPropertyString(ctx, w, "height") widget.WidthUnit = extractCustomWidgetPropertyString(ctx, w, "widthUnit") @@ -1179,11 +1179,16 @@ func extractImageProperties(ctx *ExecContext, w map[string]any, widget *rawWidge widget.Action = extractCustomWidgetPropertyAction(ctx, w, "onClick") } -// extractCustomWidgetPropertyTextTemplate extracts text from a TextTemplate property of a CustomWidget. -func extractCustomWidgetPropertyTextTemplate(ctx *ExecContext, w map[string]any, propertyKey string) string { +// extractCustomWidgetPropertyTextTemplate extracts the text of a TextTemplate +// property of a CustomWidget, together with the `{N}` parameters bound to it. +// +// The parameters are returned separately rather than folded into the text +// because MDL spells them separately: `imageUrl: '{1}', imageUrlParams: [{1} = +// PictureUrl]` (#575). +func extractCustomWidgetPropertyTextTemplate(ctx *ExecContext, w map[string]any, propertyKey string) (string, []string) { obj, ok := w["Object"].(map[string]any) if !ok { - return "" + return "", nil } propTypeKeyMap := buildPropertyTypeKeyMap(w, false) @@ -1204,22 +1209,11 @@ func extractCustomWidgetPropertyTextTemplate(ctx *ExecContext, w map[string]any, continue } // Extract text from TextTemplate - if textTemplate, ok := value["TextTemplate"].(map[string]any); ok && textTemplate != nil { - if template, ok := textTemplate["Template"].(map[string]any); ok && template != nil { - items := getBsonArrayElements(template["Items"]) - for _, item := range items { - itemMap, ok := item.(map[string]any) - if !ok { - continue - } - if text := extractString(itemMap["Text"]); text != "" { - return text - } - } - } + if text, tt := extractTextTemplateText(value); text != "" { + return text, extractTextTemplateParameters(ctx, tt) } } - return "" + return "", nil } // customWidgetPropertyActionMap returns the raw Forms$*ClientAction map stored on diff --git a/mdl/executor/validate_widget_contentparams.go b/mdl/executor/validate_widget_contentparams.go index 44f30ef83..d39240da6 100644 --- a/mdl/executor/validate_widget_contentparams.go +++ b/mdl/executor/validate_widget_contentparams.go @@ -11,6 +11,7 @@ package executor import ( "fmt" + "strings" "github.com/mendixlabs/mxcli/mdl/ast" "github.com/mendixlabs/mxcli/mdl/linter" @@ -44,3 +45,54 @@ func validatePluggableContentParams(w *ast.WidgetV3, locationPrefix string) []li "the contentparams — a single attribute can also be written inline as `'{AttrName}'`", }} } + +// validatePluggableTemplateParams reports (MDL-WIDGET21) a `Params` +// companion whose own text property carries no `{N}` placeholder to consume it. +// +// The companion binds ONE text-template property (#575), so unlike the +// widget-wide `contentparams:` above there is a specific property to look at: +// `headerCaptionParams` is consumed by `headerCaption` and by nothing else. A +// companion written beside a literal caption is the shape the issue was filed +// for, one step short of the fix — the parameters are dropped and the literal +// still renders on every row. +func validatePluggableTemplateParams(w *ast.WidgetV3, locationPrefix string) []linter.Violation { + if w == nil { + return nil + } + var out []linter.Violation + for _, key := range sortedPropertyKeys(w) { + base, ok := strings.CutSuffix(key, "Params") + if !ok || base == "" { + continue + } + // ContentParams / CaptionParams are the widget-wide spelling, judged + // against every property by the check above. + switch strings.ToLower(key) { + case "contentparams", "captionparams": + continue + } + raw, ok := lookupProperty(w.Properties, key) + if !ok { + continue + } + if params, isParams := raw.([]ast.ParamAssignmentV3); !isParams || len(params) == 0 { + continue + } + text, _ := lookupProperty(w.Properties, base) + if s, isStr := text.(string); isStr && numericTemplatePlaceholderRe.MatchString(s) { + continue + } + out = append(out, linter.Violation{ + RuleID: "MDL-WIDGET21", + Severity: linter.SeverityWarning, + Message: fmt.Sprintf( + "%s: widget `%s` (%s) has `%s` but `%s` contains no `{1}`-style placeholder to use "+ + "it, so the binding is dropped on write and the text renders literally", + locationPrefix, w.Name, w.Type, key, base, + ), + Suggestion: fmt.Sprintf( + "Write the text as a template, e.g. `%s: '{1}', %s: [{1} = ]`", base, key), + }) + } + return out +} diff --git a/mdl/executor/validate_widgets.go b/mdl/executor/validate_widgets.go index 59dbde8d5..961ce40a6 100644 --- a/mdl/executor/validate_widgets.go +++ b/mdl/executor/validate_widgets.go @@ -176,6 +176,8 @@ func validateWidgetTreeIn(widgets []*ast.WidgetV3, registry *WidgetRegistry, loc // #928: contentparams with no `{N}` placeholder to consume them. if lookupWidgetDef(w, registry) != nil { out = append(out, validatePluggableContentParams(w, locationPrefix)...) + // #575: the same drop, per text-template property. + out = append(out, validatePluggableTemplateParams(w, locationPrefix)...) } out = append(out, validateWidgetVisibility(w, registry, locationPrefix)...) // An IMAGE with nothing to show — the default source needs an image @@ -1267,6 +1269,30 @@ func addMappingNames(add func(string), m PropertyMapping) { } } +// addTemplateParamsNames records the `Params` companion of a text-template +// property. A `{1}`-style template needs its parameters bound beside it, and the +// companion's name is the widget's own property name — so it cannot have a token +// of its own, and the validator has to derive it from the definition the same way +// the engine does. +// +// Scoped to texttemplate mappings on purpose: a blanket "anything ending in +// Params" would take MDL-WIDGET01's job away from a typo (#575). +func addTemplateParamsNames(add func(string), m PropertyMapping) { + if m.Operation != "texttemplate" { + return + } + for _, n := range []string{m.PropertyKey, m.Source} { + if n != "" { + add(n + "Params") + } + } + for _, a := range m.MdlAliases { + if a != "" { + add(a + "Params") + } + } +} + // readsFixedASTSlot reports whether an operation's value is resolved from a // dedicated AST accessor rather than from a property looked up by name. // @@ -1409,6 +1435,7 @@ func allowedWidgetProperties(def *WidgetDefinition) (map[string]bool, []string) for _, m := range def.PropertyMappings { addMappingNames(add, m) + addTemplateParamsNames(add, m) } for _, m := range def.ChildSlots { add(m.PropertyKey) @@ -1419,6 +1446,7 @@ func allowedWidgetProperties(def *WidgetDefinition) (map[string]bool, []string) for _, mode := range def.Modes { for _, m := range mode.PropertyMappings { addMappingNames(add, m) + addTemplateParamsNames(add, m) } for _, m := range mode.ChildSlots { add(m.PropertyKey) diff --git a/mdl/executor/widget_engine.go b/mdl/executor/widget_engine.go index 98c2c205f..5dd93bfb8 100644 --- a/mdl/executor/widget_engine.go +++ b/mdl/executor/widget_engine.go @@ -598,12 +598,14 @@ func (e *PluggableWidgetEngine) Build(def *WidgetDefinition, w *ast.WidgetV3) (* case "Expression": builder.SetExpression(propName, strVal) case "TextTemplate": - // `contentparams:` supplies the parameters for a `{1}`-style template. - // Without this branch the numeric spelling had no route on a pluggable - // widget — the template was written with an empty parameter list and - // mxbuild answered CE0720. The named `{AttrName}` spelling keeps its - // own path, which derives parameters from the entity context. (#928) - if params := e.pageBuilder.buildClientTemplateParams(w.GetContentParams()); len(params) > 0 && + // `Params:` — or, failing that, the widget-wide + // `contentparams:` — supplies the parameters for a `{1}`-style + // template. Without this branch the numeric spelling had no route on a + // pluggable widget — the template was written with an empty parameter + // list and mxbuild answered CE0720. The named `{AttrName}` spelling + // keeps its own path, which derives parameters from the entity + // context. (#928, #575) + if params := e.textTemplateParams(w, propName); len(params) > 0 && numericTemplatePlaceholderRe.MatchString(strVal) { builder.SetTextTemplateWithClientParams(propName, strVal, params) break @@ -1019,6 +1021,72 @@ func namedPropValue(mapping PropertyMapping, w *ast.WidgetV3) string { return "" } +// namedPropValueWithKey is namedPropValue plus the NAME the value was found +// under. A text-template property's parameters live beside it under that same +// name + "Params", and the script may have written either the schema key or one +// of the MDL aliases — so the companion cannot be looked up until it is known +// which one was used. +func namedPropValueWithKey(mapping PropertyMapping, w *ast.WidgetV3) (string, string) { + if v, ok := lookupProperty(w.Properties, mapping.PropertyKey); ok { + return stringifyAny(v), mapping.PropertyKey + } + for _, alias := range mapping.MdlAliases { + if v, ok := lookupProperty(w.Properties, alias); ok { + return stringifyAny(v), alias + } + } + return "", "" +} + +// templateParamNames lists the names a text-template mapping's `Params` +// companion may be written under, most specific first: the name the template +// text itself was authored under, then the schema key, then the aliases. An +// alias-authored caption keeps working when the companion is written with the +// schema key and vice versa. +func templateParamNames(mapping PropertyMapping, matched string) []string { + names := make([]string, 0, len(mapping.MdlAliases)+2) + if matched != "" { + names = append(names, matched) + } + if mapping.PropertyKey != "" && mapping.PropertyKey != matched { + names = append(names, mapping.PropertyKey) + } + for _, a := range mapping.MdlAliases { + if a != "" && a != matched && a != mapping.PropertyKey { + names = append(names, a) + } + } + return names +} + +// textTemplateParams resolves the ClientTemplateParameters bound to ONE +// text-template property: its own `Params` companion, falling back to the +// widget-wide `contentparams:`. +// +// The companion is the same convention an object-list item already used (#956). +// Stopping it at the item boundary left a widget-level textTemplate — a TreeNode +// `headerCaption`, a Timeline `title`/`description`/`timeIndication` — with one +// shared parameter list for every template on the widget, which cannot express +// "this caption binds Name and that one binds Remarks". A literal was the only +// thing left that worked, so every row rendered the same string (#575). +func (e *PluggableWidgetEngine) textTemplateParams(w *ast.WidgetV3, names ...string) []*pages.ClientTemplateParameter { + for _, name := range names { + if name == "" { + continue + } + raw, ok := lookupProperty(w.Properties, name+"Params") + if !ok { + continue + } + astParams, ok := raw.([]ast.ParamAssignmentV3) + if !ok || len(astParams) == 0 { + continue + } + return e.pageBuilder.buildClientTemplateParams(astParams) + } + return e.pageBuilder.buildClientTemplateParams(w.GetContentParams()) +} + // namedDataSourceValue returns the datasource a script authored under a // mapping's own schema key (or one of its aliases): // @@ -1249,10 +1317,10 @@ func (e *PluggableWidgetEngine) resolveMapping(mapping PropertyMapping, w *ast.W // A required widget-level caption authored by a named MDL property // (e.g. PieChart `SeriesName: '...'` → `seriesName`). applyOperation // "texttemplate" writes ctx.PrimitiveVal as the ClientTemplate text. - if v := namedPropValue(mapping, w); v != "" { + if v, matched := namedPropValueWithKey(mapping, w); v != "" { ctx.PrimitiveVal = v if numericTemplatePlaceholderRe.MatchString(v) { - ctx.ClientParams = e.pageBuilder.buildClientTemplateParams(w.GetContentParams()) + ctx.ClientParams = e.textTemplateParams(w, templateParamNames(mapping, matched)...) } } @@ -1371,6 +1439,16 @@ func (e *PluggableWidgetEngine) resolveMapping(mapping PropertyMapping, w *ast.W val = mapping.Default } ctx.PrimitiveVal = val + // A text-template mapping addressed by its SOURCE name — an Image's + // `ImageUrl:`, whose schema key is `imageUrl` — lands here rather than in + // the "TextTemplate" case above, and reached applyOperation with no + // parameters at all: neither its own `Params` companion nor the + // widget-wide `contentparams:` (#575, and the half of #928 that spelling + // never got). + if mapping.Operation == "texttemplate" && numericTemplatePlaceholderRe.MatchString(val) { + ctx.ClientParams = e.textTemplateParams(w, + append([]string{source}, templateParamNames(mapping, "")...)...) + } } return ctx, nil diff --git a/mdl/executor/widget_texttemplate_named_params_575_test.go b/mdl/executor/widget_texttemplate_named_params_575_test.go new file mode 100644 index 000000000..8dedfe865 --- /dev/null +++ b/mdl/executor/widget_texttemplate_named_params_575_test.go @@ -0,0 +1,250 @@ +// SPDX-License-Identifier: Apache-2.0 + +// ako/mxcli#575: "A pluggable widget's textTemplate property takes only literal +// text, so it renders the same string on every row." +// +// A TreeNode's `headerCaption` is a textTemplate. Written as +// +// treenode tnCustomer (headerType: 'text', headerCaption: 'Name', ...) +// +// it passes check, exec and mx check — and renders the literal word "Name" on +// every node. The companion an object-list item has took the widget's own +// property name + "Params" (#956), but that convention stopped at the item +// boundary: at the WIDGET level `headerCaptionParams` was an unknown property +// (MDL-WIDGET01) and the engine read parameters only from the single, +// widget-wide `contentparams:`. +package executor + +import ( + "strings" + "testing" + + "github.com/mendixlabs/mxcli/mdl/ast" +) + +func treeNodeDef() *WidgetDefinition { + return &WidgetDefinition{ + WidgetID: "com.mendix.widget.custom.treenode.TreeNode", + MDLName: "TREENODE", + PropertyMappings: []PropertyMapping{ + {PropertyKey: "headerCaption", Source: "TextTemplate", Operation: "texttemplate"}, + {PropertyKey: "headerType", Source: "Primitive", Operation: "primitive"}, + }, + } +} + +// A timeline exposes SEVERAL textTemplate properties, which is what the single +// widget-wide `contentparams:` cannot address: each one needs its own binding. +func timelineDef() *WidgetDefinition { + return &WidgetDefinition{ + WidgetID: "com.mendix.widget.custom.timeline.Timeline", + MDLName: "TIMELINE", + PropertyMappings: []PropertyMapping{ + {PropertyKey: "title", Source: "TextTemplate", Operation: "texttemplate"}, + {PropertyKey: "description", Source: "TextTemplate", Operation: "texttemplate"}, + }, + } +} + +func TestIssue575_TextTemplateTakesItsOwnParamsCompanion(t *testing.T) { + def := treeNodeDef() + e := &PluggableWidgetEngine{pageBuilder: &pageBuilder{entityContext: "Sales.Customer"}, currentDef: def} + w := &ast.WidgetV3{Name: "tn", Properties: map[string]any{ + "headerType": "text", + "headerCaption": "{1}", + "headerCaptionParams": []ast.ParamAssignmentV3{{Index: 1, Value: "Name"}}, + }} + + ctx, err := e.resolveMapping(def.PropertyMappings[0], w) + if err != nil { + t.Fatal(err) + } + if ctx.PrimitiveVal != "{1}" { + t.Fatalf("PrimitiveVal = %q, want %q", ctx.PrimitiveVal, "{1}") + } + if len(ctx.ClientParams) != 1 { + t.Fatalf("ClientParams = %d, want 1 — a {1} with no parameter renders nothing "+ + "and is CE0720 at build time", len(ctx.ClientParams)) + } + if got := ctx.ClientParams[0].AttributeRef; got != "Sales.Customer.Name" { + t.Errorf("parameter attribute = %q, want Sales.Customer.Name", got) + } +} + +// Each textTemplate property binds its OWN parameters. The widget-wide +// `contentparams:` cannot express this: one list, several templates. +func TestIssue575_EachTextTemplatePropertyBindsSeparately(t *testing.T) { + def := timelineDef() + e := &PluggableWidgetEngine{pageBuilder: &pageBuilder{entityContext: "Sales.Order"}, currentDef: def} + w := &ast.WidgetV3{Name: "tl", Properties: map[string]any{ + "title": "{1}", + "titleParams": []ast.ParamAssignmentV3{{Index: 1, Value: "OrderNumber"}}, + "description": "{1}", + "descriptionParams": []ast.ParamAssignmentV3{{Index: 1, Value: "Remarks"}}, + }} + + want := map[string]string{"title": "Sales.Order.OrderNumber", "description": "Sales.Order.Remarks"} + for _, m := range def.PropertyMappings { + ctx, err := e.resolveMapping(m, w) + if err != nil { + t.Fatal(err) + } + if len(ctx.ClientParams) != 1 { + t.Fatalf("%s: ClientParams = %d, want 1", m.PropertyKey, len(ctx.ClientParams)) + } + if got := ctx.ClientParams[0].AttributeRef; got != want[m.PropertyKey] { + t.Errorf("%s: parameter attribute = %q, want %q", m.PropertyKey, got, want[m.PropertyKey]) + } + } +} + +// The widget-wide `contentparams:` keeps working for single-template widgets. +func TestIssue575_ContentParamsStillTheFallback(t *testing.T) { + def := treeNodeDef() + e := &PluggableWidgetEngine{pageBuilder: &pageBuilder{entityContext: "Sales.Customer"}, currentDef: def} + w := &ast.WidgetV3{Name: "tn", Properties: map[string]any{ + "headerCaption": "{1}", + "ContentParams": []ast.ParamAssignmentV3{{Index: 1, Value: "Name"}}, + }} + ctx, err := e.resolveMapping(def.PropertyMappings[0], w) + if err != nil { + t.Fatal(err) + } + if len(ctx.ClientParams) != 1 { + t.Fatalf("ClientParams = %d, want 1 — contentparams must remain the fallback", len(ctx.ClientParams)) + } +} + +func TestIssue575_ParamsCompanionIsNotAnUnknownProperty(t *testing.T) { + reg := &WidgetRegistry{byMDLName: map[string]*WidgetDefinition{"TREENODE": treeNodeDef()}} + w := &ast.WidgetV3{Name: "tn", Type: "treenode", Properties: map[string]any{ + "headerCaption": "{1}", + "headerCaptionParams": []ast.ParamAssignmentV3{{Index: 1, Value: "Name"}}, + }} + got := ruleIDs(validatePluggableWidgetProperties(w, reg, "page P")) + if msg, ok := got["MDL-WIDGET01"]; ok { + t.Errorf("headerCaptionParams reported as unknown: %s", msg) + } +} + +// Control: the allowance is per text-template property, not a blanket "*Params". +func TestIssue575_ParamsCompanionForANonTemplatePropertyIsStillUnknown(t *testing.T) { + reg := &WidgetRegistry{byMDLName: map[string]*WidgetDefinition{"TREENODE": treeNodeDef()}} + w := &ast.WidgetV3{Name: "tn", Type: "treenode", Properties: map[string]any{ + "headerTypeParams": []ast.ParamAssignmentV3{{Index: 1, Value: "Name"}}, + }} + got := ruleIDs(validatePluggableWidgetProperties(w, reg, "page P")) + if _, ok := got["MDL-WIDGET01"]; !ok { + t.Errorf("a Params companion for a non-template property must stay MDL-WIDGET01; got %v", keysOf(got)) + } +} + +// Parameters with no `{N}` placeholder to fill are dropped on write — the same +// silent class MDL-WIDGET21 already reports for the widget-wide contentparams. +func TestIssue575_OrphanedParamsCompanionIsReported(t *testing.T) { + reg := &WidgetRegistry{byMDLName: map[string]*WidgetDefinition{"TREENODE": treeNodeDef()}} + w := &ast.WidgetV3{Name: "tn", Type: "treenode", Properties: map[string]any{ + "headerCaption": "Name", + "headerCaptionParams": []ast.ParamAssignmentV3{{Index: 1, Value: "Name"}}, + }} + found := false + for _, v := range validateWidgetTree([]*ast.WidgetV3{w}, reg, "page P") { + if v.RuleID == "MDL-WIDGET21" { + found = true + } + } + if !found { + t.Fatal("headerCaptionParams with no {1} in headerCaption must be reported (MDL-WIDGET21)") + } +} + +// The built-in pluggable Image is the shape reachable without a Marketplace +// widget: two text-template properties (`imageUrl`, `alternativeText`), which is +// exactly what one widget-wide `contentparams:` cannot address. Its mappings are +// named by SOURCE (`ImageUrl`), so they resolve through a different branch of +// resolveMapping than the TreeNode's — and that branch carried no parameters at +// all, contentparams included. +func TestIssue575_ImageBindsEachTemplateSeparately(t *testing.T) { + reg := LoadWidgetRegistry("") + if reg == nil { + t.Fatal("built-in widget registry not available") + } + def := reg.byMDLName["IMAGE"] + if def == nil { + t.Fatal("no built-in IMAGE definition") + } + + for _, spelling := range []struct { + name string + url, urlParam string + alt, altParam string + }{ + {"source names", "ImageUrl", "ImageUrlParams", "AlternativeText", "AlternativeTextParams"}, + {"schema keys", "imageUrl", "imageUrlParams", "alternativeText", "alternativeTextParams"}, + } { + t.Run(spelling.name, func(t *testing.T) { + e := &PluggableWidgetEngine{pageBuilder: &pageBuilder{entityContext: "Sales.Product"}, currentDef: def} + w := &ast.WidgetV3{Name: "img", Type: "image", Properties: map[string]any{ + spelling.url: "{1}", + spelling.urlParam: []ast.ParamAssignmentV3{{Index: 1, Value: "PhotoUrl"}}, + spelling.alt: "{1}", + spelling.altParam: []ast.ParamAssignmentV3{{Index: 1, Value: "Name"}}, + }} + + want := map[string]string{"imageUrl": "Sales.Product.PhotoUrl", "alternativeText": "Sales.Product.Name"} + for _, m := range def.PropertyMappings { + if m.Operation != "texttemplate" { + continue + } + ctx, err := e.resolveMapping(m, w) + if err != nil { + t.Fatal(err) + } + if len(ctx.ClientParams) != 1 { + t.Fatalf("%s: ClientParams = %d, want 1", m.PropertyKey, len(ctx.ClientParams)) + } + if got := ctx.ClientParams[0].AttributeRef; got != want[m.PropertyKey] { + t.Errorf("%s: parameter attribute = %q, want %q", m.PropertyKey, got, want[m.PropertyKey]) + } + } + + if got := ruleIDs(validatePluggableWidgetProperties(w, reg, "page P")); got["MDL-WIDGET01"] != "" { + t.Errorf("params companion reported as unknown: %s", got["MDL-WIDGET01"]) + } + }) + } +} + +// DESCRIBE must emit the companion, or the round trip unbinds what it copied: a +// bound `ImageUrl: '{1}'` described back without its parameter re-executes into +// CE0720 ("Place holder index 1 is greater than 0, the number of parameter(s)"). +func TestIssue575_DescribeEmitsTheParamsCompanion(t *testing.T) { + w := rawWidget{ + Name: "cardImage", + ImageType: "imageUrl", + ImageUrl: "{1}", + ImageUrlParams: []string{"Bug575.Product.PictureUrl"}, + AlternativeText: "{1}", + AlternativeTextParams: []string{"Bug575.Product.Name"}, + } + got := strings.Join(describeImageWidgetProps(w), ", ") + for _, want := range []string{ + "ImageUrlParams: [{1} = Bug575.Product.PictureUrl]", + "AlternativeTextParams: [{1} = Bug575.Product.Name]", + } { + if !strings.Contains(got, want) { + t.Errorf("describe output missing %q; got: %s", want, got) + } + } +} + +// CONTROL: an unbound template emits no companion. An empty `[]` would not parse +// back, and a companion on a literal caption is what MDL-WIDGET21 warns about. +func TestIssue575_DescribeOmitsAnAbsentParamsCompanion(t *testing.T) { + w := rawWidget{Name: "cardImage", ImageType: "imageUrl", ImageUrl: "https://example.com/x.png"} + for _, p := range describeImageWidgetProps(w) { + if strings.HasSuffix(strings.SplitN(p, ":", 2)[0], "Params") { + t.Errorf("emitted %q for a template with no parameters", p) + } + } +} From 240c5aa6680d719354576cf21a9247c4a7324b33 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 21 Sep 2026 18:20:53 +0000 Subject: [PATCH 09/15] fix(executor): keep validation rules `create or modify entity` cannot spell MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The entity body authors exactly two rule types — Required from `not null` and Unique from `unique`. RegEx and Range are authored by a separate `create validation rule` statement and have no spelling in the entity body at all, but the merge listed ValidationRules among the fields the statement is authoritative about, so every rewrite overwrote the whole list and deleted them. The visible symptom was churn rather than loss, because the scripts that hit it re-create the rule in the next statement: an idempotent script reported `Modified entity: X` + `Created RegEx validation rule on X.Attr` on every run forever. Measured on the v1 fixture, the unit came back the same size (1,368 bytes) with 62 bytes differing, all of them element $IDs under ValidationRules/1. Splitting the two statements says which one moves — the entity rewrite ALONE takes the unit from 1,368 to 947 bytes. The loss is the real defect, and it is silent. Measured on a real 11.14.0 project, `create or modify entity` adding one attribute deletes the entity's RegEx rule and `mx check` reports "The app contains: 0 errors" — a re-created, or absent, rule is a valid model either way. Carrying an unreadable rule (MaxLength, EqualsTo) matters for a second reason: the model has no payload type for either, so it is UpdateEntity's guard-don't-drop refusal that must see it. Dropping it here skipped the guard entirely and turned "mxcli will not rewrite this entity" into a silent loss. A rule is dropped only on positive evidence that this rewrite removed its attribute — the stored entity owned one of that name and the rebuilt one does not. "Not among the declared attributes" is the wrong predicate and wrong in the direction that loses data, since an entity's rules can name inherited members that never appear in its own Attributes list (the trap pruneMemberAccessesForDroppedAttributes already paid for). ValidationRules is now in neither category of the field drift guard, so entityFieldsMergedWithStored is the third, and listing a field there is the deliberate act of taking it out of that guard's reach. `create validation rule` also reports through ReportMutation now, so a statement whose write was elided says Unchanged like every other one. Refs: ako/mxcli#556 Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01QWZjWZQhk3cNQCcy1Z2xzH --- mdl/executor/cmd_entities.go | 150 ++++++++++++++++- mdl/executor/cmd_validationrules.go | 7 +- mdl/executor/entity_modify_preserves_test.go | 164 +++++++++++++++++++ 3 files changed, 318 insertions(+), 3 deletions(-) diff --git a/mdl/executor/cmd_entities.go b/mdl/executor/cmd_entities.go index 74fca5ff2..2c22aa5f9 100644 --- a/mdl/executor/cmd_entities.go +++ b/mdl/executor/cmd_entities.go @@ -484,7 +484,7 @@ func mergeDeclaredOntoStoredEntity(stored, declared *domainmodel.Entity, s *ast. merged.Persistable = declared.Persistable merged.Location = declared.Location merged.Attributes = declared.Attributes - merged.ValidationRules = declared.ValidationRules + merged.ValidationRules = mergeValidationRules(stored, declared) merged.Indexes = declared.Indexes merged.EventHandlers = declared.EventHandlers merged.HasOwner = declared.HasOwner @@ -524,7 +524,6 @@ var entityFieldsDeclaredByStatement = map[string]bool{ "Location": true, "Persistable": true, "Attributes": true, - "ValidationRules": true, "Indexes": true, "EventHandlers": true, "HasOwner": true, @@ -537,6 +536,153 @@ var entityFieldsDeclaredByStatement = map[string]bool{ // ContainerID and the embedded BaseElement identify the stored element, so // they are preserved. `entity.ID = existingEntity.ID` at the call site says // the same thing about the ID; the merge is where it now comes from. + // + // ValidationRules is in neither group — see entityFieldsMergedWithStored. +} + +// entityFieldsMergedWithStored names the fields the statement is authoritative +// about only IN PART, so neither "take the declared value" nor "keep the stored +// one" is right and the merge has to decide per element. +// +// There is exactly one, and it earned its place: the entity body can spell +// `not null` and `unique` and nothing else, so it declares Required and Unique +// rules and says nothing at all about RegEx, Range, MaxLength or EqualsTo — +// which are authored by `create validation rule`, a separate statement. +// Overwriting the whole list deleted those on every rewrite (ako/mxcli#556). +// +// A field listed here is exempt from the value check in +// TestMergeDeclaredOntoStoredEntity_EveryFieldHasADecision, so adding one means +// writing the tests for its semantics by hand. +var entityFieldsMergedWithStored = map[string]bool{ + "ValidationRules": true, +} + +// mergeValidationRules keeps the rules the entity body cannot express and lets +// the statement own the ones it can. +// +// `create [or modify] entity` authors exactly two rule types — Required from +// `not null` and Unique from `unique` — so for those an omission is a removal, +// the same contract an omitted attribute has. Every other type (RegEx, Range, +// and the MaxLength/EqualsTo that do not survive the read at all) has no +// spelling in the statement, so an omission carries no meaning and the stored +// rule stands. +// +// Dropping them was not merely lossy, it churned: the following +// `create validation rule` statement put an identically-shaped rule back under +// four fresh element identities, so an idempotent script reported +// `Modified entity: …` forever and the unit never came back byte-identical +// (ako/mxcli#556, measured at 62 differing bytes in a 1,368-byte unit). +// +// Carrying a MaxLength or EqualsTo rule matters for a second reason: the model +// carries no payload for either, so it is UpdateEntity's refusal — not this +// function — that must see it. Dropping it here skipped the guard entirely and +// turned "mxcli will not rewrite this entity" into a silent loss of the +// constraint that no build reports (guard-don't-drop, ADR-0005). +// +// Stored order is kept, with a re-declared Required/Unique rule taking the +// stored one's position, so the rewrite is a minimal diff rather than a +// reshuffle. +func mergeValidationRules(stored, declared *domainmodel.Entity) []*domainmodel.ValidationRule { + attrNameByID := make(map[model.ID]string, len(declared.Attributes)) + declaredAttrNames := make(map[string]bool, len(declared.Attributes)) + for _, a := range declared.Attributes { + if a == nil { + continue + } + attrNameByID[a.ID] = a.Name + declaredAttrNames[a.Name] = true + } + storedAttrNames := make(map[string]bool, len(stored.Attributes)) + for _, a := range stored.Attributes { + if a != nil { + storedAttrNames[a.Name] = true + } + } + + declaredBySlot := make(map[string]*domainmodel.ValidationRule, len(declared.ValidationRules)) + for _, vr := range declared.ValidationRules { + if vr == nil { + continue + } + declaredBySlot[validationRuleSlot(vr, attrNameByID)] = vr + } + + out := make([]*domainmodel.ValidationRule, 0, len(stored.ValidationRules)+len(declared.ValidationRules)) + taken := make(map[*domainmodel.ValidationRule]bool, len(declared.ValidationRules)) + for _, vr := range stored.ValidationRules { + if vr == nil { + continue + } + // A rule may only be dropped on POSITIVE evidence that this rewrite + // removed its attribute: the stored entity owned one of that name and the + // rebuilt one does not. "Not among the declared attributes" is the wrong + // predicate and wrong in the direction that loses data — an entity's rules + // can name INHERITED members, which never appear in its own Attributes + // list, and an entity that owns no attributes at all would lose every rule. + // (The same trap cost a day in pruneMemberAccessesForDroppedAttributes.) + name := validationRuleAttributeName(vr, attrNameByID) + if storedAttrNames[name] && !declaredAttrNames[name] { + continue + } + if !entityBodyDeclaresRuleType(vr.Type) { + out = append(out, vr) + continue + } + if d, ok := declaredBySlot[validationRuleSlot(vr, attrNameByID)]; ok && !taken[d] { + taken[d] = true + out = append(out, d) + } + } + for _, vr := range declared.ValidationRules { + if vr == nil || taken[vr] { + continue + } + out = append(out, vr) + } + if len(out) == 0 { + return nil + } + return out +} + +// entityBodyDeclaresRuleType reports whether `create [or modify] entity` has a +// spelling for this rule type, and so is authoritative about its presence. +// +// The empty string is Required: a rule whose RuleInfo the reader could not name +// is written back as RequiredRuleInfo (ruleInfoToGen's `case "Required", ""`), +// so treating it as anything else here would let the same rule be both carried +// and re-declared. +func entityBodyDeclaresRuleType(t string) bool { + return t == "Required" || t == "Unique" || t == "" +} + +// validationRuleSlot identifies the (attribute, rule type) pair a rule occupies. +// Mendix allows one rule of a type per attribute, so this is what decides that a +// declared rule REPLACES a stored one rather than joining it. +func validationRuleSlot(vr *domainmodel.ValidationRule, attrNameByID map[model.ID]string) string { + t := vr.Type + if t == "" { + t = "Required" + } + return validationRuleAttributeName(vr, attrNameByID) + "\x00" + t +} + +// validationRuleAttributeName resolves the attribute a rule constrains, across +// the two spellings the field carries. +// +// The modelsdk reader puts a QUALIFIED NAME in AttributeID (validationRuleFromGen +// — the writer accepts either and this is the lossless one), while a rule the +// executor just built from the statement holds a real attribute ID. Reading only +// one would make this function a no-op on half its inputs. +func validationRuleAttributeName(vr *domainmodel.ValidationRule, attrNameByID map[model.ID]string) string { + s := string(vr.AttributeID) + if i := strings.LastIndex(s, "."); i >= 0 { + return s[i+1:] + } + if n, ok := attrNameByID[vr.AttributeID]; ok { + return n + } + return s } // pruneMemberAccessesForDroppedAttributes removes the member entries of the diff --git a/mdl/executor/cmd_validationrules.go b/mdl/executor/cmd_validationrules.go index b45f00f42..1fcf142e1 100644 --- a/mdl/executor/cmd_validationrules.go +++ b/mdl/executor/cmd_validationrules.go @@ -85,7 +85,12 @@ func execCreateValidationRule(ctx *ExecContext, s *ast.CreateValidationRuleStmt) invalidateHierarchy(ctx) invalidateDomainModelsCache(ctx) - fmt.Fprintf(ctx.Output, "Created %s validation rule on %s\n", ruleType, attrQN) + // Through ReportMutation, not Fprintf: re-running a script that already + // matches the project offers the write and has it elided, and reporting + // "Created" there is how ako/mxcli#556 read as churn long after the bytes + // had stopped moving. The verb is only downgraded on positive evidence — + // writes offered, none landed. + ctx.ReportMutation("Created", "%s validation rule on %s", ruleType, attrQN) return nil } diff --git a/mdl/executor/entity_modify_preserves_test.go b/mdl/executor/entity_modify_preserves_test.go index 0626f2186..31e7bb113 100644 --- a/mdl/executor/entity_modify_preserves_test.go +++ b/mdl/executor/entity_modify_preserves_test.go @@ -246,6 +246,14 @@ func TestMergeDeclaredOntoStoredEntity_EveryFieldHasADecision(t *testing.T) { if name == "BaseElement" || name == "ContainerID" { continue // element identity; preserved, and re-asserted at the call site } + if entityFieldsMergedWithStored[name] { + // Neither side owns the whole field, so there is no value to compare + // against. The semantics are covered by the named tests above + // (KeepsRulesTheEntityBodyCannotSpell and its three controls), and + // listing a field here is the deliberate act of taking it out of this + // guard's reach. + continue + } got, fromStored, fromDeclared := mv.Field(i), sv.Field(i), dv.Field(i) wantDeclared := entityFieldsDeclaredByStatement[name] @@ -295,3 +303,159 @@ func fillDistinctly(v reflect.Value, seed int) { } } } + +// --------------------------------------------------------------------------- +// Validation rules the entity body has no words for (ako/mxcli#556) +// --------------------------------------------------------------------------- + +// The reported symptom: re-running an idempotent script reports +// `Modified entity: …` + `Created RegEx validation rule on …` on EVERY run, and +// the unit comes back the same size with a handful of 16-byte runs changed. +// +// MEASURED on the v1 fixture, two statements that say nothing new: +// +// create or modify entity Demo.Widget ( SerialNumber: String(100) ); +// create validation rule for Demo.Widget.SerialNumber regex Demo.SerialPattern ...; +// +// unit 1368 bytes before and after, 62 bytes differ, at +// ValidationRules/1/$ID, .../Message/$ID, .../Message/Items/1/$ID, .../RuleInfo/$ID +// +// Splitting the two statements says which one moves: the entity rewrite alone +// takes the unit from 1368 to 947 bytes — it DELETES the rule — and the +// validation-rule statement then puts an identically-shaped one back under four +// fresh identities. The validation-rule statement on its own is already +// idempotent to the byte (canon transplants the ids back), so the churn is +// entirely the drop. +// +// The drop is the bug, not the re-mint. `create [or modify] entity` can only +// spell Required (`not null`) and Unique (`unique`); RegEx and Range have no +// spelling in the entity body at all, so an omission carries no meaning — the +// same reasoning that already preserves access rules here. +func storedEntityWithARegexRule() *domainmodel.Entity { + e := storedEntityWithARule() + e.ValidationRules = []*domainmodel.ValidationRule{{ + BaseElement: model.BaseElement{ID: "vr-regex"}, + AttributeID: "Fx.Probe.Code", + Type: "RegEx", + Rule: &domainmodel.RegexValidationRuleInfo{RegularExpressionQualifiedName: "Fx.SerialPattern"}, + }} + return e +} + +func TestMergeDeclaredOntoStoredEntity_KeepsRulesTheEntityBodyCannotSpell(t *testing.T) { + stored := storedEntityWithARegexRule() + merged := mergeDeclaredOntoStoredEntity(stored, declaredEntity("Name", "Code"), &ast.CreateEntityStmt{}) + + if len(merged.ValidationRules) != 1 || merged.ValidationRules[0].Type != "RegEx" { + t.Fatalf("the rewrite kept %+v, want the stored RegEx rule — `create or modify entity` "+ + "has no spelling for one, so dropping it makes every re-run re-create it under "+ + "fresh identities (ako/mxcli#556)", merged.ValidationRules) + } + if merged.ValidationRules[0].ID != "vr-regex" { + t.Errorf("the carried rule lost its identity: %q", merged.ValidationRules[0].ID) + } +} + +// A MaxLength or EqualsTo rule does not survive the READ (the model carries no +// payload type for it), so carrying it is what lets UpdateEntity REFUSE the +// rewrite. Dropping it here is worse than refusing: the constraint is gone and +// the build still passes. +func TestMergeDeclaredOntoStoredEntity_KeepsAnUnreadableRuleSoTheWriteCanRefuse(t *testing.T) { + stored := storedEntityWithARule() + stored.ValidationRules = []*domainmodel.ValidationRule{{ + BaseElement: model.BaseElement{ID: "vr-maxlength"}, + AttributeID: "Fx.Probe.Code", + Type: "MaxLength", + }} + merged := mergeDeclaredOntoStoredEntity(stored, declaredEntity("Name", "Code"), &ast.CreateEntityStmt{}) + if len(merged.ValidationRules) != 1 { + t.Fatalf("a MaxLength rule was silently dropped by the rewrite (%d rules kept) — "+ + "UpdateEntity's guard-don't-drop refusal never even sees it", len(merged.ValidationRules)) + } +} + +// The other half of the contract, and the control that a merge broken into +// "keep everything stored" would fail: Required and Unique DO have a spelling, +// so the statement stays authoritative about them. Omitting `not null` removes +// the Required rule. +func TestMergeDeclaredOntoStoredEntity_StatementStillOwnsRequiredAndUnique(t *testing.T) { + stored := storedEntityWithARegexRule() + stored.ValidationRules = append(stored.ValidationRules, &domainmodel.ValidationRule{ + BaseElement: model.BaseElement{ID: "vr-required"}, + AttributeID: "Fx.Probe.Name", + Type: "Required", + }) + + merged := mergeDeclaredOntoStoredEntity(stored, declaredEntity("Name", "Code"), &ast.CreateEntityStmt{}) + + for _, vr := range merged.ValidationRules { + if vr.Type == "Required" { + t.Fatalf("omitting `not null` left the Required rule in place — the statement is " + + "authoritative about the rule types it can spell") + } + } + if len(merged.ValidationRules) != 1 { + t.Fatalf("kept %d rules, want only the RegEx one", len(merged.ValidationRules)) + } +} + +// A re-declared Required rule is taken from the STATEMENT, not carried — its +// error message is part of what the statement says. +func TestMergeDeclaredOntoStoredEntity_RedeclaredRequiredComesFromTheStatement(t *testing.T) { + stored := storedEntityWithARule() + stored.ValidationRules = []*domainmodel.ValidationRule{{ + BaseElement: model.BaseElement{ID: "vr-stored"}, + AttributeID: "Fx.Probe.Name", + Type: "Required", + ErrorMessage: &model.Text{Translations: map[string]string{"en_US": "old message"}}, + }} + declared := declaredEntity("Name", "Code") + declared.Attributes[0].ID = "attr-name" + declared.ValidationRules = []*domainmodel.ValidationRule{{ + BaseElement: model.BaseElement{ID: "vr-declared"}, + AttributeID: "attr-name", + Type: "Required", + ErrorMessage: &model.Text{Translations: map[string]string{"en_US": "new message"}}, + }} + + merged := mergeDeclaredOntoStoredEntity(stored, declared, &ast.CreateEntityStmt{}) + if len(merged.ValidationRules) != 1 { + t.Fatalf("kept %d rules, want 1", len(merged.ValidationRules)) + } + if got := merged.ValidationRules[0].ErrorMessage.Translations["en_US"]; got != "new message" { + t.Errorf("the re-declared Required rule kept the stored message %q", got) + } +} + +// A rule whose attribute this rewrite REMOVED must go with it, or it outlives +// the attribute it constrains — CE1613, the same class reconcileDroppedIndexes +// and pruneMemberAccessesForDroppedAttributes exist to prevent. +func TestMergeDeclaredOntoStoredEntity_DropsARuleWhoseAttributeWentAway(t *testing.T) { + stored := storedEntityWithARegexRule() + merged := mergeDeclaredOntoStoredEntity(stored, declaredEntity("Name"), &ast.CreateEntityStmt{}) + if len(merged.ValidationRules) != 0 { + t.Errorf("the rule on the dropped Code attribute survived: %+v", merged.ValidationRules) + } +} + +// CONTROL for that, and the specialisation trap pruneMemberAccessesForDroppedAttributes +// already paid for: a rule naming a member the entity does not own itself +// (inherited, or an attribute the reader spelled differently) is kept unless the +// STORED entity owned an attribute of that name and this rewrite removed it. +func TestMergeDeclaredOntoStoredEntity_KeepsARuleNamingAMemberTheEntityDoesNotOwn(t *testing.T) { + stored := &domainmodel.Entity{ + Name: "ExportDocument", + GeneralizationRef: "System.FileDocument", + ValidationRules: []*domainmodel.ValidationRule{{ + BaseElement: model.BaseElement{ID: "vr-inherited"}, + AttributeID: "System.FileDocument.Name", + Type: "RegEx", + Rule: &domainmodel.RegexValidationRuleInfo{RegularExpressionQualifiedName: "Fx.P"}, + }}, + } + declared := &domainmodel.Entity{Name: "ExportDocument", GeneralizationRef: "System.FileDocument"} + merged := mergeDeclaredOntoStoredEntity(stored, declared, &ast.CreateEntityStmt{}) + if len(merged.ValidationRules) != 1 { + t.Errorf("a rule on an inherited member was dropped by an entity that owns no attributes") + } +} From 3ed95a4337cf98d506468356a9fd2ef8a185fc66 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 21 Sep 2026 18:21:11 +0000 Subject: [PATCH 10/15] fix(mpr): reconcile a unit re-inserted under the ID of one just deleted MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Several `create or modify` handlers are implemented as delete + create under the preserved unit ID rather than as an update. canon.Reconcile lives in updateUnit, so those writes never reached it and the rebuild's fresh element $IDs went straight to disk. Measured on a real 11.14.0 project, two consecutive identical runs of one `create or modify rest client` statement: the 1,128-byte unit came back the same size with 143 bytes differing, in 9 runs, every one an element $ID — /BaseUrl/$ID, /Operations/1/$ID, its Method, Path, Headers, Parameters and ResponseHandling. Feeding those two stored versions straight to canon.TransplantIDs makes them byte-identical, so the policy was never wrong here. It was never called. This is CLAUDE.md's "a new write path means wiring it to canon.Reconcile" in the shape that hides: the new path is not a new function, it is insertUnit reached by way of deleteUnit. deleteUnit now remembers what it removed (bounded at 1024 entries, past which the carry degrades to today's behaviour) and insertUnit reconciles a re-insert against it. The carry cannot elide — the row and the file are already gone — so a recreate that put back exactly what it removed also restores _Transaction.LastTransactionID, which both the delete and the insert bumped. That second half is load-bearing: with only the byte carry, every row of the SQLite db was logically identical and the .mpr still showed as modified, because that one UUID had moved. `create or modify rest client` is also fixed at its own layer, which is better than relying on the safety net: when the statement leaves the service where it is, it now calls UpdateConsumedRestService and the write is elided outright. Delete+create survives only for a folder move, which lives in the unit's row rather than in its contents. Deferring the delete to the write point is also strictly safer than doing it while scanning — building the service from the AST can fail, and the old code had already deleted the stored one by then. Measured end to end on 11.14.0, three consecutive re-runs of an identical statement: `Modified rest client` and 2 changed files every time before, `Unchanged rest client` and a clean tree after. A real edit (changing an operation's path) still lands, still reports Modified, and still builds at 0 errors. Refs: ako/mxcli#556 Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01QWZjWZQhk3cNQCcy1Z2xzH --- CLAUDE.md | 22 +- mdl/executor/cmd_rest_clients.go | 61 +++-- mdl/executor/cmd_rest_openapi_mock_test.go | 26 +- modelsdk/mpr/writer_core.go | 117 +++++++++ modelsdk/mpr/writer_recreate_identity_test.go | 243 ++++++++++++++++++ 5 files changed, 443 insertions(+), 26 deletions(-) create mode 100644 modelsdk/mpr/writer_recreate_identity_test.go diff --git a/CLAUDE.md b/CLAUDE.md index 63a1ff71f..36b6bee63 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -335,10 +335,24 @@ is stored ([ADR-0008](docs/13-decisions/0008-identity-and-idempotence.md)). The comparison is on a canonical form — every element `$ID` replaced by its index in a containment walk — because a rebuild mints a fresh random `$ID` per sub-element, so comparing bytes would skip nothing. The policy lives in `modelsdk/canon` -(`Reconcile`) and is called at the single write choke point of **both** engines: -`modelsdk/mpr/writer_core.go` (`updateUnit` *and* `WriteTransaction.WriteUnit` — -`codec.Store` reaches storage through the latter). There is one engine, so that is -the whole list. +(`Reconcile`) and is called at every write choke point in +`modelsdk/mpr/writer_core.go`: `updateUnit`, `WriteTransaction.WriteUnit` +(`codec.Store` reaches storage through this one) and — since ako/mxcli#556 — +`insertUnit`, for the case below. + +**A delete followed by an insert is a write path too**, and it is the one that +hides. Several `create or modify` handlers are implemented as delete + create +under the preserved unit ID rather than as an update, and an insert has nothing +stored to reconcile against, so the rebuild's fresh `$ID`s went straight to disk: +`create or modify rest client` rewrote 9 element `$ID`s in a 1,128-byte unit on +every run, forever. `deleteUnit` now remembers what it removed and `insertUnit` +reconciles a re-insert against it (`carryIdentityFromRemovedUnit`). That carry +cannot *elide* — the row and the file are already gone — so a no-op recreate also +restores `_Transaction.LastTransactionID`, which both the delete and the insert +bumped; without it the `.mpr` still showed as modified after every `.mxunit` had +gone quiet. Prefer an in-place update where the handler can do one: the REST +client's own fix is to call `UpdateConsumedRestService` and keep delete+create +only for a folder move, which lives in the unit's row rather than its contents. When something *has* changed, `Reconcile` still does not let the rebuild's fresh `$ID`s reach disk: `canon.TransplantIDs` matches the incoming document against the diff --git a/mdl/executor/cmd_rest_clients.go b/mdl/executor/cmd_rest_clients.go index 664ee5ece..bb756d189 100644 --- a/mdl/executor/cmd_rest_clients.go +++ b/mdl/executor/cmd_rest_clients.go @@ -374,9 +374,9 @@ func createRestClient(ctx *ExecContext, stmt *ast.CreateRestClientStmt) error { } var preservedID model.ID - // Where the service currently sits. A rest client is rewritten as - // delete+create, so its container is re-applied on every statement and an - // unset value files a foldered service back into the module root (#932). + // Where the service currently sits. Its container is re-applied on every + // statement, and an unset value files a foldered service back into the + // module root (#932). var preservedContainerID model.ID var preservedDocumentation string wasModified := false @@ -388,13 +388,10 @@ func createRestClient(ctx *ExecContext, stmt *ast.CreateRestClientStmt) error { // Preserve the existing ID so SEND REST REQUEST references stay valid after replace. preservedID = existing.ID preservedContainerID = existing.ContainerID - // The rewrite is a delete+create, so the stored documentation - // has to be captured before the delete or it is gone (#1018). + // A rewrite that carried no doc comment keeps the stored one, so + // it has to be read before anything replaces the unit (#1018). preservedDocumentation = existing.Documentation wasModified = true - if err := ctx.Backend.DeleteConsumedRestService(existing.ID); err != nil { - return mdlerrors.NewBackend("delete existing rest client", err) - } } else { return mdlerrors.NewAlreadyExistsMsg("rest client", moduleName+"."+stmt.Name.Name, fmt.Sprintf("rest client already exists: %s.%s (use create or modify to overwrite)", moduleName, stmt.Name.Name)) } @@ -478,7 +475,7 @@ func createRestClient(ctx *ExecContext, stmt *ast.CreateRestClientStmt) error { } // Write to project - if err := ctx.Backend.CreateConsumedRestService(svc); err != nil { + if err := saveConsumedRestService(ctx, svc, preservedID, preservedContainerID); err != nil { return mdlerrors.NewBackend("create rest client", err) } @@ -486,10 +483,42 @@ func createRestClient(ctx *ExecContext, stmt *ast.CreateRestClientStmt) error { if wasModified { verb = "Modified" } - fmt.Fprintf(ctx.Output, "%s rest client: %s.%s (%d operations)\n", verb, moduleName, stmt.Name.Name, len(svc.Operations)) + ctx.ReportMutation(verb, "rest client: %s.%s (%d operations)", + moduleName, stmt.Name.Name, len(svc.Operations)) return nil } +// saveConsumedRestService writes a consumed REST client, rewriting the stored +// unit IN PLACE when the statement is a modification that leaves it where it is. +// +// The obvious implementation — delete the old unit, insert a new one under the +// preserved ID — is what made `create or modify rest client` churn. An insert +// does not pass through the storage layer's reconciliation, so an identical +// re-run rewrote nine element $IDs in the document, dirtied the .mpr, and did it +// again on every run (ako/mxcli#556: 143 bytes of a 1,128-byte unit, all of them +// $IDs). An update does pass through it, and is elided outright when the +// rebuild means the same thing as what is stored. +// +// Delete+create survives for the one thing an in-place rewrite cannot express: +// moving the document to another folder, which lives in the unit's ROW rather +// than in its contents. +// +// Deferring the delete to here is also strictly safer than doing it while +// scanning: building the service from the AST can fail, and the old code had +// already deleted the stored one by then. +func saveConsumedRestService(ctx *ExecContext, svc *model.ConsumedRestService, preservedID, preservedContainerID model.ID) error { + if preservedID == "" { + return ctx.Backend.CreateConsumedRestService(svc) + } + if svc.ContainerID == preservedContainerID { + return ctx.Backend.UpdateConsumedRestService(svc) + } + if err := ctx.Backend.DeleteConsumedRestService(preservedID); err != nil { + return fmt.Errorf("delete existing rest client: %w", err) + } + return ctx.Backend.CreateConsumedRestService(svc) +} + // buildRestClientOperation converts an AST RestOperationDef to a model RestClientOperation. func buildRestClientOperation(opDef *ast.RestOperationDef) (*model.RestClientOperation, error) { if err := checkInlineMappingBody(opDef); err != nil { @@ -847,6 +876,7 @@ func createRestClientFromSpec(ctx *ExecContext, stmt *ast.CreateRestClientStmt) return mdlerrors.NewBackend("build hierarchy", err) } openAPIWasModified := false + var openAPIPreservedID, openAPIPreservedContainerID model.ID for _, existing := range existingServices { existModID := h.FindModuleID(existing.ContainerID) existModName := h.GetModuleName(existModID) @@ -860,9 +890,8 @@ func createRestClientFromSpec(ctx *ExecContext, stmt *ast.CreateRestClientStmt) svc.ContainerID = existing.ContainerID } openAPIWasModified = true - if err := ctx.Backend.DeleteConsumedRestService(existing.ID); err != nil { - return mdlerrors.NewBackend("delete existing rest client", err) - } + openAPIPreservedID = existing.ID + openAPIPreservedContainerID = existing.ContainerID } else { return mdlerrors.NewAlreadyExistsMsg("rest client", moduleName+"."+stmt.Name.Name, fmt.Sprintf("rest client already exists: %s.%s (use create or modify to overwrite)", moduleName, stmt.Name.Name)) @@ -870,7 +899,7 @@ func createRestClientFromSpec(ctx *ExecContext, stmt *ast.CreateRestClientStmt) } } - if err := ctx.Backend.CreateConsumedRestService(svc); err != nil { + if err := saveConsumedRestService(ctx, svc, openAPIPreservedID, openAPIPreservedContainerID); err != nil { return mdlerrors.NewBackend("create rest client", err) } @@ -878,8 +907,8 @@ func createRestClientFromSpec(ctx *ExecContext, stmt *ast.CreateRestClientStmt) if openAPIWasModified { openAPIVerb = "Modified" } - fmt.Fprintf(ctx.Output, "%s rest client: %s.%s (%d operations from OpenAPI spec)\n", - openAPIVerb, moduleName, stmt.Name.Name, len(svc.Operations)) + ctx.ReportMutation(openAPIVerb, "rest client: %s.%s (%d operations from OpenAPI spec)", + moduleName, stmt.Name.Name, len(svc.Operations)) return nil } diff --git a/mdl/executor/cmd_rest_openapi_mock_test.go b/mdl/executor/cmd_rest_openapi_mock_test.go index dc83ea165..b611cbb89 100644 --- a/mdl/executor/cmd_rest_openapi_mock_test.go +++ b/mdl/executor/cmd_rest_openapi_mock_test.go @@ -181,6 +181,11 @@ func TestCreateRestClientFromSpec_OrModifyPreservesID(t *testing.T) { created = svc return nil } + var updated *model.ConsumedRestService + mb.UpdateConsumedRestServiceFunc = func(svc *model.ConsumedRestService) error { + updated = svc + return nil + } ctx, buf := newMockCtx(t, withBackend(mb), withHierarchy(h)) stmt := &ast.CreateRestClientStmt{ @@ -190,14 +195,23 @@ func TestCreateRestClientFromSpec_OrModifyPreservesID(t *testing.T) { } assertNoError(t, createRestClient(ctx, stmt)) - if deletedID != existingID { - t.Errorf("expected existing service to be deleted, got deletedID=%v", deletedID) + // A re-import that leaves the service where it is now REWRITES the unit + // rather than deleting and re-inserting it. The insert path skips the + // storage layer's reconciliation, so the old shape re-minted every element + // $ID on every run (ako/mxcli#556); the update path is elided when nothing + // changed. The ID preservation this test was written for is unaffected — it + // is now preservation of the whole unit. + if deletedID != "" { + t.Errorf("the stored service was deleted (%v); a same-folder rewrite must update in place", deletedID) } - if created == nil { - t.Fatal("CreateConsumedRestService was not called") + if created != nil { + t.Errorf("CreateConsumedRestService was called for an existing service") + } + if updated == nil { + t.Fatal("UpdateConsumedRestService was not called") } - if created.ID != existingID { - t.Errorf("expected recreated service to reuse existing ID %v, got %v", existingID, created.ID) + if updated.ID != existingID { + t.Errorf("expected the rewrite to reuse existing ID %v, got %v", existingID, updated.ID) } assertContainsStr(t, buf.String(), "Modified rest client") } diff --git a/modelsdk/mpr/writer_core.go b/modelsdk/mpr/writer_core.go index 9496486dc..43316fd12 100644 --- a/modelsdk/mpr/writer_core.go +++ b/modelsdk/mpr/writer_core.go @@ -51,8 +51,31 @@ type Writer struct { // non-unit writes (generated .java/.js source) to this total. writesOffered int writesLanded int + + // removedUnits holds what this session deleted, keyed by unit ID, so a unit + // RE-INSERTED under the same ID can be reconciled against what it replaced. + // See carryIdentityFromRemovedUnit. + removedUnits map[string]removedUnit +} + +// removedUnit is everything about a deleted unit that a re-insert has to be +// compared against: its bytes, its row, and the project's transaction id at the +// moment it went. The last one is what lets a delete+insert that nets to nothing +// leave the .mpr alone entirely. +type removedUnit struct { + contents []byte + containerID []byte + containmentName string + transactionID string } +// maxRemovedUnitsRemembered caps the delete→insert carry. A script that drops a +// module deletes thousands of units and re-inserts none of them, so the map is +// bounded rather than allowed to hold a whole project in memory. Past the cap +// nothing further is recorded and the carry degrades to the behaviour that +// existed before it — a churned re-insert, not a wrong one. +const maxRemovedUnitsRemembered = 1024 + // WriteStats reports how many unit writes this session offered to storage and // how many were not elided as no-ops. func (w *Writer) WriteStats() (offered, written int) { @@ -512,6 +535,9 @@ func (w *Writer) insertUnit(unitID, containerID, containmentName, unitType strin return fmt.Errorf("invalid container ID (not a valid UUID): %q", containerID) } + contents, restoreTransactionID := w.carryIdentityFromRemovedUnit( + unitID, containerIDBlob, containmentName, contents) + if w.reader.version == MPRVersionV2 { // Get swapped UUID for file path swappedUUID := blobToUUIDSwapped(unitIDBlob) @@ -544,6 +570,7 @@ func (w *Writer) insertUnit(unitID, containerID, containmentName, unitType strin } w.reader.InvalidateCache() w.updateTransactionID() + w.restoreTransactionID(restoreTransactionID) return nil } @@ -656,6 +683,95 @@ func (w *Writer) reconcileWithStored(unitID string, contents []byte, opts ...can return out, unchanged } +// rememberRemovedUnit captures a unit's bytes on the way out, so an insert of +// the same unit ID later in this session can be reconciled against them. +// +// Read before the row and file go, obviously, and best-effort: a unit whose +// bytes cannot be read is simply not remembered, which costs a churned +// re-insert and never a wrong one. +func (w *Writer) rememberRemovedUnit(unitID string) { + if len(w.removedUnits) >= maxRemovedUnitsRemembered { + return + } + contents, err := w.reader.GetRawUnitBytes(unitID) + if err != nil || len(contents) == 0 { + return + } + rec := removedUnit{contents: append([]byte(nil), contents...)} + _ = w.reader.db.QueryRow( + `SELECT ContainerID, ContainmentName FROM Unit WHERE UnitID = ?`, uuidToBlob(unitID), + ).Scan(&rec.containerID, &rec.containmentName) + _ = w.reader.db.QueryRow(`SELECT LastTransactionID FROM _Transaction`).Scan(&rec.transactionID) + + if w.removedUnits == nil { + w.removedUnits = make(map[string]removedUnit) + } + w.removedUnits[unitID] = rec +} + +// carryIdentityFromRemovedUnit applies the shared reconciliation policy to a +// unit being re-inserted under the ID of one this session deleted. +// +// # Why an insert needs this at all +// +// Several `create or modify` handlers are implemented as delete + insert under +// the preserved unit ID rather than as an update — the consumed REST client is +// the measured one (ako/mxcli#556): re-running an identical statement rewrote 9 +// element $IDs in a 1,128-byte unit, on every run, forever. The document was +// never different; canon was simply never asked, because updateUnit is where +// Reconcile is called and a delete+insert does not go through it. +// +// # What it does and does not do +// +// It carries identity, translations and element $IDs exactly as updateUnit +// does, so a re-inserted unit that means the same thing lands byte-identical. +// It does NOT elide: the row and the file are already gone, so something has to +// be written back whatever the bytes say. That asymmetry is the whole reason +// this is a separate function from reconcileWithStored rather than a flag on it. +// +// The write is still counted, and counted as landed only when the bytes moved, +// so ReportMutation can tell a real rewrite from one that restored what it +// removed. +// +// A $Type that changed means the new unit is not the old one; canon's pairing +// refuses to carry anything across that, so nothing special is needed here. +func (w *Writer) carryIdentityFromRemovedUnit( + unitID string, containerID []byte, containmentName string, contents []byte, +) (out []byte, restoreTransactionID string) { + prev, ok := w.removedUnits[unitID] + if !ok { + return contents, "" + } + delete(w.removedUnits, unitID) + + w.writesOffered++ + out, _ = canon.Reconcile(contents, prev.contents) + + // A delete+insert that put back exactly what it removed changed nothing, so + // the project's transaction id — which is how Studio Pro decides there is + // something to re-sync — must not move either. The test is deliberately the + // strict one (identical bytes and an identical row), not canon's "unchanged": + // the file is written back regardless, so the only question worth asking is + // whether what landed is what was there. + if bytes.Equal(out, prev.contents) && + bytes.Equal(containerID, prev.containerID) && + containmentName == prev.containmentName { + return out, prev.transactionID + } + w.writesLanded++ + return out, "" +} + +// restoreTransactionID puts back the transaction id a no-op delete+insert bumped +// twice. Best-effort, like updateTransactionID itself, and a no-op for v1 +// projects and for any insert that was not a recreate. +func (w *Writer) restoreTransactionID(id string) { + if id == "" || w.reader.version != MPRVersionV2 { + return + } + _, _ = w.reader.db.Exec(`UPDATE _Transaction SET LastTransactionID = ?`, id) +} + // UpdateRawUnit saves raw BSON bytes for a unit, bypassing deserialization. // Used by ALTER PAGE to modify the BSON widget tree directly. func (w *Writer) UpdateRawUnit(unitID string, contents []byte) error { @@ -728,6 +844,7 @@ func (w *Writer) deleteUnit(unitID string) error { if unitIDBlob == nil { return fmt.Errorf("invalid unit ID: %s", unitID) } + w.rememberRemovedUnit(unitID) if w.reader.version == MPRVersionV2 { // Get swapped UUID for file path diff --git a/modelsdk/mpr/writer_recreate_identity_test.go b/modelsdk/mpr/writer_recreate_identity_test.go new file mode 100644 index 000000000..b4cc7f506 --- /dev/null +++ b/modelsdk/mpr/writer_recreate_identity_test.go @@ -0,0 +1,243 @@ +// SPDX-License-Identifier: Apache-2.0 + +package mpr + +import ( + "os" + "testing" + + "go.mongodb.org/mongo-driver/v2/bson" +) + +// ako/mxcli#556, the second named instance. `create or modify rest client` is a +// DELETE followed by an INSERT under the preserved unit ID, so the write never +// reaches updateUnit and therefore never reaches canon.Reconcile. Measured on a +// real 11.14.0 project, two consecutive identical re-runs of one statement: +// +// Rest$ConsumedRestService unit, 1,128 bytes before and after +// 143 bytes differ, in 9 runs, every one an element $ID: +// /BaseUrl/$ID, /Operations/1/$ID, .../Method/$ID, .../Path/$ID, +// .../Headers/1/$ID, .../Headers/1/Value/$ID, .../Parameters/1/$ID, +// .../Parameters/1/DataType/$ID, .../ResponseHandling/$ID +// +// Nothing about the document changed — only which UUIDs the rebuild happened to +// mint. Feeding those same two units to canon.TransplantIDs makes them +// byte-identical, so the policy was never wrong here; it simply was not called. +// +// This is CLAUDE.md's second rule about elision, in the shape that is easiest to +// miss: the new write path is not a new function, it is an existing insert +// reached by a delete. + +// recreatedDoc builds a two-element document whose inner $ID differs per call, +// standing in for a rebuild that mints fresh identities for everything under the +// root. The root $ID is fixed, because a recreate preserves the unit ID. +func recreatedDoc(t *testing.T, name, innerID string) []byte { + t.Helper() + b, err := bson.Marshal(bson.D{ + {Key: "$Type", Value: "Rest$ConsumedRestService"}, + {Key: "$ID", Value: bson.Binary{Subtype: 0x00, Data: uuidToBlob("11111111-1111-1111-1111-111111111111")}}, + {Key: "Name", Value: name}, + {Key: "BaseUrl", Value: bson.D{ + {Key: "$Type", Value: "Rest$ValueTemplate"}, + {Key: "$ID", Value: bson.Binary{Subtype: 0x00, Data: uuidToBlob(innerID)}}, + {Key: "Template", Value: "https://example.test/api"}, + }}, + }) + if err != nil { + t.Fatalf("marshal unit: %v", err) + } + return b +} + +func TestRecreatedUnitKeepsTheIdentitiesItWasDeletedWith(t *testing.T) { + const unitID = "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee" + const containerID = "22222222-2222-2222-2222-222222222222" + + stored := recreatedDoc(t, "PartsAvailabilityAPI", "33333333-3333-3333-3333-333333333333") + rebuilt := recreatedDoc(t, "PartsAvailabilityAPI", "44444444-4444-4444-4444-444444444444") + + w, unitPath := newV2WriterForCommitTest(t, unitID, stored) + seedTransactionTable(t, w, "the-original-transaction") + seedUnitRow(t, w, unitID, containerID, "Documents") + + if err := w.deleteUnit(unitID); err != nil { + t.Fatalf("delete unit: %v", err) + } + if err := w.insertUnit(unitID, containerID, "Documents", "Rest$ConsumedRestService", rebuilt); err != nil { + t.Fatalf("insert unit: %v", err) + } + + got, err := os.ReadFile(unitPath) + if err != nil { + t.Fatalf("read unit back: %v", err) + } + if string(got) != string(stored) { + t.Errorf("a delete+insert of identical content rewrote %d of %d bytes — the re-inserted "+ + "unit minted fresh element $IDs, so `git status` never comes back clean and an "+ + "MDL-generated project is not reviewable in version control (ako/mxcli#556)", + differingBytes(got, stored), len(stored)) + } + // The bytes are only half of it. `_Transaction.LastTransactionID` is how + // Studio Pro decides an external change needs re-syncing, and both the + // delete and the insert bump it — so without restoring it the .mpr itself + // still shows up in `git status` even once every .mxunit is stable. That is + // exactly where the fix stood when it was measured on a real project: one + // changed file left, and the only differing row was this one. + if got := transactionID(t, w); got != "the-original-transaction" { + t.Errorf("LastTransactionID = %q after a delete+insert that changed nothing — "+ + "the .mpr is dirty although no unit moved", got) + } +} + +// seedUnitRow fills in the ContainerID and ContainmentName the shared harness +// leaves null. They are load-bearing here: a recreate is only a no-op when the +// ROW came back the same too, so a unit that moved folders must still land. +func seedUnitRow(t *testing.T, w *Writer, unitID, containerID, containmentName string) { + t.Helper() + if _, err := w.reader.db.Exec( + `UPDATE Unit SET ContainerID = ?, ContainmentName = ? WHERE UnitID = ?`, + uuidToBlob(containerID), containmentName, uuidToBlob(unitID), + ); err != nil { + t.Fatalf("seed unit row: %v", err) + } +} + +func seedTransactionTable(t *testing.T, w *Writer, id string) { + t.Helper() + if _, err := w.reader.db.Exec(`CREATE TABLE _Transaction (LastTransactionID TEXT)`); err != nil { + t.Fatalf("create _Transaction: %v", err) + } + if _, err := w.reader.db.Exec(`INSERT INTO _Transaction (LastTransactionID) VALUES (?)`, id); err != nil { + t.Fatalf("seed _Transaction: %v", err) + } +} + +func transactionID(t *testing.T, w *Writer) string { + t.Helper() + var got string + if err := w.reader.db.QueryRow(`SELECT LastTransactionID FROM _Transaction`).Scan(&got); err != nil { + t.Fatalf("read LastTransactionID: %v", err) + } + return got +} + +// CONTROL: a recreate that really did change something still lands. Without +// this, a carry broken into "always write back the deleted bytes" would pass the +// test above and silently discard the user's edit. +func TestRecreatedUnitStillLandsARealChange(t *testing.T) { + const unitID = "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee" + const containerID = "22222222-2222-2222-2222-222222222222" + + stored := recreatedDoc(t, "PartsAvailabilityAPI", "33333333-3333-3333-3333-333333333333") + renamed := recreatedDoc(t, "PartsAvailabilityAPIv2", "44444444-4444-4444-4444-444444444444") + + w, unitPath := newV2WriterForCommitTest(t, unitID, stored) + seedTransactionTable(t, w, "the-original-transaction") + seedUnitRow(t, w, unitID, containerID, "Documents") + if err := w.deleteUnit(unitID); err != nil { + t.Fatalf("delete unit: %v", err) + } + if err := w.insertUnit(unitID, containerID, "Documents", "Rest$ConsumedRestService", renamed); err != nil { + t.Fatalf("insert unit: %v", err) + } + + got, err := os.ReadFile(unitPath) + if err != nil { + t.Fatalf("read unit back: %v", err) + } + var doc bson.D + if err := bson.Unmarshal(got, &doc); err != nil { + t.Fatalf("unmarshal: %v", err) + } + for _, e := range doc { + if e.Key == "Name" && e.Value != "PartsAvailabilityAPIv2" { + t.Fatalf("the rename was discarded: Name = %v", e.Value) + } + } + if storedHash(t, w, unitID) != hashOf(got) { + t.Error("ContentsHash does not describe the bytes on disk") + } + if got := transactionID(t, w); got == "the-original-transaction" { + t.Error("LastTransactionID was restored although the recreate changed the document — " + + "Studio Pro would not notice the rewrite") + } +} + +// CONTROL: an ordinary insert — no preceding delete — is untouched. The carry +// must key on a unit this session actually removed, not on a unit ID that +// happens to have been seen. +func TestFreshInsertIsNotAffected(t *testing.T) { + const unitID = "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee" + const otherID = "bbbbbbbb-bbbb-cccc-dddd-eeeeeeeeeeee" + const containerID = "22222222-2222-2222-2222-222222222222" + + stored := recreatedDoc(t, "Existing", "33333333-3333-3333-3333-333333333333") + fresh := recreatedDoc(t, "Fresh", "44444444-4444-4444-4444-444444444444") + + w, _ := newV2WriterForCommitTest(t, unitID, stored) + if err := w.insertUnit(otherID, containerID, "Documents", "Rest$ConsumedRestService", fresh); err != nil { + t.Fatalf("insert unit: %v", err) + } + blob := uuidToBlob(otherID) + swapped := blobToUUIDSwapped(blob) + got, err := os.ReadFile(w.reader.contentsDir + "/" + swapped[0:2] + "/" + swapped[2:4] + "/" + swapped + ".mxunit") + if err != nil { + t.Fatalf("read new unit: %v", err) + } + if string(got) != string(fresh) { + t.Error("a fresh insert was altered") + } +} + +func differingBytes(a, b []byte) int { + n := 0 + for i := 0; i < len(a) && i < len(b); i++ { + if a[i] != b[i] { + n++ + } + } + return n + abs(len(a)-len(b)) +} + +func abs(n int) int { + if n < 0 { + return -n + } + return n +} + +// CONTROL for the row half of the no-op test: identical bytes but a different +// container is a MOVE, and it has to land. `create or modify rest client +// ... in folder X` is exactly that statement. +func TestRecreatedUnitInAnotherContainerStillLands(t *testing.T) { + const unitID = "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee" + const oldContainer = "22222222-2222-2222-2222-222222222222" + const newContainer = "55555555-5555-5555-5555-555555555555" + + stored := recreatedDoc(t, "PartsAvailabilityAPI", "33333333-3333-3333-3333-333333333333") + rebuilt := recreatedDoc(t, "PartsAvailabilityAPI", "44444444-4444-4444-4444-444444444444") + + w, _ := newV2WriterForCommitTest(t, unitID, stored) + seedTransactionTable(t, w, "the-original-transaction") + seedUnitRow(t, w, unitID, oldContainer, "Documents") + + if err := w.deleteUnit(unitID); err != nil { + t.Fatalf("delete unit: %v", err) + } + if err := w.insertUnit(unitID, newContainer, "Documents", "Rest$ConsumedRestService", rebuilt); err != nil { + t.Fatalf("insert unit: %v", err) + } + + var got []byte + if err := w.reader.db.QueryRow( + `SELECT ContainerID FROM Unit WHERE UnitID = ?`, uuidToBlob(unitID), + ).Scan(&got); err != nil { + t.Fatalf("read ContainerID: %v", err) + } + if string(got) != string(uuidToBlob(newContainer)) { + t.Error("the move was discarded") + } + if id := transactionID(t, w); id == "the-original-transaction" { + t.Error("LastTransactionID was restored although the unit moved folders") + } +} From 568f7be0ae8fec0d8725dcb55d54540463140d75 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 21 Sep 2026 18:21:11 +0000 Subject: [PATCH 11/15] test: regression case and finding for re-exec identity stability MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit One idempotent script covering both named instances of ako/mxcli#556. Running it twice must leave the project byte-identical. The control is a binary built with both fixes stashed out, run on the same project: 3 changed files on every re-run, and `mx check` reporting "The app contains: 0 errors" each time — which is why the version-control diff was the only signal this class ever produced. Refs: ako/mxcli#556 Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01QWZjWZQhk3cNQCcy1Z2xzH --- .../fix-issue/findings/mdl-executor.jsonl | 1 + .../556-reexec-identity-stability.mdl | 61 +++++++++++++++++++ 2 files changed, 62 insertions(+) create mode 100644 mdl-examples/bug-tests/556-reexec-identity-stability.mdl diff --git a/.claude/skills/fix-issue/findings/mdl-executor.jsonl b/.claude/skills/fix-issue/findings/mdl-executor.jsonl index 057547703..dcb2eff1a 100644 --- a/.claude/skills/fix-issue/findings/mdl-executor.jsonl +++ b/.claude/skills/fix-issue/findings/mdl-executor.jsonl @@ -663,3 +663,4 @@ {"area": "mdl/executor", "date": "2026-09-20", "symptom": "`describe page` → `exec` over a **Studio Pro-authored** page silently drops six things, `mx check` 0 errors throughout. The one that matters: `IsPasswordBox True → False` — a **password field round-trips into a plaintext text box**, and describe → rename → exec is mxcli's copy operation. Also `Validation.Expression` blanked, a DataView's `ReadOnlyStyle Text → Control`, `PopupCloseAction` wiped, and two typed-array markers", "cause": "Four different causes behind one symptom, which is why triage came first: (1) IsPasswordBox — model and writer carried it, nothing parsed it, nothing emitted it; (2) Validation — `widgetValidationToGen()` wrote a DEFAULT EMPTY Forms$WidgetValidation over whatever was stored, on five widget types; (3) ReadOnlyStyle — wired for CheckBox only, and a DataView's draws no MDL-WIDGET07 warning because `staticWidgetKnownProps` is deliberately a union across widget types; (4) PopupCloseAction — `pageToGen` wrote \"\" unconditionally. Plus ParameterMappings/OutputMappings markers", "file": "`mdl/executor/cmd_pages_describe_parse.go` + `_output.go` (extract/emit), `cmd_pages_builder_v3_widgets.go` (consume), `cmd_pages_builder_v3.go`, `mdl/visitor/visitor_page_v3.go`, `mdl/ast/ast_page_v3.go`, `sdk/pages/*`, `mdl/backend/modelsdk/widget_write.go` + `page_write.go`, `mdl/executor/validate_widgets.go` (describe vocabulary)", "insight": "**Triage the layer before writing anything** — describer / grammar / builder have different fixes and this one issue had all three. The quickest probe is to run the property through `mxcli check`: MDL-WIDGET07 names an unrecognised one, and *silence is not acceptance* — the known-props list is a union across widget types, so a DataView's ReadOnlyStyle passed check and was dropped anyway. **Emit an expression QUOTED, not bracketed**: `[...]` is the XPath-constraint spelling and `propertyValueV3` parses it as an ARRAY, so `GetStringProp` yields \"\" — the emitter's own unit test was green while the real round trip still lost the value (storage form is not input form). **Measure the default before keeping it**: a DataView's ReadOnlyStyle is Control on 47 of 56, never Inherit, so the 'obvious' Inherit that every other input widget uses would have been wrong. Markers likewise measured, not assumed: ParameterMappings is marker 2 on 220 of 220 lists in every parent type, OutputMappings present on 91 of 91 — and an EMPTY list needs `MandatoryListMarkers` since `RegisterListMarker` keys on a child that is not there. Result 17 → 9 differences, the 9 being ako/mxcli#549", "refs": ["#550", "#541", "#549", "#490"]} {"area": "mdl/executor", "date": "2026-09-20", "symptom": "MDL-PAGEARG01 refused a list widget's OWN row action: `datagrid dg (DataSource: DATABASE M.E, onClick: SHOW_PAGE M.Edit(E: $currentObject))` was rejected at `check` with \"widget `dg` is not inside a data view, list view or grid row\" \u2014 and since exec refuses a script whose check errors, the slice could not be applied at all. On a `listview` the message contradicted itself. mxbuild 11.14.0 accepts the stored pages at 0 errors.", "cause": "The #1029 guard judged EVERY widget's own action in the context its PARENT supplies: `argContextForSubtreeOf` returns the parent context for a childless widget and `validate_widgets.go` passed the inherited `argCtx` to `validateShowPageArguments`. Right for a button, wrong for the widget that ESTABLISHES the context \u2014 a list widget's onClick is row-scoped, so the row it renders is the context object. Added `argContextForOwnAction`: a widget that binds a source of its own supplies the context for its own action; a source in a shape the pass cannot read (the bare-entity shorthand) degrades to UNKNOWN so the guard stands down rather than refusing what it cannot prove is discarded.", "file": "`mdl/executor/cmd_pages_showpage_args.go` (argContextForOwnAction, argContextForSubtreeOf), `mdl/executor/validate_widgets.go`", "insight": "**A false refusal costs more than a missing rule now that exec refuses on a check error** \u2014 the blast radius is 'this project cannot be built with this mxcli', not 'a warning is noisy'. Two things would have caught it before release: judging the rule against the widget kinds it NAMES in its own message (the listview refusal reads 'lvA is not inside a \u2026 list view'), and running it against mxbuild rather than against intuition. The mxbuild run paid for itself twice: it also showed that `DataSource: M.E` (bare-entity shorthand) on a datagrid is silently dropped, so that case is CE0488 + a REAL CE1571 \u2014 the stand-down is still correct, but the shorthand case must not be written into a bug test as mxbuild-clean (#576). Control the fix with the widget kinds STILL refused (a foreign variable, a sibling button beside the grid), or it is indistinguishable from deleting the rule.", "refs": ["#552", "#576", "mendixlabs/mxcli#1029", "#939"]} {"area": "mdl/executor", "date": "2026-09-21", "symptom": "A page's image-collection reference passed `mxcli check --references` and failed the build. Reported as \"no MDL syntax for a StaticImageViewer inside a Selection helper custom state\" — the authoring half was already closed by #1057; what was left is that nothing RESOLVED the name it made writable. Measured on a blank Mendix 11.14.0 project: `staticimage imgAll (Image: 'Atlas_UI_Resources.Atlas_Icons.checkbox_checked')` in a custom state -> check passed, exec created the page, `mx check` -> 3x CE1613 \"The selected image … no longer exists.\"", "cause": "TWO independent holes, and either alone leaves the reported script unchecked. (1) widgetRefCollector keyed the image reference on the widget TYPE — `if w.Type == \"image\"` — so the pluggable widget was collected and `staticimage` (which #1057 had just given the SAME `Image:` property) and `dynamicimage`'s `DefaultImage` were not; replaced with an imageRefProps table. (2) A page's widgets live in two AST fields: `Widgets` is the bare body, `Placeholders` holds `placeholder X { … }` content (#532). validate.go passed `s.Widgets` alone to validateWidgetReferences, validatePageContextTree AND validateFlowArguments, so EVERY reference inside a placeholder block — microflow, nanoflow, page, snippet, entity, image — was validated by nothing; added allPageWidgets to collect both roots once.", "file": "`mdl/executor/helpers.go` (widgetRefCollector.collectFromWidget, imageRefProps), `mdl/executor/validate.go` (allPageWidgets)", "insight": "**When a capability gets a new spelling, grep for who RESOLVES the old one.** #1057 added `Image:` to a second and third widget and moved on; the resolver keyed on the type name, so the new spellings were unchecked from the day they shipped. A property list and a resolver list that describe the same property are two copies — `validate_widgets.go` already accepted `Image`/`DefaultImage` for these widgets and DESCRIBE already emitted them, and only the resolver disagreed. **The placeholder hole is the more useful lesson: it was the THIRD copy of one walk.** validateIconRefs (#1008) and forEachWidget had each grown the `Placeholders` arm separately, with a comment saying a missed walk is silent both ways — and the three validators next door still had not. When a fix is 'add the missing arm to this walker', the question is how many walkers there are; collect the roots once instead. **Do not reason about a bug report from the issue text alone when the version is older than the fix** — the reported symptom did not reproduce on main at all, and running the reporter's own script end to end is what turned 'already fixed, close it' into two real defects. **Control both directions**: a reference that resolves must stay silent, because a walker that can suddenly see a whole new region of the tree is as likely to report correct scripts as broken ones.", "refs": ["mendixlabs/mxcli#1149", "mendixlabs/mxcli#1057", "mendixlabs/mxcli#1008", "#532"]} +{"area": "mdl/executor + modelsdk/mpr", "date": "2026-09-21", "symptom": "Re-running an identical MDL script never left `git status` clean: 15 of 485 units differed between two consecutive runs, every differing byte an identifier mxcli had re-minted. Two named instances — `Modified entity: X` + `Created RegEx validation rule on X.Attr` on every exec (unit the same size, ~62 bytes differing in 16-byte runs), and `Modified rest client: X` (1,128-byte unit, 143 bytes differing in 9 runs). Both `mx check`-silent: measured on 11.14.0 the faulted build reports 0 errors every run, because a re-created rule is a valid rule.", "cause": "TWO different mechanisms wearing one symptom, and the issue's own diagnosis (\"nested GUIDs are re-minted\", \"canon.CarryIdentity only patches top-level binaries\") was right about neither. Feeding both units' before/after bytes straight to canon.TransplantIDs made them byte-identical, so the policy was never wrong — it was never reached. (1) `create or modify entity` listed ValidationRules among the fields the statement is authoritative about, but the entity body can only spell Required (`not null`) and Unique (`unique`); RegEx/Range are authored by a separate `create validation rule` statement, so the rewrite DELETED the rule and the next statement put an identical one back under four fresh $IDs. Splitting the two statements is what shows it: the entity rewrite alone took the unit from 1,368 to 947 bytes. (2) `create or modify rest client` was implemented as delete + insert under the preserved unit ID — and insertUnit does not go through canon.Reconcile, which lives in updateUnit.", "file": "`mdl/executor/cmd_entities.go` (mergeValidationRules, entityFieldsMergedWithStored), `mdl/executor/cmd_rest_clients.go` (saveConsumedRestService), `modelsdk/mpr/writer_core.go` (rememberRemovedUnit / carryIdentityFromRemovedUnit)", "insight": "**Measure which STATEMENT moves the bytes before theorising about which PROPERTY does.** Both instances were reported as GUID re-minting; both were `$ID` re-minting, and the `$ID`s were re-minted because the document had been deleted and rebuilt, not because any carry was missing. Running each statement of the script on its own copy of the project, and diffing the unit's BSON with the offsets mapped back to key paths, took ten minutes and pointed at a completely different layer from the report. **The cheap second probe is to hand canon the two stored versions directly** — if TransplantIDs already makes them equal, the bug is a write path that skipped canon, never canon itself. **A delete+insert is a write path.** CLAUDE.md's rule is \"a new choke point that writes directly will silently churn\", and this one is not a new function — it is insertUnit reached by way of deleteUnit, which is why nobody noticed it bypassed elision. Fixing it needed BOTH halves: carrying identities makes the .mxunit stable, and restoring `_Transaction.LastTransactionID` is what stops the .mpr itself showing as modified (after the carry, every row of the SQLite db was logically identical and only that one UUID differed). **`mx check` is not a control for this class at all** — 0 errors on every variant, including the one that silently deletes the user's validation rule, so the version-control diff is the only signal. **The honest control is a faulted binary**, not a reverted test: build one with the fix stashed out and run the same script on the same project, or the \"it's clean now\" claim rests on nothing.", "refs": ["ako/mxcli#556", "ako/mxcli#553", "mendixlabs/mxcli#910", "mendixlabs/mxcli#949", "ADR-0008", "ADR-0005"]} diff --git a/mdl-examples/bug-tests/556-reexec-identity-stability.mdl b/mdl-examples/bug-tests/556-reexec-identity-stability.mdl new file mode 100644 index 000000000..2e4310049 --- /dev/null +++ b/mdl-examples/bug-tests/556-reexec-identity-stability.mdl @@ -0,0 +1,61 @@ +-- ============================================================================ +-- ako/mxcli#556 — re-exec is not identity-stable +-- ============================================================================ +-- +-- The regression case for the two named instances. Both are IDEMPOTENT scripts: +-- running this file twice must leave the project byte-identical, so a `git +-- status` after the second run comes back clean. Before the fix neither did. +-- +-- 1) A VALIDATION RULE on an entity that a later `create or modify entity` also +-- names. The entity body can only spell `not null` and `unique`, so it was +-- overwriting the whole ValidationRules list and DELETING the RegEx rule; +-- the next statement put an identical one back under four fresh element +-- identities. Measured on the v1 fixture: the unit came back the same size +-- (1,368 bytes) with 62 bytes differing, at +-- ValidationRules/1/$ID, .../Message/$ID, .../Message/Items/1/$ID, +-- .../RuleInfo/$ID +-- Splitting the two statements is what says which one moves: the entity +-- rewrite ALONE takes the unit from 1,368 to 947 bytes. +-- +-- 2) A REST CLIENT, which `create or modify` implemented as delete + insert +-- under the preserved unit ID. An insert does not reach the storage layer's +-- reconciliation, so the rebuild's fresh $IDs went straight to disk. +-- Measured on a real 11.14.0 project: 143 bytes of a 1,128-byte unit, in 9 +-- runs, every one an element $ID. +-- +-- Both are `mx check`-silent. Measured on 11.14.0, the faulted build reports +-- "The app contains: 0 errors." on every run — a recreated rule is a valid +-- rule, so nothing downstream ever complains. That is what makes the version- +-- control diff the only signal there was. +-- ============================================================================ + +CREATE MODULE BugReexecIdentity; + +CREATE OR MODIFY PERSISTENT ENTITY BugReexecIdentity.LithoSystem ( + SerialNumber: String(100), + Model: String(50) +); + +CREATE OR MODIFY REGULAR EXPRESSION BugReexecIdentity.SerialPattern ( + pattern: '^[A-Z]{3}-[0-9]{4}$', + caseSensitive: true +); + +-- Instance 1: the entity above is rewritten on every run, and must not take +-- this rule with it. +CREATE VALIDATION RULE FOR BugReexecIdentity.LithoSystem.SerialNumber + REGEX BugReexecIdentity.SerialPattern + FEEDBACK 'Serial must look like ABC-1234'; + +-- Instance 2: a consumed REST client, rewritten in place rather than recreated. +CREATE OR MODIFY REST CLIENT BugReexecIdentity.PartsAvailabilityAPI ( + BaseUrl: 'https://parts.example.com/api', + Authentication: NONE +) +{ + OPERATION GetPart { + Method: GET, + Path: '/parts/{id}', + Parameters: ($id: String) + } +}; From 65d7981809ec7cbadac18959626330d3df435d64 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 21 Sep 2026 18:32:28 +0000 Subject: [PATCH 12/15] fix(view entity): write the OQL document in place instead of replacing it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `CREATE OR MODIFY VIEW ENTITY` that changed ONLY the OQL printed Unchanged view entity: Mod.Totals while `describe entity` showed the new query stored. Changing the attribute list as well reported Modified correctly, which is why it hid. A view entity's OQL lives in a separate DomainModels$ViewEntitySourceDocument unit, and the executor DELETED that unit and INSERTED a fresh one on every write. Two costs, measured: - `ReportMutation` downgrades the verb when writes were offered and none landed, but the counters live at the update choke points and an InsertUnit is not one. The domain-model unit was offered and correctly elided, the OQL write was invisible, and the report believed the half it could see. - the unit was replaced under a fresh GUID on EVERY run, including a byte-identical one, so an MDL-generated project could never come back clean in version control. #556 counted these units. WriteViewEntitySourceDocument keeps the stored unit's id and goes through UpdateRawUnit, which reconciles against what is stored (ADR-0008): an identical query is elided, a changed one lands and is counted, duplicates are still cleared. Create and update now share one encoder so they cannot drift into writing different documents for the same query. Counting InsertUnit instead would have swapped a false "Unchanged" for a false "Modified" — measured: a byte-identical re-run re-minted the unit id, so every view-entity statement would have reported Modified forever. That measurement is why the fix is the reconcile wiring rather than the counter. Verbs on the bug-test, in order, all four correct: Created / Modified / Modified (OQL only) / Unchanged (identical re-run) Unit id now stable across an identical AND a changed re-run; mxbuild 11.14.0 reports 0 errors on the result. Control: with the old delete-then-create restored inside the new method, the test fails with the reported symptom — "a changed OQL landed no counted write (Written stayed 0)". Residual, deliberately not chased here: any content reaching storage through InsertUnit is still invisible to the elision check, so a statement whose only landing write is a NEW unit can be mis-reported. MoveUnit carries a comment explaining it was counted for exactly this reason; insert and delete were missed. Noted in #583 rather than changed unmeasured. Closes #583 Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01L2aYb3zDscezm874CDTH6R --- .../fix-issue/findings/mdl-backend.jsonl | 1 + ...y-583-oql-only-change-reports-modified.mdl | 71 ++++++++++++++++ mdl/backend/domainmodel.go | 4 + mdl/backend/mcp/unsupported_gen.go | 5 ++ mdl/backend/mock/backend.go | 1 + mdl/backend/mock/mock_domainmodel.go | 11 +++ mdl/backend/modelsdk/move_view_write.go | 73 ++++++++++++++-- .../modelsdk/view_source_doc_write_test.go | 84 +++++++++++++++++++ mdl/executor/cmd_entities.go | 18 ++-- 9 files changed, 254 insertions(+), 14 deletions(-) create mode 100644 mdl-examples/bug-tests/viewentity-583-oql-only-change-reports-modified.mdl create mode 100644 mdl/backend/modelsdk/view_source_doc_write_test.go diff --git a/.claude/skills/fix-issue/findings/mdl-backend.jsonl b/.claude/skills/fix-issue/findings/mdl-backend.jsonl index 5f26332fd..e081d8099 100644 --- a/.claude/skills/fix-issue/findings/mdl-backend.jsonl +++ b/.claude/skills/fix-issue/findings/mdl-backend.jsonl @@ -122,3 +122,4 @@ {"area": "mdl/backend", "date": "2026-09-20", "symptom": "An access rule on an entity carrying `AutoOwner` (or `AutoChangedBy`) made mxbuild report the whole module as **CE0066** \"Entity access is out of date\" \u2014 and `UPDATE SECURITY`, the documented repair for exactly that error, printed `Reconciled 1 access rule(s) in module Mod` and left the error standing. Four lines reproduce it on a clean production-security app: `alter entity M.Fab add attribute Owner: AutoOwner;` + `update security;`. The original reporter bisected 9 entities and 36 rules one grant at a time behind a ~40s `mx check` to find it, because CE0066 names only the module.", "cause": "mxcli wrote a MemberAccess for the implicit `System.owner` / `System.changedBy` association, in TWO places that had to agree: the GRANT handler (`cmd_security_write.go`) and `ReconcileMemberAccesses`. Mendix maintains those members itself and treats a rule naming one as out of date. The audit DATE members were already known to work this way (issuetracker #20) \u2014 the owner/changedBy pair was assumed to be the opposite case because they are associations rather than attributes, and Mendix really does add them implicitly. Fixed by writing no entry for any of the four, and by REMOVING a stored one in the reconcile (an explicit case before the foreign-module branch, which otherwise preserves `System.*` forever on the grounds that System is not loaded).", "file": "`mdl/backend/modelsdk/domainmodel_security_write.go` (isAuditMemberRef, ReconcileMemberAccesses), `mdl/executor/cmd_security_write.go`", "insight": "**The decisive probe was removing the entry, not adding anything.** CE0066 says 'out of date', which reads as 'something is missing' and sends you looking for a member to add; the model had one too many. A build flag (`MXCLI_PROBE_NO_SYSOWNER`) that dropped the entry took the module from CE0066 to 0 errors in one mxbuild run and settled it. The same repo's earlier finding had already written the rule down \u2014 *'Ask mxbuild what it wants instead of inferring symmetry'* \u2014 and this defect is that exact inference, made in the same file for the sibling members. **A fix here is not done when the new writes are correct**: `update security` exists to repair a project an older mxcli damaged, so the reconcile has to remove the entry, not merely stop adding it. Measured separately: a stale `System.owner` entry survived even after the flag was turned off, because `!assocRefBelongsTo` preserved it as an unverifiable foreign-module reference.", "refs": ["#554", "#524", "issuetracker #20"]} {"area": "mdl/backend", "date": "2026-09-20", "symptom": "`DROP ENTITY` left every CROSS-MODULE association pointing at the deleted entity in place. Dropping the local BY-ID (FROM) end made mxbuild 11.14.0 unable to LOAD the project: `System.AggregateException \u2026 (The given key '' was not present in the dictionary.)` at `StreamingBsonUnitReader.ResolvePostponedProperties()` \u2014 no CE code, no document named, so the obvious reading is 'the project is corrupt, restore from git'. Dropping the BY-NAME (TO) end is milder and still wrong: CE1613 at the cross-module association. `show associations` shows a raw GUID where the parent entity should be.", "cause": "`removeAssocsReferencing` swept `dm.AssociationsItems()` and asserted `*genDm.Association` per item, so the SEPARATE `CrossAssociations` collection was never looked at. Fixed with `removeCrossAssocsReferencing`, matching BOTH ends because a cross-module association addresses them differently \u2014 FROM by element id (local), TO by qualified name (another module) \u2014 called in DeleteEntity locally and in its cascade over the other domain models.", "file": "`mdl/backend/modelsdk/domainmodel_alter.go` (removeCrossAssocsReferencing, DeleteEntity)", "insight": "**Reported against a view entity; nothing about it was view-entity specific.** The reporter met it dropping view entities (whose associations are DERIVED from OQL, so there is no CREATE ASSOCIATION to undo) and filed it that way. The first probe \u2014 a view entity and its source entity in the SAME module \u2014 did not reproduce at all, and that negative is the useful one: it says the variable is cross-module, not view-ness. A plain `create association A.X from A.X to B.Y` plus `drop entity A.X` reproduces the identical crash. Two lessons: when a repro fails, vary the dimension the report did not mention before doubting the report, and treat a collection-typed `.(*T)` assertion in a cascade as a place where a sibling type hides. mxbuild's diagnostic distinguishes the two ends for free \u2014 a dangling 16-byte pointer is a LOAD crash, a dangling qualified name is CE1613 \u2014 so testing only one end proves half the fix.", "refs": ["#553", "#556"]} {"area": "mdl/backend", "date": "2026-09-21", "symptom": "`alter settings workflows add group 'Auditors'` reports \"Added workflow group: Auditors (3 group(s))\" and writes nothing \u2014 `show workflow groups` still lists 2, and `mx check` is 0 errors either way", "cause": "`UpdateProjectSettings` overlays the workflows part field by field onto the PRESERVED raw part, so a child LIST that nothing rebuilds is carried through from disk unchanged. Adding `Groups` to the semantic model and to the read path is not enough; the write needs `settingsoverlay.WorkflowGroups(ws, rawPart)`. Identical shape to the enabled-language list the same function already documents", "file": "`mdl/backend/modelsdk/settings_write.go` + `mdl/settingsoverlay/settingsoverlay.go` (`WorkflowGroups`)", "insight": "For anything under Settings$ProjectSettings, the executor's success message proves NOTHING \u2014 it reports the in-memory model, and the overlay is where a list quietly fails to land. Assert on the re-read document, not the handler's output. Two more things a reference project settles in one dump and a guess gets wrong: the `Groups` typed-array marker is 2, not the 3 every other settings child list uses (`ArrayMarker` preserves a stored one, but the fallback matters on a fresh list), and the element's `$ID` is the RUNTIME's identity \u2014 a booted 11.13.0 app keys `system$workflowgroup.modelguid` on it, byte-identical once the .NET GUID field order is undone, so re-minting it on a description edit would orphan every group membership with a perfectly valid model. Control: deleting the one overlay call reproduces the symptom verbatim. mendixlabs/mxcli#272", "refs": ["mendixlabs/mxcli#272"]} +{"area": "mdl/backend", "date": "2026-09-21", "symptom": "`CREATE OR MODIFY VIEW ENTITY` that changed ONLY the OQL printed `Unchanged view entity: \u2026` while `describe entity` showed the new query stored. Changing the attribute list as well reported `Modified` correctly, which is why it hid. Also: the OQL document's unit was replaced under a FRESH GUID on every run, even a byte-identical one, so an MDL-generated project could never come back clean in git (one of the four units #556 measured).", "cause": "A view entity's OQL lives in a separate `DomainModels$ViewEntitySourceDocument` unit, and the executor DELETED it and INSERTED a fresh one on every write. `ReportMutation` downgrades the verb when writes were offered and none landed, but the counters are incremented only at the update choke points (`writer_core.go` reconcileWithStored / MoveUnit) \u2014 `InsertUnit` is not counted at all. So the domain-model unit was offered and correctly elided, the OQL write was invisible, and the report believed the half it could see. Fixed with `WriteViewEntitySourceDocument`, which keeps the stored unit's id and goes through `UpdateRawUnit` \u2192 reconcile: an identical query is elided, a changed one lands and is counted, duplicates are still cleared.", "file": "`mdl/backend/modelsdk/move_view_write.go` (WriteViewEntitySourceDocument, encodeViewEntitySourceDocument), `mdl/executor/cmd_entities.go`", "insight": "**The first fix that comes to mind \u2014 count InsertUnit \u2014 would have swapped a false \"Unchanged\" for a false \"Modified\".** Measuring before changing is what caught it: re-running a BYTE-IDENTICAL script still re-minted the source document's unit id, so counting inserts would have made every view-entity statement report Modified forever. The right fix was the one ADR-0008 already mandates (wire the write path to canon.Reconcile), and it fixes the churn and the verb together. Generalisation worth remembering: any content that reaches storage through `InsertUnit` is invisible to the elision check, so a statement whose only landing write is a NEW unit can still be mis-reported \u2014 `MoveUnit` has a comment explaining it was counted for exactly this reason, and insert/delete were missed. Control the fix on the identical re-run, not just the changed one.", "refs": ["#583", "#556", "#910"]} diff --git a/mdl-examples/bug-tests/viewentity-583-oql-only-change-reports-modified.mdl b/mdl-examples/bug-tests/viewentity-583-oql-only-change-reports-modified.mdl new file mode 100644 index 000000000..1383e93a1 --- /dev/null +++ b/mdl-examples/bug-tests/viewentity-583-oql-only-change-reports-modified.mdl @@ -0,0 +1,71 @@ +-- ako/mxcli#583 — `CREATE OR MODIFY VIEW ENTITY` that changes ONLY the OQL +-- printed `Unchanged view entity: …` while the new query WAS stored. +-- +-- The OQL lives in a separate DomainModels$ViewEntitySourceDocument unit. The +-- executor deleted that unit and inserted a fresh one on every write, and an +-- InsertUnit is not one of the choke points WriteStats counts — so the elision +-- check saw the (genuinely unchanged) domain-model unit, saw no counted write, +-- and downgraded the verb. `describe entity` showed the new query all along. +-- +-- Two things this file demonstrates, and the second is the control: +-- +-- statement 3 changes only the OQL -> must report Modified +-- statement 4 is byte-identical to 3 -> must report Unchanged +-- +-- Without the control a fix is indistinguishable from deleting the downgrade. +-- Run with `mxcli exec` and read the verbs; `mxcli docker check` must be 0 errors. +-- +-- The same fix stops the unit being replaced under a new GUID on every run, +-- which is one of the four units #556 measured as un-reviewable in git. + +create module Ve583; + +create persistent entity Ve583.Meter (MeterName: String(100)); +create persistent entity Ve583.Reading (Kwh: Decimal); +create association Ve583.Reading_Meter from Ve583.Reading to Ve583.Meter type reference; + +-- 1. Created. +create or modify view entity Ve583.Totals ( + TotalKwh: Decimal +) as ( + from Ve583.Reading as r + join r/Ve583.Reading_Meter/Ve583.Meter as m + group by m.MeterName + select sum(r.Kwh) as TotalKwh +); + +-- 2. Attribute list AND query change: reported Modified even before the fix, +-- because the domain-model unit changed. Here to keep the two cases apart. +create or modify view entity Ve583.Totals ( + TotalKwh: Decimal, + ReadingCount: Integer +) as ( + from Ve583.Reading as r + join r/Ve583.Reading_Meter/Ve583.Meter as m + group by m.MeterName + select sum(r.Kwh) as TotalKwh, count(r.ID) as ReadingCount +); + +-- 3. ONLY the query changes — the bug. Must report Modified. +create or modify view entity Ve583.Totals ( + TotalKwh: Decimal, + ReadingCount: Integer +) as ( + from Ve583.Reading as r + join r/Ve583.Reading_Meter/Ve583.Meter as m + where r.Kwh > 0 + group by m.MeterName + select sum(r.Kwh) as TotalKwh, count(r.ID) as ReadingCount +); + +-- 4. The control: identical to 3. Must report Unchanged. +create or modify view entity Ve583.Totals ( + TotalKwh: Decimal, + ReadingCount: Integer +) as ( + from Ve583.Reading as r + join r/Ve583.Reading_Meter/Ve583.Meter as m + where r.Kwh > 0 + group by m.MeterName + select sum(r.Kwh) as TotalKwh, count(r.ID) as ReadingCount +); diff --git a/mdl/backend/domainmodel.go b/mdl/backend/domainmodel.go index cf07155ee..7ff7ef16b 100644 --- a/mdl/backend/domainmodel.go +++ b/mdl/backend/domainmodel.go @@ -45,6 +45,10 @@ type DomainModelBackend interface { // View entities CreateViewEntitySourceDocument(moduleID model.ID, moduleName, docName, oqlQuery, documentation string) (model.ID, error) + // WriteViewEntitySourceDocument stores the OQL document, updating the unit + // already there rather than replacing it — see the modelsdk implementation + // for why the difference is load-bearing (ako/mxcli#583). + WriteViewEntitySourceDocument(moduleID model.ID, moduleName, docName, oqlQuery, documentation string) (model.ID, error) DeleteViewEntitySourceDocument(id model.ID) error DeleteViewEntitySourceDocumentByName(moduleName, docName string) error FindViewEntitySourceDocumentID(moduleName, docName string) (model.ID, error) diff --git a/mdl/backend/mcp/unsupported_gen.go b/mdl/backend/mcp/unsupported_gen.go index 24c4dd589..e8e5fc652 100644 --- a/mdl/backend/mcp/unsupported_gen.go +++ b/mdl/backend/mcp/unsupported_gen.go @@ -264,6 +264,11 @@ func (unsupportedBackend) CreateViewEntitySourceDocument(_ model.ID, _ string, _ return } +func (unsupportedBackend) WriteViewEntitySourceDocument(_ model.ID, _ string, _ string, _ string, _ string) (r0 model.ID, err1 error) { + err1 = errUnsupported("WriteViewEntitySourceDocument") + return +} + func (unsupportedBackend) CreateWorkflow(_ *workflows.Workflow) (err0 error) { err0 = errUnsupported("CreateWorkflow") return diff --git a/mdl/backend/mock/backend.go b/mdl/backend/mock/backend.go index 9b0798f64..971df1944 100644 --- a/mdl/backend/mock/backend.go +++ b/mdl/backend/mock/backend.go @@ -77,6 +77,7 @@ type MockBackend struct { DeleteAssociationFunc func(domainModelID model.ID, assocID model.ID) error DeleteCrossAssociationFunc func(domainModelID model.ID, assocID model.ID) error CreateViewEntitySourceDocumentFunc func(moduleID model.ID, moduleName, docName, oqlQuery, documentation string) (model.ID, error) + WriteViewEntitySourceDocumentFunc func(moduleID model.ID, moduleName, docName, oqlQuery, documentation string) (model.ID, error) DeleteViewEntitySourceDocumentFunc func(id model.ID) error DeleteViewEntitySourceDocumentByNameFunc func(moduleName, docName string) error FindViewEntitySourceDocumentIDFunc func(moduleName, docName string) (model.ID, error) diff --git a/mdl/backend/mock/mock_domainmodel.go b/mdl/backend/mock/mock_domainmodel.go index 9b8747d8f..eb2c47681 100644 --- a/mdl/backend/mock/mock_domainmodel.go +++ b/mdl/backend/mock/mock_domainmodel.go @@ -128,6 +128,17 @@ func (m *MockBackend) CreateViewEntitySourceDocument(moduleID model.ID, moduleNa return "", nil } +func (m *MockBackend) WriteViewEntitySourceDocument(moduleID model.ID, moduleName, docName, oqlQuery, documentation string) (model.ID, error) { + if m.WriteViewEntitySourceDocumentFunc != nil { + return m.WriteViewEntitySourceDocumentFunc(moduleID, moduleName, docName, oqlQuery, documentation) + } + // Mirrors CreateViewEntitySourceDocument above rather than the checklist's + // "not configured" error: a view-entity handler calls this on every write, so + // an error default would fail every executor test that does not care where + // the OQL document went. + return "", nil +} + func (m *MockBackend) DeleteViewEntitySourceDocument(id model.ID) error { if m.DeleteViewEntitySourceDocumentFunc != nil { return m.DeleteViewEntitySourceDocumentFunc(id) diff --git a/mdl/backend/modelsdk/move_view_write.go b/mdl/backend/modelsdk/move_view_write.go index 3b28359bf..48b140e43 100644 --- a/mdl/backend/modelsdk/move_view_write.go +++ b/mdl/backend/modelsdk/move_view_write.go @@ -123,6 +123,72 @@ func (b *Backend) CreateViewEntitySourceDocument(moduleID model.ID, moduleName, return "", fmt.Errorf("CreateViewEntitySourceDocument: not connected for writing") } docID := model.ID(mmpr.GenerateID()) + contents, err := encodeViewEntitySourceDocument(docID, docName, oqlQuery, documentation) + if err != nil { + return "", fmt.Errorf("CreateViewEntitySourceDocument: %w", err) + } + if err := b.writer.InsertUnit(string(docID), string(moduleID), "Documents", "DomainModels$ViewEntitySourceDocument", contents); err != nil { + return "", fmt.Errorf("CreateViewEntitySourceDocument: insert: %w", err) + } + return docID, nil +} + +// WriteViewEntitySourceDocument stores the OQL source document backing a view +// entity, KEEPING the unit already there when there is one. +// +// The executor used to delete the stored document and insert a fresh one on every +// CREATE OR MODIFY, which cost two things at once (ako/mxcli#583): +// +// - the unit was replaced under a new GUID on every run, even when the query +// was byte-identical, so an MDL-generated project could never come back clean +// in version control (#556 counted these units among the four that are +// replaced rather than rewritten); +// - an InsertUnit is not one of the write choke points WriteStats counts, so +// `exec` reported `Unchanged view entity: …` for a statement that had just +// rewritten the OQL — the domain-model unit was genuinely unchanged (the +// attribute list did not move) and the only write that landed was invisible. +// +// Updating in place puts the write back on the reconciling path (ADR-0008): an +// identical query is elided, a changed one lands and is counted, and the unit id +// survives either way. +// +// Extra documents under the same name — which the old delete-then-create existed +// to clear up — are removed, keeping the oldest as the one to update. A duplicate +// OQL document is a real hazard: the entity references its source by qualified +// name, so a second one under that name is ambiguous. +func (b *Backend) WriteViewEntitySourceDocument(moduleID model.ID, moduleName, docName, oqlQuery, documentation string) (model.ID, error) { + if b.writer == nil { + return "", fmt.Errorf("WriteViewEntitySourceDocument: not connected for writing") + } + + ids, err := b.FindAllViewEntitySourceDocumentIDs(moduleName, docName) + if err != nil { + return "", fmt.Errorf("WriteViewEntitySourceDocument: find existing: %w", err) + } + if len(ids) == 0 { + return b.CreateViewEntitySourceDocument(moduleID, moduleName, docName, oqlQuery, documentation) + } + for _, extra := range ids[1:] { + if err := b.DeleteViewEntitySourceDocument(extra); err != nil { + return "", fmt.Errorf("WriteViewEntitySourceDocument: remove duplicate %s: %w", extra, err) + } + } + + docID := ids[0] + contents, err := encodeViewEntitySourceDocument(docID, docName, oqlQuery, documentation) + if err != nil { + return "", err + } + if err := b.writer.UpdateRawUnit(string(docID), contents); err != nil { + return "", fmt.Errorf("WriteViewEntitySourceDocument: update: %w", err) + } + return docID, nil +} + +// encodeViewEntitySourceDocument builds the stored form of an OQL source +// document. Shared by the create and update paths so the two cannot drift into +// writing different documents for the same query. +func encodeViewEntitySourceDocument(docID model.ID, docName, oqlQuery, documentation string) ([]byte, error) { d := genDm.NewViewEntitySourceDocument() d.SetID(element.ID(docID)) d.SetName(docName) @@ -132,12 +198,9 @@ func (b *Backend) CreateViewEntitySourceDocument(moduleID model.ID, moduleName, d.SetOql(oqlQuery) contents, err := (&codec.Encoder{}).Encode(d) if err != nil { - return "", fmt.Errorf("CreateViewEntitySourceDocument: encode: %w", err) + return nil, fmt.Errorf("encode view entity source document: %w", err) } - if err := b.writer.InsertUnit(string(docID), string(moduleID), "Documents", "DomainModels$ViewEntitySourceDocument", contents); err != nil { - return "", fmt.Errorf("CreateViewEntitySourceDocument: insert: %w", err) - } - return docID, nil + return contents, nil } // MoveViewEntitySourceDocument reparents the OQL source document backing a moved diff --git a/mdl/backend/modelsdk/view_source_doc_write_test.go b/mdl/backend/modelsdk/view_source_doc_write_test.go new file mode 100644 index 000000000..4572ef5b8 --- /dev/null +++ b/mdl/backend/modelsdk/view_source_doc_write_test.go @@ -0,0 +1,84 @@ +// SPDX-License-Identifier: Apache-2.0 + +// ako/mxcli#583 — a view entity's OQL document was DELETED and RE-INSERTED on +// every CREATE OR MODIFY, under a fresh unit id, whether or not the query had +// changed. Two consequences, both measured against a real project: +// +// - `exec` printed `Unchanged view entity: …` for a statement that rewrote the +// OQL, because the elision check counts writes at the update choke points +// and an InsertUnit is not one. The domain-model unit really was unchanged +// (same attribute list), the OQL document really was written, and the report +// believed the half it could see. +// - the unit was replaced under a new GUID on every run, so the .mpr could +// never come back clean in version control (#556 counted these units). +// +// Writing the document in place fixes both: the update path reconciles against +// what is stored (ADR-0008), so an unchanged query is elided and a changed one +// is counted. +package modelsdkbackend + +import ( + "testing" +) + +func TestWriteViewEntitySourceDocument_KeepsItsUnitAndReconciles(t *testing.T) { + proj := copyFixture(t) + b := New() + if err := b.Connect(proj); err != nil { + t.Fatalf("connect: %v", err) + } + t.Cleanup(func() { _ = b.Disconnect() }) + + mod, err := b.GetModuleByName("MyFirstModule") + if err != nil || mod == nil { + t.Fatalf("GetModuleByName: %v", err) + } + + const oql = "from MyFirstModule.Thing as t select t.Name as Name" + first, err := b.WriteViewEntitySourceDocument(mod.ID, "MyFirstModule", "ZzTotals", oql, "") + if err != nil { + t.Fatalf("WriteViewEntitySourceDocument (create): %v", err) + } + + // Writing the SAME query again must keep the unit and land nothing. + before := b.WriteStats() + again, err := b.WriteViewEntitySourceDocument(mod.ID, "MyFirstModule", "ZzTotals", oql, "") + if err != nil { + t.Fatalf("WriteViewEntitySourceDocument (no-op): %v", err) + } + if again != first { + t.Errorf("an unchanged rewrite replaced the unit: %s -> %s — the .mpr can never come back "+ + "clean in version control (ako/mxcli#556)", first, again) + } + if after := b.WriteStats(); after.Written != before.Written { + t.Errorf("an unchanged rewrite landed a write (Written %d -> %d) — `exec` would report it as "+ + "Modified", before.Written, after.Written) + } + + // Changing the query must keep the unit AND land a write, or ReportMutation + // downgrades the verb to "Unchanged" for a statement that changed the model. + before = b.WriteStats() + changed, err := b.WriteViewEntitySourceDocument(mod.ID, "MyFirstModule", "ZzTotals", + oql+" where t.Name != ''", "") + if err != nil { + t.Fatalf("WriteViewEntitySourceDocument (change): %v", err) + } + if changed != first { + t.Errorf("a changed rewrite replaced the unit: %s -> %s", first, changed) + } + after := b.WriteStats() + if after.Written == before.Written { + t.Errorf("a changed OQL landed no counted write (Written stayed %d) — this is the reported "+ + "symptom: `exec` prints \"Unchanged view entity\" while `describe entity` shows the new query", + before.Written) + } + + // And the new query is what is stored, not just what was offered. + id, err := b.FindViewEntitySourceDocumentID("MyFirstModule", "ZzTotals") + if err != nil || id == "" { + t.Fatalf("FindViewEntitySourceDocumentID: %v (id=%q)", err, id) + } + if id != first { + t.Errorf("the stored document is a different unit: %s, want %s", id, first) + } +} diff --git a/mdl/executor/cmd_entities.go b/mdl/executor/cmd_entities.go index 74fca5ff2..6ebc37426 100644 --- a/mdl/executor/cmd_entities.go +++ b/mdl/executor/cmd_entities.go @@ -802,15 +802,15 @@ func execCreateViewEntity(ctx *ExecContext, s *ast.CreateViewEntityStmt) error { location = model.Point{X: 100 + len(dm.Entities)*150, Y: 100} } - // Create or update ViewEntitySourceDocument (separate document for OQL query) + // Create or update ViewEntitySourceDocument (separate document for OQL query). + // Written IN PLACE: this used to delete the stored document and insert a fresh + // one every time, which replaced the unit under a new GUID on every run and + // made `exec` report `Unchanged view entity` for a statement that had just + // rewritten the OQL — an insert is not a counted write, so the elision check + // saw only the (genuinely unchanged) domain-model unit (ako/mxcli#583). + // Duplicate documents, which the delete existed to clear, are still removed. sourceDocRef := s.Name.Module + "." + s.Name.Name - // Always delete any existing ViewEntitySourceDocument before creating a new one. - // This prevents duplicate OQL documents from accumulating (e.g., from re-running - // scripts or after a previous DROP that didn't clean up properly). - if err := ctx.Backend.DeleteViewEntitySourceDocumentByName(s.Name.Module, s.Name.Name); err != nil { - return mdlerrors.NewBackend("delete existing ViewEntitySourceDocument", err) - } - _, err = ctx.Backend.CreateViewEntitySourceDocument( + _, err = ctx.Backend.WriteViewEntitySourceDocument( module.ID, s.Name.Module, s.Name.Name, @@ -818,7 +818,7 @@ func execCreateViewEntity(ctx *ExecContext, s *ast.CreateViewEntityStmt) error { s.Documentation, ) if err != nil { - return mdlerrors.NewBackend("create ViewEntitySourceDocument", err) + return mdlerrors.NewBackend("write ViewEntitySourceDocument", err) } // Create view attributes with OqlViewValue references. From 839e6c282ef4605f7ee0d9a0b1a1f4b76af6966d Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 21 Sep 2026 18:41:07 +0000 Subject: [PATCH 13/15] Read text templates in the generic pluggable DESCRIBE path (#575) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit TreeNode and Timeline — the widgets the issue was filed against — do have a generic describe path: every pluggable widget outside isKnownCustomWidgetType's list of nine goes through extractExplicitProperties. That path read AttributeRef and PrimitiveValue only, so EVERY text-template property of those widgets was absent from `describe page` whether it was bound or literal, and describe -> exec dropped the caption outright. Measured on a blank 11.14.0 project, before this commit: treenode tnProduct ( advancedMode: false, headerType: 'text', openNodeOn: 'headerClick', hasChildren: true, startExpanded: false, animate: true, showIcon: 'left', animateIcon: true ) Ten primitives and no caption — which is why it looked complete. After: treenode tnProduct ( DataSource: database from Bug575.Product, advancedMode: false, headerType: 'text', openNodeOn: 'headerClick', headerCaption: '{1}', headerCaptionParams: [{1} = Name], ... ) and the same for a Timeline's title / description. Both pages round-trip: describe -> exec reports "Unchanged page", mx check 0 errors. Also corrects the bug test, which used the built-in Image on the stated grounds that the reported widgets needed a Marketplace download. They do not: TreeNode.mpk and Timeline.mpk ship in a blank app's widgets/ and their defs are generated by `mxcli init`. The test now exercises all three — the Image earns its place because its def.json names the mappings by SOURCE (`ImageUrl`) where the TreeNode's names them by schema key, and the two take different branches of resolveMapping. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01DXYNJwiutu5AjxLmG7Fgsu --- .../fix-issue/findings/mdl-executor.jsonl | 2 +- docs-site/src/language/widget-types.md | 7 +- ...gets-575-texttemplate-params-companion.mdl | 57 +++++++++-- mdl/executor/cmd_pages_describe.go | 7 ++ mdl/executor/cmd_pages_describe_output.go | 6 ++ mdl/executor/cmd_pages_describe_pluggable.go | 13 +++ ...dget_texttemplate_named_params_575_test.go | 99 +++++++++++++++++++ 7 files changed, 178 insertions(+), 13 deletions(-) diff --git a/.claude/skills/fix-issue/findings/mdl-executor.jsonl b/.claude/skills/fix-issue/findings/mdl-executor.jsonl index 8a88d70c3..7eeef32f3 100644 --- a/.claude/skills/fix-issue/findings/mdl-executor.jsonl +++ b/.claude/skills/fix-issue/findings/mdl-executor.jsonl @@ -663,4 +663,4 @@ {"area": "mdl/executor", "date": "2026-09-20", "symptom": "`describe page` → `exec` over a **Studio Pro-authored** page silently drops six things, `mx check` 0 errors throughout. The one that matters: `IsPasswordBox True → False` — a **password field round-trips into a plaintext text box**, and describe → rename → exec is mxcli's copy operation. Also `Validation.Expression` blanked, a DataView's `ReadOnlyStyle Text → Control`, `PopupCloseAction` wiped, and two typed-array markers", "cause": "Four different causes behind one symptom, which is why triage came first: (1) IsPasswordBox — model and writer carried it, nothing parsed it, nothing emitted it; (2) Validation — `widgetValidationToGen()` wrote a DEFAULT EMPTY Forms$WidgetValidation over whatever was stored, on five widget types; (3) ReadOnlyStyle — wired for CheckBox only, and a DataView's draws no MDL-WIDGET07 warning because `staticWidgetKnownProps` is deliberately a union across widget types; (4) PopupCloseAction — `pageToGen` wrote \"\" unconditionally. Plus ParameterMappings/OutputMappings markers", "file": "`mdl/executor/cmd_pages_describe_parse.go` + `_output.go` (extract/emit), `cmd_pages_builder_v3_widgets.go` (consume), `cmd_pages_builder_v3.go`, `mdl/visitor/visitor_page_v3.go`, `mdl/ast/ast_page_v3.go`, `sdk/pages/*`, `mdl/backend/modelsdk/widget_write.go` + `page_write.go`, `mdl/executor/validate_widgets.go` (describe vocabulary)", "insight": "**Triage the layer before writing anything** — describer / grammar / builder have different fixes and this one issue had all three. The quickest probe is to run the property through `mxcli check`: MDL-WIDGET07 names an unrecognised one, and *silence is not acceptance* — the known-props list is a union across widget types, so a DataView's ReadOnlyStyle passed check and was dropped anyway. **Emit an expression QUOTED, not bracketed**: `[...]` is the XPath-constraint spelling and `propertyValueV3` parses it as an ARRAY, so `GetStringProp` yields \"\" — the emitter's own unit test was green while the real round trip still lost the value (storage form is not input form). **Measure the default before keeping it**: a DataView's ReadOnlyStyle is Control on 47 of 56, never Inherit, so the 'obvious' Inherit that every other input widget uses would have been wrong. Markers likewise measured, not assumed: ParameterMappings is marker 2 on 220 of 220 lists in every parent type, OutputMappings present on 91 of 91 — and an EMPTY list needs `MandatoryListMarkers` since `RegisterListMarker` keys on a child that is not there. Result 17 → 9 differences, the 9 being ako/mxcli#549", "refs": ["#550", "#541", "#549", "#490"]} {"area": "mdl/executor", "date": "2026-09-20", "symptom": "MDL-PAGEARG01 refused a list widget's OWN row action: `datagrid dg (DataSource: DATABASE M.E, onClick: SHOW_PAGE M.Edit(E: $currentObject))` was rejected at `check` with \"widget `dg` is not inside a data view, list view or grid row\" \u2014 and since exec refuses a script whose check errors, the slice could not be applied at all. On a `listview` the message contradicted itself. mxbuild 11.14.0 accepts the stored pages at 0 errors.", "cause": "The #1029 guard judged EVERY widget's own action in the context its PARENT supplies: `argContextForSubtreeOf` returns the parent context for a childless widget and `validate_widgets.go` passed the inherited `argCtx` to `validateShowPageArguments`. Right for a button, wrong for the widget that ESTABLISHES the context \u2014 a list widget's onClick is row-scoped, so the row it renders is the context object. Added `argContextForOwnAction`: a widget that binds a source of its own supplies the context for its own action; a source in a shape the pass cannot read (the bare-entity shorthand) degrades to UNKNOWN so the guard stands down rather than refusing what it cannot prove is discarded.", "file": "`mdl/executor/cmd_pages_showpage_args.go` (argContextForOwnAction, argContextForSubtreeOf), `mdl/executor/validate_widgets.go`", "insight": "**A false refusal costs more than a missing rule now that exec refuses on a check error** \u2014 the blast radius is 'this project cannot be built with this mxcli', not 'a warning is noisy'. Two things would have caught it before release: judging the rule against the widget kinds it NAMES in its own message (the listview refusal reads 'lvA is not inside a \u2026 list view'), and running it against mxbuild rather than against intuition. The mxbuild run paid for itself twice: it also showed that `DataSource: M.E` (bare-entity shorthand) on a datagrid is silently dropped, so that case is CE0488 + a REAL CE1571 \u2014 the stand-down is still correct, but the shorthand case must not be written into a bug test as mxbuild-clean (#576). Control the fix with the widget kinds STILL refused (a foreign variable, a sibling button beside the grid), or it is indistinguishable from deleting the rule.", "refs": ["#552", "#576", "mendixlabs/mxcli#1029", "#939"]} {"area": "mdl/executor", "date": "2026-09-21", "symptom": "A page's image-collection reference passed `mxcli check --references` and failed the build. Reported as \"no MDL syntax for a StaticImageViewer inside a Selection helper custom state\" — the authoring half was already closed by #1057; what was left is that nothing RESOLVED the name it made writable. Measured on a blank Mendix 11.14.0 project: `staticimage imgAll (Image: 'Atlas_UI_Resources.Atlas_Icons.checkbox_checked')` in a custom state -> check passed, exec created the page, `mx check` -> 3x CE1613 \"The selected image … no longer exists.\"", "cause": "TWO independent holes, and either alone leaves the reported script unchecked. (1) widgetRefCollector keyed the image reference on the widget TYPE — `if w.Type == \"image\"` — so the pluggable widget was collected and `staticimage` (which #1057 had just given the SAME `Image:` property) and `dynamicimage`'s `DefaultImage` were not; replaced with an imageRefProps table. (2) A page's widgets live in two AST fields: `Widgets` is the bare body, `Placeholders` holds `placeholder X { … }` content (#532). validate.go passed `s.Widgets` alone to validateWidgetReferences, validatePageContextTree AND validateFlowArguments, so EVERY reference inside a placeholder block — microflow, nanoflow, page, snippet, entity, image — was validated by nothing; added allPageWidgets to collect both roots once.", "file": "`mdl/executor/helpers.go` (widgetRefCollector.collectFromWidget, imageRefProps), `mdl/executor/validate.go` (allPageWidgets)", "insight": "**When a capability gets a new spelling, grep for who RESOLVES the old one.** #1057 added `Image:` to a second and third widget and moved on; the resolver keyed on the type name, so the new spellings were unchecked from the day they shipped. A property list and a resolver list that describe the same property are two copies — `validate_widgets.go` already accepted `Image`/`DefaultImage` for these widgets and DESCRIBE already emitted them, and only the resolver disagreed. **The placeholder hole is the more useful lesson: it was the THIRD copy of one walk.** validateIconRefs (#1008) and forEachWidget had each grown the `Placeholders` arm separately, with a comment saying a missed walk is silent both ways — and the three validators next door still had not. When a fix is 'add the missing arm to this walker', the question is how many walkers there are; collect the roots once instead. **Do not reason about a bug report from the issue text alone when the version is older than the fix** — the reported symptom did not reproduce on main at all, and running the reporter's own script end to end is what turned 'already fixed, close it' into two real defects. **Control both directions**: a reference that resolves must stay silent, because a walker that can suddenly see a whole new region of the tree is as likely to report correct scripts as broken ones.", "refs": ["mendixlabs/mxcli#1149", "mendixlabs/mxcli#1057", "mendixlabs/mxcli#1008", "#532"]} -{"area": "mdl/executor", "date": "2026-09-21", "symptom": "A pluggable widget's textTemplate property took only literal text, so it rendered the same string on every row. `treenode tn (headerType: 'text', headerCaption: 'Name')` passed check, exec and `mx check` and printed the word \"Name\" on every node; the companion the report reached for did not exist — \"widget `tn` (treenode) has no property `headerCaptionParams` — did you mean `headerCaption`? [MDL-WIDGET01]\".", "cause": "The `Params` convention was already how an object-list ITEM bound its text templates (#956, buildObjectListItem looks up matchedAlias+\"Params\"), and it simply stopped at the item boundary. At the WIDGET level the engine read parameters from ONE place: the widget-wide `contentparams:`. Three separate holes: (1) `resolveMapping` case \"TextTemplate\" consulted only `w.GetContentParams()`; (2) a texttemplate mapping addressed by its def.json SOURCE name (an Image's `ImageUrl:`, schema key `imageUrl`) fell through to the `default:` branch, which set no ClientParams at all — not even contentparams, so #928's fix covered only the schema-key spelling; (3) `allowedWidgetProperties` did not know the companion, so writing it was MDL-WIDGET01 and exec refused the script. Added `textTemplateParams` (own companion, then contentparams as fallback), `namedPropValueWithKey`/`templateParamNames` so the companion is found under whichever spelling the template was written in, `addTemplateParamsNames` in the validator, `validatePluggableTemplateParams` (MDL-WIDGET21) for an orphaned companion, and the DESCRIBE side for the Image's two templates.", "file": "`mdl/executor/widget_engine.go` (textTemplateParams, namedPropValueWithKey, templateParamNames, resolveMapping TextTemplate + default branches), `mdl/executor/validate_widgets.go` (addTemplateParamsNames), `mdl/executor/validate_widget_contentparams.go` (validatePluggableTemplateParams), `mdl/executor/cmd_pages_describe_pluggable.go` + `cmd_pages_describe_output.go`", "insight": "**The built-in pluggable Image is the cheap stand-in for any Marketplace widget with this shape** — it has TWO texttemplate properties (`imageUrl`, `alternativeText`), which is exactly the case one widget-wide `contentparams:` cannot express, and it needs no download. The silent control is what makes the bug visible: with the companions deleted and one shared `contentparams: [{1} = PictureUrl]` left, `check`, `exec` and `mx check` are all clean and BOTH stored ClientTemplates bind PictureUrl — measured in the BSON, on a blank 11.14.0 project. Do not reach for the MDL-WIDGET01 message as the whole bug: the reported spelling being rejected is the loud half, and fixing only that leaves the shared-list aliasing intact. Also worth knowing before theorising: a texttemplate mapping's def.json `source` is sometimes a source KIND (\"TextTemplate\") and sometimes a real MDL property name (\"ImageUrl\"), and the two take different branches of resolveMapping — a fix applied to one branch looks complete and covers half the widgets.", "refs": ["ako/mxcli#575", "mendixlabs/mxcli#928", "mendixlabs/mxcli#956"], "rules": ["MDL-WIDGET01", "MDL-WIDGET21"], "ce": ["CE0720"]} +{"area": "mdl/executor", "date": "2026-09-21", "symptom": "A pluggable widget's textTemplate property took only literal text, so it rendered the same string on every row. `treenode tn (headerType: 'text', headerCaption: 'Name')` passed check, exec and `mx check` and printed the word \"Name\" on every node; the companion the report reached for did not exist — \"widget `tn` (treenode) has no property `headerCaptionParams` — did you mean `headerCaption`? [MDL-WIDGET01]\".", "cause": "The `Params` convention was already how an object-list ITEM bound its text templates (#956, buildObjectListItem looks up matchedAlias+\"Params\"), and it simply stopped at the item boundary. At the WIDGET level the engine read parameters from ONE place: the widget-wide `contentparams:`. Three separate holes: (1) `resolveMapping` case \"TextTemplate\" consulted only `w.GetContentParams()`; (2) a texttemplate mapping addressed by its def.json SOURCE name (an Image's `ImageUrl:`, schema key `imageUrl`) fell through to the `default:` branch, which set no ClientParams at all — not even contentparams, so #928's fix covered only the schema-key spelling; (3) `allowedWidgetProperties` did not know the companion, so writing it was MDL-WIDGET01 and exec refused the script. (4) on the READ side `extractExplicitProperties` — the GENERIC describe path, used by every pluggable widget with no dedicated extractor, TreeNode and Timeline included — read AttributeRef and PrimitiveValue only, so every text-template property was absent from `describe page` whether bound or literal, and describe → exec dropped the caption outright. Added `textTemplateParams` (own companion, then contentparams as fallback), `namedPropValueWithKey`/`templateParamNames` so the companion is found under whichever spelling the template was written in, `addTemplateParamsNames` in the validator, `validatePluggableTemplateParams` (MDL-WIDGET21) for an orphaned companion, and the DESCRIBE side for the Image's two templates.", "file": "`mdl/executor/widget_engine.go` (textTemplateParams, namedPropValueWithKey, templateParamNames, resolveMapping TextTemplate + default branches), `mdl/executor/validate_widgets.go` (addTemplateParamsNames), `mdl/executor/validate_widget_contentparams.go` (validatePluggableTemplateParams), `mdl/executor/cmd_pages_describe_pluggable.go` + `cmd_pages_describe_output.go`, `mdl/executor/cmd_pages_describe.go` (rawExplicitProp.Params)", "insight": "**There IS a generic describe path for pluggable widgets and it has a hole, which is not the same as there being no path** — `extractExplicitProperties` handles every widget outside `isKnownCustomWidgetType`'s list of nine, and it read AttributeRef and PrimitiveValue only. Reading the dispatcher rather than running it gives the wrong answer here: the describe LOOKS complete (it emits ten primitives for a TreeNode) and is silently missing the caption. Run `describe page` on the widget before concluding anything about its read side. **Also do not assume the reported widget needs a Marketplace download**: TreeNode.mpk and Timeline.mpk both ship in a blank 11.14.0 app's `widgets/`, and their defs are generated by `mxcli init`, so the exact widgets from a report are usually reproducible directly — check `/widgets/` first. The silent control is what makes the write-side bug visible: with the companions deleted and one shared `contentparams: [{1} = PictureUrl]` left, `check`, `exec` and `mx check` are all clean and BOTH of an Image's stored ClientTemplates bind PictureUrl. Do not treat the MDL-WIDGET01 message as the whole bug — that is the loud half, and fixing only it leaves the shared-list aliasing intact. Last trap: a texttemplate mapping's def.json `source` is sometimes a source KIND (\"TextTemplate\", as on TreeNode/Timeline) and sometimes a real MDL property name (\"ImageUrl\"), and the two take different branches of resolveMapping — a fix applied to one branch looks complete and covers half the widgets.", "refs": ["ako/mxcli#575", "mendixlabs/mxcli#928", "mendixlabs/mxcli#956"], "rules": ["MDL-WIDGET01", "MDL-WIDGET21"], "ce": ["CE0720"]} diff --git a/docs-site/src/language/widget-types.md b/docs-site/src/language/widget-types.md index 9fadab07b..e697de795 100644 --- a/docs-site/src/language/widget-types.md +++ b/docs-site/src/language/widget-types.md @@ -541,8 +541,11 @@ image cardImage ( The companion is the property's own name + `Params`, in whichever spelling the template itself was written. It takes the same per-parameter `format (...)` -block a `dynamictext` does, and `DESCRIBE PAGE` emits it, so describe → exec -keeps the binding. +block a `dynamictext` does, and `DESCRIBE PAGE` emits both the template and its +companion — for the Image and for every other pluggable widget — so describe → +exec keeps the binding. Before ako/mxcli#575 the generic describe path read +attribute references and primitives only, so a TreeNode's `headerCaption` and a +Timeline's `title` were missing from its output altogether. Two shorter spellings remain: diff --git a/mdl-examples/bug-tests/widgets-575-texttemplate-params-companion.mdl b/mdl-examples/bug-tests/widgets-575-texttemplate-params-companion.mdl index abd7047a5..5762ff4d4 100644 --- a/mdl-examples/bug-tests/widgets-575-texttemplate-params-companion.mdl +++ b/mdl-examples/bug-tests/widgets-575-texttemplate-params-companion.mdl @@ -20,17 +20,21 @@ -- every template on the widget, which cannot say "this caption binds Name and -- that one binds Remarks". -- --- This file uses the built-in pluggable Image rather than a Marketplace --- TreeNode, because it has the same shape and needs no download: TWO --- text-template properties, `imageUrl` and `alternativeText`, which is exactly --- what one shared contentparams cannot address. +-- The reported widgets are BUNDLED in a blank app -- TreeNode.mpk and +-- Timeline.mpk both ship in 11.14.0's `widgets/`, and `mxcli init` generates +-- their defs -- so this file exercises them directly. The built-in pluggable +-- Image is here too: it has the same shape (TWO text templates, `imageUrl` and +-- `alternativeText`) and reaches the engine through a DIFFERENT branch, because +-- its def.json names the mappings by SOURCE (`ImageUrl`) where the TreeNode's +-- names them by schema key. A fix applied to one branch looks complete. -- -- Measured on a blank Mendix 11.14.0 project (mxbuild 11.14.0): -- --- pre-fix check -> widget `cardImage` (IMAGE) has no property --- `imageUrlParams` -- did you mean `imageUrl`? --- (x4: imageUrlParams, alternativeTextParams, and the --- same two in the source-name spelling) [MDL-WIDGET01] +-- pre-fix check -> widget `tnProduct` (TREENODE) has no property +-- `headerCaptionParams` -- did you mean +-- `headerCaption`? [MDL-WIDGET01], and the same for +-- titleParams / descriptionParams / imageUrlParams / +-- alternativeTextParams -- -> exec refuses; nothing is written. -- -- with the companions deleted and one shared `contentparams:` @@ -39,10 +43,17 @@ -- parameter {1} out of the same list. That is the reported class: -- valid, buildable, and wrong. -- +-- describe of the treenode/timeline emitted every primitive and +-- NO caption at all (`extractExplicitProperties` read AttributeRef +-- and PrimitiveValue only), so describe -> exec lost it outright. +-- -- fixed check -> Check passed! --- exec -> Created page Bug575.Catalogue +-- exec -> Created page Bug575.Catalogue / Bug575.Tree -- mx check -> 0 errors --- describe -> each template round-trips with its own binding +-- BSON -> headerCaption {1}=[Name]; title {1}=[Name]; +-- description {1}=[PictureUrl]; imageUrl +-- {1}=[PictureUrl]; alternativeText {1}=[Name] +-- describe -> exec -> "Unchanged page" (round trip stable) -- -- Usage: -- mxcli exec mdl-examples/bug-tests/widgets-575-texttemplate-params-companion.mdl -p app.mpr @@ -95,3 +106,29 @@ create or modify page Bug575.Catalogue ( } } / + +-- The widgets the issue was filed against, both bundled in a blank app. Neither +-- has a dedicated DESCRIBE extractor, so they also cover the generic read path: +-- before the fix their captions were absent from `describe page` entirely. +create or modify page Bug575.Tree ( + Title: 'Tree', + Layout: Atlas_Core.Atlas_Default +) +{ + TREENODE tnProduct ( + datasource: DATABASE Bug575.Product, + headerType: 'text', + headerCaption: '{1}', headerCaptionParams: [{1} = Name] + ) + + -- Three templates on one widget, of which two are bound to DIFFERENT + -- attributes. This is the case the widget-wide `contentparams:` cannot + -- express at all; `timeIndication` is left unset and stays unset. + TIMELINE tl ( + data: DATABASE Bug575.Product, + groupEvents: false, + title: '{1}', titleParams: [{1} = Name], + description: '{1}', descriptionParams: [{1} = PictureUrl] + ) +} +/ diff --git a/mdl/executor/cmd_pages_describe.go b/mdl/executor/cmd_pages_describe.go index 1248b2ada..f46bfd8dc 100644 --- a/mdl/executor/cmd_pages_describe.go +++ b/mdl/executor/cmd_pages_describe.go @@ -776,6 +776,13 @@ type rawExplicitProp struct { // String property holding "30" or "true" still has to come back quoted. // Empty when the widget's schema is not in the document (ledger #104). ValueType string + // Params holds a text-template property's `{N}` bindings, emitted as the + // `Params` companion. The generic extractor read AttributeRef and + // PrimitiveValue only, so EVERY text-template property of every widget + // without a dedicated extractor — a TreeNode's headerCaption, a Timeline's + // title/description — was dropped from DESCRIBE whether it was bound or + // literal (ako/mxcli#575). + Params []string } // rawDesignProp represents a parsed design property from BSON. diff --git a/mdl/executor/cmd_pages_describe_output.go b/mdl/executor/cmd_pages_describe_output.go index 905836818..c174cb389 100644 --- a/mdl/executor/cmd_pages_describe_output.go +++ b/mdl/executor/cmd_pages_describe_output.go @@ -727,6 +727,12 @@ func outputWidgetMDLV3(ctx *ExecContext, w rawWidget, indent int) { props = appendWidgetDataSources(props, w) for _, ep := range w.ExplicitProperties { props = append(props, fmt.Sprintf("%s: %s", ep.Key, explicitPropValue(ep))) + // A `{1}` re-executed without its parameter is CE0720, so the + // companion travels with the text it belongs to (#575). + if len(ep.Params) > 0 { + props = append(props, fmt.Sprintf("%sParams: [%s]", + ep.Key, strings.Join(formatParametersV3(ep.Params), ", "))) + } } // onClick action (ledger #67 — reported on CustomChart) if w.OnClick != "" { diff --git a/mdl/executor/cmd_pages_describe_pluggable.go b/mdl/executor/cmd_pages_describe_pluggable.go index d5ddb2683..74bcc9211 100644 --- a/mdl/executor/cmd_pages_describe_pluggable.go +++ b/mdl/executor/cmd_pages_describe_pluggable.go @@ -1146,6 +1146,19 @@ func extractExplicitProperties(ctx *ExecContext, w map[string]any) []rawExplicit } } + // A text-template property: its text, plus the `{N}` bindings under the + // `Params` companion. An unset or widget-hidden template stores a + // null or an empty ClientTemplate and yields "", so it emits nothing. + if text, tt := extractTextTemplateText(value); text != "" { + result = append(result, rawExplicitProp{ + Key: propKey, + Value: text, + ValueType: valueTypes[typePointerID], + Params: extractTextTemplateParameters(ctx, tt), + }) + continue + } + // Check for a PrimitiveValue. // // Booleans used to be dropped here as "common defaults". They are not: diff --git a/mdl/executor/widget_texttemplate_named_params_575_test.go b/mdl/executor/widget_texttemplate_named_params_575_test.go index 8dedfe865..9c4c07ce9 100644 --- a/mdl/executor/widget_texttemplate_named_params_575_test.go +++ b/mdl/executor/widget_texttemplate_named_params_575_test.go @@ -248,3 +248,102 @@ func TestIssue575_DescribeOmitsAnAbsentParamsCompanion(t *testing.T) { } } } + +// treeNodeStoredWidget is one stored CustomWidget in the shape a TreeNode takes: +// a text-template property (`headerCaption`) bound to `{1}` = attr, beside a +// primitive (`headerType`). TreeNode and Timeline have no dedicated DESCRIBE +// extractor, so they go through extractExplicitProperties — which read +// AttributeRef and PrimitiveValue only. +func treeNodeStoredWidget(attr string) map[string]any { + template := map[string]any{ + "$Type": "Forms$ClientTemplate", + "Template": map[string]any{ + "$Type": "Texts$Text", + "Items": []any{ + map[string]any{"$Type": "Texts$Translation", "LanguageCode": "en_US", "Text": "{1}"}, + }, + }, + "Parameters": []any{ + map[string]any{ + "$Type": "Forms$ClientTemplateParameter", + "AttributeRef": map[string]any{"$Type": "DomainModels$AttributeRef", "Attribute": attr}, + "Expression": "", + }, + }, + } + propType := func(id, key, valueType string) map[string]any { + return map[string]any{ + "$ID": id, "$Type": "CustomWidgets$WidgetPropertyType", + "PropertyKey": key, "ValueType": valueType, + } + } + prop := func(ptr string, value map[string]any) map[string]any { + return map[string]any{"$Type": "CustomWidgets$WidgetProperty", "TypePointer": ptr, "Value": value} + } + return map[string]any{ + "Type": map[string]any{"ObjectType": map[string]any{"PropertyTypes": []any{ + propType("pt-1", "headerCaption", "TextTemplate"), + propType("pt-2", "headerType", "Enumeration"), + }}}, + "Object": map[string]any{"Properties": []any{ + prop("pt-1", map[string]any{"$Type": "CustomWidgets$WidgetValue", "TextTemplate": template}), + prop("pt-2", map[string]any{"$Type": "CustomWidgets$WidgetValue", "PrimitiveValue": "text"}), + }}, + } +} + +// The generic extractor must read text templates. Without this every +// text-template property of every widget with no dedicated extractor — a +// TreeNode's headerCaption, a Timeline's title/description/timeIndication — was +// absent from DESCRIBE whether it was bound OR literal, so describe → exec +// dropped the caption entirely and the copy rendered blank. +func TestIssue575_GenericDescribeReadsTextTemplates(t *testing.T) { + ctx, _ := newMockCtx(t) + props := extractExplicitProperties(ctx, treeNodeStoredWidget("Sales.Customer.Name")) + + var caption *rawExplicitProp + for i := range props { + if props[i].Key == "headerCaption" { + caption = &props[i] + } + } + if caption == nil { + t.Fatalf("headerCaption absent from the generic describe; got %+v", props) + } + if caption.Value != "{1}" { + t.Errorf("headerCaption = %q, want {1}", caption.Value) + } + if len(caption.Params) != 1 || caption.Params[0] != "Name" { + t.Errorf("headerCaption params = %v, want [Name] — a `{1}` re-executed with no "+ + "parameter is CE0720", caption.Params) + } + + // The primitive beside it is untouched: the new branch must not swallow + // properties the extractor already handled. + var seenHeaderType bool + for _, p := range props { + if p.Key == "headerType" && p.Value == "text" { + seenHeaderType = true + } + } + if !seenHeaderType { + t.Errorf("headerType lost from the generic describe; got %+v", props) + } +} + +// CONTROL: an unset or widget-hidden template stores a null / empty +// ClientTemplate and must emit nothing — a bare `headerCaption: ”` would +// re-execute into an empty caption where the widget's own default belongs. +func TestIssue575_GenericDescribeOmitsAnEmptyTextTemplate(t *testing.T) { + ctx, _ := newMockCtx(t) + w := treeNodeStoredWidget("Sales.Customer.Name") + obj := w["Object"].(map[string]any) + obj["Properties"].([]any)[0].(map[string]any)["Value"] = map[string]any{ + "$Type": "CustomWidgets$WidgetValue", "TextTemplate": nil, + } + for _, p := range extractExplicitProperties(ctx, w) { + if p.Key == "headerCaption" { + t.Errorf("emitted %q for an unset template", p.Value) + } + } +} From 1237b5cd5ad529b6e4042216995b5b0af4469c14 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 21 Sep 2026 18:42:57 +0000 Subject: [PATCH 14/15] fix(view entity): do not judge a pass-through length mxcli does not know MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit MDL031's pass-through length rule refused EVERY declaration of a column taken from a System.* string attribute — including the one that builds — and suggested the one that does not. Measured on mxbuild 11.14.0. System.User.Name is String(100), not the 200 the field report and my own issue text both assumed: declared mxcli check (before) mx check String(200) refused CE6770 String(100) refused 0 errors <- the correct one String passed CE6770 The inferred length was 0 because the System metadata carries no lengths (0 of 115 attributes, #584), and the rule treated 0 as a length to match rather than as "not known". formatDataTypeForMDL then substituted a hardcoded String(200) for the unknown length, so the message refused String(200) and prescribed String(200) in one sentence — and on this attribute that value is precisely what mxbuild rejects. passthroughStringLengthMismatch now declines a source length of 0. The rule still fires wherever mxcli has the length, with its original message and exact advice, verified end to end: attribute 'MeterName': declared as String(200) but pass-through column 'm.MeterName' inherits length 100 from source attribute Mod.Meter.MeterName — Mendix requires an exact length match (CE6770 …). Fix: change to 'MeterName: String(100)' The hardcoded String(200) in formatDataTypeForMDL is untouched: a DERIVED string column is String(200) by rule whatever its source, so the two type-mismatch suggestion sites want it. Only the pass-through branch, where the number belongs to the source, is affected. The trade, stated rather than hidden: mxcli no longer catches a wrong length over a System.* attribute. It did not really catch one before — it refused everything, which is why the reporter rewrote their column as cast(u.Name as string) — but the gap is now honest, and #584 closes it by supplying the lengths. Control: with the unknown case forced back through the old wording, the test fails with the reported message verbatim, including "inherits length 0" and the invented suggestion. The known-length case is asserted alongside it, so a build that simply deleted the rule fails too. Closes #585 Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01L2aYb3zDscezm874CDTH6R --- .../fix-issue/findings/mdl-executor.jsonl | 1 + ...viewentity-585-system-attribute-length.mdl | 64 ++++++++++++++ .../oql_passthrough_unknown_length_test.go | 84 +++++++++++++++++++ mdl/executor/oql_type_inference.go | 46 ++++++++-- 4 files changed, 186 insertions(+), 9 deletions(-) create mode 100644 mdl-examples/bug-tests/viewentity-585-system-attribute-length.mdl create mode 100644 mdl/executor/oql_passthrough_unknown_length_test.go diff --git a/.claude/skills/fix-issue/findings/mdl-executor.jsonl b/.claude/skills/fix-issue/findings/mdl-executor.jsonl index 057547703..c9149b2c3 100644 --- a/.claude/skills/fix-issue/findings/mdl-executor.jsonl +++ b/.claude/skills/fix-issue/findings/mdl-executor.jsonl @@ -663,3 +663,4 @@ {"area": "mdl/executor", "date": "2026-09-20", "symptom": "`describe page` → `exec` over a **Studio Pro-authored** page silently drops six things, `mx check` 0 errors throughout. The one that matters: `IsPasswordBox True → False` — a **password field round-trips into a plaintext text box**, and describe → rename → exec is mxcli's copy operation. Also `Validation.Expression` blanked, a DataView's `ReadOnlyStyle Text → Control`, `PopupCloseAction` wiped, and two typed-array markers", "cause": "Four different causes behind one symptom, which is why triage came first: (1) IsPasswordBox — model and writer carried it, nothing parsed it, nothing emitted it; (2) Validation — `widgetValidationToGen()` wrote a DEFAULT EMPTY Forms$WidgetValidation over whatever was stored, on five widget types; (3) ReadOnlyStyle — wired for CheckBox only, and a DataView's draws no MDL-WIDGET07 warning because `staticWidgetKnownProps` is deliberately a union across widget types; (4) PopupCloseAction — `pageToGen` wrote \"\" unconditionally. Plus ParameterMappings/OutputMappings markers", "file": "`mdl/executor/cmd_pages_describe_parse.go` + `_output.go` (extract/emit), `cmd_pages_builder_v3_widgets.go` (consume), `cmd_pages_builder_v3.go`, `mdl/visitor/visitor_page_v3.go`, `mdl/ast/ast_page_v3.go`, `sdk/pages/*`, `mdl/backend/modelsdk/widget_write.go` + `page_write.go`, `mdl/executor/validate_widgets.go` (describe vocabulary)", "insight": "**Triage the layer before writing anything** — describer / grammar / builder have different fixes and this one issue had all three. The quickest probe is to run the property through `mxcli check`: MDL-WIDGET07 names an unrecognised one, and *silence is not acceptance* — the known-props list is a union across widget types, so a DataView's ReadOnlyStyle passed check and was dropped anyway. **Emit an expression QUOTED, not bracketed**: `[...]` is the XPath-constraint spelling and `propertyValueV3` parses it as an ARRAY, so `GetStringProp` yields \"\" — the emitter's own unit test was green while the real round trip still lost the value (storage form is not input form). **Measure the default before keeping it**: a DataView's ReadOnlyStyle is Control on 47 of 56, never Inherit, so the 'obvious' Inherit that every other input widget uses would have been wrong. Markers likewise measured, not assumed: ParameterMappings is marker 2 on 220 of 220 lists in every parent type, OutputMappings present on 91 of 91 — and an EMPTY list needs `MandatoryListMarkers` since `RegisterListMarker` keys on a child that is not there. Result 17 → 9 differences, the 9 being ako/mxcli#549", "refs": ["#550", "#541", "#549", "#490"]} {"area": "mdl/executor", "date": "2026-09-20", "symptom": "MDL-PAGEARG01 refused a list widget's OWN row action: `datagrid dg (DataSource: DATABASE M.E, onClick: SHOW_PAGE M.Edit(E: $currentObject))` was rejected at `check` with \"widget `dg` is not inside a data view, list view or grid row\" \u2014 and since exec refuses a script whose check errors, the slice could not be applied at all. On a `listview` the message contradicted itself. mxbuild 11.14.0 accepts the stored pages at 0 errors.", "cause": "The #1029 guard judged EVERY widget's own action in the context its PARENT supplies: `argContextForSubtreeOf` returns the parent context for a childless widget and `validate_widgets.go` passed the inherited `argCtx` to `validateShowPageArguments`. Right for a button, wrong for the widget that ESTABLISHES the context \u2014 a list widget's onClick is row-scoped, so the row it renders is the context object. Added `argContextForOwnAction`: a widget that binds a source of its own supplies the context for its own action; a source in a shape the pass cannot read (the bare-entity shorthand) degrades to UNKNOWN so the guard stands down rather than refusing what it cannot prove is discarded.", "file": "`mdl/executor/cmd_pages_showpage_args.go` (argContextForOwnAction, argContextForSubtreeOf), `mdl/executor/validate_widgets.go`", "insight": "**A false refusal costs more than a missing rule now that exec refuses on a check error** \u2014 the blast radius is 'this project cannot be built with this mxcli', not 'a warning is noisy'. Two things would have caught it before release: judging the rule against the widget kinds it NAMES in its own message (the listview refusal reads 'lvA is not inside a \u2026 list view'), and running it against mxbuild rather than against intuition. The mxbuild run paid for itself twice: it also showed that `DataSource: M.E` (bare-entity shorthand) on a datagrid is silently dropped, so that case is CE0488 + a REAL CE1571 \u2014 the stand-down is still correct, but the shorthand case must not be written into a bug test as mxbuild-clean (#576). Control the fix with the widget kinds STILL refused (a foreign variable, a sibling button beside the grid), or it is indistinguishable from deleting the rule.", "refs": ["#552", "#576", "mendixlabs/mxcli#1029", "#939"]} {"area": "mdl/executor", "date": "2026-09-21", "symptom": "A page's image-collection reference passed `mxcli check --references` and failed the build. Reported as \"no MDL syntax for a StaticImageViewer inside a Selection helper custom state\" — the authoring half was already closed by #1057; what was left is that nothing RESOLVED the name it made writable. Measured on a blank Mendix 11.14.0 project: `staticimage imgAll (Image: 'Atlas_UI_Resources.Atlas_Icons.checkbox_checked')` in a custom state -> check passed, exec created the page, `mx check` -> 3x CE1613 \"The selected image … no longer exists.\"", "cause": "TWO independent holes, and either alone leaves the reported script unchecked. (1) widgetRefCollector keyed the image reference on the widget TYPE — `if w.Type == \"image\"` — so the pluggable widget was collected and `staticimage` (which #1057 had just given the SAME `Image:` property) and `dynamicimage`'s `DefaultImage` were not; replaced with an imageRefProps table. (2) A page's widgets live in two AST fields: `Widgets` is the bare body, `Placeholders` holds `placeholder X { … }` content (#532). validate.go passed `s.Widgets` alone to validateWidgetReferences, validatePageContextTree AND validateFlowArguments, so EVERY reference inside a placeholder block — microflow, nanoflow, page, snippet, entity, image — was validated by nothing; added allPageWidgets to collect both roots once.", "file": "`mdl/executor/helpers.go` (widgetRefCollector.collectFromWidget, imageRefProps), `mdl/executor/validate.go` (allPageWidgets)", "insight": "**When a capability gets a new spelling, grep for who RESOLVES the old one.** #1057 added `Image:` to a second and third widget and moved on; the resolver keyed on the type name, so the new spellings were unchecked from the day they shipped. A property list and a resolver list that describe the same property are two copies — `validate_widgets.go` already accepted `Image`/`DefaultImage` for these widgets and DESCRIBE already emitted them, and only the resolver disagreed. **The placeholder hole is the more useful lesson: it was the THIRD copy of one walk.** validateIconRefs (#1008) and forEachWidget had each grown the `Placeholders` arm separately, with a comment saying a missed walk is silent both ways — and the three validators next door still had not. When a fix is 'add the missing arm to this walker', the question is how many walkers there are; collect the roots once instead. **Do not reason about a bug report from the issue text alone when the version is older than the fix** — the reported symptom did not reproduce on main at all, and running the reporter's own script end to end is what turned 'already fixed, close it' into two real defects. **Control both directions**: a reference that resolves must stay silent, because a walker that can suddenly see a whole new region of the tree is as likely to report correct scripts as broken ones.", "refs": ["mendixlabs/mxcli#1149", "mendixlabs/mxcli#1057", "mendixlabs/mxcli#1008", "#532"]} +{"area": "mdl/executor", "date": "2026-09-21", "symptom": "MDL031's pass-through length rule refused a view entity column over a `System.*` string attribute AND suggested exactly the declaration it had just rejected: \"declared as String(200) but pass-through column 'u.Name' inherits length 0 from source attribute System.User.Name \u2026 Fix: change to 'EngineerName: String(200)'\". Worse than self-contradictory: it refused EVERY declaration except `String` (unlimited), which is the one spelling that fails the build.", "cause": "The System metadata carries no attribute lengths (0 of 115, #584), so the inferred length is 0, and the rule treated 0 as a length to match. `formatDataTypeForMDL` then substituted a hardcoded `String(200)` for any unknown length, which is correct for a DERIVED column (String(200) by rule) and wrong here, where the number is the source's. Fixed by having the predicate decline an unknown length; the message collapses to the one case it can judge.", "file": "`mdl/executor/oql_type_inference.go` (passthroughStringLengthMismatch, passthroughLengthError)", "insight": "**Ask mxbuild for the number rather than trusting either side's belief about it.** The field report said System.User.Name is String(200) and I repeated it in the issue; two one-column view entities settled it in two builds \u2014 String(200) is CE6770, String(100) is 0 errors, so it is String(100). That single measurement overturned the report, my issue text, AND my first proposed fix. The decisive third probe was checking what `check` did to the CORRECT declaration: it refused that too, which turned 'confusing message' into 'the rule blocks the right answer and waves through the wrong one' and changed the fix from re-wording to not judging. When a rule compares against a value your own metadata supplies, test the case where that value is MISSING \u2014 a 0 that means 'unknown' silently becomes a 0 that means 'zero'. Probe recipe, reusable and needing no decompiler: one small view entity per attribute, two builds, bisect.", "refs": ["#585", "#584", "#583"]} diff --git a/mdl-examples/bug-tests/viewentity-585-system-attribute-length.mdl b/mdl-examples/bug-tests/viewentity-585-system-attribute-length.mdl new file mode 100644 index 000000000..22de21a68 --- /dev/null +++ b/mdl-examples/bug-tests/viewentity-585-system-attribute-length.mdl @@ -0,0 +1,64 @@ +-- ako/mxcli#585 — MDL031's pass-through length rule refused EVERY declaration +-- of a column taken from a System.* string attribute, including the one that +-- builds, and suggested the one that does not. +-- +-- Measured on mxbuild 11.14.0 (System.User.Name is String(100), not the 200 the +-- field report and the issue both assumed): +-- +-- declared mxcli check (before) mx check +-- String(200) refused CE6770 +-- String(100) refused 0 errors <- the correct one +-- String passed CE6770 +-- +-- The inferred length was 0 because the System metadata carries no lengths +-- (0 of 115 attributes — #584), and the rule treated 0 as a length to match. +-- It now declines a length it does not know; #584 restores the judgement. +-- +-- WHAT THIS FILE CHECKS, AND WHAT IT DOES NOT. `make check-mdl` runs +-- `mxcli check` with NO project, so it only proves these statements parse. The +-- length rule needs the project to resolve System.User at all, so run it with +-- `-p .mpr` to exercise the fix: +-- +-- mxcli check mdl-examples/bug-tests/viewentity-585-system-attribute-length.mdl -p app.mpr +-- -> Check passed! (both view entities below) +-- +-- The assertion that survives CI is the unit test, +-- mdl/executor/oql_passthrough_unknown_length_test.go, which pins the predicate +-- directly AND keeps the known-length case refused. + +create module Ve585; + +-- 1. The declaration mxbuild accepts. Refused before the fix — this is the +-- statement the reporter could not write, which drove them to cast(). +create view entity Ve585.UserNames ( + UserName: String(100) +) as ( + from System.User as u + select u.Name as UserName +); + +-- 2. The declaration mxbuild rejects (CE6770). Also refused before the fix, and +-- also accepted now: with no length for System.User.Name, mxcli cannot tell +-- these two apart. That is the trade this fix makes, and #584 closes it. +-- `mxcli docker check` is the arbiter until then. +create view entity Ve585.UserNames200 ( + UserName: String(200) +) as ( + from System.User as u + select u.Name as UserName +); + +-- 3. The control: a source whose length mxcli DOES know. The rule must still +-- refuse this, with the exact fix named — a build where the rule had simply +-- been deleted would accept it. +-- +-- create persistent entity Ve585.Meter (MeterName: String(100)); +-- create view entity Ve585.MeterNames (MeterName: String(200)) as ( +-- from Ve585.Meter as m select m.MeterName as MeterName +-- ); +-- -> attribute 'MeterName': declared as String(200) but pass-through column +-- 'm.MeterName' inherits length 100 from source attribute +-- Ve585.Meter.MeterName … Fix: change to 'MeterName: String(100)' +-- +-- Left commented because this file must pass `make check-mdl`; the live +-- version of this control is in the unit test named above. diff --git a/mdl/executor/oql_passthrough_unknown_length_test.go b/mdl/executor/oql_passthrough_unknown_length_test.go new file mode 100644 index 000000000..860490cf1 --- /dev/null +++ b/mdl/executor/oql_passthrough_unknown_length_test.go @@ -0,0 +1,84 @@ +// SPDX-License-Identifier: Apache-2.0 + +// ako/mxcli#585 — MDL031's pass-through length error prescribed the very +// declaration mxbuild rejects. +// +// `formatDataTypeForMDL` substitutes a hardcoded `String(200)` when the inferred +// length is 0, so a source attribute whose length mxcli does not know produced: +// +// declared as String(200) but pass-through column 'u.Name' inherits length 0 +// from source attribute System.User.Name … Fix: change to 'X: String(200)' +// +// — refusing String(200) and prescribing String(200) in one sentence. Measured on +// mxbuild 11.14.0, `System.User.Name` is String(100): the refusal was right and +// the advice was actively wrong, which is why the reporter rewrote the column as +// `cast(u.Name as string)` instead. +// +// The hardcoded 200 is CORRECT for a derived string column, which is String(200) +// by rule whatever its source — so the fix is confined to the pass-through +// branch, where the number is the SOURCE's and mxcli does not have it. +package executor + +import ( + "strings" + "testing" + + "github.com/mendixlabs/mxcli/mdl/ast" +) + +func TestPassthroughLength_UnknownSourceLengthIsNotJudged(t *testing.T) { + // Measured on mxbuild 11.14.0, System.User.Name is String(100): + // + // String(200) refused by check CE6770 from mxbuild + // String(100) refused by check 0 errors from mxbuild ← the correct one + // String passed by check CE6770 from mxbuild + // + // So judging against a length of 0 blocked the right answer and waved through + // the wrong one. Neither declaration may be refused until mxcli knows the + // source length (#584). + for _, declaredLen := range []int{100, 200} { + if passthroughStringLengthMismatch( + ast.DataType{Kind: ast.TypeString, Length: declaredLen}, + ast.DataType{Kind: ast.TypeString, Length: 0}, + "Name") { + t.Errorf("String(%d) over a source of unknown length was refused — the rule cannot tell "+ + "a correct declaration from a wrong one here, and refusing blocks the one that builds", + declaredLen) + } + } + + // The control: a KNOWN source length is the case the rule exists for, and it + // must still fire, with advice that names the exact declaration. + t.Run("a known source length is still judged", func(t *testing.T) { + declared := ast.DataType{Kind: ast.TypeString, Length: 200} + inferred := ast.DataType{Kind: ast.TypeString, Length: 100} + if !passthroughStringLengthMismatch(declared, inferred, "Name") { + t.Fatal("String(200) over a String(100) source was not refused — this is CE6770 at build time") + } + msg := passthroughLengthError("UserName", declared, inferred, "u.Name", "System.User", "Name") + for _, want := range []string{"inherits length 100", "change to 'UserName: String(100)'", "CE6770"} { + if !strings.Contains(msg, want) { + t.Errorf("message does not mention %q: %s", want, msg) + } + } + }) + + // And an equal known length is not a mismatch at all. + if passthroughStringLengthMismatch( + ast.DataType{Kind: ast.TypeString, Length: 100}, + ast.DataType{Kind: ast.TypeString, Length: 100}, + "Name") { + t.Error("a matching declaration was refused") + } +} + +// A derived string column is String(200) by rule whatever its source, so the +// default in formatDataTypeForMDL must stay for the type-mismatch suggestions. +func TestFormatDataTypeForMDL_KeepsTheDerivedStringDefault(t *testing.T) { + if got := formatDataTypeForMDL(ast.DataType{Kind: ast.TypeString, Length: 0}); got != "String(200)" { + t.Errorf("derived string suggestion = %q, want String(200) — MDL031's rule for a derived column", got) + } + if got := formatDataTypeForMDL(ast.DataType{Kind: ast.TypeString, Length: 100}); got != "String(100)" { + t.Errorf("suggestion with a known length = %q, want String(100)", got) + } +} diff --git a/mdl/executor/oql_type_inference.go b/mdl/executor/oql_type_inference.go index 78fc62280..c52555ed2 100644 --- a/mdl/executor/oql_type_inference.go +++ b/mdl/executor/oql_type_inference.go @@ -422,9 +422,44 @@ func inferCaseType(expr string) ast.DataType { func passthroughStringLengthMismatch(declared, inferred ast.DataType, sourceAttr string) bool { return sourceAttr != "" && declared.Kind == ast.TypeString && inferred.Kind == ast.TypeString && + // A source length of 0 means mxcli does not KNOW the length, not that the + // source is unbounded — today that is every System.* string, because the + // metadata carries none (ako/mxcli#584). Judging against 0 refused every + // declaration except `String`, which is the one spelling that definitely + // fails the build. Measured on mxbuild 11.14.0 against System.User.Name, + // which is String(100): + // + // String(200) check: refused mxbuild: CE6770 + // String(100) check: refused mxbuild: 0 errors ← the correct one + // String check: passed mxbuild: CE6770 + // + // So the rule blocked the right answer and waved through the wrong one. + // It now declines to judge what it cannot measure; #584 restores the + // judgement by giving it the lengths. + inferred.Length > 0 && declared.Length != inferred.Length } +// passthroughLengthError words the pass-through length refusal, for the only +// case the rule now fires in: a source length mxcli actually knows. +// +// The unknown case used to be worded from the same string, reporting "inherits +// length 0" as though 0 were the source's length and then prescribing a +// hardcoded String(200) from formatDataTypeForMDL — which, for System.User.Name +// on 11.14.0, is precisely the value mxbuild rejects (ako/mxcli#585). It is +// no longer reachable: passthroughStringLengthMismatch declines an unknown +// length rather than guessing at it. +// +// The hardcoded String(200) in formatDataTypeForMDL is untouched and still +// right where it is used: a DERIVED string column is String(200) by rule +// whatever its source. +func passthroughLengthError(attrName string, declared, inferred ast.DataType, expression, sourceEntity, sourceAttr string) string { + return fmt.Sprintf( + "attribute '%s': declared as %s but pass-through column '%s' inherits length %d from source attribute %s.%s — Mendix requires an exact length match (CE6770 \"View Entity out of sync\"). Fix: change to '%s: %s'", + attrName, formatDataTypeForError(declared), expression, inferred.Length, + sourceEntity, sourceAttr, attrName, formatDataTypeForMDL(inferred)) +} + // validateViewEntityTypes validates that declared attribute types match inferred OQL types. func validateViewEntityTypes(ctx *ExecContext, stmt *ast.CreateViewEntityStmt) []string { var errors []string @@ -466,15 +501,8 @@ func validateViewEntityTypes(ctx *ExecContext, stmt *ast.CreateViewEntityStmt) [ // set only for direct attribute references, never for aggregates/derived // expressions. (ledger finding #36) if passthroughStringLengthMismatch(attr.Type, col.InferredType, col.SourceAttr) { - errors = append(errors, fmt.Sprintf( - "attribute '%s': declared as %s but pass-through column '%s' inherits length %d from source attribute %s.%s — Mendix requires an exact length match (CE6770 \"View Entity out of sync\"). Fix: change to '%s: %s'", - attr.Name, - formatDataTypeForError(attr.Type), - col.Expression, - col.InferredType.Length, - col.SourceEntity, col.SourceAttr, - attr.Name, - formatDataTypeForMDL(col.InferredType))) + errors = append(errors, passthroughLengthError( + attr.Name, attr.Type, col.InferredType, col.Expression, col.SourceEntity, col.SourceAttr)) continue } From 616e2d238e422ae697ccbadb0e17147907938d29 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 22 Sep 2026 03:09:54 +0000 Subject: [PATCH 15/15] Measure the System module's String lengths (#584) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `meta.SystemAttrDef` declared a `Length` and not one of the 115 String attributes populated it, so `systemAttrType` built every System string as `StringAttributeType{Length: 0}` — unlimited — and every length comparison against a System attribute was made against 0. The cost was not cosmetic. A view entity selecting `u.Name` from System.User could not be declared in any way that both passed check and built: declared mxcli check (before) mxcli check (after) mx check 11.14.0 String(100) refused passes 0 errors String(200) passed (after #585) refused, "use 100" CE6770 String passed refused, "use 100" CE6770 The reporter worked round it with `cast(u.Name as string)`, where the derived column rule fixes the length at 200 whatever the source. The lengths are measured, from the System module's domain model inside the `deployment/model/model.mdp` mxbuild writes — the model the runtime builds the System tables from, so the same number CE6770 is decided by. That is one `--target=deploy` for all 115, against ~40s per attribute for the view-entity probe; the Model SDK does not carry them at all (its gen/ describes metamodel types) and the modeler's copy is inside Mendix.Modeler.Core.dll. Whether the numbers are version-specific was measured rather than assumed: 10.24.4.77222 and 11.14.0 agree on all 216 attributes they share, types and lengths alike, so one table serves every supported version. - testdata/system_string_lengths.txt holds the measurement, with the command that regenerates it. - TestSystemStringLengths holds SystemEntities to it in BOTH directions, so a String attribute added without a measurement fails rather than defaulting to unlimited. That is what makes a stored 0 Mendix's "unlimited" rather than "nobody looked" — 46 of the 115 genuinely are unlimited. - TestSystemStringLengthsUpdate is the measuring half: point it at a built model.mdp with -mdp, add -update to rewrite the golden. Control: reverting the populated lengths fails both new tests with the reported symptom ("System.User.Name: read as String(0), mxbuild builds it as String(100)"). End to end on mxbuild 11.14.0, the String(100) view entity is 0 errors and the String(200) one is CE6770, matching mxcli's new verdicts. The 29 System members 11.14.0 has that this table does not are left alone: a member absent from the target version is CE1613, which is a version-gating question a length never is. Closes #584 Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01L2aYb3zDscezm874CDTH6R --- .../skills/fix-issue/findings/modelsdk.jsonl | 1 + CLAUDE.md | 1 + .../viewentity-584-system-string-lengths.mdl | 56 ++++++ ...viewentity-585-system-attribute-length.mdl | 33 ++-- .../modelsdk/system_module_read_test.go | 57 +++++- modelsdk/meta/system_module.go | 166 ++++++++++-------- .../system_string_lengths_measure_test.go | 160 +++++++++++++++++ modelsdk/meta/system_string_lengths_test.go | 128 ++++++++++++++ .../meta/testdata/system_string_lengths.txt | 139 +++++++++++++++ 9 files changed, 655 insertions(+), 86 deletions(-) create mode 100644 mdl-examples/bug-tests/viewentity-584-system-string-lengths.mdl create mode 100644 modelsdk/meta/system_string_lengths_measure_test.go create mode 100644 modelsdk/meta/system_string_lengths_test.go create mode 100644 modelsdk/meta/testdata/system_string_lengths.txt diff --git a/.claude/skills/fix-issue/findings/modelsdk.jsonl b/.claude/skills/fix-issue/findings/modelsdk.jsonl index 414ecedec..59c630c7b 100644 --- a/.claude/skills/fix-issue/findings/modelsdk.jsonl +++ b/.claude/skills/fix-issue/findings/modelsdk.jsonl @@ -16,3 +16,4 @@ {"area": "modelsdk/canon", "date": "2026-08-28", "symptom": "`mx check` fails with **`InvalidOperationException: … Duplicate Guid in unit page 'M.P'. Object types: Translation, Translation`** and the project will not LOAD at all — and every later edit of that page reports the same thing", "cause": "Two elements in one unit share an `$ID`. \"Guid\" in Mendix's message is the element `$ID`, not a `GUID` property — a `Texts$Translation` stores exactly four keys (`$ID`, `$Type`, `LanguageCode`, `Text`) and has no GUID of its own", "file": "`modelsdk/canon/duplicates.go` (`DuplicateElementIDs`, `DuplicateElementIDError`), wired at all four write choke points: `modelsdk/mpr/writer_core.go` (`insertUnit`, `updateUnit`) and `sdk/mpr/writer_units.go` (same two)", "insight": "**A guard is worth shipping without a root cause.** The reporter lost ~1h to one of these, proposed two causes, tested both, and withdrew both (ako/mxcli-captrack #2) — the trigger is still unknown. Refusing at the write turns an unopenable project into one message, and breaks the thing that made it expensive: the corruption is **STICKY**, so once a unit carries duplicate ids every later edit inherits them and each subsequent write looks like the culprit. Restore before re-testing, or the second experiment measures the first one's damage. Check BEFORE `canon.Reconcile`, not after — an elided write is not a safe one, only one that did not happen this time. Two controls are mandatory and both are cheap: **pointers are not duplicates** (an element's id is referenced by primitive properties like `ParentPointer` all over a normal document — counting those refuses every write), and a **normal translated page** with two translations differing only in id must still be accepted. False-positive control run at scale: **0 flagged out of 33,645 units across 90 projects**. Do NOT \"repair\" by deduplicating — an `$ID` is a pointer target, so choosing which element keeps its identity silently re-points references (ADR-0008).", "refs": ["#2"]} {"area": "modelsdk", "date": "2026-09-02", "symptom": "Every in-place edit of a page is refused with `refusing to write unit \u2026: 1 element id(s) are used more than once \u2026 held by [Texts$Translation \u00d78]` \u2014 `GRANT VIEW ON PAGE`, `ALTER PAGE \u2026 INSERT` \u2014 while a full `CREATE OR REPLACE PAGE` still works, so the page looks correct and only UPDATES are blocked. Surfaces after a second language is enabled.", "cause": "`canon.CarryTranslations` pairs a rebuilt text to its stored translations BY SOURCE STRING when the two documents' text paths differ, and `mergeText` appended the stored `Texts$Translation` element **verbatim** \u2014 deliberately, because keeping the stored `$ID` is what lets no-op elision fire. When several rebuilt texts share one source string (eight copies of the literal `'{1}'` on a page is ordinary), all of them resolve to the SAME stored set and every one got the same element, id included. `reuseSafeID` now gives the first use the stored id and derives a fresh deterministic one (SHA-256 of stored id + containment path + language) for each further copy; the visit order is sorted rather than map order, or which text keeps the stored id would vary per run and the document would churn.", "file": "`modelsdk/canon/translations.go` (`reuseSafeID`, `derivedID`, `elementIDs`, `sortedPaths`, `mergeText`), `modelsdk/canon/duplicates.go` (comment corrected \u2014 it recorded the cause as unestablished)", "insight": "**Re-identifying a copy is safe here in a way that deduplicating ids in general is not, and that distinction is the whole argument.** An `$ID` is a pointer target and rewriting one means finding every reference (ADR-0008) \u2014 which is exactly why `duplicates.go` refuses rather than repairs. Nothing references a `Texts$Translation`: it is a leaf child of a `Texts$Text` with four keys and no identity anything resolves by, so there are no references to miss. Only the COPIES are re-identified; the first use keeps the stored id, so an unchanged document still compares equal. **Verify elision explicitly after touching this** \u2014 the fix trades against the exact property the verbatim append existed for: measured, a second identical run still reports `Unchanged page` with the same sha and mtime. Controls, end-to-end on a real 11.13 project with de_DE enabled and three widgets sharing a caption: the pre-fix binary writes one id used 3\u00d7 and the next `ALTER PAGE` is refused with the reporter's message verbatim; the fixed binary writes 27 distinct ids for 27 elements, the `ALTER PAGE` succeeds, the German translation survives (the control against a 'fix' that just stops carrying), and `mx check` is 0 errors. Reported as CapTrackV2 FINDINGS \u00a730/\u00a717."} {"area": "mdl/backend/modelsdk", "date": "2026-09-07", "symptom": "`mxcli lint` QUAL002 reported \"Page 'X' has no documentation\" against a page carrying a javadoc comment; the catalog's Description column was blank for every page and snippet; `describe page` emitted no documentation. The comment looked, from every angle, like it had been dropped (ako/CapTrackV4 R12).", "cause": "Nothing was dropped: the AST, executor and writer all carry it, and `mxcli bson dump --type page` shows Documentation with the right value. pageFromGen and the ListSnippets constructor in mdl/backend/modelsdk/page.go simply did not read it back, so on the DEFAULT engine every symptom downstream of the read was wrong at once. Fixed by carrying Documentation in both. Separately, QUAL002 stopped sweeping modules: a Mendix module HAS no documentation property (generated/metamodel's ProjectsModule declares none, modelsdk/gen's Module has no accessor, no stored Projects$ModuleImpl carries the key).", "file": "`mdl/backend/modelsdk/page.go` (pageFromGen, ListSnippets); `mdl/linter/context.go` (documentableSources); `.claude/lint-rules/missing_documentation.star`; tests `mdl/backend/modelsdk/page_documentation_test.go`", "insight": "When a value looks absent everywhere, check the WRITE first: `bson dump` showed it stored correctly and localised the bug to the read in one step, where chasing the reported symptom would have started at the visitor. The engine split is the second cheap discriminator — the legacy reader parsed it fine, so the defect was in the default engine alone. A stale catalog nearly hid that: an earlier per-engine comparison reused a cached catalog.db and showed both engines empty, so DELETE the catalog between engine comparisons rather than trusting `refresh catalog full`. Finally, page and snippet were 2 of 5 sibling readers in one file — layout, building block and page template all carried Documentation — which is the shape to look for when one document type behaves differently from its neighbours. And a rule asking for a property the platform does not have is not a gap in the language: three sources agreed before that row was removed."} +{"area": "modelsdk/meta", "date": "2026-09-22", "symptom": "A view entity selecting `u.Name` from System.User could not be declared in any way that both passed `mxcli check` and built: `String(100)` (the correct length) was refused, `String` (unlimited) passed check and then failed mxbuild with CE6770 \"View Entity is out of sync with the OQL Query\". `describe entity System.User` reported `Name: String(unlimited)`. Reported in ako/ChipCoV4 FINDINGS.md against Mendix 11.14.0 (ako/mxcli#584, with #585 the other half).", "cause": "meta.SystemAttrDef declared a Length field and NOT ONE of the 115 String attributes in modelsdk/meta/system_module.go populated it, so systemAttrType built every System string as StringAttributeType{Length: 0} — which mxcli reads as unlimited. Every length comparison against a System attribute was therefore made against 0. Fixed by measuring all 115 and populating them, with a golden table (modelsdk/meta/testdata/system_string_lengths.txt) and TestSystemStringLengths holding the two in step.", "file": "`modelsdk/meta/system_module.go` (SystemEntities, SystemAttrDef.Length); `modelsdk/meta/testdata/system_string_lengths.txt`; tests `modelsdk/meta/system_string_lengths_test.go`, `modelsdk/meta/system_string_lengths_measure_test.go`, `mdl/backend/modelsdk/system_module_read_test.go`", "insight": "The System module's attribute lengths are IN THE BUILD OUTPUT: `deployment/model/model.mdp` is a stream of BSON documents (each with its own 4-byte length prefix — unmarshalling the file whole fails with \"invalid document length\"), the System module arrives as a Projects$ModuleImpl carrying only a Name with its DomainModels$DomainModel immediately after, and every entity's attributes are there with their StringAttributeType.Length. That is ONE `mxbuild --target=deploy` for all 115, and it is the model the runtime builds the tables from, so it is the same number CE6770 is decided by. Two searches not worth repeating, both spent on this issue: the Mendix Model SDK does not carry them (its gen/ describes metamodel TYPES, so System.User.Name is not in it) and the modeler's own copy is inside Mendix.Modeler.Core.dll, i.e. a decompiler. The one-view-entity-per-attribute mxbuild probe works but is ~40s each. The version question answers itself the same way: building 10.24.4.77222 as well showed all 216 shared attributes identical in type AND length to 11.14.0, so one table serves every supported version instead of a per-version registry — measure the second version rather than reasoning about it, it is one more build. Finally, 0 is Mendix's own encoding of \"unlimited\" (46 of the 115), so populating the table does not make 0 safe to read as a length — what makes it safe is that the golden enumerates every String attribute, so 'unmeasured' cannot exist without failing a test."} diff --git a/CLAUDE.md b/CLAUDE.md index 36b6bee63..780027be9 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -916,6 +916,7 @@ Full syntax tables for all MDL statements (microflows, pages, security, navigati - `sdk/widgets/templates/` - Embedded widget templates for pluggable widgets (ComboBox, DataGrid2, etc.) - `sdk/widgets/templates/README.md` - **Critical**: Template extraction requirements (must include both `type` AND `object`) - `generated/metamodel/enums.go` - All Mendix enumeration types +- `modelsdk/meta/system_module.go` - The virtual System module's entities, attributes and associations. String lengths are **measured**, from the System module's domain model inside a built `deployment/model/model.mdp` (a BSON document stream, one `mxbuild --target=deploy` for all 115 at once) — not from the Model SDK, which describes metamodel types and does not contain them. `modelsdk/meta/testdata/system_string_lengths.txt` is the measurement and `TestSystemStringLengths` holds the table to it; a `Length` of 0 is Mendix's "unlimited", never "unmeasured". Measured identical across 10.24.4 and 11.14.0, which is why there is one table and not a per-version registry - `mdl/grammar/MDL.g4` - ANTLR4 grammar for MDL syntax (production) - `mdl/executor/executor.go` - MDL statement execution logic - `reference/mdl-grammar/` - Comprehensive MDL grammar reference diff --git a/mdl-examples/bug-tests/viewentity-584-system-string-lengths.mdl b/mdl-examples/bug-tests/viewentity-584-system-string-lengths.mdl new file mode 100644 index 000000000..63cbbb48d --- /dev/null +++ b/mdl-examples/bug-tests/viewentity-584-system-string-lengths.mdl @@ -0,0 +1,56 @@ +-- ako/mxcli#584 — the System module metadata declared a Length and populated it +-- for none of its 115 String attributes, so every System string read as +-- unlimited and mxcli's length checks disagreed with mxbuild's. +-- +-- Measured on mxbuild 11.14.0 against a view entity over System.User.Name, +-- which Mendix builds as String(100): +-- +-- declared mxcli check (before) mxcli check (after) mx check +-- String(100) refused PASSES 0 errors +-- String(200) passed refused, "use 100" CE6770 +-- String passed refused, "use 100" CE6770 +-- +-- Before, no declaration both passed check and built; the reporter worked round +-- it with `cast(u.Name as string)`, where the derived-column rule fixes the +-- length at 200 regardless of source. After, mxcli's verdict matches mxbuild's +-- on all three. +-- +-- WHAT THIS FILE CHECKS, AND WHAT IT DOES NOT. `make check-mdl` runs +-- `mxcli check` with NO project, so it only proves these statements parse. +-- Resolving System.User needs the project, so run it with -p to exercise the fix: +-- +-- mxcli check mdl-examples/bug-tests/viewentity-584-system-string-lengths.mdl -p app.mpr --references +-- -> Check passed! +-- +-- The assertion that survives CI is modelsdk/meta/system_string_lengths_test.go, +-- which holds SystemEntities to the measured table, plus +-- mdl/backend/modelsdk/system_module_read_test.go, which proves the length +-- reaches the model the rest of mxcli reads. + +create module Ve584; + +-- The declaration mxbuild accepts, and the one mxcli could not express before. +create view entity Ve584.UserNames ( + UserName: String(100) +) as ( + from System.User as u + select u.Name as UserName +); + +-- A second System source with a different length, so a build that hardcoded one +-- number would not pass this file. System.FileDocument.Name is String(400). +create view entity Ve584.FileNames ( + FileName: String(400) +) as ( + from System.FileDocument as f + select f.Name as FileName +); + +-- The wrong declarations are NOT in this file, because they must fail +-- `mxcli check -p`. They are: +-- +-- create view entity Ve584.UserNames200 ( UserName: String(200) ) as ( +-- from System.User as u select u.Name as UserName ); +-- -> declared as String(200) but pass-through column 'u.Name' inherits +-- length 100 from source attribute System.User.Name … change to +-- 'UserName: String(100)' [and CE6770 from mxbuild, measured] diff --git a/mdl-examples/bug-tests/viewentity-585-system-attribute-length.mdl b/mdl-examples/bug-tests/viewentity-585-system-attribute-length.mdl index 22de21a68..b4b13d0a0 100644 --- a/mdl-examples/bug-tests/viewentity-585-system-attribute-length.mdl +++ b/mdl-examples/bug-tests/viewentity-585-system-attribute-length.mdl @@ -10,9 +10,11 @@ -- String(100) refused 0 errors <- the correct one -- String passed CE6770 -- --- The inferred length was 0 because the System metadata carries no lengths +-- The inferred length was 0 because the System metadata carried no lengths -- (0 of 115 attributes — #584), and the rule treated 0 as a length to match. --- It now declines a length it does not know; #584 restores the judgement. +-- It now declines a length it does not know. #584 then measured the lengths, so +-- System.User.Name is no longer such a case: after both fixes, mxcli's verdict +-- matches mxbuild's on all three declarations above. -- -- WHAT THIS FILE CHECKS, AND WHAT IT DOES NOT. `make check-mdl` runs -- `mxcli check` with NO project, so it only proves these statements parse. The @@ -20,7 +22,7 @@ -- `-p .mpr` to exercise the fix: -- -- mxcli check mdl-examples/bug-tests/viewentity-585-system-attribute-length.mdl -p app.mpr --- -> Check passed! (both view entities below) +-- -> Check passed! -- -- The assertion that survives CI is the unit test, -- mdl/executor/oql_passthrough_unknown_length_test.go, which pins the predicate @@ -38,15 +40,22 @@ create view entity Ve585.UserNames ( ); -- 2. The declaration mxbuild rejects (CE6770). Also refused before the fix, and --- also accepted now: with no length for System.User.Name, mxcli cannot tell --- these two apart. That is the trade this fix makes, and #584 closes it. --- `mxcli docker check` is the arbiter until then. -create view entity Ve585.UserNames200 ( - UserName: String(200) -) as ( - from System.User as u - select u.Name as UserName -); +-- briefly accepted after it: with no length for System.User.Name, mxcli +-- could not tell these two apart. #584 supplied the length (String(100), +-- measured from the deployed model), so `mxcli check -p` refuses THIS +-- statement again — this time for the right reason, naming 100. That means +-- the file below no longer passes `mxcli check -p …` as a whole; run +-- statement 1 alone to see the #585 fix, or use +-- bug-tests/viewentity-584-system-string-lengths.mdl, which is the +-- both-halves-fixed version. It is commented out here for the same reason +-- statement 3 is: this file has to pass `mxcli check`. +-- +-- create view entity Ve585.UserNames200 ( +-- UserName: String(200) +-- ) as ( +-- from System.User as u +-- select u.Name as UserName +-- ); -- 3. The control: a source whose length mxcli DOES know. The rule must still -- refuse this, with the exact fix named — a build where the rule had simply diff --git a/mdl/backend/modelsdk/system_module_read_test.go b/mdl/backend/modelsdk/system_module_read_test.go index 2531f22f7..a16eace6b 100644 --- a/mdl/backend/modelsdk/system_module_read_test.go +++ b/mdl/backend/modelsdk/system_module_read_test.go @@ -2,7 +2,11 @@ package modelsdkbackend -import "testing" +import ( + "testing" + + "github.com/mendixlabs/mxcli/sdk/domainmodel" +) // TestSystemDomainModel_UniqueAttrIDs guards the catalog PK invariant: the // virtual System module's attributes must each have a unique, non-empty ID. @@ -24,3 +28,54 @@ func TestSystemDomainModel_UniqueAttrIDs(t *testing.T) { } } } + +// TestSystemDomainModel_StringLengthsReachTheModel is ako/mxcli#584 at the layer +// the reporter met it: the length has to survive meta.SystemAttrDef -> +// domainmodel.StringAttributeType, because that is what `describe entity` +// prints and what the view-entity pass-through rule (MDL031) compares a +// declaration against. +// +// Before #584 every one of these read as 0 — "unlimited" — so `describe entity +// System.User` said `Name: String(unlimited)` while mxbuild builds the table at +// String(100), and a view entity selecting u.Name could not be declared in any +// way that both passed check and built. +func TestSystemDomainModel_StringLengthsReachTheModel(t *testing.T) { + // Measured on the 11.14.0 deployed model; see + // modelsdk/meta/testdata/system_string_lengths.txt. + want := map[string]int{ + "User.Name": 100, + "FileDocument.Name": 400, + "Image.PublicThumbnailPath": 500, + "Language.Code": 20, + "HttpMessage.HttpVersion": 10, + // 0 here is Mendix's "unlimited", and is measured too — it must not be + // mistaken for the unset value the whole table used to carry. + "Error.Message": 0, + } + + dm := buildSystemDomainModel() + seen := map[string]bool{} + for _, e := range dm.Entities { + for _, a := range e.Attributes { + key := e.Name + "." + a.Name + expected, ok := want[key] + if !ok { + continue + } + seen[key] = true + st, isString := a.Type.(*domainmodel.StringAttributeType) + if !isString { + t.Errorf("System.%s: want a string attribute, got %T", key, a.Type) + continue + } + if st.Length != expected { + t.Errorf("System.%s: read as String(%d), mxbuild builds it as String(%d)", key, st.Length, expected) + } + } + } + for key := range want { + if !seen[key] { + t.Errorf("System.%s is not in the domain model at all", key) + } + } +} diff --git a/modelsdk/meta/system_module.go b/modelsdk/meta/system_module.go index d4b34f8ff..158be20e9 100644 --- a/modelsdk/meta/system_module.go +++ b/modelsdk/meta/system_module.go @@ -10,9 +10,22 @@ const ( // SystemAttrDef defines an attribute in a System entity. type SystemAttrDef struct { - Name string - Type string // "String", "Integer", "Decimal", "Boolean", "DateTime", "Enumeration", "Long", "Binary", "HashedString", "AutoNumber" - Length int // for String type + Name string + Type string // "String", "Integer", "Decimal", "Boolean", "DateTime", "Enumeration", "Long", "Binary", "HashedString", "AutoNumber" + + // Length is the maximum string length for Type "String", carrying Mendix's + // own meaning: 0 is UNLIMITED, not unknown. Every String attribute here is + // measured from a deployed model (testdata/system_string_lengths.txt), and + // TestSystemStringLengths fails on a String attribute that has no + // measurement — so a 0 is a finding, never a gap. + // + // It went three versions unpopulated, which made every System string read + // as unlimited and put mxcli's length checks into direct disagreement with + // mxbuild: a view entity over System.User.Name had its correct String(100) + // refused and its wrong String waved through, to fail later with CE6770 + // (ako/mxcli#584). + Length int + EnumQN string // for Enumeration type, qualified name } @@ -87,15 +100,22 @@ func ModelerSystemAssociations() []SystemAssocDef { } // SystemEntities lists all entities in the System module. -// Extracted from Mendix Studio Pro 11.6.4 via DummySystem module. +// +// Entities, attributes and types were extracted from Mendix Studio Pro 11.6.4 +// via a DummySystem module. The String LENGTHS come from a different and more +// direct source — the System module's domain model inside the +// `deployment/model/model.mdp` mxbuild writes — because that is the model the +// runtime builds the System tables from, and so the one Mendix judges a length +// against. See testdata/system_string_lengths.txt for the measurement and how +// to redo it. var SystemEntities = []SystemEntityDef{ {Name: "UserRole", Persistable: true, Attributes: []SystemAttrDef{ - {Name: "ModelGUID", Type: "String"}, - {Name: "Name", Type: "String"}, - {Name: "Description", Type: "String"}, + {Name: "ModelGUID", Type: "String", Length: 36}, + {Name: "Name", Type: "String", Length: 100}, + {Name: "Description", Type: "String", Length: 1000}, }}, {Name: "User", Persistable: true, Attributes: []SystemAttrDef{ - {Name: "Name", Type: "String"}, + {Name: "Name", Type: "String", Length: 100}, {Name: "Password", Type: "HashedString"}, {Name: "LastLogin", Type: "DateTime"}, {Name: "Blocked", Type: "Boolean"}, @@ -107,42 +127,42 @@ var SystemEntities = []SystemEntityDef{ }}, {Name: "FileDocument", Persistable: true, Attributes: []SystemAttrDef{ {Name: "FileID", Type: "AutoNumber"}, - {Name: "Name", Type: "String"}, + {Name: "Name", Type: "String", Length: 400}, {Name: "DeleteAfterDownload", Type: "Boolean"}, {Name: "Contents", Type: "Binary"}, {Name: "HasContents", Type: "Boolean"}, {Name: "Size", Type: "Long"}, }}, {Name: "Image", Persistable: true, Generalization: "System.FileDocument", Attributes: []SystemAttrDef{ - {Name: "PublicThumbnailPath", Type: "String"}, + {Name: "PublicThumbnailPath", Type: "String", Length: 500}, {Name: "EnableCaching", Type: "Boolean"}, }}, {Name: "XASInstance", Persistable: true, Attributes: []SystemAttrDef{ - {Name: "XASId", Type: "String"}, + {Name: "XASId", Type: "String", Length: 50}, {Name: "LastUpdate", Type: "DateTime"}, {Name: "AllowedNumberOfConcurrentUsers", Type: "Integer"}, - {Name: "PartnerName", Type: "String"}, - {Name: "CustomerName", Type: "String"}, + {Name: "PartnerName", Type: "String", Length: 200}, + {Name: "CustomerName", Type: "String", Length: 200}, }}, {Name: "Session", Persistable: true, Attributes: []SystemAttrDef{ - {Name: "SessionId", Type: "String"}, - {Name: "CSRFToken", Type: "String"}, + {Name: "SessionId", Type: "String", Length: 50}, + {Name: "CSRFToken", Type: "String", Length: 36}, {Name: "LastActive", Type: "DateTime"}, }}, {Name: "ScheduledEventInformation", Persistable: true, Attributes: []SystemAttrDef{ - {Name: "Name", Type: "String"}, + {Name: "Name", Type: "String", Length: 200}, {Name: "Description", Type: "String"}, {Name: "StartTime", Type: "DateTime"}, {Name: "EndTime", Type: "DateTime"}, {Name: "Status", Type: "Enumeration", EnumQN: "System.EventStatus"}, }}, {Name: "Language", Persistable: true, Attributes: []SystemAttrDef{ - {Name: "Code", Type: "String"}, - {Name: "Description", Type: "String"}, + {Name: "Code", Type: "String", Length: 20}, + {Name: "Description", Type: "String", Length: 200}, }}, {Name: "TimeZone", Persistable: true, Attributes: []SystemAttrDef{ - {Name: "Code", Type: "String"}, - {Name: "Description", Type: "String"}, + {Name: "Code", Type: "String", Length: 50}, + {Name: "Description", Type: "String", Length: 100}, {Name: "RawOffset", Type: "Integer"}, }}, {Name: "Error", Persistable: false, Attributes: []SystemAttrDef{ @@ -163,7 +183,7 @@ var SystemEntities = []SystemEntityDef{ {Name: "UserAgent", Type: "String"}, }}, {Name: "HttpMessage", Persistable: false, Attributes: []SystemAttrDef{ - {Name: "HttpVersion", Type: "String"}, + {Name: "HttpVersion", Type: "String", Length: 10}, {Name: "Content", Type: "String"}, }}, {Name: "HttpHeader", Persistable: false, Attributes: []SystemAttrDef{ @@ -172,7 +192,7 @@ var SystemEntities = []SystemEntityDef{ }}, {Name: "UserReportInfo", Persistable: true, Attributes: []SystemAttrDef{ {Name: "UserType", Type: "Enumeration", EnumQN: "System.UserType"}, - {Name: "Hash", Type: "String"}, + {Name: "Hash", Type: "String", Length: 64}, }}, {Name: "HttpRequest", Persistable: true, Generalization: "System.HttpMessage", Attributes: []SystemAttrDef{ {Name: "Uri", Type: "String"}, @@ -184,28 +204,28 @@ var SystemEntities = []SystemEntityDef{ {Name: "Paging", Persistable: false, Attributes: []SystemAttrDef{ {Name: "PageNumber", Type: "Long"}, {Name: "IsSortable", Type: "Boolean"}, - {Name: "SortAttribute", Type: "String"}, + {Name: "SortAttribute", Type: "String", Length: 200}, {Name: "SortAscending", Type: "Boolean"}, {Name: "HasMoreData", Type: "Boolean"}, }}, {Name: "SynchronizationError", Persistable: true, Attributes: []SystemAttrDef{ {Name: "Reason", Type: "String"}, - {Name: "ObjectId", Type: "String"}, - {Name: "ObjectType", Type: "String"}, + {Name: "ObjectId", Type: "String", Length: 200}, + {Name: "ObjectType", Type: "String", Length: 1000}, {Name: "ObjectContent", Type: "String"}, }}, {Name: "SynchronizationErrorFile", Persistable: true, Generalization: "System.FileDocument"}, {Name: "ProcessedQueueTask", Persistable: true, Attributes: []SystemAttrDef{ {Name: "Sequence", Type: "Long"}, {Name: "Status", Type: "Enumeration", EnumQN: "System.QueueTaskStatus"}, - {Name: "QueueId", Type: "String"}, - {Name: "QueueName", Type: "String"}, + {Name: "QueueId", Type: "String", Length: 36}, + {Name: "QueueName", Type: "String", Length: 200}, {Name: "ContextType", Type: "Enumeration", EnumQN: "System.ContextType"}, {Name: "ContextData", Type: "String"}, - {Name: "MicroflowName", Type: "String"}, - {Name: "UserActionName", Type: "String"}, + {Name: "MicroflowName", Type: "String", Length: 200}, + {Name: "UserActionName", Type: "String", Length: 200}, {Name: "Arguments", Type: "String"}, - {Name: "XASId", Type: "String"}, + {Name: "XASId", Type: "String", Length: 50}, {Name: "ThreadId", Type: "Long"}, {Name: "Created", Type: "DateTime"}, {Name: "StartAt", Type: "DateTime"}, @@ -214,39 +234,39 @@ var SystemEntities = []SystemEntityDef{ {Name: "Duration", Type: "Long"}, {Name: "Retried", Type: "Long"}, {Name: "ErrorMessage", Type: "String"}, - {Name: "ScheduledEventName", Type: "String"}, + {Name: "ScheduledEventName", Type: "String", Length: 200}, }}, {Name: "QueuedTask", Persistable: true, Attributes: []SystemAttrDef{ {Name: "Sequence", Type: "AutoNumber"}, {Name: "Status", Type: "Enumeration", EnumQN: "System.QueueTaskStatus"}, - {Name: "QueueId", Type: "String"}, - {Name: "QueueName", Type: "String"}, + {Name: "QueueId", Type: "String", Length: 36}, + {Name: "QueueName", Type: "String", Length: 200}, {Name: "ContextType", Type: "Enumeration", EnumQN: "System.ContextType"}, {Name: "ContextData", Type: "String"}, - {Name: "MicroflowName", Type: "String"}, - {Name: "UserActionName", Type: "String"}, + {Name: "MicroflowName", Type: "String", Length: 200}, + {Name: "UserActionName", Type: "String", Length: 200}, {Name: "Arguments", Type: "String"}, - {Name: "XASId", Type: "String"}, + {Name: "XASId", Type: "String", Length: 50}, {Name: "ThreadId", Type: "Long"}, {Name: "Created", Type: "DateTime"}, {Name: "StartAt", Type: "DateTime"}, {Name: "Started", Type: "DateTime"}, {Name: "Retried", Type: "Long"}, - {Name: "Retry", Type: "String"}, - {Name: "ScheduledEventName", Type: "String"}, + {Name: "Retry", Type: "String", Length: 200}, + {Name: "ScheduledEventName", Type: "String", Length: 200}, }}, {Name: "WorkflowDefinition", Persistable: true, Attributes: []SystemAttrDef{ - {Name: "Name", Type: "String"}, - {Name: "Title", Type: "String"}, + {Name: "Name", Type: "String", Length: 200}, + {Name: "Title", Type: "String", Length: 200}, {Name: "IsObsolete", Type: "Boolean"}, {Name: "IsLocked", Type: "Boolean"}, }}, {Name: "WorkflowUserTaskDefinition", Persistable: true, Attributes: []SystemAttrDef{ - {Name: "Name", Type: "String"}, + {Name: "Name", Type: "String", Length: 200}, {Name: "IsObsolete", Type: "Boolean"}, }}, {Name: "Workflow", Persistable: true, Attributes: []SystemAttrDef{ - {Name: "Name", Type: "String"}, + {Name: "Name", Type: "String", Length: 200}, {Name: "Description", Type: "String"}, {Name: "StartTime", Type: "DateTime"}, {Name: "EndTime", Type: "DateTime"}, @@ -263,13 +283,13 @@ var SystemEntities = []SystemEntityDef{ {Name: "StartTime", Type: "DateTime"}, {Name: "DueDate", Type: "DateTime"}, {Name: "EndTime", Type: "DateTime"}, - {Name: "Outcome", Type: "String"}, + {Name: "Outcome", Type: "String", Length: 200}, {Name: "State", Type: "Enumeration", EnumQN: "System.WorkflowUserTaskState"}, {Name: "CompletionType", Type: "Enumeration", EnumQN: "System.WorkflowUserTaskCompletionType"}, }}, {Name: "TaskQueueToken", Persistable: true, Attributes: []SystemAttrDef{ - {Name: "QueueName", Type: "String"}, - {Name: "XASId", Type: "String"}, + {Name: "QueueName", Type: "String", Length: 200}, + {Name: "XASId", Type: "String", Length: 50}, {Name: "ValidUntil", Type: "DateTime"}, }}, {Name: "ODataResponse", Persistable: false, Attributes: []SystemAttrDef{ @@ -282,18 +302,18 @@ var SystemEntities = []SystemEntityDef{ {Name: "Action", Type: "Enumeration", EnumQN: "System.WorkflowCurrentActivityAction"}, }}, {Name: "WorkflowActivityDetails", Persistable: false, Attributes: []SystemAttrDef{ - {Name: "ActivityId", Type: "String"}, + {Name: "ActivityId", Type: "String", Length: 50}, {Name: "ActivityCaption", Type: "String"}, {Name: "ActivityType", Type: "Enumeration", EnumQN: "System.WorkflowActivityType"}, {Name: "ExistsInCurrentVersion", Type: "Boolean"}, }}, {Name: "WorkflowUserTaskOutcome", Persistable: true, Attributes: []SystemAttrDef{ - {Name: "Outcome", Type: "String"}, + {Name: "Outcome", Type: "String", Length: 200}, {Name: "Time", Type: "DateTime"}, }}, {Name: "WorkflowRecord", Persistable: false, Attributes: []SystemAttrDef{ - {Name: "WorkflowKey", Type: "String"}, - {Name: "Name", Type: "String"}, + {Name: "WorkflowKey", Type: "String", Length: 200}, + {Name: "Name", Type: "String", Length: 200}, {Name: "Description", Type: "String"}, {Name: "State", Type: "Enumeration", EnumQN: "System.WorkflowState"}, {Name: "StartTime", Type: "DateTime"}, @@ -302,22 +322,22 @@ var SystemEntities = []SystemEntityDef{ {Name: "Reason", Type: "String"}, }}, {Name: "WorkflowActivityRecord", Persistable: false, Attributes: []SystemAttrDef{ - {Name: "ModelGUID", Type: "String"}, - {Name: "ActivityKey", Type: "String"}, - {Name: "PreviousActivityKey", Type: "String"}, + {Name: "ModelGUID", Type: "String", Length: 200}, + {Name: "ActivityKey", Type: "String", Length: 200}, + {Name: "PreviousActivityKey", Type: "String", Length: 200}, {Name: "ActivityType", Type: "Enumeration", EnumQN: "System.WorkflowActivityType"}, {Name: "Caption", Type: "String"}, {Name: "State", Type: "Enumeration", EnumQN: "System.WorkflowActivityExecutionState"}, {Name: "StartTime", Type: "DateTime"}, {Name: "EndTime", Type: "DateTime"}, - {Name: "Outcome", Type: "String"}, - {Name: "MicroflowName", Type: "String"}, + {Name: "Outcome", Type: "String", Length: 200}, + {Name: "MicroflowName", Type: "String", Length: 200}, {Name: "TaskName", Type: "String"}, {Name: "TaskDescription", Type: "String"}, {Name: "TaskDueDate", Type: "DateTime"}, {Name: "TaskCompletionType", Type: "Enumeration", EnumQN: "System.WorkflowUserTaskCompletionType"}, {Name: "TaskRequiredUsers", Type: "Integer"}, - {Name: "TaskKey", Type: "String"}, + {Name: "TaskKey", Type: "String", Length: 200}, {Name: "Reason", Type: "String"}, }}, {Name: "WorkflowEvent", Persistable: false, Attributes: []SystemAttrDef{ @@ -338,27 +358,27 @@ var SystemEntities = []SystemEntityDef{ {Name: "StartTime", Type: "DateTime"}, {Name: "DueDate", Type: "DateTime"}, {Name: "EndTime", Type: "DateTime"}, - {Name: "Outcome", Type: "String"}, + {Name: "Outcome", Type: "String", Length: 200}, {Name: "State", Type: "Enumeration", EnumQN: "System.WorkflowUserTaskState"}, {Name: "CompletionType", Type: "Enumeration", EnumQN: "System.WorkflowUserTaskCompletionType"}, - {Name: "UserTaskKey", Type: "String"}, + {Name: "UserTaskKey", Type: "String", Length: 200}, }}, {Name: "WorkflowEndedUserTaskOutcome", Persistable: true, Attributes: []SystemAttrDef{ - {Name: "Outcome", Type: "String"}, + {Name: "Outcome", Type: "String", Length: 200}, {Name: "Time", Type: "DateTime"}, }}, {Name: "WorkflowGroup", Persistable: true, Attributes: []SystemAttrDef{ - {Name: "Name", Type: "String"}, + {Name: "Name", Type: "String", Length: 200}, {Name: "Description", Type: "String"}, }}, // --- Entities below extracted from MDP (Phase 3, 2026-04-24) --- {Name: "WorkflowVersion", Persistable: true, RuntimeOnly: true, Attributes: []SystemAttrDef{ - {Name: "VersionHash", Type: "String"}, + {Name: "VersionHash", Type: "String", Length: 200}, {Name: "ModelJSON", Type: "String"}, }}, {Name: "WorkflowActivity", Persistable: true, RuntimeOnly: true, Attributes: []SystemAttrDef{ - {Name: "ModelGUID", Type: "String"}, - {Name: "ActivityGUID", Type: "String"}, + {Name: "ModelGUID", Type: "String", Length: 36}, + {Name: "ActivityGUID", Type: "String", Length: 36}, {Name: "Caption", Type: "String"}, {Name: "DetailsJson", Type: "String"}, {Name: "State", Type: "Enumeration", EnumQN: "System.WorkflowActivityState"}, @@ -366,13 +386,13 @@ var SystemEntities = []SystemEntityDef{ {Name: "EndTime", Type: "DateTime"}, {Name: "ActionTime", Type: "DateTime"}, {Name: "Reason", Type: "String"}, - {Name: "ActivityHash", Type: "String"}, + {Name: "ActivityHash", Type: "String", Length: 200}, {Name: "IsDerivedActivity", Type: "Boolean"}, - {Name: "Outcome", Type: "String"}, - {Name: "OutcomeModelGUID", Type: "String"}, + {Name: "Outcome", Type: "String", Length: 200}, + {Name: "OutcomeModelGUID", Type: "String", Length: 36}, }}, {Name: "WorkflowActivityUserTaskOutcome", Persistable: true, RuntimeOnly: true, Attributes: []SystemAttrDef{ - {Name: "Outcome", Type: "String"}, + {Name: "Outcome", Type: "String", Length: 200}, {Name: "Time", Type: "DateTime"}, }}, {Name: "PrivateFileDocument", Persistable: true, RuntimeOnly: true, Generalization: "System.FileDocument"}, @@ -385,24 +405,24 @@ var SystemEntities = []SystemEntityDef{ {Name: "Successful", Type: "Boolean"}, }}, {Name: "AutoCommitEntry", Persistable: true, RuntimeOnly: true, Attributes: []SystemAttrDef{ - {Name: "SessionId", Type: "String"}, + {Name: "SessionId", Type: "String", Length: 36}, {Name: "ObjectId", Type: "Long"}, }}, {Name: "UnreferencedFile", Persistable: true, RuntimeOnly: true, Attributes: []SystemAttrDef{ - {Name: "FileKey", Type: "String"}, + {Name: "FileKey", Type: "String", Length: 36}, {Name: "State", Type: "Enumeration", EnumQN: "System.UnreferencedFileState"}, - {Name: "TransactionId", Type: "String"}, + {Name: "TransactionId", Type: "String", Length: 36}, }}, {Name: "OfflineCreatedGuids", Persistable: true, RuntimeOnly: true, Attributes: []SystemAttrDef{ - {Name: "Guid", Type: "String"}, + {Name: "Guid", Type: "String", Length: 200}, }}, {Name: "OfflineSynchronizationHistory", Persistable: true, RuntimeOnly: true, Attributes: []SystemAttrDef{ - {Name: "SyncId", Type: "String"}, + {Name: "SyncId", Type: "String", Length: 200}, }}, {Name: "ChangeHash", Persistable: true, RuntimeOnly: true, Attributes: []SystemAttrDef{ {Name: "ObjectId", Type: "Long"}, - {Name: "Attribute", Type: "String"}, - {Name: "Hash", Type: "String"}, + {Name: "Attribute", Type: "String", Length: 200}, + {Name: "Hash", Type: "String", Length: 200}, }}, } diff --git a/modelsdk/meta/system_string_lengths_measure_test.go b/modelsdk/meta/system_string_lengths_measure_test.go new file mode 100644 index 000000000..d7c254459 --- /dev/null +++ b/modelsdk/meta/system_string_lengths_measure_test.go @@ -0,0 +1,160 @@ +// SPDX-License-Identifier: Apache-2.0 + +// The measuring half of ako/mxcli#584's guard: reads the System module's domain +// model out of a built `deployment/model/model.mdp` and rewrites the golden. +// +// The .mdp is not one BSON document — it is a STREAM of them, each prefixed by +// its own 4-byte length, so unmarshalling the file whole fails with "invalid +// document length". Modules arrive as `Projects$ModuleImpl` documents carrying +// only a Name, and each module's documents follow it; the domain model is the +// first `DomainModels$DomainModel` after the System module's own document. +// +// Run it only when re-measuring — without -mdp it skips. +package meta + +import ( + "encoding/binary" + "os" + "strings" + "testing" + + "go.mongodb.org/mongo-driver/bson" +) + +func TestSystemStringLengthsUpdate(t *testing.T) { + if *mdpPath == "" { + t.Skip("no -mdp given; this test re-measures from a built deployment/model/model.mdp") + } + raw, err := os.ReadFile(*mdpPath) + if err != nil { + t.Fatalf("read %s: %v", *mdpPath, err) + } + measured, err := systemStringLengthsFromMDP(raw) + if err != nil { + t.Fatalf("parse %s: %v", *mdpPath, err) + } + if len(measured) == 0 { + t.Fatalf("%s carries no System module domain model — is it a deploy build?", *mdpPath) + } + + // Only the attributes SystemEntities declares: the deployed model carries + // more (11.14.0 has 29 members mxcli's 11.6.4-era table does not know), and + // adding those is a separate change with a version-gating question of its + // own — a member that does not exist in the target version is CE1613, which + // a length can never be. + keep := map[string]int{} + var missing []string + for _, e := range SystemEntities { + for _, a := range e.Attributes { + if a.Type != "String" { + continue + } + key := e.Name + "." + a.Name + length, ok := measured[key] + if !ok { + missing = append(missing, key) + continue + } + keep[key] = length + } + } + if len(missing) > 0 { + t.Errorf("declared as String but absent from %s: %s\n"+ + " Either the attribute is gone in this Mendix version, or the build is not a deploy build.", + *mdpPath, strings.Join(missing, ", ")) + } + + if !*updateGold { + t.Logf("measured %d String attributes; pass -update to rewrite %s", len(keep), goldenPath) + golden, err := readGoldenLengths(goldenPath) + if err != nil { + t.Fatalf("read golden: %v", err) + } + for key, want := range keep { + if got, ok := golden[key]; !ok || got != want { + t.Errorf("System.%s: golden says %d, %s says %d", key, golden[key], *mdpPath, want) + } + } + return + } + + old, err := os.ReadFile(goldenPath) + if err != nil { + t.Fatalf("read golden: %v", err) + } + header, _, found := strings.Cut(string(old), "# entity\tattribute\tlength\n") + if !found { + t.Fatalf("%s has lost its header line; restore it before regenerating", goldenPath) + } + body := header + "# entity\tattribute\tlength\n" + formatGoldenLengths(keep) + if err := os.WriteFile(goldenPath, []byte(body), 0o644); err != nil { + t.Fatalf("write golden: %v", err) + } + t.Logf("rewrote %s with %d measured lengths — update the header's version line by hand", goldenPath, len(keep)) +} + +// systemStringLengthsFromMDP returns "Entity.Attribute" -> String length for the +// System module's domain model in a deployed model.mdp. +func systemStringLengthsFromMDP(raw []byte) (map[string]int, error) { + out := map[string]int{} + inSystem := false + for off := 0; off+4 <= len(raw); { + size := int(binary.LittleEndian.Uint32(raw[off : off+4])) + if size < 5 || off+size > len(raw) { + break + } + var doc bson.M + if err := bson.Unmarshal(raw[off:off+size], &doc); err == nil { + switch ty, _ := doc["$Type"].(string); ty { + case "Projects$ModuleImpl": + name, _ := doc["Name"].(string) + inSystem = name == "System" + case "DomainModels$DomainModel": + if inSystem { + collectStringLengths(doc, out) + inSystem = false + } + } + } + off += size + } + return out, nil +} + +func collectStringLengths(domainModel bson.M, out map[string]int) { + entities, _ := domainModel["Entities"].(bson.A) + for _, ev := range entities { + entity, _ := ev.(bson.M) + entityName, _ := entity["UnqualifiedName"].(string) + attrs, _ := entity["Attributes"].(bson.A) + for _, av := range attrs { + attr, _ := av.(bson.M) + attrType, _ := attr["Type"].(bson.M) + if ty, _ := attrType["$Type"].(string); ty != "DomainModels$StringAttributeType" { + continue + } + attrName, _ := attr["Name"].(string) + // A String attribute with no Length property is unlimited, the same + // as an explicit 0 — Mendix omits the default. The width of the + // stored integer is not fixed (mxcli has already been bitten by a + // gen-declared int32 stored as int64, #585), so accept either rather + // than reading every length as 0 through a failed assertion. + out[entityName+"."+attrName] = bsonInt(attrType["Length"]) + } + } +} + +// bsonInt reads an integer property whatever width it was stored at. +func bsonInt(v any) int { + switch n := v.(type) { + case int32: + return int(n) + case int64: + return int(n) + case int: + return n + case float64: + return int(n) + } + return 0 +} diff --git a/modelsdk/meta/system_string_lengths_test.go b/modelsdk/meta/system_string_lengths_test.go new file mode 100644 index 000000000..73c6f9d9d --- /dev/null +++ b/modelsdk/meta/system_string_lengths_test.go @@ -0,0 +1,128 @@ +// SPDX-License-Identifier: Apache-2.0 + +// ako/mxcli#584 — SystemAttrDef declared a Length and not one of the 115 String +// attributes populated it, so every System string read as unlimited. The cost +// was not cosmetic: a view entity selecting `u.Name` from System.User had its +// correct declaration (`String(100)`) refused and its wrong one (`String`) +// waved through, to fail the build later with CE6770. +// +// The lengths are MEASURED, from the System module's domain model inside the +// `deployment/model/model.mdp` mxbuild writes — the same model the runtime +// creates the System tables from. testdata/system_string_lengths.txt is that +// measurement; this test holds SystemEntities to it. +// +// Two searches this saves repeating, both already spent on #584: the Mendix +// Model SDK does NOT carry these (its gen/ describes metamodel TYPES, so +// System.User.Name is simply not in it), and reading them out of +// Mendix.Modeler.Core.dll means a decompiler. The .mdp costs one build for all +// 115 at once. +package meta + +import ( + "bufio" + "flag" + "fmt" + "os" + "sort" + "strconv" + "strings" + "testing" +) + +var ( + mdpPath = flag.String("mdp", "", "path to a built deployment/model/model.mdp to re-measure from") + updateGold = flag.Bool("update", false, "rewrite testdata/system_string_lengths.txt from -mdp") +) + +const goldenPath = "testdata/system_string_lengths.txt" + +func TestSystemStringLengths(t *testing.T) { + if *updateGold { + t.Skip("-update regenerates the golden; see TestSystemStringLengthsUpdate") + } + golden, err := readGoldenLengths(goldenPath) + if err != nil { + t.Fatalf("read golden: %v", err) + } + + declared := map[string]int{} + for _, e := range SystemEntities { + for _, a := range e.Attributes { + if a.Type == "String" { + declared[e.Name+"."+a.Name] = a.Length + } + } + } + + for key, want := range golden { + got, ok := declared[key] + if !ok { + t.Errorf("System.%s is measured at length %d but SystemEntities no longer declares it as a String", key, want) + continue + } + if got != want { + t.Errorf("System.%s: SystemEntities says String(%d), the deployed model says String(%d)\n"+ + " Mendix decides CE6770 by ITS number, so mxcli disagreeing with it is a wrong answer, not a stale one.\n"+ + " Re-measure with -mdp -update rather than editing either side by hand.", + key, got, want) + } + } + // The other direction is what makes this a guard rather than a snapshot: a + // String attribute added to SystemEntities without being measured has no + // golden row, and would otherwise sit at length 0 — reading as "unlimited", + // which is a claim, not an absence. + for key := range declared { + if _, ok := golden[key]; !ok { + t.Errorf("System.%s is declared as a String but has no measured length.\n"+ + " Build a project on the target Mendix version and re-measure:\n"+ + " mxcli new Probe --version --theme none --layout none --skip-init\n"+ + " go test ./modelsdk/meta -run TestSystemStringLengths -mdp Probe/deployment/model/model.mdp -update", + key) + } + } +} + +// readGoldenLengths parses the measured table: entityattributelength, +// '#' comments, keyed as "Entity.Attribute". +func readGoldenLengths(path string) (map[string]int, error) { + f, err := os.Open(path) + if err != nil { + return nil, err + } + defer f.Close() + + out := map[string]int{} + sc := bufio.NewScanner(f) + for line := 1; sc.Scan(); line++ { + text := strings.TrimSpace(sc.Text()) + if text == "" || strings.HasPrefix(text, "#") { + continue + } + parts := strings.Split(text, "\t") + if len(parts) != 3 { + return nil, fmt.Errorf("%s:%d: want entityattributelength, got %q", path, line, text) + } + n, err := strconv.Atoi(parts[2]) + if err != nil { + return nil, fmt.Errorf("%s:%d: length: %w", path, line, err) + } + out[parts[0]+"."+parts[1]] = n + } + return out, sc.Err() +} + +// formatGoldenLengths renders the table body in the golden's stable order. +func formatGoldenLengths(lengths map[string]int) string { + keys := make([]string, 0, len(lengths)) + for k := range lengths { + keys = append(keys, k) + } + sort.Strings(keys) + + var b strings.Builder + for _, k := range keys { + entity, attr, _ := strings.Cut(k, ".") + fmt.Fprintf(&b, "%s\t%s\t%d\n", entity, attr, lengths[k]) + } + return b.String() +} diff --git a/modelsdk/meta/testdata/system_string_lengths.txt b/modelsdk/meta/testdata/system_string_lengths.txt new file mode 100644 index 000000000..da0f268c1 --- /dev/null +++ b/modelsdk/meta/testdata/system_string_lengths.txt @@ -0,0 +1,139 @@ +# System module String attribute lengths, measured -- not transcribed. +# +# Source: the System module's domain model inside `deployment/model/model.mdp`, +# the BSON document stream mxbuild writes for the runtime. It is the model the +# runtime creates the System tables from, so it is what CE6770 -- and every +# other length check Mendix applies to a System attribute -- is decided against. +# +# How to regenerate (ONE build, not one build per attribute): +# +# mxcli new Probe --version --theme none --layout none --skip-init +# go test ./modelsdk/meta -run TestSystemStringLengths \ +# -mdp Probe/deployment/model/model.mdp -update +# +# Measured on Mendix 11.14.0. Of the 115 attributes below, 108 also exist in +# 10.24.4.77222 and ALL 108 carry the same length there -- so the lengths are +# stable across a major-version boundary and one table serves every supported +# version, rather than a per-version registry. (The 7 that do not appear in +# 10.24.4 are Mendix 11 workflow additions.) +# +# A length of 0 is Mendix's own encoding of "unlimited", NOT "unmeasured". This +# file lists every String attribute `SystemEntities` declares, so an unmeasured +# one cannot exist without failing the test. +# +# entity attribute length +AutoCommitEntry SessionId 36 +BackgroundJob Result 0 +ChangeHash Attribute 200 +ChangeHash Hash 200 +ConsumedODataConfiguration ProxyHost 0 +ConsumedODataConfiguration ProxyPassword 0 +ConsumedODataConfiguration ProxyUsername 0 +ConsumedODataConfiguration ServiceUrl 0 +Error ErrorType 0 +Error Message 0 +Error Stacktrace 0 +FileDocument Name 400 +HttpHeader Key 0 +HttpHeader Value 0 +HttpMessage Content 0 +HttpMessage HttpVersion 10 +HttpRequest Uri 0 +HttpResponse ReasonPhrase 0 +Image PublicThumbnailPath 500 +Language Code 20 +Language Description 200 +OfflineCreatedGuids Guid 200 +OfflineSynchronizationHistory SyncId 200 +Paging SortAttribute 200 +ProcessedQueueTask Arguments 0 +ProcessedQueueTask ContextData 0 +ProcessedQueueTask ErrorMessage 0 +ProcessedQueueTask MicroflowName 200 +ProcessedQueueTask QueueId 36 +ProcessedQueueTask QueueName 200 +ProcessedQueueTask ScheduledEventName 200 +ProcessedQueueTask UserActionName 200 +ProcessedQueueTask XASId 50 +QueuedTask Arguments 0 +QueuedTask ContextData 0 +QueuedTask MicroflowName 200 +QueuedTask QueueId 36 +QueuedTask QueueName 200 +QueuedTask Retry 200 +QueuedTask ScheduledEventName 200 +QueuedTask UserActionName 200 +QueuedTask XASId 50 +ScheduledEventInformation Description 0 +ScheduledEventInformation Name 200 +Session CSRFToken 36 +Session SessionId 50 +SoapFault Code 0 +SoapFault Detail 0 +SoapFault Node 0 +SoapFault Reason 0 +SoapFault Role 0 +SynchronizationError ObjectContent 0 +SynchronizationError ObjectId 200 +SynchronizationError ObjectType 1000 +SynchronizationError Reason 0 +TaskQueueToken QueueName 200 +TaskQueueToken XASId 50 +TimeZone Code 50 +TimeZone Description 100 +TokenInformation UserAgent 0 +UnreferencedFile FileKey 36 +UnreferencedFile TransactionId 36 +User Name 100 +UserReportInfo Hash 64 +UserRole Description 1000 +UserRole ModelGUID 36 +UserRole Name 100 +Workflow Description 0 +Workflow Name 200 +Workflow Reason 0 +WorkflowActivity ActivityGUID 36 +WorkflowActivity ActivityHash 200 +WorkflowActivity Caption 0 +WorkflowActivity DetailsJson 0 +WorkflowActivity ModelGUID 36 +WorkflowActivity Outcome 200 +WorkflowActivity OutcomeModelGUID 36 +WorkflowActivity Reason 0 +WorkflowActivityDetails ActivityCaption 0 +WorkflowActivityDetails ActivityId 50 +WorkflowActivityRecord ActivityKey 200 +WorkflowActivityRecord Caption 0 +WorkflowActivityRecord MicroflowName 200 +WorkflowActivityRecord ModelGUID 200 +WorkflowActivityRecord Outcome 200 +WorkflowActivityRecord PreviousActivityKey 200 +WorkflowActivityRecord Reason 0 +WorkflowActivityRecord TaskDescription 0 +WorkflowActivityRecord TaskKey 200 +WorkflowActivityRecord TaskName 0 +WorkflowActivityUserTaskOutcome Outcome 200 +WorkflowDefinition Name 200 +WorkflowDefinition Title 200 +WorkflowEndedUserTask Description 0 +WorkflowEndedUserTask Name 0 +WorkflowEndedUserTask Outcome 200 +WorkflowEndedUserTask UserTaskKey 200 +WorkflowEndedUserTaskOutcome Outcome 200 +WorkflowGroup Description 0 +WorkflowGroup Name 200 +WorkflowJumpToDetails Error 0 +WorkflowRecord Description 0 +WorkflowRecord Name 200 +WorkflowRecord Reason 0 +WorkflowRecord WorkflowKey 200 +WorkflowUserTask Description 0 +WorkflowUserTask Name 0 +WorkflowUserTask Outcome 200 +WorkflowUserTaskDefinition Name 200 +WorkflowUserTaskOutcome Outcome 200 +WorkflowVersion ModelJSON 0 +WorkflowVersion VersionHash 200 +XASInstance CustomerName 200 +XASInstance PartnerName 200 +XASInstance XASId 50