diff --git a/.claude/lint-rules/conv010_act_microflow_content.star b/.claude/lint-rules/conv010_act_microflow_content.star index 323dea7ebd..c248203290 100644 --- a/.claude/lint-rules/conv010_act_microflow_content.star +++ b/.claude/lint-rules/conv010_act_microflow_content.star @@ -8,6 +8,7 @@ # - ShowMessageAction (show message) # - DownloadFileAction (download file) # - MicroflowCallAction (call sub-microflow for logic delegation) +# - NanoflowCallAction (the same delegation from an ACT_ NANOFLOW) # # Business logic should be delegated to SUB_ microflows. # Requires FULL catalog (REFRESH CATALOG FULL). @@ -37,6 +38,18 @@ ALLOWED_ACTIONS = ( "ShowMessageAction", "DownloadFileAction", "MicroflowCallAction", + # An ACT_ NANOFLOW delegates with a nanoflow call, not a microflow call. + # microflows() yields nanoflows too (the catalog's `microflows` table carries + # a MicroflowType column), so CONV010 lints them — and without this entry it + # flagged the very delegation it demands: an ACT_ nanoflow could satisfy the + # rule in no way at all. Reported from a real project, which patched its own + # copy of the rule and asked for it upstream (ako/mxcli#644). + # + # This is the third time this allowlist has been short. It has held the wrong + # vocabulary (storage names, matching nothing) and been missing an activity a + # permitted one necessarily creates (ExclusiveMerge). The pattern is the same + # each time: a rule that cannot be satisfied reads as the code being wrong. + "NanoflowCallAction", # Storage names — belt and braces; see the note above. "ShowFormAction", "CloseFormAction", diff --git a/.claude/skills/fix-issue/findings/cmd-mxcli.jsonl b/.claude/skills/fix-issue/findings/cmd-mxcli.jsonl index 1f79a2e267..2350194b7c 100644 --- a/.claude/skills/fix-issue/findings/cmd-mxcli.jsonl +++ b/.claude/skills/fix-issue/findings/cmd-mxcli.jsonl @@ -125,3 +125,4 @@ {"area":"cmd/mxcli","date":"2026-09-23","symptom":"`mxcli test`, `mxcli check`, `mxcli exec` with no arguments wrote 0 session_start records (`mxcli check /nonexistent.mdl` wrote 1), so a run that failed cobra's Args validation was invisible to `diag loop-report`.","cause":"#617 put diaglog.Init in the root's PersistentPreRun and described that as 'before argument validation'. Cobra's execute() runs ValidateArgs, and answers --help/--version, BEFORE any hook, so arity failures returned before the session opened. The same move also meant the root's -c and REPL sessions were recorded as mode \"mxcli\" instead of \"batch\"/\"repl\", because PreRun reached the singleton first.","file":"cmd/mxcli/session_start.go","insight":"A cobra hook is not 'before anything': execute() order is ParseFlags → help/version → ValidateArgs → PersistentPreRun, so anything that must see every invocation belongs in main() before Execute, keyed on rootCmd.Find(os.Args[1:]), which needs no parsed command. Moving the open that early moves the close problem with it: --help/--version return nil without running PersistentPostRun, so a close left there turns every help lookup into an 'unclosed' (failed) run. That was the false failure #617 had already paid for once. Close in main() after a nil Execute instead. When a singleton is opened earlier, whichever caller reaches it first picks the mode, so resolve the mode the later callers would have passed (batch/repl) at the early call site. The regression test runs the real main() in a re-executed test binary with MXCLI_LOG_DIR. It is the only layer that sees os.Exit paths, and both controls fail at the right place: stubbing startSession gives 0/0/0 records with the /nonexistent.mdl control still passing, and moving the close back to PostRun gives 0 session_end records for --help/--version.","refs":["ako/mxcli#633","ako/mxcli#617"]} {"area": "cmd/mxcli/diag", "date": "2026-09-23", "symptom": "`mxcli diag loop-report` showed all 5 `test` runs as 'did not close' although every test passed, and inflated the `-c` (111) and `exec` (180) counts in the same log. Reported as 'the command apparently skips the summary record on success too' \u2014 which is not what happens: `test` returns normally, PersistentPostRun fires, and the session_end IS written.", "cause": "mxcli runs mxcli. Measured from a real `mxcli test` with MXCLI_LOG_DIR pointed at a scratch dir: one parent session_start (pid 1071) followed by THREE child session_starts \u2014 `-c DESCRIBE SETTINGS`, `-c SHOW MODULES`, and an `exec` of the generated runner \u2014 before a single test executes. `new`, `eval`, `tui` and the LSP self-spawn the same way (six os.Executable() sites). buildInvocations segmented on 'next session_end OR next session_start, whichever comes first', so a child's start closed the parent's invocation and the parent's own end landed on whatever was open by then. session_end carried no pid, so pairing by process was impossible.", "file": "`mdl/diaglog/diaglog.go` (pid on session_end; parentPIDEnv marker set once in Init and inherited by every child), `cmd/mxcli/diag_loop_report.go` (buildInvocations pairs by pid with the positional rule as fallback; spawned runs excluded from the table and wall time, counted on their own line), tests `cmd/mxcli/diag_loop_report_test.go`", "insight": "The segmentation rule documented itself as exact 'for sequential invocations, which is what an agent loop produces' \u2014 and the thing that breaks that assumption is the tool itself, not concurrency by the user. When a tool can invoke itself, EVERY per-process measurement over it needs a parent link, not just a pid: a pid alone fixes the pairing but still counts three phantom agent calls per test run. The marker belongs on the ENVIRONMENT, not at each spawn site: exec.Command inherits the parent's environment (explicitly via os.Environ(), implicitly when Cmd.Env is nil), so one os.Setenv in Init covers all six self-spawn sites and any added later \u2014 six edits that would each have to be remembered become zero. Second-order trap: spawned runs must be excluded from WALL TIME too, not just the count, because a child's seconds are already inside its parent's; the test asserts 10s for a parent with three 1s children, and the reverted code says 3. Prove-by-revert done on the measured record shape: the positional rule gives Invocations=4 (want 1), Unclosed=1 for a parent whose tests all passed, Wall=3 (want 10).", "refs": ["ako/mxcli#617", "ako/mxcli#629"]} {"area": "cmd/mxcli/syntax", "date": "2026-09-23", "symptom": "`mxcli syntax page datasource` documented `DataSource: MICROFLOW Module.MF($P)`. That form is a parse error: `dataview dv (datasource: microflow M.DS_X($State))` gives 'line 2:15 no viable alternative at input datasource'. Only the NAMED form `M.DS_X(State: $State)` parses. Hit in a real build, diagnosed from the error rather than the doc.", "cause": "The syntax entry was written from the intended shape rather than from something that had been run through the parser. Nothing checks it: the Syntax and Example fields are free text.", "file": "`cmd/mxcli/syntax/features_page.go` (page.datasource entry now shows `MICROFLOW Module.MF(Param: $P)` and states that the positional form is a parse error)", "insight": "CLAUDE.md deliberately points at `mxcli syntax` instead of restating syntax, so that it cannot go stale \u2014 which makes a wrong entry there worse than a wrong entry in prose, because it is the thing consulted INSTEAD of checking. The cost lands in the agent loop: read it, write it, fail to parse, diagnose, retry. Measured before reaching for the systemic guard: 42 of 164 syntax examples fail `mxcli check` today, but the large majority are fragments by design (a microflow body like `IF \u2026`, a widget snippet like `DATAGRID \u2026`, an OQL fragment) and are legitimately not standalone top-level MDL \u2014 so a blanket 'every example must parse' test would be mostly noise, and making it useful needs a way to mark which examples are standalone. Measuring that first is what stopped a plausible-sounding guard from being built wrong.", "refs": ["ako/mxcli#630"]} +{"area": "cmd/mxcli/test", "date": "2026-09-23", "symptom": "Windows: `mxcli test tests/ -p MyApp.mpr --local` fails with `local runtime: starting mxbuild serve: mxbuild --serve did not become ready` after caching a Linux ELF in %USERPROFILE%\\.mxcli\\mxbuild, and there is \"no flag, environment variable, or mechanism to redirect mxcli to the Windows mxbuild.exe already present in the Studio Pro installation\"", "cause": "Two layers. The platform part (downloading/exec'ing the Linux binary, serving from the cache instead of the resolved binary) was already fixed by #916 and #1122, both after the reporter's v0.21.0. What remained on main: `test` never registered `--mxbuild-path` — `run` gained it in #1125, but `test --local` boots through the same `ResolveMxBuildForLocal` and prints the same 'pass --mxbuild-path' guidance while answering `unknown flag`. `RunOptions` had no field and `localAppOptions` never set `LocalAppOptions.MxBuildPath`, though StartLocalApp honoured it. No env override existed anywhere", "file": "`cmd/mxcli/main.go` + `cmd_test_run.go` (flag), `testrunner/runner.go` + `localapp_options.go` (plumbing), `docker/mxbuild_platform.go` (`MxBuildPathEnv`, read in `resolveMxBuildForLocalOn` after the flag)", "insight": "**The guard for #1125 asserted its invariant against one command** — `TestErrorGuidanceNamesAFlagThatExists` checked only `runCmd`, while the guidance it polices is emitted by a resolver two commands share. When a test pins 'the advertised flag exists', enumerate the callers of the code that ADVERTISES it, not the command the report named; it now iterates `run` and `test`. **Before fixing a platform report, date it against the fixes**: the reporter's first two suggestions ('download platform-correct binary', 'auto-discover Studio Pro') were already on main, and re-implementing them would have been churn — only the override was missing. Put the env var in the resolver, not the CLI, so `run --local` and `test --local` both get it from one line; a flag-level env read would have been one more per-command copy to drift. Control: the env test runs as goos=windows with an unmatched version, so without the override it fails fast with the 'Linux binary cannot run natively on windows' refusal instead of hitting the CDN; removing only the `MxBuildPath:` line in localAppOptions fails the plumbing test for both runners. **Unverified**: no Windows host; code-level with OS-injected tests", "refs": ["mendixlabs/mxcli#1086", "mendixlabs/mxcli#1125", "mendixlabs/mxcli#916", "mendixlabs/mxcli#1122"]} diff --git a/.claude/skills/fix-issue/findings/mdl-backend.jsonl b/.claude/skills/fix-issue/findings/mdl-backend.jsonl index 7997f2b6fc..9a677c2619 100644 --- a/.claude/skills/fix-issue/findings/mdl-backend.jsonl +++ b/.claude/skills/fix-issue/findings/mdl-backend.jsonl @@ -113,6 +113,10 @@ {"area":"mdl/backend","date":"2026-09-15","symptom":"Phase 4a took sdk/mpr from 27 importers to 0, but nothing stopped the count from creeping back — there was no build or test guard, only the plan document and a habit.","cause":"The invariant lived in prose. A single new `import \"github.com/mendixlabs/mxcli/sdk/mpr\"` compiles, passes every test, and reintroduces exactly the blind spot Phase 4a existed to close: the unimplemented-method census in mdl/backend/modelsdk lists methods with NO implementation, so a caller reaching one through a concrete *sdk/mpr.Reader never appears in it. That is what hid project_tree.go's 36 semantic reads (#477) and cmd_extract_templates.go's FindCustomWidgetType (#484) until each was found by hand.","file":"mdl/backend/sdkmpr_import_guard_test.go","fix":"TestNothingImportsTheLegacyEngine parses every .go file's imports (go/parser, ImportsOnly) and fails naming any file that imports sdk/mpr, with the remedy in the message. Controlled by dropping a one-line file importing sdk/mpr into examples/ — it fails and names the file.","insight":"A zero-count invariant needs TWO positive controls or it passes vacuously forever, and the failure mode is silent by construction: a walk rooted at the wrong directory, a skipped-dir rule that is too broad, or an import-parsing mistake all report '0 importers' and read as success. So assert (1) a plausible number of files was actually scanned (here >500; it sees 2551) and (2) the detector can see imports AT ALL, by counting a package the repo definitely does import (mdl/backend, 120 files). Only then does 0 mean zero. This is scripts/check-tunnel-deps.sh's pattern — it asserts chisel IS in the linux graph before asserting it is absent from windows/darwin — and the same reasoning as a bug-fix control: a test that only ever passes has not been shown to detect anything. Practical note: skip sdk/mpr's own directory by comparing the path to the repo root rather than by basename, or a directory named mpr elsewhere is skipped too."} {"area": "mdl/backend/modelsdk", "date": "2026-09-16", "symptom": "`describe enumeration System.WorkflowActivityType` -> \"enumeration not found\" while `describe entity System.WorkflowActivityRecord` prints `ActivityType: Enumeration(System.WorkflowActivityType)` in the same session. `show enumerations` omits every System enum; `check --references` REJECTS a valid attribute typed against one (a false positive that blocks correct scripts); `CATALOG.attributes LEFT JOIN CATALOG.enumerations` resolves 0 of 19 on a blank app. Values could only be guessed at until the build rejected one with CE1613", "cause": "`modelsdk/meta.SystemEnumerations` — all 15 System enumerations with their values — had been in the tree since #889 with ZERO non-test consumers. The System module is not stored in the .mpr at all (measured: the string `WorkflowActivityType` occurs in 0 of 370 mprcontents units and 0 bytes of the .mpr sqlite), so its entities/associations/Java actions are each synthesized by a `Build*` helper and appended to a listing. The enumeration half had the data table and neither the helper nor the wiring, so the entity attribute printed a type naming a document nothing could produce", "file": "`modelsdk/meta/system_enumerations.go` (new, `BuildSystemEnumerations`); wired in `mdl/backend/modelsdk/enumeration.go` (`ListEnumerations`, `GetEnumeration`)", "insight": "**A data table with no consumer looks exactly like a missing feature.** Before opening any file, `grep -rn --include=*.go | grep -v _test` — zero hits is the whole diagnosis, and it took one command. The virtual System module has one wiring point PER LISTING, so the question for any new System doctype is \"which listings must it appear in\", not \"is the data there\". **One append fixed four of the five reported symptoms at once** (describe, show, check --references, catalog) because the catalog builder takes `ctx.Backend` as its reader — so `ListEnumerations` is the single choke point. Two traps. (1) `search` was NOT fixed by it: the strings index is built from value CAPTIONS (`builder_modules.go`), and the meta table carries names only, so System enums produce 0 rows in `CATALOG.strings` while user enums produce 36. Defaulting a caption to the value name would be inventing text indistinguishable from a developer's own — left unfixed and filed instead. (2) A stale `.mxcli/catalog.db` made the catalog look unfixed for three measurements; delete it before concluding anything about catalog output. Diagnosis tip: the sibling guard `TestModelerSystemEntities_HaveResolvableGeneralizations` existed and its enumeration twin did not — when one half of a synthesized module has a resolvability test, check whether the others do", "refs": "mendixlabs/mxcli#1102, mendixlabs/mxcli#1071, #889", "ce": "CE1613"} {"area":"mdl/backend","date":"2026-09-16","symptom":"With sdk/mpr at zero importers, `rm -rf sdk/mpr` still would not have been safe: it had two live dependencies that an import-based check cannot see. sdk/mpr/version has six importers (two of them shipping code, cmd/mxcli/docker/build.go and patch.go), and cmd/mxcli/docker/update_widgets_test.go reads sdk/mpr/testdata/v1-project by FILESYSTEM PATH.","cause":"The zero-importer guard matched the exact string \"github.com/mendixlabs/mxcli/sdk/mpr\". A SUBPACKAGE is a different import path, and a testdata directory is not an import at all — it is an os.DirFS string. Neither shape appears in a check written against the parent package's path.","file":"sdk/mpr","fix":"Repointed the six version importers at mdl/types (sdk/mpr/version.ProjectVersion is `type ProjectVersion = types.ProjectVersion`, an ALIAS, so it is the same type rather than a compatible one — modelsdk/mpr/version declares a duplicate struct and would NOT have been), moved the v1-project fixture to modelsdk/mpr/testdata/, verified both with the package still present, and only then deleted. 163 files, 41,674 lines.","insight":"Before deleting a package, search for THREE things, not one: the package's own import path, its subpackages' import paths (`.../pkg/`), and its directory as a literal string (testdata read through os.DirFS, go:embed, scripts). The last two are invisible to any importer census. Repoint everything FIRST and prove the build and tests green while the package still exists — that separates 'the repoint was wrong' from 'the deletion was wrong', which a single combined commit cannot distinguish. Two measurements worth keeping: the shipped binary is byte-identical in SIZE before and after, confirming the linker had already dropped the package, so this deletion removes source weight and not runtime behaviour; and `sdk/widgets` dropped to zero importers as a side effect but must NOT be deleted, because modelsdk/widgets/dirty_template_test.go reads sdk/widgets/templates/mendix-11.6 by path — the same path-not-import trap, found by grepping for the directory name rather than the import."} +{"area": "mdl/backend", "date": "2026-09-17", "symptom": "Every MDL write to an EXISTING persistent entity resets each attribute's storage GUID to the attribute's own `$ID`. The next deploy against a database that already holds data drops and recreates every column of that entity: rows and associations survive, all attribute values are gone. Reported from production \u2014 28 attributes of 607 rows emptied by one edit. `mxcli check`, `exec`, `mx check` and the build are all clean, and `DESCRIBE ENTITY` is byte-identical before and after. All six ALTER forms do it, INCLUDING `SET DOCUMENTATION`, which touches no attribute", "cause": "The unclosed half of #657. `UpdateEntity` rebuilds the target with `entityToGen`, which rebuilds every ATTRIBUTE via `attributeToGen` -> fresh `NewAttribute` (`raw==nil`), so the codec's `EmitGUID` default writes `GUID = $ID`. #657 carried `orig.Raw()` onto the rebuilt ENTITY and noted that siblings survive via the list-rebuild raw passthrough \u2014 but the target's own children are all rebuilt, and nothing carried their raw. The new GUID comes out equal to the STORED `$ID` (not a random one) because `canon.TransplantIDs` then substitutes the stored `$ID` over every 16-byte occurrence of the fresh one, GUID included, since the GUID *is* that value", "file": "`mdl/backend/modelsdk/domainmodel_alter.go` -> `UpdateEntity`; fix in `mdl/backend/modelsdk/domainmodel_child_identity.go` (`carryChildIdentity`); guard in `modelsdk/canon/storageguid.go`", "insight": "Three separate guards could not see this, and the reason is worth more than the fix. (1) The corruption is IDEMPOTENT: GUID is derived from a now-stable `$ID`, so the second identical write produces byte-identical bytes, elision fires and the executor reports `Unchanged`. Damage happens exactly once, on the first write, and every later diagnostic \u2014 including re-running the same script \u2014 looks clean. Measured: same unit bytes, 0 of 16 GUIDs matching the original. (2) `TestFreshGUIDFieldsHaveAnIdentityDecision` sees only codec `FreshGUIDFields`; a GUID derived from `$ID` is `EmitGUID`, a different registration, so it was never in that guard's view \u2014 the same blind spot that let `Workflows$*.PersistentId` through in #949. (3) `identityFields`/`CarryIdentity` reach only top-level properties of the document ROOT; these GUIDs sit on nested elements. When a write 'succeeds' but the reader shows nothing changed, assert on raw BSON \u2014 and ask whether the damage REPEATS, because a one-shot corruption defeats every same-vs-same check. Pair a GUID carry on the identity the caller tracked (`domainmodel.Attribute.ID`, which `attributeFromGen` round-trips), never on a structural pairing: `TransplantIDs`' matching is deliberately tolerant because a wrong `$ID` match only makes a diff bigger, but a wrong GUID match makes the runtime adopt another column's data under a new name and type. That is also why the canon guard REFUSES rather than repairs. The guard found the one legitimate GUID writer on the first full-suite run \u2014 `marketplace.ApplyIdentities`, which transplants captured GUIDs onto a module's replacement documents, exactly as Studio Pro's update does \u2014 so it needs a named opt-out (`UpdateRawUnitOwningStorageGUIDs`), and running the whole suite is how you find out which paths a new write-path refusal breaks. Issue mendixlabs/mxcli#1119", "refs": ["mendixlabs/mxcli#1119", "#657", "#949"], "ce": []} +{"area": "mdl/backend", "date": "2026-09-17", "symptom": "END-TO-END CONFIRMATION of the #1119 data loss on a live database, plus the part nobody had measured: a recreated column with a MODEL DEFAULT is silently backfilled with that default, so the loss can look like plausible data rather than empty cells.", "cause": "Same as the #1119 record: `UpdateEntity` rebuilt each attribute with `raw==nil`, so `EmitGUID` wrote `GUID = $ID` and the runtime's synchroniser saw every attribute as deleted-and-re-added.", "file": "`mdl/backend/modelsdk/domainmodel_child_identity.go`; measured against Mendix 11.13.0 + PostgreSQL 16 via `mxcli run --local --ensure-db`", "insight": "The experiment, worth repeating for any identity-related write: blank `mxcli new` app on 11.13.0, subject `Administration.Account` (Studio Pro-authored, so `GUID != $ID` on all three attributes \u2014 an entity mxcli CREATED is immune, because its GUID equals its $ID from birth and an ALTER then reproduces the same value, which is exactly how this can be missed in testing). Seeded 607 rows. (1) `mendixsystem$attribute.id` holds the model `GUID` byte-for-byte once the .NET field order is undone \u2014 verified on all three attributes, and it matches the GUID, never the `$ID`. (2) PRE-FIX binary, one `ALTER ENTITY \u2026 SET DOCUMENTATION`: all three GUIDs became equal to their `$ID`, `mx check` reported 0 ERRORS on the result, then boot logged `ConnectionBus: Executing 14 database synchronization command(s)` and `fullname` went 607 -> 0, `email` 607 -> 0. The column ORDER in `\\d` changed, which is the cheap tell that a column was dropped and re-added rather than altered. (3) The boolean `IsLocalUser` read as intact at first \u2014 607 non-null \u2014 and that was a MEASUREMENT TRAP: it carries `default true` in the model, so the fresh column was backfilled from the default. Proved by setting all 607 to `false` first: after the GUID change they were all `true` again. So `count(col)` is not a data-loss test; seed a NON-DEFAULT value and check the values. (4) FIXED binary, same ALTER: GUIDs unchanged, and after redeploy all 607 rows kept `fullname`, `email` AND the non-default `islocaluser=false`. The control is the pre-fix binary built from the parent commit in a git worktree, not a stubbed guard, so it exercises the real shipped path.", "refs": ["mendixlabs/mxcli#1119"], "ce": []} +{"area": "mdl/backend", "date": "2026-09-22", "symptom": "`MOVE ENTITY Mod.E TO Target` either silently destroyed the entity's whole table on the next deploy, or \u2014 after the #1119 write guard landed \u2014 was REFUSED outright for any entity participating in an association, naming a `DomainModels$CrossAssociation` the user never mentioned.", "cause": "`MoveEntity` (`mdl/backend/modelsdk/association_move_write.go`) rebuilds the entity with `entityToGen` and carried NEITHER of the two identity carries `UpdateEntity` has \u2014 no `SetRaw(orig.Raw())` (#657, the entity's own GUID) and no `carryChildIdentity` (#1119, attributes and indexes). Separately `crossAssocFromGenAssoc` built a fresh `NewCrossAssociation` that preserved the association's `$ID` but not its GUID, so `EmitGUID` wrote `GUID = $ID`. The guard saw only that third one, because the conversion happens IN PLACE in the source unit while the entity lands in a different unit where its `$ID` pairs with nothing.", "file": "`mdl/backend/modelsdk/association_move_write.go` (`MoveEntity`, `crossAssocFromGenAssoc`, `crossAssocRawFromAssoc`)", "insight": "MEASURED on Mendix 11.13.0 + PostgreSQL 16, same 250-row starting state both ways: GUID re-minted -> the old table is dropped and an empty one created (250 -> 0 rows); GUID preserved -> the runtime RENAMES the table (`myfirstmodule$moveprobe` -> `administration$moveprobe`, 3 DDL commands) and every row survives with its values. So a module move preserves data IFF the GUID does \u2014 the runtime resolves the entity by GUID, not by table name \u2014 and a move is therefore the WORST case of this class, losing a whole table rather than a column. No warning is warranted; the carry is the fix. Two method notes. (1) Converting an element to a different `$Type` cannot use `SetRaw`, because the encoder passes `$Type` through from raw, and cannot use gen's `SetDataStorageGuid` either: gen binds it under key `DataStorageGuid` where Studio Pro stores `GUID` (a keyaudit row) AND types it `string` where the property is a 16-byte binary. The way through is a RAW transform of the stored document \u2014 here `$Type` rewritten, `ChildPointer` -> `Child`, the two `*Connection` waypoints dropped, everything else verbatim \u2014 with the target key set taken from `generated/metamodel`, the arbiter: exactly the 13 keys `DomainModelsCrossAssociation` declares, confirmed against the emitted document. `SetRaw` + `InitFromRaw` leaves the element clean, so the encoder's existing-element path passes it through and the registered `EmitGUID` default (fresh elements only) does not double up. (2) The MXCLI-CREATED IMMUNITY bit three times in one session and is now the first thing to check when a GUID test or repro script passes: an entity mxcli created has GUID == $ID from birth, so 'GUID = $ID' re-mints the same value and nothing is detectable. It voided a runtime control, and it makes the MDL bug-test script in `mdl-examples/bug-tests/` unable to fail \u2014 verified by running the pre-fix binary against it. A raw carry onto a MOVED entity is nonetheless safe: the unmodeled `Image`/`ImageData` are qualified names for a document that does not move with the entity (so keeping them is correct, and today's move silently DROPPED the image), and everything needing a rewrite is modeled and therefore dirty. Separate, pre-existing, NOT this fix: `MOVE ENTITY` rewrites no qualified-name references, so a move of a referenced entity leaves 33 CE1613s in a blank 11.13 app \u2014 identical count before and after this change.", "refs": ["ako/mxcli#503", "mendixlabs/mxcli#1119", "#657"], "ce": ["CE1613"]} +{"area": "mdl/backend", "date": "2026-09-23", "symptom": "Six statements were affected and only one was in the report's title. On upstream, `ALTER ASSOCIATION ... SET COMMENT|SET OWNER`, `CREATE OR MODIFY ASSOCIATION` (even an identical re-run), `RENAME ASSOCIATION` and `RENAME ENTITY` each re-minted the storage GUID of EVERY element in the module's domain-model unit — reporter measured 282 in one module (37 entities, 224 attributes, 21 associations) and a runtime crash, `Cannot invoke \"...Table.getTableName()\" because \"table\" is null`. On a branch carrying the #1119 write guard the same six statements were REFUSED instead, naming 9 elements on a blank 11.13 app, so the commands were simply unavailable for any Studio Pro-authored module.", "cause": "`Backend.UpdateDomainModel` (`mdl/backend/modelsdk/domainmodel_alter.go`) removes and rebuilds the WHOLE Entities and Associations lists, so every element arrives `raw == nil` and the codec's `EmitGUID` default writes `GUID = $ID`. Its own doc comment claimed it preserved \"each element's identity\", which was true of the `$ID` (explicitly `SetID`-ed) and false of the `GUID`. This is the same defect as #657 (entity) and #1119 (its children) on the OTHER list-rebuild shape: `UpdateEntity` swaps one entity into an otherwise raw-passthrough list, so its siblings were always safe; here there are no passthrough siblings at all.", "file": "`mdl/backend/modelsdk/domainmodel_alter.go` (`UpdateDomainModel`); the carry helper is `domainmodel_child_identity.go` (`carryChildIdentity`)", "insight": "WHAT MAKES THIS THE EXPENSIVE ONE IS THE STATEMENT NOBODY ASSOCIATED WITH IT: `RENAME ENTITY` routes through the same function, and an entity's name IS its table name. Per the #503 measurement (11.13.0 + PostgreSQL 16, same 250-row start both ways) a preserved GUID makes the runtime RENAME the table and a re-minted one makes it DROP the table and create an empty one — so a rename loses a whole table, where an ALTER loses a column. The title said ALTER ASSOCIATION; the census of call sites is what found the rest. METHOD: enumerate every caller of a GUID-bearing converter (`entityToGen`, `assocToGen`, `attributeToGen`) rather than reproducing the reported statement — that closed the class in one pass and turned up a fifth site, `UpdateAttribute`, reachable only through the public `api/` package and therefore invisible to any MDL repro. The fix is simpler than #503's: `assocToGen` is Association -> Association, so the stored and rebuilt `$Type` agree and a direct `SetRaw` works, where a cross-module move had to transform the raw because the `$Type` changes. A raw carry here also SUBSUMES #872, which patched the association line anchors property-by-property after the same 'the rebuild only carries what the semantic model models' mechanism dropped them. TWO MEASUREMENT TRAPS, both cost time. (1) A GUID census keyed on element NAME reads a RENAME as 'the old element vanished, a new one appeared' — 4 moved and 1 added where the truth was 0; key it on `$ID`. (2) `ALTER ASSOCIATION ... SET DOCUMENTATION` is a parse error; the real spelling is `SET COMMENT`, so a probe written from the wrong keyword reports the statement unaffected. THE GUARD IS NOW THE CHEAPEST ASSERTION AVAILABLE for this class: a write that lands on a Studio Pro-authored unit IS proof no GUID in it moved, which is why the end-to-end check is 'the statement succeeded', not a BSON diff — verified anyway by grepping the 16-byte binaries under key `GUID` out of the `.mxunit` before and after (9 of 9 identical, all five statements, `mx check` 0 errors).", "refs": ["mendixlabs/mxcli#1169", "#657", "mendixlabs/mxcli#1119", "ako/mxcli#503", "#872"], "ce": []} {"area": "mdl/backend", "date": "2026-09-17", "symptom": "A microflow's URL (the deep link Studio Pro shows on the microflow's properties, Mendix 10.6+, e.g. `item/{Key}`) disappears after any `CREATE OR MODIFY MICROFLOW` \u2014 including one that only edits the body. `mxcli check`, `mx check` and mxbuild all report success before and after; the loss is visible only in Studio Pro (#1120)", "cause": "`microflowToGen` wrote `out.SetUrl(\"\")` and `out.SetUrlSearchParametersQualifiedNames(nil)` unconditionally in its `major >= 10` block, `microflowFromGen` never read either back, and `sdk/microflows.Microflow` had no field to hold them \u2014 so the value had no path across a rewrite at any of the three layers", "file": "`mdl/backend/modelsdk/microflow_write.go` (microflowToGen), `mdl/backend/modelsdk/microflow.go` (microflowFromGen), `mdl/executor/cmd_microflows_build.go`, `sdk/microflows/microflows.go`", "fix": "Carry `Url`/`UrlSearchParameters` the way AllowConcurrentExecution/MarkAsUsed/ApplyEntityAccess already are: field on the semantic microflow, read in microflowFromGen, written from the model in microflowToGen, and seeded from the stored microflow in the executor's rewrite path. DESCRIBE emits a `-- URL: \u2026` note, because a describe -> rename -> exec COPY still has nothing to preserve from", "insight": "This is the fifth property in `microflows.Microflow` lost this way and the first with NO checker behind it, which is what made it a user report rather than an internal find. The earlier four were all caught by a build error eventually (CE4899 for the concurrency flags, CE0122 for Excluded) or by a security review (ApplyEntityAccess); a microflow with no URL is simply a valid microflow, so every gate stays green and only a human opening Studio Pro can see it. **Generalisable**: when auditing a rebuild for guard-don't-drop, rank the constants it writes by whether a checker would notice their absence \u2014 the ones nothing checks are the ones that reach users, and they are exactly the ones a 'does this look like configuration?' audit skips. The mechanical version is to diff a Studio Pro document key by key against the writer's output; here `grep 'out.Set.*(\"\")\\|(nil)' microflow_write.go` finds the whole remaining set in one line (`ExportLevel` pinned to \"Hidden\", `ConcurrencyErrorMicroflow`/`ConcurrencyErrorMessage` emptied \u2014 both still unguarded, though CE4899 makes the concurrency pair loud). Measured control: reverting either half (SetUrl or the read) alone fails TestMicroflowRoundTrip_DeepLinkURL with the reported symptom, so both halves are load-bearing"} {"area": "mdl/backend", "date": "2026-09-17", "symptom": "A microflow's **export level** (Studio Pro's Hidden/API switch \u2014 whether it is part of the module's public surface when the module is exported as a package) is reset to `Hidden` by any `CREATE OR MODIFY MICROFLOW`. Every checker stays green, because a hidden microflow is a valid microflow; the module's API is simply smaller", "cause": "`microflowToGen` wrote `out.SetExportLevel(\"Hidden\")` unconditionally, `microflowFromGen` never read it back, and `sdk/microflows.Microflow` had no field \u2014 the identical three-layer gap as the deep-link URL in the same function", "file": "`mdl/backend/modelsdk/microflow_write.go` (microflowToGen), `mdl/backend/modelsdk/microflow.go` (microflowFromGen), `mdl/executor/cmd_microflows_build.go`, `sdk/microflows/microflows.go`", "fix": "Same carry as the URL, plus a DEFAULT: `\"\"` is not a member of `MicroflowsExportLevel`, so an empty model value is written as `Hidden` rather than passed through (the precedent is `json_write.go`). DESCRIBE emits `-- Export level:` only when the value is not `Hidden`", "insight": "Found by running the mechanical audit the URL fix prompted \u2014 `grep 'out.Set.*(\"\\|(nil)' microflow_write.go` over the one function \u2014 which is the cheap move after any instance of this class and turned up three more constants in one line. **The measurement that shaped the fix**: three real marketplace modules (Business Events 3.12.0, External Database Connector 6.2.3/6.3.0) store `Hidden` on 3 of 3 microflows and 55 of 55 documents overall, all three exporting at module level `Source` \u2014 so the hardcoded value was not wrong, it was a default masquerading as a constant. That is the shape of the trap: the audit finds the constant, but only a reference document tells you whether to carry it, default it, or leave it alone. A marketplace `.mpk` is a free source of these \u2014 `unzip -o pkg.mpk project.mpr` gives a real Studio Pro-authored MPR to query, no Studio Pro and no network needed (`mx-modules/` holds three). **Never carry an enum-valued property straight through without a default**: a stored document that says nothing reads as `\"\"`, and writing `\"\"` back is precisely the unloadable-model write CLAUDE.md warns about \u2014 mxbuild tolerates it and Studio Pro throws at MprProperty.cs. Controls: pinning the writer back, stubbing the reader, and neutralising the executor carry each fail a different test with the reported symptom"} {"area": "mdl/backend", "date": "2026-09-17", "symptom": "Unit tests for a carried microflow property (URL, export level, concurrency) all pass, and the end-to-end behaviour against a real project is still unverified \u2014 the integration gate that would have caught it, `TestMxCheck_DoctypeScripts`, `t.Skip`s whenever `mx` is absent, which is every run in a fresh container", "cause": "Two separate measurement errors, both invisible to `go test`. (1) The test fixture paired `Url: \"item/{Key}\"` with `UrlSearchParameters: [\"\u2026.Key\"]` \u2014 the SAME parameter \u2014 which mxbuild rejects as **CE5612**: a parameter used in the URL path may not also be a search parameter. Nothing in a unit test validates the model, so the fixture described a document Mendix refuses to build. (2) `bin/mxcli` was stale: `go build ./mdl/...` and `make test` had been run after each fix, but not `make build`, so the end-to-end run exercised a binary predating two of the three commits", "file": "`mdl/backend/modelsdk/microflow_roundtrip_flags_test.go`, `mdl/executor/microflow_carried_properties_test.go`, `mdl/executor/roundtrip_doctype_test.go` (the skipping gate)", "fix": "Fixture uses a distinct `Filter` parameter and says why. End-to-end procedure that actually measures it: `mxcli setup mxbuild -p ` (~719 MB, works through the session proxy), copy `testdata/expr-checker` as the fixture, create the microflow with mxcli, seed the unauthorable properties straight into the stored unit with `mpr.NewWriter` + `UpdateRawUnit`, then `mx check` BEFORE (the fixture must be a document Mendix accepts, or it proves nothing), `mxcli exec` a body-only rewrite, read the unit back, `mx check` after", "insight": "**A skipping integration gate is worse than no gate**: `mxCheckAvailable()` + `t.Skip` means a green `make test` says nothing about mxbuild, and reading the CLAUDE.md line about #808 is not the same as checking whether it applies to your own run \u2014 `ls ~/.mxcli/mxbuild` is. **Rebuild the binary before any end-to-end run**, and check its mtime against the last commit: a stale `bin/mxcli` produced a result (URL survived, export level did not) that looked exactly like a genuine second-read-path defect, and sent me hunting for a duplicate resolver that does not exist. **Seed the fixture through the writer, not by hand-editing BSON**, and always `mx check` the seeded state first: the CE5612 error came from the seed, not from mxcli, and without the before-check it would have been misattributed to the fix. Measured, mxbuild 11.6.6: pre-fix binary rewrites the microflow to `Url=\"\"`, empty search params, `ExportLevel=\"Hidden\"`; post-fix keeps all three; `mx check` 0 errors on both the seeded control and the rewritten project"} @@ -126,3 +130,4 @@ {"area":"mdl/backend","date":"2026-09-22","symptom":"modelsdk/mpr/version.ProjectVersion declared its own struct with the same seven fields as mdl/types.ProjectVersion instead of aliasing it, so a *version.ProjectVersion could not be passed where a *types.ProjectVersion was wanted and vice versa — two unrelated Go types that both print as 'ProjectVersion'.","cause":"The deleted sdk/mpr/version aliased the canonical type (`type ProjectVersion = types.ProjectVersion`); this copy declared a duplicate. CLAUDE.md's shared-types rule asks for the alias, and nothing enforced it. The duplication survived the legacy-engine retirement because it compiles perfectly — the two declarations are field-for-field identical, so only an assignment ACROSS the boundary reveals them as different types.","file":"modelsdk/mpr/version/version.go","fix":"Made it an alias. The four methods it redeclared (IsAtLeast, IsAtLeastFull, String, IsMPRv2) were verified semantically identical to types' first — IsAtLeast differed only in early-return style, same truth table — and now come from types. IsSupported/SupportsFeature could not survive as methods on an aliased type and had ZERO callers anywhere (measured), so they went with Feature, MinVersion, featureVersions and SupportedVersionRange; that map called itself 'the fallback when the YAML registry is unavailable' and the live registry is sdk/versions/mendix-{9,10,11}.yaml via checkFeature.","insight":"A same-shape duplicate type is invisible to every signal except an assignment across the package boundary: it compiles, tests pass, and the error it eventually produces names the same type on both sides of 'want'. So the guard is a COMPILE-TIME assertion, not a runtime test — `var _ *types.ProjectVersion = (*version.ProjectVersion)(nil)` builds only under an alias and fails to build under a duplicate, which is strictly stronger than anything a test body can assert. Write it before the fix and watch it fail to compile; that failure IS the reproduction. Two measurements that made the cleanup safe rather than brave: diff the method BODIES before assuming the redeclarations are redundant (identical behaviour, different style, is the common case and the dangerous one is the near-miss), and count callers of anything the alias forces you to drop — here six exported symbols had zero. Unrelated trap hit while verifying: four cmd/mxcli tests that read skill files failed once in a full `go test ./...` interleaved with `make check-mdl`, which runs sync-skills (rsync --delete into cmd/mxcli/skills/). They pass in isolation, on clean main, and in an uninterleaved full run — do not attribute a skills-reading test failure to your change without re-running it alone."} {"area": "mdl/backend", "date": "2026-09-22", "symptom": "`create workflow … overview page X` reports `Created workflow` and exit 0 and stores NOTHING — the written unit carries no page reference and not even the page's qualified name as a string. `mx check` passes (a workflow with no overview page is valid) and `describe workflow` omits the clause, so nothing reveals the loss. Running `alter workflow … set overview page X` afterwards DOES write it, which is what makes the split visible", "cause": "Two fields for one concept, never joined: the executor set semantic `Workflow.OverviewPage` (`cmd_workflows_write.go:170`) and `workflowToGen` only ever read `Workflow.AdminPage`, which nothing set. The READ half was wrong in the mirror direction — `workflowFromGen` took `g.OverviewPageQualifiedName()`, so even the correctly-written ALTER read back empty and the catalog's overview-page reference edge never fired", "file": "`sdk/workflows/workflow.go` (the two fields collapsed to one), `mdl/backend/modelsdk/workflow_write.go` (`workflowToGen`), `mdl/backend/modelsdk/workflow_read.go` (`workflowOverviewPageName`)", "insight": "**The Model SDK's StructureVersionInfo settles which of two rival property names is real, in one grep**: `npm pack mendixmodelsdk` then `src/gen/workflows.js` gives `overviewPage: {deleted: \"9.11.0\"}` and `adminPage: {introduced: \"9.11.0\"}` — so AdminPage (a `Workflows$PageReference` CHILD, not a by-name string) is the stored property, and `generated/metamodel` agrees by declaring AdminPage and no OverviewPage. `modelsdk/gen` declares BOTH, which is how a reader and a writer ended up on opposite sides of a 9.11 rename inside one package. **The version branch CLAUDE.md's overlay rule would demand is dead here, and that is a measurement not an assumption**: `workflowToGen` writes `WorkflowV2`, introduced in 11.1.0, unconditionally — so no reachable project wants the pre-9.11 key. Write one spelling, READ both (a read fallback invents nothing). **The differential that proves it on a real build**: same script, same project, only the write suppressed — control 0 errors, fixed `CE7410 \"The selected page 'Overview' should accept a parameter of type 'Workflow'\"` on mxbuild 11.6.6. mxbuild can only validate a page it can see, so the error IS the evidence; with a valid overview page both variants are 0 errors, which is the usual weak-signal trap. Useful side-finding: an overview page takes **System.Workflow**, while a user task's page takes **System.WorkflowUserTask** — two pages, two parameters. NOT fixed: no check rule for CE7410 yet, and `WorkflowV2` being written unconditionally is questionable for a 10.x project. Same shape as the `create … comment 'text'` bug (findings/mdl-grammar.jsonl 2026-08-25): grep for `stmt.X = …` / `wf.X = …` with no matching read. Tests `mdl/backend/modelsdk/workflow_overview_page_test.go`; repro `mdl-examples/bug-tests/workflow-586b-overview-page-dropped.mdl`", "refs": ["ako/mxcli#586"], "ce": ["CE7410"]} {"area": "mdl/backend", "date": "2026-09-23", "symptom": "`describe java action` prints `ContextObject: entity <>` for a parameter declared `entity not null`; a bare type-parameter reference (`Obj: pEntity`, `returns pEntity`) reads back nameless too. The description no longer round-trips", "cause": "The stored parameter type (`CodeActions$EntityTypeParameterType` / `ParameterizedEntityType`) holds only a BY_ID pointer to the `CodeActions$TypeParameter`. `javaActionFromGen` carried the ID into the semantic type but never resolved it to the name, and the name is all the describer prints. `javascript_read.go` had always done this resolution pass; the Java reader was ported without it", "file": "`mdl/backend/modelsdk/java_read.go` (`resolveJavaActionTypeParameterNames`)", "insight": "The executor's `entity <>` fallback in `formatJavaActionType` is the tell: an empty name at DESCRIBE means the *reader* dropped a by-ID resolution, not that the writer lost it \u2014 the write path sets both ID and name, so a create\u2192read unit test in the backend reproduces it without any project. When a JS and a Java reader cover the same `CodeActions$` shapes, diff their post-processing first; any pass one has and the other lacks is a candidate. The write was never wrong (replaying the fixed description into a fresh project describes identically), so no `mx check` run is needed. Issue mendixlabs/mxcli#1034", "refs": ["mendixlabs/mxcli#1034"]} +{"area": "mdl/backend", "date": "2026-09-23", "symptom": "`CREATE OR MODIFY PERSISTENT ENTITY` was REFUSED by the #1119 storage-GUID guard on a doctype script that had been passing for months: `failed to update entity: refusing to write unit d82b0484-…: 1 element(s) kept their $ID but would be written with a different GUID — dff2ced1-… (DomainModels$Attribute): stored 4b52b36b-…, would write dff2ced1-…`. Two independent defects wore that one message.", "cause": "(1) A REAL data loss the guard caught: `mergeDeclaredOntoStoredEntity` sets `merged.Attributes = declared.Attributes` and `merged.Indexes = declared.Indexes` — the lists the STATEMENT declares, built from text by the visitor and carrying no element ID — and `carryChildIdentity` keyed entirely on that ID, reading an empty one as 'a genuinely new member, so a fresh GUID is right'. Every attribute of a re-declared entity was therefore re-minted, i.e. #1119 through a second executor path. (2) A FALSE POSITIVE in the guard itself: `canon.TransplantIDs` pairs STRUCTURALLY ($Type + shape, LCS-anchored), so on a statement that drops six differently-named attributes and adds one, it paired the NEW attribute with a REMOVED one and handed it that stored `$ID`; the codec had written `GUID = $ID` and the transplant substitutes over every 16-byte binary, so the GUID followed. The guard's premise — written into its own doc comment as 'unambiguously' — that a shared `$ID` after the transplant means the same element, is false.", "file": "`mdl/backend/modelsdk/domainmodel_child_identity.go` (`carryAttributeIdentity`, `carryIndexIdentity`), `modelsdk/canon/storageguid.go` (`sameMember`, `elementGUIDs`)", "insight": "ONE ERROR MESSAGE, TWO DEFECTS, AND FIXING EITHER ALONE LEAVES IT RED — which is why the first fix (the name fallback) changed nothing and the SAME element and GUIDs came back byte-for-byte. That repetition was the signal: an identical failure after a real fix means the reproduction is exercising a different code path than the one reasoned about. What settled it was describing the actual subject: the marketplace `PublishedBusinessEvent` has six attributes and NONE is named `EventId`, so there was no member to carry — the pairing itself was spurious. Reproduce against the real stored document before believing any theory about which elements correspond. METHOD that made this cheap: the CI failure reproduced locally in 0.5s as a backend unit test (strip the IDs off a fixture entity's attributes, call UpdateEntity) versus 26s for the integration subtest, but ONLY the integration subtest could have found the second defect, because the false pairing needs a real drop-six-add-one document. Run both. TRADE-OFF worth restating: the guard now pairs on `$ID` + `$Type` + `Name`, which loses one arm — a RENAME that re-mints a GUID is no longer refused, since the name is what changed — and that arm is covered directly by the carry tests where it is decidable. A backstop that refuses correct writes is worse than a backstop with a hole: the first makes documented statements unusable, and this one already had. Also: feeding a deliberately-approximate pairing to a guard promotes its error rate into refusals. TransplantIDs' correctness bar is low ON PURPOSE (a wrong match only makes a diff bigger); anything that reads its output as identity has to add its own test of identity.", "refs": ["mendixlabs/mxcli#1119", "mendixlabs/mxcli#1169", "ako/mxcli#643"], "ce": []} diff --git a/.claude/skills/fix-issue/findings/mdl-executor.jsonl b/.claude/skills/fix-issue/findings/mdl-executor.jsonl index cd31786cc8..1319382a89 100644 --- a/.claude/skills/fix-issue/findings/mdl-executor.jsonl +++ b/.claude/skills/fix-issue/findings/mdl-executor.jsonl @@ -636,6 +636,8 @@ {"area": "mdl/executor", "date": "2026-09-16", "symptom": "`checkbox cb (Attribute: A, Editable: Never, ReadOnlyStyle: Control)` parses, passes `mxcli check`, is reported as executed \u2014 and the stored document keeps ReadOnlyStyle \"Inherit\". DESCRIBE *does* read and emit the property, so describe -> exec on a Studio Pro-authored page silently downgraded Control to Inherit. Visible only in the browser: with Inherit a read-only check box renders the TEXT \"Yes\"/\"No\", with Control the (disabled) checkbox glyph \u2014 measured on 11.6.6 by patching the stored string by hand ('Inherit' and 'Control' are both 7 bytes, so a byte substitution in the .mxunit is a valid document; mx check 0 errors both ways).", "cause": "Three silent layers. sdk/pages.CheckBox had no ReadOnlyStyle field, so buildCheckBoxV3 had nowhere to put the property; the codec writer hardcoded g.SetReadOnlyStyle(\"Inherit\") (and a per-type constant for TextBox/TextArea/DatePicker/RadioButtons, \"Control\" for DataView); and `ReadOnlyStyle` sits in validate_widgets.go's known-property allowlist under the comment \"vocabulary describe page emits\", which silenced the MDL-WIDGET check that flags a property no builder consumes.", "file": "mdl/executor/cmd_pages_builder_v3_widgets.go", "fix": "Add ReadOnlyStyle to pages.CheckBox; read and canonicalise it in buildCheckBoxV3 (Inherit/Control/Text, case-insensitive in, Mendix casing out, unknown value refused); write orDefaultStr(x.ReadOnlyStyle, \"Inherit\") in the codec so an omitted property still produces the document it always did. ako/mxcli#490.", "insight": "An allowlist added so DESCRIBE output re-parses will also silence the warning that a property is going nowhere \u2014 the two uses are indistinguishable from inside the checker, so entries added for the first reason need a consuming builder or they become a licence to drop. The asymmetry is the tell: a describer that READS a property whose writer hardcodes a constant is a round trip that quietly rewrites the user's model, and it is worth grepping for that pairing directly (`grep SetX(\"literal\")` against what extract*/describe reads). Measurement trick worth reusing: when the model layer cannot express a value, patch the stored BSON to the candidate value and boot \u2014 a same-length string substitution keeps the document valid, which settles 'is this the property that changes the rendering?' before writing any Go. The end-to-end control afterwards is the elision verb: re-authoring through MDL over the hand-patched document reported `Unchanged page`, i.e. what mxcli now writes is byte-for-byte the document that was verified in the browser."} {"area": "mdl/executor", "date": "2026-09-16", "symptom": "A .def.json that maps a Data Grid 2 column filter's `linkedDs` produces a widget mxbuild rejects with CE0642 \"Property 'Datasource to Filter' is required\" — naming the very property the value was written into", "cause": "`linkedDs` is declared `isLinked=\"true\"` in widget.xml: the platform fills it from the containing DataGrid2, and mxbuild resolves it from the parent rather than reading what is stored. mxcli could not tell a linked datasource from an authorable one because IsLinked, though present in the template ValueType, was not carried into PropertyTypeIDEntry", "file": "`mdl/types/widget_property_type.go` (IsLinked), `modelsdk/widgets/loader.go`, `mdl/executor/widget_engine.go` (`refuseLinkedDataSourceMapping`)", "insight": "Before mapping a widget property, check `isLinked` in widget.xml — a linked property is the platform's to fill, the ADR-0005 'author only what the model owns' rule wearing a widget hat. Three cheap measurements settle it faster than reasoning: grep the shipped template's ValueType for IsLinked, dump the property off Studio Pro-authored widgets in testdata/expr-checker (5 of 5 store linkedDs empty), and mx check the correct shape (0 errors WITHOUT it). Beware the inverted signal: writing the value does NOT clear CE0642, so a failing check after writing it looks like the value is missing rather than unwanted. Across all widget packages in testdata, linkedDs is the ONLY linked datasource among the 8 multi-datasource widgets — DROPDOWNFILTER is single-source from MDL's side, ComboBox and the 6 charts are genuinely multi-source", "ce": ["CE0642"]} {"area": "mdl/executor", "date": "2026-09-16", "symptom": "A chart series given BOTH a static and a dynamic datasource writes its static x/y attributes against the DYNAMIC source's entity — mxbuild reports CE1613 \"The selected attribute 'CH.Forecast.Region' no longer exists.\"", "cause": "buildObjectListItem pre-resolves every datasource the item configures and dropped each resolved entity into the one shared pageBuilder.entityContext, so the LAST one won. The per-property link was already in hand and ignored: ItemPropertyMapping.DataSource carries widget.xml's `dataSource=\"...\"` and GenerateDefJSON already emits it for every chart dependent", "file": "`mdl/executor/widget_engine.go` (`itemEntityContextFor`, `prebuiltEntities` in `buildObjectListItem`)", "insight": "The item twin of the widget-level per-datasource context (#1109). Look for the SECOND copy whenever a context fix lands at widget level — object-list items run the same pre-resolve/resolve shape with their own loop. The shipped chart defs already map staticDataSource AND dynamicDataSource with every dependent's link, so nothing needed mapping; the links simply were not read. Note the weak in-repo signals: `mxcli check` only warns (MDL-WIDGET10, the inactive set is hidden) and the describe output looks right, so the defect is visible only in the stored BSON or from mxbuild. Charts' static/dynamic sit INSIDE the `lines` object list, not at widget level — a recursive widget.xml scan makes them look like widget properties", "ce": ["CE1613"]} +{"area": "mdl/executor", "date": "2026-09-22", "symptom": "A cross-module `MOVE ENTITY` reported success and left the project unbuildable: 33 CE1613s in a blank 11.13 app, every one a reference still naming the source module \u2014 the entity (13), its attribute paths (11) and the converted association (9) \u2014 in microflow activities, page widgets, page parameters and access rules. Nothing warned.", "cause": "TWO defects. (1) `execMove` returns early for ENTITY at `cmd_move.go:42`, before the doctype switch AND before the `isCrossModuleMove` sweep (`updateQualifiedNameRefs` \u2192 `UpdateQualifiedNameInAllUnits`) that every other doctype gets, so no project-wide rewrite happened at all. The comment there is right that an entity is not a top-level unit, but the sweep is purely name-based and applies just the same. (2) `Backend.MoveEntity` re-points the view source and each validation rule's attribute, and ACCESS RULES were the missed sibling \u2014 the moved entity's own rules kept `Source.Entity.Member`.", "file": "`mdl/executor/cmd_move.go` (`moveEntity`), `mdl/backend/modelsdk/association_move_write.go` (`MoveEntity`), `mdl/types/entity_move.go`", "insight": "The stale access rules are worse than a dangling string, and the mechanism generalises: `entityToGen`'s `syncMemberAccesses` matches existing entries BY QUALIFIED NAME, so the stale `Source.Entity.Attr` never equals the rebuilt `Target.Entity.Attr` and it APPENDS the new one while keeping the old. The entity ends up carrying every member twice, half dangling \u2014 and `DESCRIBE ENTITY` cannot show it, because it renders members bare, so the only visible trace is each member appearing twice in the grant. Whenever a sync-by-name helper meets a rename, expect duplication rather than staleness. The sweep must be driven by the OLD and NEW names the backend reports, never derived from the module names: the conversion is asymmetric, since Mendix stores an association in the module of its FROM entity, so the parent moving takes the cross-association to the target (measured: 9 of 33 errors name it) while the child moving leaves it put (measured: 0 name it). Deriving would have corrupted the second case; a `MovedAssociation{Old,New}` makes it a no-op instead of a branch. Establishing the gap was cheap because the machinery already existed and worked \u2014 `MOVE MICROFLOW` cross-module prints 'Updated references in 1 document(s)' and checks at 0 errors \u2014 so a control on a SIBLING doctype is the fastest way to tell a missing capability from a missing call. Verified end to end both directions: 33 \u2192 0. THIRD, SEPARATE, STILL OPEN: moving both endpoints of one association in sequence leaves a pre-existing CrossAssociation pointing at an element no longer in its unit, and the project will not OPEN (`System.AggregateException: The given key '' was not present in the dictionary`) \u2014 `MoveEntity`'s conversion loop only walks `AssociationsItems()` and never `CrossAssociationsItems()`. Reproduced with a binary predating both fixes, so it is not introduced by them; it is why the MDL bug-test keeps one endpoint of each pair put.", "refs": ["ako/mxcli#605", "ako/mxcli#503"], "ce": ["CE1613"]} +{"area": "mdl/executor", "date": "2026-09-23", "symptom": "`RENAME ENTITY` and `RENAME ASSOCIATION` reported success and a reference count — \"Updated 24 reference(s) in 11 document(s)\" — and then left the renamed element's OWN member references stale, as CE1613 at build time: \"The selected attribute 'Mod.OldEntity.Attr' no longer exists.\" at Access rule / Validation rule of the entity that had just been renamed. 4 errors on a Studio Pro-authored 11.13 module, 3 on an mxcli-created one.", "cause": "A CLOBBER, not a missed sweep. `execRenameEntity` / `execRenameAssociation` (`mdl/executor/cmd_rename.go`) read the domain model, run the project-wide `RenameReferences` pass — which DOES rewrite those names in the raw unit — and then call `UpdateDomainModel` with the semantic model read BEFORE the sweep, putting the stale names back. Access rules name a member `Module.Entity.Attr`, validation rules hold the same string in `AttributeID`, and an association member is `Module.Association`: all three embed a name the rename changes, and all three live in the unit the persist overwrites.", "file": "`mdl/executor/cmd_rename.go` (`execRenameEntity`, `execRenameAssociation`, `repointEntitySelfRefs`, `repointAssociationMemberRefs`)", "insight": "THE REPORT AND THE ERRORS LINING UP IS THE DIAGNOSIS: \"Updated 3 reference(s) in 1 document(s)\" followed by exactly 3 CE1613s naming those 3 members means the sweep found them and something undid it — so look for a later write to the same unit, not for a gap in the scanner. A sweep that reports work it then discards is indistinguishable from a sweep that never ran, except by that count. This is the RENAME sibling of ako/mxcli#605 (same stale member names on a cross-module MOVE ENTITY, stale on the module prefix instead of the entity name), and the reason both matter more than a dangling string is `entityToGen`'s `syncMemberAccesses`, which matches existing entries BY qualified name: a stale `Mod.Old.Attr` never equals the rebuilt `Mod.New.Attr`, so it APPENDS the new entry and keeps the old, and the entity carries every member twice with half the entries dangling. DESCRIBE renders members bare, so duplication is the only visible trace. THE ASSOCIATION ARM IS ASYMMETRIC AND A GREEN BUILD DOES NOT COVER IT: against an access rule mxcli wrote, the stale association name raises NO error (the MemberAccess still carries a valid element pointer; the qualified name is what a reader shows, not what mxbuild resolves), while the same rename against a STUDIO PRO-authored rule is CE1613 — so the synthetic repro proves the attribute half only, and the association half has to be measured on a real module. The exact-vs-prefix distinction is load-bearing in the other direction too: an association member is `Module.Association` with nothing after it, so a prefix match would rewrite a differently-named association that merely starts with the same text. Discovered while validating the #1169 GUID fix, and pre-existing — established by building a binary with only that fix stashed out and getting an identical error count both ways, which is the only way to tell 'my change did this' from 'my change let me finally reach this'.", "refs": ["ako/mxcli#605", "mendixlabs/mxcli#1169"], "ce": ["CE1613", "CE0066"]} {"area": "mdl/executor", "date": "2026-09-17", "symptom": "`CREATE OR MODIFY MICROFLOW` re-enables concurrent execution on a microflow that disallowed it \u2014 the running app's concurrency protection removed \u2014 and drops the concurrency error message (all translations) and error microflow, plus `MarkAsUsed`. Every checker is green: **CE4899 fires only on disallow-without-a-message, never on allow**, so the one error that exists in this area is exactly the one the reset switches off", "cause": "`buildMicroflowFromStmt` built the rebuild struct with `AllowConcurrentExecution: true` and `MarkAsUsed: false` literals, and `microflowToGen` wrote `SetConcurrencyErrorMicroflowQualifiedName(\"\")` + a bare `genTexts.NewText()`. The backend already READ the two flags back (the #723 \u00a7A fix), so the round-trip test passed while the bug was live \u2014 the executor overwrote them before the backend ever saw them", "file": "`mdl/executor/cmd_microflows_build.go` (buildMicroflowFromStmt), `mdl/backend/modelsdk/microflow_write.go` (microflowToGen), `mdl/backend/modelsdk/microflow.go` (microflowFromGen), `sdk/microflows/microflows.go`", "fix": "Carry all four from the stored microflow, seeding the locals with the NEW-microflow defaults (true/false) so no separate preserve flag is needed. The error message reuses the existing `textFromGen`/`textToGen` pair, so translations survive; nil still emits the bare empty `Texts$Text` the writer always wrote", "insight": "**A passing round-trip test at one layer says nothing about the layer above it.** `TestMicroflowRoundTrip_ConcurrentExecutionFlags` had guarded these two flags since #723 and was green throughout, because the executor's rebuild struct overwrites them before calling the backend. When a property is reset, locate the LAST writer on the path, not the first one that looks responsible. **And check which way a reset goes**: #723's backend bug wrote the Go zero value (allow -> disallow) and hit CE4899 immediately; the executor's literal writes the opposite (disallow -> allow), and the same CE4899 that caught the first direction is structurally blind to the second. A checker that catches a property's loss in one direction is not coverage for that property. Two methodological traps in the test itself, both hit: `bytes.Equal` on two encodes of the same microflow ALWAYS differs (fresh random sub-element `$ID`s \u2014 the reason `canon` exists), and `canon.Equal` on a whole microflow always differs too, because `StableId` is a fresh GUID *value* per encode and `Equal` does not mask \u2014 only `Reconcile` may be asked that question. Compare the sub-element under test, or use Reconcile. Controls: hardcoding the executor literals back, emptying the writer's pair, and stubbing the reader each fail a different test with the reported symptom"} {"area":"mdl/executor","date":"2026-09-17","symptom":"`DESCRIBE ENUMERATION Mod.E` prints every value with an empty caption (`MyValue ''`) although Studio Pro shows them. Re-executing that output then DESTROYS the real captions (exec reports \"Modified enumeration\" and the stored Texts$Translation goes empty). Reported on Windows, single-language project, v0.18.0 and v0.22.0","cause":"The read asked `v.Caption.GetTranslation(\"en_US\")`. Mendix has no language-neutral text: a project whose DefaultLanguageCode is nl_NL stores the caption under nl_NL and nothing else, so the lookup misses and returns \"\". #970 fixed the WRITE side to use the project language and #702 fixed the widget READ side; the enumeration/validation-rule/message-template reads were the sites neither sweep reached","file":"`mdl/executor/cmd_enumerations.go` (describeEnumeration), `cmd_diff_mdl.go` (enumerationToMDL), `describe_language.go` (pickTextTranslation's fallback now sorts), `mdl/catalog/language.go` + `builder_modules.go`","insight":"**Reproduce it with mxcli alone — no Studio Pro and no non-English project needed.** `ALTER SETTINGS LANGUAGE ADD OR MODIFY 'nl_NL' (...); ALTER SETTINGS LANGUAGE DefaultLanguageCode = 'nl_NL';` on a copy of any fixture, then CREATE the enumeration: the write side already honours the project language, so the captions land under nl_NL and DESCRIBE reads '' immediately. That also gives the impact control for free — feed the '' output back through exec and grep the .mxunit for `Texts$Translation LanguageCode nl_NL Text ` with nothing after it. **The plausible wrong turn to skip**: suspecting the codec or a gen storage-name mismatch. `EnumerationValue.Caption` is NOT in keyaudit_test.go and the strings are plainly visible in the unit — dump the .mxunit with a printable-ASCII regex FIRST (one command) and the language code tells you it is a read-side language bug, not a decode bug. **The fallback has to sort**: `for _, v := range t.Translations` returns a different language per run, so a multi-language project's DESCRIBE output was undiffable — a bug that a single-language repro can never show.","refs":["mendixlabs/mxcli#1113","mendixlabs/mxcli#970","mendixlabs/mxcli#702"],"ce":[]} {"area": "mdl/executor", "date": "2026-09-17", "symptom": "`CREATE OR MODIFY EXTERNAL ENTITIES FROM` imports an OData entity with **none** of its ComplexType properties — `describe entity` lists only the key. `exec` reports `1 created, 0 failed` and prints nothing; `mx check` says 0 errors. The loss surfaces much later as CE1613 on a page written against the attributes Studio Pro would have made. `DESCRIBE CONTRACT ENTITY` compounded it by reporting the complex property as `String(200)`", "cause": "`mdl/types/edmx.go` never parsed `` at all, so a property typed `Shared.Uom.Quantity` was indistinguishable from one of an unknown type, and `createExternalEntities`' `if !strings.HasPrefix(p.Type, \"Edm.\")` dropped it with no `continue` message. `String(200)` was `edmToMendixType`'s default branch", "file": "`mdl/types/edmx.go` (EdmComplexType, FindComplexType, FlattenProperties, EdmProperty.RemotePath/Path), `mdl/executor/cmd_contract.go` (createExternalEntities, describeContractEntity, outputContractEntityMDL)", "insight": "**The local name and the remote name differ by SEPARATOR, and that is the core of the fix.** Studio Pro names the attribute `MaxQty_UoMNId` and reads it over the OData path `MaxQty/UoMNId`; assuming RemoteName == attribute name is the obvious wrong turn and it is silent in the model. Measured on mxbuild 11.12.1, three copies of one project: RemoteName `MaxQty/UoMNId` -> 0 errors; `MaxQty_UoMNId` -> 4x **CE6615** \"Attribute 'X' of external entity 'Definition' does not exist in the OData service\"; a deliberately bogus path -> the same 4x CE6615. So mxbuild resolves the path INTO the complex type and genuinely validates it — the 0-error run is evidence, not a rubber stamp, and CE6615 is the detector to reach for on any external-entity remote-name question. **Do not stop at a synthetic fixture.** A two-property complex type in its own namespace passed `mx check` clean and the fix still shipped four defects, all caught by the integration suite's live **TripPin** contract (`10-odata-examples.mdl`) at 11 errors. TripPin is the fixture to reach for: it has a base complex type, two types derived from it, a nested complex property, an Edm.GeographyPoint, and both a top-level entity set and types derived from it. What it taught, each measured: (1) Mendix imports a complex type's **own** properties only — flattening `AirportLocation`'s inherited `Address` is CE6615, while the same `Address` via `Person.HomeAddress` (typed `Location` directly) is accepted, so the line is inheritance, not path syntax; (2) `!strings.HasPrefix(t, \"Edm.\")` is not the supported-type test — `Edm.GeographyPoint` passes it and is **CE6622** \"The type of attribute 'Location_Loc' … is not supported\", so the importable primitives must be a closed set; (3) Creatable/Updatable are always false on a flattened attribute (against a contract annotated Insertable=true AND Updatable=true, Mendix still says False — 2x **CE6630** per attribute, matching the doc's \"can only be read or deleted\"), but **Filterable/Sortable are not**: they follow the ENTITY SET, and CE6630 fires in BOTH directions, so neither blanket answer survives. On TripPin, `Person` (entity set `People`) wants True and `Employee`/`Manager`/`Event` (derived, no entity set) want False; `Manager.BossOffice` is Manager's own property and still False, which rules out inheritance as the explanation. A test for a two-directional rule needs **both** controls — stamping false everywhere passes the derived case and fails People. Resolve complex types by QUALIFIED name: one document may declare `Quantity` in two namespaces, and FindEntityType's short-name fallback would silently hand over the other schema's properties. Also: the pre-fix control (attributes simply absent) is **0 errors**, so the build never catches the drop itself — a regression test asserting `mx check` clean would have passed against the bug. The report said 'only cross-namespace'; in fact every complex type was dropped, since nothing named `Edm.*` is complex. Repro `mdl-examples/bug-tests/1118-odata-complextype-flattening.mdl`", "file_refs": ["mdl/types/edmx.go", "mdl/executor/cmd_contract.go"], "refs": ["mendixlabs/mxcli#1118"], "ce": ["CE6615", "CE6622", "CE6630", "CE1613"]} @@ -678,5 +680,9 @@ {"area": "mdl/executor", "date": "2026-09-23", "symptom": "`currentDeviceType()` in a nanoflow passes `mxcli check` (\"All references valid.\") and `mxcli exec` (\"Created nanoflow: Test.NF_Dev\"), then the build fails `[error] [CE0117] \"Error(s) in expression.\" at Log message activity 'Log message (info)'`", "cause": "Two gaps stacked. MDL044 lived in ValidateMicroflow, which only CREATE MICROFLOW reaches — check (validate_program.go), the LSP and exec (buildNanoflowFromStmt) never ran it on a nanoflow, so the #828 fix covered half the flow types. And walkBody listed MDL044's expression sites inline per statement (return/if/declare/set/create/change) with no `log` case, so the reported repro — the call in a LOG message — was missed in a microflow too", "file": "`mdl/executor/validate_microflow.go` (`checkStmtExprFunctions`), `mdl/executor/validate_nanoflow.go` (new: `ValidateNanoflow`, `validateNanoflowRules`), `mdl/executor/cmd_microflows_build.go` (`buildNanoflowFromStmt`), `mdl/executor/validate_program.go`, `cmd/mxcli/lsp_diagnostics.go`", "insight": "**When a rule fix names a statement type, grep every entry point for the sibling type** — `CreateMicroflowStmt` and `CreateNanoflowStmt` share a body grammar but have separate wiring in check, LSP and exec, and #828 wired only the microflow side of all three. **Do not run ValidateMicroflow over a nanoflow to close it**: MDL057 refuses `synchronize`, which is nanoflow-only, so a wholesale reuse turns a gap into a false-positive write barrier; run only the rule that holds (one shared `checkStmtExprFunctions` site list, so the two walkers cannot drift). **Test the reported repro verbatim, not the prior fix's shape**: #828's test used `declare`, the report used `log`, and that difference was a second, independent gap. **Build the report's workaround before repeating it**: `[%CurrentDeviceType%]` is CE0117 on 11.13.0 in a nanoflow AND a microflow (control `[%CurrentDateTime%]` builds 0 errors) — and `check` does not validate token names at all, a separate gap. Operational trap: a project from `mxcli new` links `./mxcli` to `bin/mxcli` by shared inode, so a project created before rebuilding runs the PRE-fix binary — the first 'fixed' run here wrote the nanoflow; call `bin/mxcli` after `make build`", "refs": ["mendixlabs/mxcli#1033", "mendixlabs/mxcli#828"], "ce": ["CE0117"], "rules": ["MDL044"]} {"area": "mdl/executor", "date": "2026-09-23", "symptom": "`jump to A;` inside a `boundary event … { }` body: `check --references` clean, `exec` written, DESCRIBE reads back `jump to A;`, then native `mx check`: [CE0495] \"Duplicate name 'A'.\" at User task 'A', Jump 'A' and [CE6680] \"The 'Target' property is required.\" at Jump 'A'", "cause": "Same as mendixlabs/mxcli#1005: `buildJumpTo` named the jump after its target, and in v0.20.0 name deduplication did not reach boundary-event bodies, so the jump kept the target's name. The report was filed against v0.20.0 (2026-08-28), three days before 825873d6 fixed it", "file": "`mdl/executor/cmd_workflows_write.go` (`buildJumpTo`, `deduplicateActivityNamesInFlow`); regression test `mdl/executor/issue1024_boundary_jump_test.go`", "insight": "**Before fixing, rebuild the reporter's version and main, and run the repro on both.** Main gave 0 errors on 11.14.0, and a build of `825873d6^` reproduced both errors verbatim, so the fix was a regression test, not a code change. It looked like a different bug because the report said the outcome-body form worked. That was a flow-order accident from #1005: dedup renames the SECOND activity with a name, and an outcome body comes after its task, while boundary bodies were then outside the dedup walk entirely. CE6680 'Target required' is Mendix's wording when the target name resolves to the jump itself, not a sign that TargetActivity was empty. It was set to `A` all along. **The #1005 fix has two independent guards** (jump named `JumpTo`, and jumps deduplicated last). Reverting either one alone leaves the test green, so a control has to stub both, or run the unchanged test against the pre-fix commit (with a shim for helpers added later). Otherwise the control 'passes' and proves nothing. mendixlabs/mxcli#1024", "refs": ["mendixlabs/mxcli#1005", "mendixlabs/mxcli#1024"], "ce": ["CE0495", "CE6680"]} {"area": "mdl/executor", "date": "2026-09-23", "symptom": "`describe page` on a page with several galleries emits MDL that mxcli's own `check --references` rejects: `duplicate widget name 'template1' (used 4 times) — Mendix requires unique widget names per page (CE0495)` (and `filter1` when the galleries have filters). The row/col half of the same report was already fixed; this third name survived it", "cause": "`template`/`filter` inside a gallery are CHILD SLOTS of its definition (gallery.def.json childSlots). `applyChildSlots` builds only the block's children into the slot property and discards the block's name, so DESCRIBE has to synthesise `template1` for every gallery. `checkDuplicateWidgetNames` already exempted the parent's object-list containers but not its child-slot containers", "file": "`mdl/executor/validate_page_context.go` (`unstoredContainerKinds`, was `objectListContainerKinds`; `checkDuplicateWidgetNames`); test `mdl/executor/validate_page_slot_containers_test.go`", "insight": "**When a multi-symptom report is marked fixed, re-run every symptom in it, not the first** — the earlier fix covered `row1`/`col1`/`FullName` and its test never mentioned `template1`, so the issue stayed open with a third name nobody had reproduced. Cheapest proof a name is not stored: author a distinctive one (`template contentZebra`), `grep -rl` it in `mprcontents/` (0 files) and DESCRIBE it back (`template1`). Don't exempt the keyword `template` globally — outside a slot-declaring widget, `template` is `buildTemplateV3`, a `Forms$DivContainer` that DOES store its name; resolve it from the enclosing widget's def (both `ChildSlots` and every mode's), and cover the `container ` spelling that `applyChildSlots` routes by name. Measured on 11.12.2: four galleries each `template template1` → `mx check` 0 errors", "refs": ["mendixlabs/mxcli#978"], "ce": ["CE0495"]} +{"area":"mdl/executor","date":"2026-09-23","symptom":"`call javascript action NanoflowCommons.RefreshEntity(` / `EntityToRefresh = Mod.Entity` / `);` — argument on its own line — passes `mxcli check --references` and exec, then mxbuild fails. Reported as CE0115 on v0.23.0 (that was #1137, fixed after the release); with #1137 in, the same MDL fails **CE1613** \"The selected entity 'Mod.Entity\\n' no longer exists\" at the Call JavaScript action activity. Java-action entity-type parameters had the same hole","cause":"The visitor deliberately keeps each call argument's trailing whitespace (so an expression round-trips as written). The entity-type branch of both code-action builders took `exprToString(arg.Value)` — expression text — as a qualified name, storing `Entity: \"Mod.Entity\\n\"`; a `$var` argument likewise missed the varTypes lookup and stored `\"$var\\n \"`","file":"`mdl/executor/cmd_microflows_builder_calls.go` (`entityTypeArgument`, used by `addCallJavaActionAction` and `addCallJavaScriptActionAction`)","insight":"**A test that builds the AST by hand cannot see anything the visitor adds.** #1137's tests used `&ast.IdentifierExpr{Name: \"Mod.Entity\"}` and were right about the $Type while blind to the text; parsing the reporter's MDL verbatim (visitor.Build) reproduced it in one run. Whenever a value that is an expression elsewhere is stored as a *name* (entity, microflow, page), normalise it — expression text carries layout. **Diff the reporter's version against the fix before calling it a duplicate**: the finding for #1137 matched the symptom word for word, and the first run of the exact MDL on main 'looked fixed' by $Type; only dumping the BSON value (`mxcli bson dump … | grep -A1 Entity`) showed the newline. The CE code changing (CE0115 → CE1613) was the tell that two defects were stacked. Controls: one-line spelling passes before and after; two builds of testdata/expr-checker/minimal.mpr (copy widgets/ too, or CE0462 noise appears only on the clean copy) gave CE1613 faulty vs 0 errors fixed on mxbuild 11.6.6. Tests `mdl/executor/cmd_microflows_builder_entity_type_arg_test.go`; example `mdl-examples/bug-tests/javascript-action-1171-entity-argument-own-line.mdl`","refs":["mendixlabs/mxcli#1171","mendixlabs/mxcli#1137"],"ce":["CE0115","CE1613"]} {"area":"mdl/executor","date":"2026-09-23","symptom":"\"List views are written with `Editable: false` by mxcli, so every input inside a list view renders disabled — even with `editable: Always` on the text box, and even inside a nested data view.\" Valid BSON, `mxcli check` clean, build succeeds; only the rendered app shows it","cause":"Not a writer bug: false IS Mendix's default for Pages$ListView.editable. The 2026-09-06 fix made an explicit `editable: true` reach the document, but nothing told an author who never wrote it that the list view's read-only context overrides the inputs' own `editable:`","file":"`mdl/executor/validate_listview_editable_inputs.go` (MDL-WIDGET31, hooked in `validate_widgets.go`); test `validate_listview_editable_inputs_test.go`; example `mdl-examples/bug-tests/631-listview-inputs-not-editable.mdl`","insight":"**Measure the default before changing it — and the measurement is cheap.** The issue parked \"default to true\" on a Studio Pro comparison it could not run; `npm pack mendixmodelsdk` and reading the class in `src/gen/pages.js` settles it in a minute: `new PrimitiveProperty(ListView, this, \"editable\", false, …)` and `_initializeDefaultProperties` never sets it, so flipping the default would have made mxcli disagree with every Studio Pro-authored list view. A Studio Pro project confirms it (ako/TestApp, 11.14.0): 30 of 32 list views stored false, none holding an input, and the one with inputs set to true by its author; `describe` of that page checks clean, deleting `Editable: true` from it fires MDL-WIDGET31, and describe -> exec preserves it. The fix is therefore a check for the COMBINATION (not-editable list view + input not marked `editable: Never`). Read editability with the SAME accessor the builder uses (`GetBoolProp`): a quoted `editable: 'true'` parses as a string, the builder writes false, and a rule that read it leniently would stay quiet on a page that renders disabled. Stop the descent at a nested list view — its own Editable governs its inputs and it is visited on its own — but NOT at a nested data view, which the reporter measured does not escape the context","refs":["ako/mxcli#631"],"rules":["MDL-WIDGET31"]} {"area": "mdl/executor", "date": "2026-09-23", "symptom": "`captionparams` on an action button is accepted by `mxcli check` but not written; a container with `onclick` and a dynamic text does the job. Measured: `actionbutton b (caption: 'Save {1}', captionparams: [{1} = Title])` in a dataview wrote the LITERAL 'Title' (describe: `ContentParams: [{1} = 'Title']`), check --references and mx check both clean; and describe -> exec of any button with params dropped them all -> 4x CE0720 'Place holder index 1 is greater than 0' at mx check.", "cause": "buildButtonV3 carried its own copy of the template-parameter resolver (bare name without `.`/`$` -> quoted literal) instead of the shared buildClientTemplateParams that dynamictext/datagrid columns use; and DESCRIBE emitted a button's params as `ContentParams:`, a key the button builder never read. MDL-WIDGET04's orphan-placeholder check covered dynamictext only, so the round-tripped `{1}` with no parameter reached the build.", "file": "`mdl/executor/cmd_pages_builder_v3_widgets.go` (buildButtonV3 -> buildClientTemplateParams, ContentParams fallback), `mdl/executor/cmd_pages_describe_output.go` (ActionButton emits CaptionParams), `mdl/executor/validate_widgets.go` (validateButtonCaptionPlaceholders)", "insight": "**The report's workaround is the diagnosis**: 'a dynamic text does the job' means the SAME parameter text works on one widget and not another, so diff the two builders before touching grammar or the writer -- the writer was fine (clientTemplateToGen is shared). Two duplicate resolvers had drifted: the shared one gained SourceVariable, toString, association steps and format blocks over a year of fixes; the button copy got none. Do not trust 'check passed' for the literal case -- a literal holding the attribute's name is a valid model, so nothing short of describe (or looking at the rendered button) shows it. The describe->exec round trip is the cheap detector: run it once and the ContentParams/CaptionParams mismatch drops every param and mx check reports CE0720. Keep reading `ContentParams:` on buttons -- scripts dumped before this fix use it. Remaining gap, NOT this bug: `check --references` does not flag an unknown BARE attribute in contentparams/captionparams on either widget (`[{1} = Titel]` passes).", "refs": ["ako/mxcli#632"], "ce": ["CE0720"], "rules": ["MDL-WIDGET04"]} +{"area": "mdl/executor", "date": "2026-09-23", "symptom": "`CALL MICROFLOW M.F(…) IN QUEUE M.Q` where `F` returns **Boolean**: `mxcli check -p --references` says `Check passed!`, `mx check` says **CE7033** \"A microflow used for background execution must have a Microflow return type of 'Nothing'.\" (at Call microflow activity 'F'). Reported with the CE0142 after-startup sibling, which MDL073 had already closed.", "cause": "The --references pass resolved the call target and the queue name separately, and both resolve. Nothing compared the binding (`in queue`) against the signature of the flow it names. Added MDL088: a project-less pass (ValidateQueuedCallReturnType) for a target the script creates, and validateQueuedMicroflowTargets on the --references path for a stored target, which skips script-defined targets so the fault is not printed twice. Stored void microflows read back as ReturnType \"Void\", not \"\" — both must mean Nothing.", "file": "`mdl/executor/validate_queued_call_return.go` (queuedMicroflowCalls, checkQueuedMicroflowReturnsNothing, validateQueuedMicroflowTargets, ValidateQueuedCallReturnType), wired in `validate_program.go` and `validate.go` (validateFlowBodyReferences); examples `mdl-examples/bug-tests/1064-queued-microflow-must-return-nothing{,.fail}.mdl`", "insight": "Same class as MDL073 (\"the reference resolves\" ≠ \"the reference is usable\"): any binding that names a flow carries a constraint on that flow's signature, and a resolver checks only the name. When one such check lands, sweep for its siblings at other binding sites. The queued CALL JAVA ACTION twin (CE7038) is still unchecked and was deliberately left out of scope. Two things that cost time: (1) `mxcli exec` of a script that CREATEs a queue and then binds a call to it refuses with 'task queue not found' — validateFlowBodyReferences checks queues against the project only, not the script context — so the repro has to create the queue in a separate exec; (2) walk call statements by reflection, not by the flowRefCollector switch, which does not descend into WHILE bodies. Measured on mxbuild 11.12.0 with two projects: Boolean target → CE7033, void target → 0 errors.", "refs": ["mendixlabs/mxcli#1064"], "ce": ["CE7033"], "rules": ["MDL088"]} +{"area": "mdl/executor/microflow-layout", "date": "2026-09-23", "symptom": "MPR011 fires on EVERY `while` loop mxcli writes \u2014 'first activity at (50,80) lies outside the loop box' \u2014 single-level loops included. `mx check` passes and the app runs; the flow just renders wrong in Studio Pro. Reported from a real project as 'looks like an mxcli layout issue', with 3 MPR011 warnings still in its final lint run. mxcli's own lint rule was correctly flagging mxcli's own output.", "cause": "One missing term in the WHILE builder. addWhileStatement had `innerStartX := LoopPadding` (50) where addLoopStatement has `LoopPadding + iteratorSpace + ActivityWidth/2` (210). A microflow object's Position is its CENTRE \u2014 the builder says so itself ('Position is the CENTER point (RelativeMiddlePoint in Mendix)') \u2014 so a centre at x=50 with ActivityWidth=120 puts the left edge at -10. The doc comment says the while layout 'matches addLoopStatement but without iterator icon space': dropping the iterator space (100) was right, taking ActivityWidth/2 with it was not, because that term is not iterator space, it is what converts a centre to a left edge. The very next line, `innerStartY := LoopPadding + ActivityHeight/2`, adds the half-height for exactly this reason \u2014 so the omission was accidental, not a choice. Reported (50,80) matches term for term: 50 = LoopPadding, 80 = LoopPadding + ActivityHeight/2.", "file": "`mdl/executor/cmd_microflows_builder_control.go` (addWhileStatement: `innerStartX := LoopPadding + ActivityWidth/2`), tests `mdl/executor/loop_containment_test.go` (TestWhileLoopBox_ContainsDefaultLaidOutChildren, TestWhileLoopFirstChildLeftEdgeIsInsideTheBox)", "insight": "The containment invariant WAS already tested \u2014 loop_containment_test.go exists from #884 and asserts exactly this \u2014 but every fixture in it built a FOREACH loop. There are two loop builders; one was covered and the uncovered one shipped the violation into every project that writes a `while`. An invariant is worth what its COVERAGE is, and a file named for an invariant reads as if it covers the invariant, which is how a second code path goes unexamined for months. When a rule flags the tool's own output, believe the rule first: the reporter hedged with 'looks like an mxcli layout issue' and was exactly right. Cheap tell for this class: a term present on one axis and absent on the other in adjacent lines (`+ ActivityHeight/2` on Y, nothing on X) is almost always an omission rather than a decision. Failing test written first; it reproduced the reported geometry to the pixel, x[-10,...] at 1, 2, 4 and 7 activities. Still uncovered: addManualWhileTrueStatement, the third loop builder.", "refs": ["ako/mxcli#884", "ako/mxcli#645"]} +{"date": "2026-09-23", "area": "mdl-executor", "symptom": "upstream #1176: DESCRIBE prints `all` on an import activity that returns ONE object — `$objectResponse = import from mapping M.IMM($s) all;` — which reads as a list import. Reported on v0.23.0 / Studio Pro 11.12.3, after #881 was believed to have settled import ranges", "cause": "#881 made `formatImportMappingRange` always emit a range keyword, because at the time a missing keyword let the range fall back to the variable's cardinality and store First. The later runtime fix (unauthored range written as All explicitly) made bare and `all` build the same activity, but the describe side was never revisited, so `all` kept printing where it was only noise", "file": "`mdl/executor/cmd_microflows_format_action.go` (`formatImportMappingRange`: return \"\" for All against SingleObject); tests `mdl/executor/cmd_microflows_import_range_test.go` (`TestImportRange_ObjectResultDescribesWithoutAll`); example `mdl-examples/bug-tests/1176-import-mapping-object-describes-without-all.mdl`", "insight": "**This was not #881 regressing — it was #881's own workaround outliving its reason.** 'DESCRIBE must never emit nothing' was a guard against the builder's then-broken default; once the builder wrote a missing keyword as All explicitly, the guard became pure noise, and nothing linked the two sites. When a formatter emits something 'because the builder would otherwise infer X', put that reason in a test that asserts the builder equivalence (bare vs keyword build the same activity), so fixing the builder flags the formatter. Proving the omission safe needs that equivalence on a real project, not just the unit test: on 11.12.3 both spellings store byte-identical ResultHandling (ConstantRange{SingleObject:false} + ObjectType), `mx check` 0 errors, and exec'ing the described text reports 'Unchanged microflow'. Wrong turn to skip: a JSON diff of two EMPTY extractions prints 'IDENTICAL' — `bson dump` emits ordered Key/Value lists, not objects; check the extraction is non-empty before trusting a diff", "refs": ["#881", "#1176"]} diff --git a/.claude/skills/fix-issue/findings/mdl-other.jsonl b/.claude/skills/fix-issue/findings/mdl-other.jsonl index 2bc072fa2a..a271382b25 100644 --- a/.claude/skills/fix-issue/findings/mdl-other.jsonl +++ b/.claude/skills/fix-issue/findings/mdl-other.jsonl @@ -69,3 +69,4 @@ {"area":"mdl/catalog","date":"2026-09-18","symptom":"A Starlark lint rule written from the bundled write-lint-rules skill matches zero rows and reports a clean pass — or, with an allowlist, inverts into flagging everything (138 of 282 ACT_ microflows on one real project, 49% false positives)","cause":"The skill's example tables are the only documentation of the lint API and nothing tied them to the values the catalog emits. action_type listed Mendix BSON *storage* names (CreateChangeAction, CommitAction, ShowFormAction, CloseFormAction, ShowHomeFormAction) against a catalog that labels an action with its SDK name via getMicroflowActionType; source_type/target_type/element_type/access_type were lower-cased against an upper-case vocabulary; data_type was lower-cased against TitleCase from AttributeType.GetTypeName","file":"`.claude/skills/mendix/write-lint-rules/SKILL.md` (six rows), `mdl/catalog/builder_references.go` (RefObject* constants + RefSourceObjectTypes/RefTargetObjectTypes), `mdl/catalog/builder_permissions.go` (PermissionElement*/AccessType* constants), test `mdl/catalog/lint_rule_doc_vocabulary_test.go`","insight":"**Fix the documentation the rule author reads, not just the rule that was reported.** The identical defect was found and fixed in CONV010's allowlist a month earlier (finding 2026-08-17, pinned by lint_rule_vocabulary_test.go) — and recurred, because the *source* the author copied from was never corrected. A rule pinned to the labeller and a doc that is not is one fix, not two. **Check the sibling rows before believing the report's scope**: element_type, access_type and data_type had the same lower-casing and nobody had reported them; data_type ('string' vs 'String') is the most-used filter in a lint rule, so it was the most expensive one. **A doc value is only pinnable against a named vocabulary**, so the fix is half refactor: the emitters' scattered ALL-CAPS literals became RefObject*/PermissionElement*/AccessType* constants with published lists, mirroring the SourceObjectTypes precedent already in this package. action_type needs no list — the label IS the Go type name (%T), so the test reads the isMicroflowAction marker methods out of sdk/microflows/microflows_actions.go with go/ast. **Three legs, not one, when proving a filter fix**: old values -> 7/7 flagged, corrected -> 0, corrected-minus-one -> exactly that one. Leg C is the control; without it a silent rule and a correct rule both report zero, which is the bug itself","refs":["mendixlabs/mxcli#1027"],"rules":["CONV010"]} {"area":"mdl/executor","date":"2026-09-18","symptom":"A page parameter passed as an argument to a nanoflow/microflow BUTTON action is not wired — Studio Pro reports CE1571 \"No argument has been selected for parameter 'X' and no default is available\" on opening the page, while `mx check`, `mxcli check --references` and `mxcli lint` are all clean. Reported as an asymmetry: of two arguments, the one matching the enclosing dataview's DataSource 'works' and the other does not","cause":"Mendix stores a flow argument in one of TWO slots of Forms$MicroflowParameterMapping / Forms$NanoflowParameterMapping: a reference to a page parameter, snippet parameter or page variable goes in `Variable` as a Forms$PageVariable; a literal or expression goes in `Expression`. mxcli only ever wrote `Expression: \"$Name\"`, which binds nothing. The read side was wrong in the mirror image — the three action describers and flowSourceArgs looked for a `Name` key on that sub-document, which Forms$PageVariable does not have","file":"`sdk/pages/pages_widgets_action.go` (VariableKind on both mapping types), `mdl/executor/cmd_pages_flow_args.go` (new: classifyFlowArgValue + pageVariableArgValue), `mdl/executor/cmd_pages_builder_v3.go` (3 of the 4 copies of the $-rule), `mdl/backend/modelsdk/widget_write.go` (bindParameterMappingValue), `mdl/executor/cmd_pages_describe_output.go` + `cmd_pages_describe_datasource.go` (read)","insight":"**The reported asymmetry is a red herring — both arguments were written identically and NEITHER was bound.** Studio Pro supplies a default for the one that is the dataview's object and reports the other; 'and no default is available' in CE1571 says exactly that. Time spent on why $Dto worked is wasted. **mxbuild is not a detector here**: `mx check` on the reported project is 0 errors before AND after the fix, so the usual two-copies-of-a-real-project run proves nothing and the reporter is right that it only shows in Studio Pro. **Get the reference from a Marketplace .mpk — it contains a whole Studio Pro-authored `project.mpr`**: `mxcli marketplace download --output x.mpk && unzip -o x.mpk project.mpr`, then `mxcli bson dump` it. A blank app is useless for this (every mapping list in it is empty); Workflow Commons 4.11.0 gave 101 flow parameter mappings, of which 95 bind through Variable and 6 through Expression — and all 6 of those are Boolean literals, so the $-prefixed Expression mxcli wrote occurs ZERO times. `marketplace install` refuses that package (javasource path guard), so extract rather than install. **The PageVariable slot follows what the name refers to** (PageParameter 20, SnippetParameter 58, Widget 17) — a snippet is the COMMON case, not the corner, and `paramScope` is the right oracle because it holds only entity-typed parameters, which is the same set Mendix binds this way. **Leave $currentObject alone**: no reference for the bare form was measured and show_page already depends on the context object being inferred (MDL-PAGEARG01), so changing it on a guess risks the case that works. **The read bug hid the write bug**: describe printed `Action: microflow M.F` with no arguments for Studio Pro content, so a round-trip looked lossless and the missing binding never showed up as a diff","refs":["mendixlabs/mxcli#1140","mendixlabs/mxcli#835"],"ce":["CE1571"]} {"area": "mdl/versions", "date": "2026-09-21", "symptom": "A version gate copied from the issue text (\"Workflow Groups are GA from Mendix 11.6\") is wrong by four minors", "cause": "Mendix's release notes date the FEATURE's general availability; the metamodel floor is when the type and its property were introduced, and that is what decides whether the document loads. `Settings$WorkflowGroup` and `WorkflowsProjectSettingsPart.groups` are both `introduced: \"11.2.0\"`", "file": "`sdk/versions/mendix-11.yaml` (`workflows.groups`)", "insight": "The arbiter for a metamodel floor is the Model SDK's own StructureVersionInfo: `npm pack mendixmodelsdk && tar xzf \u2026 && grep -n '' package/src/gen/.js`, then read BOTH the class's `versionInfo.introduced` and its `properties..introduced` \u2014 a property can arrive later than its type. Release notes, proposal text and a number already written down in this repo are all downstream of it (same trap as mendixlabs/mxcli#1121). Corroborate it against two real projects rather than trusting one source: `mxcli new` at a version either side of the floor and diff the document's keys \u2014 an 11.1.0 workflows settings part has no `Groups` key at all, an 11.13.0 one carries `Groups: [2]`, which also proves the refusal is right rather than over-cautious (writing the property below the floor would be inventing a key). mendixlabs/mxcli#272", "refs": ["mendixlabs/mxcli#272"]} +{"area": "mdl/linter/rules", "date": "2026-09-23", "symptom": "CONV010 flagged an ACT_ nanoflow that delegated to a sub-flow \u2014 the very thing the rule demands. A real project patched its own copy of the rule and asked for the fix upstream. An ACT_ nanoflow could satisfy CONV010 in NO way: delegate and be flagged, or inline the logic and be flagged.", "cause": "ALLOWED_ACTIONS held `MicroflowCallAction` but not `NanoflowCallAction`. `microflows()` yields nanoflows too \u2014 the catalog's `microflows` table carries a MicroflowType column \u2014 so CONV010 lints ACT_ nanoflows, and a nanoflow delegates with a nanoflow call.", "file": "`.claude/lint-rules/conv010_act_microflow_content.star` (NanoflowCallAction added to ALLOWED_ACTIONS; cmd/mxcli/lint-rules/ is gitignored and regenerated by `make sync-lint-rules`), test `mdl/catalog/lint_rule_vocabulary_test.go` (added to the `permitted` list)", "insight": "Third time this one allowlist has been short, and the rule's own comments record the previous two: the wrong vocabulary entirely (storage names vs SDK names, matching nothing, 11 false positives of 13 findings) and a missing ExclusiveMerge that a permitted ExclusiveSplit necessarily creates (122 hits on one project). The recurring shape is an UNSATISFIABLE rule, and its cost is asymmetric: a rule that cannot be satisfied does not read as a broken rule, it reads as broken CODE, so users refactor around it or patch the rule locally and the defect never comes back upstream \u2014 which is exactly what happened here until someone wrote 'report upstream' in their findings. A vocabulary pin test (TestCONV010AllowsWhatTheCatalogCallsUIActions) already existed to stop this class and did not, because its `permitted` list is hand-maintained and was itself incomplete: pinning a rule to a hand-written list of what SHOULD be allowed only moves the completeness problem. Worth considering: enumerate the delegation actions from the type system rather than listing them.", "refs": ["ako/mxcli#644"]} diff --git a/.claude/skills/fix-issue/findings/modelsdk.jsonl b/.claude/skills/fix-issue/findings/modelsdk.jsonl index 61eef19a39..e58615d476 100644 --- a/.claude/skills/fix-issue/findings/modelsdk.jsonl +++ b/.claude/skills/fix-issue/findings/modelsdk.jsonl @@ -19,3 +19,4 @@ {"area": "modelsdk/meta", "date": "2026-09-22", "symptom": "A view entity selecting `u.Name` from System.User could not be declared in any way that both passed `mxcli check` and built: `String(100)` (the correct length) was refused, `String` (unlimited) passed check and then failed mxbuild with CE6770 \"View Entity is out of sync with the OQL Query\". `describe entity System.User` reported `Name: String(unlimited)`. Reported in ako/ChipCoV4 FINDINGS.md against Mendix 11.14.0 (ako/mxcli#584, with #585 the other half).", "cause": "meta.SystemAttrDef declared a Length field and NOT ONE of the 115 String attributes in modelsdk/meta/system_module.go populated it, so systemAttrType built every System string as StringAttributeType{Length: 0} — which mxcli reads as unlimited. Every length comparison against a System attribute was therefore made against 0. Fixed by measuring all 115 and populating them, with a golden table (modelsdk/meta/testdata/system_string_lengths.txt) and TestSystemStringLengths holding the two in step.", "file": "`modelsdk/meta/system_module.go` (SystemEntities, SystemAttrDef.Length); `modelsdk/meta/testdata/system_string_lengths.txt`; tests `modelsdk/meta/system_string_lengths_test.go`, `modelsdk/meta/system_string_lengths_measure_test.go`, `mdl/backend/modelsdk/system_module_read_test.go`", "insight": "The System module's attribute lengths are IN THE BUILD OUTPUT: `deployment/model/model.mdp` is a stream of BSON documents (each with its own 4-byte length prefix — unmarshalling the file whole fails with \"invalid document length\"), the System module arrives as a Projects$ModuleImpl carrying only a Name with its DomainModels$DomainModel immediately after, and every entity's attributes are there with their StringAttributeType.Length. That is ONE `mxbuild --target=deploy` for all 115, and it is the model the runtime builds the tables from, so it is the same number CE6770 is decided by. Two searches not worth repeating, both spent on this issue: the Mendix Model SDK does not carry them (its gen/ describes metamodel TYPES, so System.User.Name is not in it) and the modeler's own copy is inside Mendix.Modeler.Core.dll, i.e. a decompiler. The one-view-entity-per-attribute mxbuild probe works but is ~40s each. The version question answers itself the same way: building 10.24.4.77222 as well showed all 216 shared attributes identical in type AND length to 11.14.0, so one table serves every supported version instead of a per-version registry — measure the second version rather than reasoning about it, it is one more build. Finally, 0 is Mendix's own encoding of \"unlimited\" (46 of the 115), so populating the table does not make 0 safe to read as a length — what makes it safe is that the golden enumerates every String attribute, so 'unmeasured' cannot exist without failing a test."} {"area": "docs / modelsdk/mpr", "date": "2026-09-22", "symptom": "Four reference pages described an MPR v1 `UnitContents` table holding the BSON blobs, and a v1/v2 detection recipe that probes for it. No .mpr has ever had that table: a v1 file has exactly `Unit` and `_MetaData`, and contents are the `Unit.Contents` blob. `grep -rn UnitContents --include=*.go` is 0 hits. Reported by an outside reader building an independent format reader (mendixlabs/mxcli#1072).", "cause": "Never-measured prose. The pages also invented `UnitType` and `Name` columns on `Unit` (there are seven columns and neither is among them — type and name come out of the BSON `$Type`/`Name`), and drew `mprcontents/` flat when it is sharded `//.mxunit`. Fixed by rewriting the four pages from the SQLite catalogs of two real fixtures, and adding modelsdk/mpr/docs_schema_test.go to hold them there.", "file": "`docs-site/src/internals/mpr-format.md`, `docs-site/src/internals/mpr-v1-v2.md`, `docs-site/src/appendixes/version-compatibility.md`, `docs/05-mdl-specification/10-bson-mapping.md`; test `modelsdk/mpr/docs_schema_test.go`", "insight": "Prose cannot be type-checked but the IDENTIFIERS in it can, and the rule that makes it zero-maintenance is a prefix rule, not an allowlist: check only names BEGINNING with a real table name (`Unit`, `_MetaData`, `_Transaction`) against the union of the fixtures' tables and columns. `UnitContents` and `UnitType` are caught; the catalog tables these same pages mention (`REFS` and friends) never start with a real .mpr table name, so they need no exemption and no one has to maintain a list. The page set is discovered by content (any .md under docs-site/src or docs/05-mdl-specification mentioning `.mpr`/`mprcontents`), so a page added later is covered without anyone remembering. One consequence worth stating in the docs themselves: a page that wants to say a column does NOT exist must say it in PROSE — the first fix wrote \"there is no `UnitType` column\" and the test flagged its own remedy, which is correct, because the old pages' \"no `UnitContents`\" at mpr-v1-v2.md:35 read as a v1/v2 difference rather than as a fiction and an exemption for denials would have masked it. The control is cheap and exact here: `git stash` the doc edits with the test file kept, and the failures reproduce the reporter's line list verbatim (version-compatibility.md:31, mpr-format.md:21,23, mpr-v1-v2.md:12,35,69,73,74,84,94, 10-bson-mapping.md:30) plus the two they had not found. No Mendix tool runs in this fix's argument — the claims are about SQLite schema and are read straight off `sqlite_master`/`PRAGMA table_info`, which is the primary source, so the usual 'build two apps' rule does not apply.", "refs": ["mendixlabs/mxcli#1072"]} {"area": "docs / modelsdk/mpr", "date": "2026-09-22", "symptom": "The MPR reference pages' \"Unit Types\" tables mapped BSON `$Type` to document kinds, and 15 of the rows named a spelling no unit carries: `Pages$Page`/`Pages$Layout`/`Pages$Snippet`/`Pages$BuildingBlock` (real units say `Forms$*`), and docs/05-mdl-specification/10-bson-mapping.md lowercased eleven more (`microflows$microflow`, `pages$page`, `security$ProjectSecurity`…). It also listed `CustomWidgets$customwidget` as a document type. Found while fixing mendixlabs/mxcli#1072, filed and fixed separately.", "cause": "The tables were written from the TypeScript SDK's QUALIFIED names rather than the storage names Mendix writes — the same split CLAUDE.md documents for `ShowPageAction`/`ShowFormAction`, never applied here. `CustomWidgets$CustomWidget` is a widget element inside a page's tree (mdl/catalog/builder_widget_refs.go), never a unit, so that row was removed rather than corrected.", "file": "`docs-site/src/internals/mpr-format.md`, `docs/05-mdl-specification/10-bson-mapping.md`; test `modelsdk/mpr/docs_schema_test.go` (TestDocumentedUnitTypesUseStorageNames)", "insight": "Measuring the real set is one command and settles the whole table at once: decode every `mprcontents/*/*/*.mxunit` (and every v1 `Unit.Contents` blob) and count `$Type` — 28 distinct values across a blank 11.6.6 app and a 9.24.30 one. Do NOT try to verify rows one at a time against gen, which carries BOTH spellings: `model/types.go` defines `DocumentTypePage = \"Pages$Page\"` and mdl/catalog/builder_xpath.go defensively matches `Forms$Page` AND `Pages$Page`, so grepping the codebase 'confirms' the wrong name. The fixture is the arbiter; the codebase is not. The test rule that makes this checkable without a maintenance burden keys on the LOCAL name after the `$`, case-insensitively: a fixture cannot prove a type ABSENT (a blank project has no business-event service), so demanding every documented type be present would fail correct rows — but when the fixture has a type with the same local name, the documented row must equal it exactly. That catches all four `Pages$` rows and all eleven lowercase ones with zero false positives. Its stated limit is real and cost a manual fix: a row whose local name appears nowhere in the fixtures is not checked at all, which is how `CustomWidgets$customwidget` slipped past and had to be removed by hand. One editing trap, not a Mendix one: anchoring a section replacement on `'---'` matches a markdown TABLE SEPARATOR (`|---|---|`) long before the horizontal rule you meant — the edit silently no-ops on the table you were replacing. Anchor on `'\\n---\\n'`.", "refs": ["mendixlabs/mxcli#1072"]} +{"area": "modelsdk/canon", "date": "2026-09-23", "symptom": "The storage-GUID write guard (`canon.StorageGUIDChanges`) stopped refusing the MOVE ENTITY data loss it had exposed (ako/mxcli#503). With MoveEntity's carries removed, moving an association's TO side re-minted the in-place converted cross-association's GUID and the write went through silently, where the issue records a refusal.", "cause": "`sameMember` (added in 86927852 to stop the guard refusing transplant mis-pairings) required an equal `$Type` as well as an equal `Name`. MoveEntity converts `DomainModels$Association` to `DomainModels$CrossAssociation` IN PLACE, keeping `$ID` and `Name`, so the type clause made the guard skip the pair. The type clause excluded nothing the transplant can produce: `pairDoc` stops at a `$Type` mismatch (TestTransplantIgnoresMismatchedTypes).", "file": "`modelsdk/canon/storageguid.go` (`sameMember`, the note above `GUIDChange`)", "insight": "Before adding a clause to an identity test that sits on an approximate pairing, ask what error of THAT pairing the clause excludes. The transplant only mis-pairs same-type, different-name elements, so `$Type` excluded none of its errors. Its only effect was to exclude the one writer that keeps an `$ID` across a type change deliberately. Rule now: Name when both sides have one; `$Type` only when neither does; a pair with a name on one side only is not a match. How the gap was found: stub the three MoveEntity carries on main and run TestIssue503. The child-side case returned no error where the issue quotes a refusal. That mismatch between the recorded refusal and the observed silence was the tell. A guard's quiet is not evidence of a clean write, so a guard's comment must list every hole it leaves; this one listed only renames. Controls: (1) the new canon test fails on the old `sameMember` with `got 0 change(s)`; (2) with the MoveEntity carries stubbed the child-side move is refused again with the issue's exact message, and the parent-side move still goes through, because the moved element changes unit and pairs with nothing (a documented hole); (3) the 86927852 false positive does not return: `marketplace install --file mx-modules/BusinessEvents_3.12.0.mpk` into a copy of testdata/expr-checker, then `create or modify persistent entity BusinessEvents.PublishedBusinessEvent (EventId: long)` is accepted, while a build with an `$ID`-only rule refuses it (EventId paired with a removed attribute). The existing table case `DifferentType_NotAChange` pinned the wrong decision with the justification 'nothing authors this today', which was false the day it was written. Grep for the writers (`SetID(x.ID())` next to `New()`) before claiming nothing authors a shape.", "refs": ["ako/mxcli#503", "mendixlabs/mxcli#1119"]} diff --git a/.claude/skills/mendix/json-structures-and-mappings/SKILL.md b/.claude/skills/mendix/json-structures-and-mappings/SKILL.md index 48558f26e3..ca84f5b6c8 100644 --- a/.claude/skills/mendix/json-structures-and-mappings/SKILL.md +++ b/.claude/skills/mendix/json-structures-and-mappings/SKILL.md @@ -476,8 +476,11 @@ import from mapping Module.IMM_Pet($JsonContent); #### Range — how much of the result to bind Optional trailing clause, matching Studio Pro's **All / First / Custom** setting -on the activity. Omit it and mxcli infers from the mapping's own root shape, as -it always has. +on the activity. Omitting it means **All**; whether the variable is an object or +a list is inferred from the mapping's own root shape, as it always has. +`describe` leaves `all` off an object result (writing it there reads as "returns +a list") and prints it for a list result — the two spellings store the same +activity. ```sql $Pets = import from mapping Module.IMM_Pets($Json) all; -- All (the default) diff --git a/.claude/skills/mendix/run-local/SKILL.md b/.claude/skills/mendix/run-local/SKILL.md index c3f7129e84..847c09fb89 100644 --- a/.claude/skills/mendix/run-local/SKILL.md +++ b/.claude/skills/mendix/run-local/SKILL.md @@ -85,7 +85,9 @@ association catalog only at startup; behavioural changes are hot-reloaded. - `--mxbuild-path` overrides both. It is honoured by the local loop (#916) *and* accepted by `run --local` (#1125) — between those two fixes the skill said the first and the command rejected the flag, so the advertised workaround did not - exist on the platform that needed it. + exist on the platform that needed it. `test --local` takes the same flag + (#1086), and `MXCLI_MXBUILD_PATH` sets the override for both from the + environment; the flag wins when both are set. If nothing runnable is found, the command says so up front instead of failing with `fork/exec …: exec format error`: diff --git a/.claude/skills/mendix/scheduled-events-and-queues/SKILL.md b/.claude/skills/mendix/scheduled-events-and-queues/SKILL.md index fe0fb800ae..7cd34cca59 100644 --- a/.claude/skills/mendix/scheduled-events-and-queues/SKILL.md +++ b/.claude/skills/mendix/scheduled-events-and-queues/SKILL.md @@ -171,7 +171,21 @@ end; `describe microflow` renders the clause back, so the binding round-trips. -### Two traps, both verified on mxbuild 11.13.0 +### Three traps, verified on mxbuild 11.13.0 (the microflow one on 11.12.0) + +**A queued microflow must return nothing.** A `call microflow … in queue …` whose +target declares `returns …` fails the build with **CE7033** *"A microflow used for +background execution must have a Microflow return type of 'Nothing'."*, reported at +the call activity. `mxcli check` reports it as **MDL088** — without a project when +the script creates the microflow, and under `--references` for one already stored. +Drop the `returns` clause (and the `return` value) from the worker microflow: + +```sql +create microflow Ops.ACT_Work ($Note: String) -- no `returns` +begin + log info node 'Ops' 'working'; +end; +``` **A queued Java action must return Nothing.** Anything else fails the build with **CE7038** *"A Java action used for background execution must have a return type diff --git a/.claude/skills/mendix/write-microflows/SKILL.md b/.claude/skills/mendix/write-microflows/SKILL.md index a7252620fb..28be44b05f 100644 --- a/.claude/skills/mendix/write-microflows/SKILL.md +++ b/.claude/skills/mendix/write-microflows/SKILL.md @@ -462,8 +462,9 @@ call java action Module.RefreshData(Url = $Url) in queue Module.RefreshQueue; ``` **Queued calls** — the queue must already exist (`create queue Module.RefreshQueue -(Parallelism: 2)`), and a queued **Java action must `returns void`** or the build -fails with CE7038. Rewriting a microflow that has a queued call must restate the +(Parallelism: 2)`), and the called flow must return nothing: a queued +**microflow with a `returns` clause** fails the build with CE7033 (`mxcli check`: +MDL088), a queued **Java action must `returns void`** or it fails with CE7038. Rewriting a microflow that has a queued call must restate the `in queue` clause; a rewrite that omits it is refused rather than silently dropping the binding. See `.claude/skills/mendix/scheduled-events-and-queues`. diff --git a/CHANGELOG.md b/CHANGELOG.md index 06ab4bc998..38127e9acb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). ### Fixed - **An input inside a list view rendered disabled with every gate green** (ako/mxcli#631) — a `listview` that does not say `editable: true` is written `Editable: false`, and the list view's read-only context wins over `editable: Always` on the input, including inside a nested data view. The BSON is valid, `mx check` is clean and the build succeeds; only the running app shows it. The default is not changed: false is Mendix's own (mendixmodelsdk 4.115.0, `Pages$ListView.editable` defaults to false). `mxcli check` now warns **MDL-WIDGET31** on a list view that will be written read-only while it holds an input not marked `editable: Never`, and names `editable: true` as the fix. A quoted `editable: 'true'` is a string and is written false, so it warns too. +- **A queued call to a microflow that returns a value passed `mxcli check --references` and failed the build with CE7033** (mendixlabs/mxcli#1064) — `call microflow M.F() in queue M.Q` where `F` returns Boolean said "Check passed!", and `mx check` then reported `[CE7033] "A microflow used for background execution must have a Microflow return type of 'Nothing'."`. Both the call and the queue resolve; the constraint is on the flow the call names. **MDL088** now reports it — with no project when the script creates the microflow itself, and under `--references` for one already stored. Measured on mxbuild 11.12.0: a Boolean target → CE7033, the same call on a void target → 0 errors. - **Two bundled lint rules reported nothing, on every project, since they were written** — `ARCH002` (entities without a data-change microflow) and `ARCH003` (entities without a business key) both skip an entity with `if entity.entity_type != "PERSISTENT"`. The catalog stores `PERSISTENT`, but `LintContext.Entities` normalizes the kind to `Persistent` before a Starlark rule reads it, so the guard held for every entity and the body never ran. There was nothing to notice: no error, no output, and `--list-rules` still listed both. Measured on a generated app of five persistent entities, `ARCH003` went from 0 findings to 5 once the spelling matched. The `write-lint-rules` skill had documented a third spelling, `"persistent"`, in its field table and in its worked example; both now say what the API returns, and a test loads the two shipped rule files and fails if either stops seeing a persistent entity. - **A Barcode Scanner authored by mxcli failed the build with CE0463** (mendixlabs/mxcli#1161) — `barcodescanner bsCode (datasource: Module.Entity.Code)` was accepted by `mxcli check`, accepted by `mx check`, and then rejected by headless `mxbuild` with `[CE0463] The definition of this widget has changed`, once per instance. The app would not deploy, and the only documented repair was Studio Pro's right-click → "Update widget". diff --git a/CLAUDE.md b/CLAUDE.md index 9f9d1ea499..7384391049 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -147,6 +147,11 @@ same attributes — makes the runtime treat it as a different entity and **destr its rows**. An unchanged reboot is the control, and preserves them. See [PROPOSAL_marketplace_module_upgrade.md §8](docs/11-proposals/PROPOSAL_marketplace_module_upgrade.md). +Re-measured per **attribute** (#1119): the columns are dropped and recreated, and a +recreated column with a model default **comes back filled with that default** — so +`count(col)` proves nothing; seed a non-default value and compare values. Method: +[rewrite-drops-unauthored-state](docs-wiki/bug-patterns/rewrite-drops-unauthored-state.md). + Consequences for any write path: 1. **Preserve the stored `GUID` when rewriting an existing element.** A codec that @@ -154,14 +159,43 @@ Consequences for any write path: on the next deploy — a failure that no `mx check` and no build will catch, because the model is perfectly valid. This is the same class as the identity properties in `canon.identityFields` and belongs in that decision. + + **The write path refuses it** — `canon.StorageGUIDError`, pairing on `$ID` + + `Name` (`$Type` only if nameless — comparing it hid #503's in-place re-type): the + transplant's pairing is structural, so a shared `$ID` alone is NOT one member, and + reading it as one made the guard refuse correct writes. It + refuses rather than repairs; the carry belongs with the write, which knows which + element is which (`carryChildIdentity`) — keyed on name as well as `$ID`, since + `CREATE OR MODIFY` declares members with no ID. + The one deliberate GUID transplant, the marketplace module update, opts out by + name via `UpdateRawUnitOwningStorageGUIDs` — passing that because "the guard was + in the way" is how #1119 ships again. + + The carry is fixed per rebuild **shape**, not per element type: swapping one element + into a list leaves its siblings passing through as stored bytes (#657, #1119), while + emptying the list and rebuilding all of it has no safe siblings (#1169). The reported + statement is rarely the blast radius — enumerate the converter's **call sites**. 2. **`$ID` renumbering is irrelevant to data safety** — the inverse of the natural - assumption. Studio Pro renumbers every `$ID` in a module on update (94 of 94) - and preserves every `GUID` (9 of 9), which is exactly why its update does not - lose data. `$ID` matters for *intra-unit pointer consistency* (see below); - `GUID` matters for the database. + assumption. Studio Pro renumbers every `$ID` in a module on update and preserves + every `GUID`, which is exactly why its update does not lose data. `$ID` matters + for *intra-unit pointer consistency* (see below); `GUID` for the database. 3. **A new element must get a fresh `GUID`**, and an element copied from another model must not keep the source's — two elements sharing a `GUID` are one entity as far as the runtime is concerned. +4. **Moving an element between modules is the most expensive case, not a lesser one.** + Measured, same 250-row start both ways: `GUID` preserved → the runtime **renames** + the table, all 250 rows survive; re-minted → the table is dropped and an empty one + created. The runtime resolves the entity by `GUID`, not by table name, so a move + loses a whole **table** where an ALTER loses a column (#503) — and `RENAME ENTITY` + is the same case, the name being the table name (#1169). A `$Type` change on the + way needs a **raw transform**: not `SetRaw` (it passes the stored `$Type` through), + not gen's `SetDataStorageGuid` (wrong key, `string` where the property is binary). + +**Before trusting any GUID test, check the subject.** An element **mxcli created** +has `GUID == $ID` from birth, so re-minting `GUID = $ID` reproduces the same value +and the defect is undetectable — only a **Studio Pro-authored** element can fail. +This has already voided a live-database control and an MDL repro script. Suspect it +first whenever a GUID test passes. ### The Tunnel Is Linux-Only, On Purpose — Do Not "Restore" It diff --git a/cmd/mxcli/cmd_run_mxbuildpath_test.go b/cmd/mxcli/cmd_run_mxbuildpath_test.go index d108d728e3..5246208e52 100644 --- a/cmd/mxcli/cmd_run_mxbuildpath_test.go +++ b/cmd/mxcli/cmd_run_mxbuildpath_test.go @@ -7,6 +7,8 @@ import ( "path/filepath" "strings" "testing" + + "github.com/spf13/cobra" ) // `--mxbuild-path` was documented as the override for the local loop by the @@ -47,7 +49,10 @@ func TestRunMxBuildPathFlagIsAccepted(t *testing.T) { // TestErrorGuidanceNamesAFlagThatExists is the general form of the defect: an // error telling the user to pass an option the command does not have is worse // than no guidance, because it reads as the user's mistake. Every --flag any -// mxbuild-resolution message recommends must be registered on `run`. +// mxbuild-resolution message recommends must be registered on every command that +// reaches that resolution — `run --local` and `test --local` both go through +// ResolveMxBuildForLocal, so both print the same guidance. Checking only `run` +// is how `test` shipped without the flag (issue #1086). func TestErrorGuidanceNamesAFlagThatExists(t *testing.T) { for _, src := range []string{ filepath.Join("docker", "mxbuild_platform.go"), @@ -61,8 +66,10 @@ func TestErrorGuidanceNamesAFlagThatExists(t *testing.T) { if !strings.Contains(string(b), "--mxbuild-path") { continue } - if runCmd.Flags().Lookup("mxbuild-path") == nil { - t.Errorf("%s tells users to pass --mxbuild-path, which `run` does not accept", src) + for _, c := range []*cobra.Command{runCmd, testRunCmd} { + if c.Flags().Lookup("mxbuild-path") == nil { + t.Errorf("%s tells users to pass --mxbuild-path, which `%s` does not accept", src, c.Name()) + } } } } diff --git a/cmd/mxcli/cmd_test_mxbuildpath_test.go b/cmd/mxcli/cmd_test_mxbuildpath_test.go new file mode 100644 index 0000000000..3c7e8df99c --- /dev/null +++ b/cmd/mxcli/cmd_test_mxbuildpath_test.go @@ -0,0 +1,29 @@ +// SPDX-License-Identifier: Apache-2.0 + +package main + +import "testing" + +// On Windows `mxcli test --local` ran the Linux mxbuild from ~/.mxcli/mxbuild and +// died with "local runtime: starting mxbuild serve: mxbuild --serve did not +// become ready", and the reporter found "no flag, environment variable, or +// mechanism to redirect mxcli to the Windows mxbuild.exe already present in the +// Studio Pro installation". `run` gained --mxbuild-path in #1125; `test` boots +// the same app through the same resolver and was left without it. (issue #1086) + +// TestTestMxBuildPathFlagIsAccepted is the reporter's command line plus the +// override, resolved through the real command tree so `-p` (persistent, on +// root) is in scope. +func TestTestMxBuildPathFlagIsAccepted(t *testing.T) { + const override = `C:\Program Files\Mendix\11.11.0\modeler\mxbuild.exe` + cmd, args, err := rootCmd.Find([]string{"test", "tests/", "-p", "MyApp.mpr", "--local", "--mxbuild-path", override}) + if err != nil { + t.Fatalf("finding test: %v", err) + } + if err := cmd.ParseFlags(args); err != nil { + t.Fatalf("mxcli test tests/ -p MyApp.mpr --local --mxbuild-path …: %v", err) + } + if got, _ := cmd.Flags().GetString("mxbuild-path"); got != override { + t.Errorf("flag parsed to %q, want %q", got, override) + } +} diff --git a/cmd/mxcli/cmd_test_run.go b/cmd/mxcli/cmd_test_run.go index de05418f3b..9794067769 100644 --- a/cmd/mxcli/cmd_test_run.go +++ b/cmd/mxcli/cmd_test_run.go @@ -140,6 +140,7 @@ Examples: skipAppStartup, _ := cmd.Flags().GetBool("skip-app-startup") configuration, _ := cmd.Flags().GetString("configuration") constantArgs, _ := cmd.Flags().GetStringArray("constant") + mxbuildPath, _ := cmd.Flags().GetString("mxbuild-path") verbose, _ := cmd.Flags().GetBool("verbose") color, _ := cmd.Flags().GetBool("color") timeoutStr, _ := cmd.Flags().GetString("timeout") @@ -176,6 +177,7 @@ Examples: Watch: watch, Attach: attach, SkipAppStartup: skipAppStartup, + MxBuildPath: mxbuildPath, Timeout: timeout, JUnitOutput: junitOutput, RequireAssertions: requireAssertions, diff --git a/cmd/mxcli/docker/mxbuild_platform.go b/cmd/mxcli/docker/mxbuild_platform.go index 80bc147680..8244f5907b 100644 --- a/cmd/mxcli/docker/mxbuild_platform.go +++ b/cmd/mxcli/docker/mxbuild_platform.go @@ -20,12 +20,17 @@ import ( // did neither: it called DownloadMxBuild directly, so on macOS and Windows it // executed whatever Linux binary the cache held. (issue #916) +// MxBuildPathEnv names the environment variable that overrides mxbuild +// resolution for the local loop, as --mxbuild-path does. The flag wins when both +// are set. (issue #1086) +const MxBuildPathEnv = "MXCLI_MXBUILD_PATH" + // ResolveMxBuildForLocal picks the mxbuild that `run --local` / `test --local` // can actually execute on this host, and reports why when none can. // // Order, and why: // -// 1. An explicit --mxbuild-path wins. It was documented as an override and was +// 1. An explicit --mxbuild-path wins, then MXCLI_MXBUILD_PATH. It was documented as an override and was // silently ignored by the local path, so a user hitting the platform // mismatch had no way out. // 2. On a non-Linux host, Studio Pro BEFORE the cache. The cache may legitimately @@ -44,6 +49,9 @@ func ResolveMxBuildForLocal(explicitPath, version string, w io.Writer) (string, // so the macOS and Windows branches are testable from any host — the platform // mismatch this fixes cannot otherwise be exercised in CI. func resolveMxBuildForLocalOn(goos, explicitPath, version string, w io.Writer) (string, error) { + if explicitPath == "" { + explicitPath = os.Getenv(MxBuildPathEnv) + } if explicitPath != "" { resolved, err := resolveMxBuild(explicitPath, version) if err != nil { diff --git a/cmd/mxcli/docker/mxbuild_platform_test.go b/cmd/mxcli/docker/mxbuild_platform_test.go index e0bc6dc18d..4dd83d0c32 100644 --- a/cmd/mxcli/docker/mxbuild_platform_test.go +++ b/cmd/mxcli/docker/mxbuild_platform_test.go @@ -142,3 +142,37 @@ func TestResolveMxBuildForLocal_NonLinuxWithoutStudioProRefuses(t *testing.T) { t.Errorf("nothing should have been downloaded, but progress was written:\n%s", out.String()) } } + +// TestResolveMxBuildForLocal_EnvOverride — MXCLI_MXBUILD_PATH is the override +// for callers that cannot add a flag (an IDE task, a CI step, a wrapper script). +// Issue #1086 asked for "--mxbuild-path flag / MXCLI_MXBUILD_PATH environment +// variable"; without it the Linux-only CDN was the only road left on Windows. +func TestResolveMxBuildForLocal_EnvOverride(t *testing.T) { + // The reporter's host: Windows, and a version no Studio Pro here matches, so + // without the override resolution has nothing it may use. + fromEnv := writeBinary(t, t.TempDir(), "mxbuild.exe", peMagic) + t.Setenv(MxBuildPathEnv, fromEnv) + + got, err := resolveMxBuildForLocalOn("windows", "", "99.99.99", &bytes.Buffer{}) + if err != nil { + t.Fatalf("env override rejected: %v", err) + } + if got != fromEnv { + t.Errorf("resolved %q, want %s=%q", got, MxBuildPathEnv, fromEnv) + } +} + +// TestResolveMxBuildForLocal_FlagBeatsEnv — the flag is the more specific +// statement of intent, so it wins over a variable set in the shell's profile. +func TestResolveMxBuildForLocal_FlagBeatsEnv(t *testing.T) { + t.Setenv(MxBuildPathEnv, writeBinary(t, t.TempDir(), "mxbuild", elfMagic)) + flag := writeBinary(t, t.TempDir(), "mxbuild", elfMagic) + + got, err := resolveMxBuildForLocalOn("linux", flag, "11.12.0", &bytes.Buffer{}) + if err != nil { + t.Fatal(err) + } + if got != flag { + t.Errorf("resolved %q, want the flag's %q", got, flag) + } +} diff --git a/cmd/mxcli/main.go b/cmd/mxcli/main.go index e4db312399..13bb1c81ec 100644 --- a/cmd/mxcli/main.go +++ b/cmd/mxcli/main.go @@ -399,6 +399,7 @@ func init() { testRunCmd.Flags().Bool("attach", false, "Run against an app already started with 'mxcli run --local --test-endpoint' instead of booting one (tests hit that app's database)") testRunCmd.Flags().String("configuration", "", "With --local, which project configuration's constant values to run the tests with (default: the only one, or \"Default\") — the same resolution 'mxcli run --local' uses, so a suite sees the same constants either way") testRunCmd.Flags().StringArray("constant", nil, "With --local, set a constant for THIS RUN only: Module.Name=value (repeatable). Never written to the project. The value is visible in shell history and in `ps` — for a value that must not be, see docs/11-proposals/PROPOSAL_constant_values.md") + testRunCmd.Flags().String("mxbuild-path", "", "With --local, the mxbuild to build with, overriding resolution (Studio Pro's bundled mxbuild on macOS/Windows, the cached CDN download on Linux); also settable as MXCLI_MXBUILD_PATH") testRunCmd.Flags().BoolP("verbose", "v", false, "Show all runtime log output") testRunCmd.Flags().BoolP("color", "", false, "Use colored output") testRunCmd.Flags().StringP("timeout", "t", "5m", "Timeout for runtime startup and test execution") diff --git a/cmd/mxcli/marketplace/identity.go b/cmd/mxcli/marketplace/identity.go index 5685d21797..fc854c8216 100644 --- a/cmd/mxcli/marketplace/identity.go +++ b/cmd/mxcli/marketplace/identity.go @@ -255,7 +255,13 @@ func ApplyIdentities(mprPath, moduleName string, ids Identities) (applied int, m } defer writer.Disconnect() for _, w := range writes { - if err := writer.UpdateRawUnit(w.id, w.contents); err != nil { + // OwningStorageGUIDs, not the plain UpdateRawUnit: moving a GUID onto an + // element that kept its $ID is the pattern the write path refuses by + // default, because everywhere else it means an element's database identity + // was dropped and the next deploy will drop its column (#1119). Here it is + // the entire point of the write — these GUIDs were captured from the stored + // model and are being put back. + if err := writer.UpdateRawUnitOwningStorageGUIDs(w.id, w.contents); err != nil { return 0, nil, fmt.Errorf("write identities into unit %s: %w", w.id, err) } } diff --git a/cmd/mxcli/testrunner/localapp_options.go b/cmd/mxcli/testrunner/localapp_options.go index 122a758170..9e336a8de7 100644 --- a/cmd/mxcli/testrunner/localapp_options.go +++ b/cmd/mxcli/testrunner/localapp_options.go @@ -41,6 +41,7 @@ func localAppOptions(opts RunOptions, logPath string, env []string, w io.Writer) SkipBuild: opts.SkipBuild, Env: env, ConstantOverrides: opts.ConstantOverrides, + MxBuildPath: opts.MxBuildPath, RuntimeLogPath: logPath, Stdout: w, Stderr: w, diff --git a/cmd/mxcli/testrunner/localapp_options_test.go b/cmd/mxcli/testrunner/localapp_options_test.go index fea8117ac1..612e720356 100644 --- a/cmd/mxcli/testrunner/localapp_options_test.go +++ b/cmd/mxcli/testrunner/localapp_options_test.go @@ -63,3 +63,20 @@ func TestLocalAppOptions_UsesAScratchDatabase(t *testing.T) { t.Error("EnsureDB is false; the scratch database would have to exist already") } } + +// TestLocalAppOptions_CarriesTheMxBuildPath — `test --local --mxbuild-path` must +// reach StartLocalApp, whose resolver honours it; a flag the boot never sees is +// the "no mechanism to redirect mxcli" of issue #1086 with extra steps. +func TestLocalAppOptions_CarriesTheMxBuildPath(t *testing.T) { + const override = `C:\Program Files\Mendix\11.11.0\modeler\mxbuild.exe` + opts := RunOptions{ProjectPath: "/tmp/app/App.mpr", MxBuildPath: override} + + for name, got := range map[string]string{ + "endpoint": localAppOptions(opts, "log", []string{endpointTokenEnv + "=tok"}, io.Discard).MxBuildPath, + "legacy": localAppOptions(opts, "log", nil, io.Discard).MxBuildPath, + } { + if got != override { + t.Errorf("%s runner: MxBuildPath = %q, want %q", name, got, override) + } + } +} diff --git a/cmd/mxcli/testrunner/runner.go b/cmd/mxcli/testrunner/runner.go index 939ca86a62..94cd2cf6fe 100644 --- a/cmd/mxcli/testrunner/runner.go +++ b/cmd/mxcli/testrunner/runner.go @@ -79,6 +79,11 @@ type RunOptions struct { // and inherits ITS constants, and the Docker path configures the container. ConstantOverrides map[string]string + // MxBuildPath overrides mxbuild resolution for a --local run, as + // `run --local --mxbuild-path` does. Empty means resolve: Studio Pro's + // bundled mxbuild on macOS/Windows, the cached CDN download on Linux. + MxBuildPath string + // Timeout for runtime startup and test execution. Timeout time.Duration diff --git a/docs-site/src/tools/run-local.md b/docs-site/src/tools/run-local.md index 6ed2c883c4..348a028e4c 100644 --- a/docs-site/src/tools/run-local.md +++ b/docs-site/src/tools/run-local.md @@ -77,7 +77,7 @@ so structural changes need a restart; behavioural changes do not. | `--app-port` | 8080 | App HTTP port | | `--admin-port` | 8090 | M2EE admin API port | | `--serve-port` | 6543 | `mxbuild --serve` port | -| `--mxbuild-path` | resolved for this host | The mxbuild to build with, overriding resolution (Studio Pro on macOS/Windows, the cached CDN download on Linux) | +| `--mxbuild-path` | resolved for this host | The mxbuild to build with, overriding resolution (Studio Pro on macOS/Windows, the cached CDN download on Linux). `MXCLI_MXBUILD_PATH` sets the same override from the environment; the flag wins | | `--db-host` | 127.0.0.1:5432 | Database `host:port`; bracket IPv6 endpoints (`[::1]:5432`) | | `--db-name` | derived from project | Database name | | `--db-user` / `--db-password` | mendix / mendix | Database credentials | diff --git a/docs-site/src/tools/running-tests.md b/docs-site/src/tools/running-tests.md index e0cec390d1..b7d034c1cb 100644 --- a/docs-site/src/tools/running-tests.md +++ b/docs-site/src/tools/running-tests.md @@ -90,6 +90,17 @@ mxcli run --local --test-endpoint -p app.mpr # terminal 1 mxcli test tests/ -p app.mpr --attach # terminal 2 ``` +`--local` builds with the same mxbuild `run --local` resolves: Studio Pro's +bundled one on macOS and Windows, the cached CDN download on Linux (the CDN +publishes Linux binaries only). To point it somewhere else — a Studio Pro +installed outside `Program Files`, say — pass `--mxbuild-path`, or set +`MXCLI_MXBUILD_PATH` for a wrapper or CI step that cannot add a flag. The flag +wins when both are set. + +```bash +mxcli test tests/ -p app.mpr --local --mxbuild-path "C:\Program Files\Mendix\11.11.0\modeler\mxbuild.exe" +``` + `--watch` is the everyday loop: edit a test *or* the microflow under test, and the verdict lands in about two seconds. diff --git a/docs-wiki/bug-patterns/rewrite-drops-unauthored-state.md b/docs-wiki/bug-patterns/rewrite-drops-unauthored-state.md index 1e66788bbf..b857fbc417 100644 --- a/docs-wiki/bug-patterns/rewrite-drops-unauthored-state.md +++ b/docs-wiki/bug-patterns/rewrite-drops-unauthored-state.md @@ -85,6 +85,122 @@ survived, values gone. Nothing in the model was wrong. This is the same concern as `GUID` preservation and the reason `canon.Reconcile` exists; a codec that mints fresh identities on rebuild is a data-loss bug wearing a clean `mx check`. +**Fixing the rewritten element does not fix its children.** The carry that saves +an ALTER target's own identity reaches the element and its untouched siblings — +siblings pass through as stored bytes, so they were never at risk — and stops +there. Everything the rebuild constructs *inside* the target arrives fresh, and +the identity default fires on each one. So the same defect returns one level +down, and the second time it is the attributes, which is where the database's +identity actually lives. + +**The carry has to be fixed per rebuild SHAPE, not per element type.** There are +two, and fixing one says nothing about the other. A rebuild that swaps one element +into an otherwise untouched list leaves every sibling passing through as stored +bytes, which is why sibling elements kept reading as safe and why the first two +rounds of this only ever touched the target. A rebuild that empties the list and +reconstructs all of it has no passthrough siblings at all, so a statement naming one +association re-minted the identity of every entity, attribute, index and association +in the module — hundreds of elements for a one-word edit. The second shape is the +more dangerous by a wide margin and looked like the same bug already fixed. + +**The statement in the report is rarely the whole blast radius; the call site is.** +The reported symptom named one command. What actually shared the defective rebuild +were six, and the costliest of them was a `RENAME`, which nobody connects to +data loss — an entity's name is its table name, so a re-minted identity makes the +platform drop the table and create an empty one instead of renaming it, losing a +whole table where the reported command lost a column. Enumerating every caller of +the converter closes a class in one pass and finds the sites no reproduction can +reach, including ones exposed only through a library API and not through the +command language at all. Reproducing the reported statement and stopping there +finds one of six. + +**A guard built on an approximate pairing promotes that pairing's error rate into +refusals.** The identity transplant matches elements structurally and its correctness +bar is deliberately low, because a wrong match only makes a diff bigger. A guard that +reads the same pairing as "these are the same element" inherits every wrong match as a +blocked write: a statement that dropped six differently-named members and added one had +the new member paired with a removed one, and the guard refused a write that corrupted +nothing. Anything consuming an approximate correspondence has to add its own test of +identity — here the member's name — and the cost of that test is a narrower guard, +which is the right trade: a backstop that refuses correct work makes documented +operations unusable, while a backstop with a hole still catches everything it did +before. + +**Test only for what the approximate pairing can actually get wrong.** The first +version of that identity test also compared `$Type`. The transplant never pairs +across a `$Type`, so that half caught nothing the pairing could produce. What it did +catch was the one writer that keeps an `$ID` through a type change on purpose: the +move that converts an association to a cross-association in place. That was the +guard's only view of that data loss, and the same arm had exposed it in the first +place. A restriction that excludes no error of the input only removes coverage, and +it does so silently, because a quiet guard reads as a clean write. Ask what each +clause of the test excludes, measured against the pairing's real failure modes, and +list every hole the test leaves. + +**The same error message can carry two defects, and fixing one leaves it byte-identical.** +A refusal naming one element persisted unchanged after a real fix to the carry, same +element and same values, which reads as "the fix did nothing" and is actually "there are +two". The tell is that an identical failure after a genuine change means the +reproduction exercises a path the diagnosis did not. What settled it was describing the +real stored document rather than the statement: the member the statement declared shared +no name with anything stored, so there was nothing to carry and the pairing itself was +spurious. + +**A fast local reproduction and a slow realistic one find different defects.** The unit +reproduction of this ran in half a second against a fixture with the identities stripped +the way the statement strips them; the integration reproduction took half a minute. Only +the slow one could expose the second defect, because a spurious pairing needs a real +document where several members are dropped and one is added. Build the fast one to +iterate and keep running the slow one to decide. + +**A sweep that reports work it then discards looks exactly like a sweep that never +ran.** A rename printed "Updated 3 reference(s) in 1 document(s)" and the build then +raised exactly three errors naming exactly those three references. That coincidence +is the diagnosis: the scanner found them and a later write to the same unit put the +old values back — here the same read-modify-write persisting a model captured +*before* the sweep. When a fix-up and a rewrite both touch one unit, the rewrite +wins, so look for the second writer rather than for a gap in the first. + +**A green build can cover one arm of a fix and not another, depending on who +authored the document.** The same stale qualified name raised an error on a +platform-authored element and none on one the tool had written itself, because the +tool's version still carried a valid element pointer beside the name and the +platform resolves the pointer. So a synthetic reproduction proved half the fix and +silently skipped the rest; the other half only shows against a document the real +modeler wrote. This is the same subject-dependence as the identity case, where an +element the tool created has its two identities equal from birth and cannot detect +a re-mint at all. + +**An identity derived from another identity is invisible to every same-vs-same +check.** A fresh random value makes a document differ from itself, which elision +notices and a churn test catches. A value computed from a property that is itself +held stable does not: the first write corrupts, and the second produces +byte-identical bytes, so the write is elided and the run reports *Unchanged*. The +damage is a one-shot, and every diagnostic after it — re-running the script +included — agrees that nothing is wrong. The question that separates the two is +not "does this write change anything?" but "does it change the same thing twice?", +and only the first write answers it. Compare against a copy taken before any +write, never against the previous run. + +**Restored data can be the default, not the data.** Where the platform recreates +a store rather than altering it, a column, field or setting that carries a default +comes back filled with that default. Counting non-empty values therefore reports +the loss as no loss, and the reconstructed value is plausible enough that nobody +looks again — a boolean with `default true` read back as fully populated on every +row, and only a run seeded entirely with the non-default value showed it had been +overwritten. A destructive-rewrite test has to compare values that could not have +been guessed, not presence. + +**A carry keyed on structure is not the same tool as a carry keyed on identity.** +The pairing behind `$ID` transplantation is deliberately tolerant, because a +wrong match there only makes a diff larger. Reusing it to carry a database +identity converts that tolerance into data loss of the opposite kind: instead of +a dropped column, a new member silently adopts a removed one's data under a name +and type that no longer describe it. Carry a database identity only on the +identity the caller itself tracked through the statement; where no such identity +is in hand, refuse the write rather than guessing, and keep the repair in the +layer that knows which element is which. + **Delete-then-create defeats every protection.** One replace path removed the stored document before writing the new one, so nothing was left for identity preservation or elision to reconcile against, and translated captions in every diff --git a/docs/01-project/MDL_QUICK_REFERENCE.md b/docs/01-project/MDL_QUICK_REFERENCE.md index f90564b3e8..d4125f88d1 100644 --- a/docs/01-project/MDL_QUICK_REFERENCE.md +++ b/docs/01-project/MDL_QUICK_REFERENCE.md @@ -519,7 +519,7 @@ it is for pages. | Reduce a list | `$Folded = reduce($list, expression, initial: value, returns: Type);` | `$currentResult` is the accumulator. `initial` and `returns` are **required** — Mendix stores both and neither is inferable, so MDL will not guess (#1004) | | Call microflow | `$Result = call microflow Module.Name (Param = $value);` | A Mendix **expression** cannot call anything — `declare $r Boolean = Module.Name(...)` is CE0117 (MDL066) | | Call a rule | `if Module.SomeRule (Param = $value) then ... end if;` | A decision is the **only** place a rule can be evaluated; there is no call activity for one. The name must resolve to a rule — a microflow there is CE0117 | -| Call microflow on a queue | `call microflow Module.Name (Param = $value) in queue Module.Queue;` | Background execution; the queue must exist (CE1613) | +| Call microflow on a queue | `call microflow Module.Name (Param = $value) in queue Module.Queue;` | Background execution; the queue must exist (CE1613), and the called microflow must return nothing, else CE7033 (**MDL088**) | | Call Java action on a queue | `call java action Module.Name (Param = $value) in queue Module.Queue;` | The Java action must `returns void`, else CE7038 | | Call nanoflow | `$Result = call nanoflow Module.Name (Param = $value);` | | | Call JS action | `$Result = call javascript action Module.Name (Param = $value);` | JavaScript action (nanoflow/microflow) | diff --git a/docs/11-proposals/PROPOSAL_agent_loop_efficiency.md b/docs/11-proposals/PROPOSAL_agent_loop_efficiency.md index e17ae38eb5..68b879983c 100644 --- a/docs/11-proposals/PROPOSAL_agent_loop_efficiency.md +++ b/docs/11-proposals/PROPOSAL_agent_loop_efficiency.md @@ -13,6 +13,11 @@ related: # Agent loop efficiency — making an mxcli session cost what the work costs +**Status:** Draft +**Date:** 2026-09-22 (initial), revised 2026-09-23 — three complete builds +measured with `diag loop-report`, overturning the `check`-dominant assumption and +promoting the app restart to the wall-time lever (§"Three projects, measured"). + ## The report A side-by-side test had Opus build an app twice: once on Vercel (TypeScript files @@ -54,6 +59,57 @@ That arithmetic sets the priority order, and it is not the intuitive one: lever 1, so the two compound. 3. Output tokens (500 k of 228 M) are a rounding error. Do not optimise here. +## Three projects, measured — and what they overturn + +This proposal's first draft reasoned from one session. `diag loop-report` has +since run against three complete builds, and the result contradicts two things +the draft assumed. Both corrections are kept here rather than quietly edited +away, because the way each assumption survived is the reusable part. + +| project | Mendix | session | invocations | top commands by CALLS | +|---|---|---|---:|---| +| mxcli-ledger | — | 5 days | 301 | `check` 250 (83%) | +| CapTrackV6 | 11.14.0 | 2 h 45 | 585 | `exec` 180, `-c` 111, `check` 90 | +| mxcli-demo-2 | 11.13.0 | 5 h 07 | 408 | `-c` 120, `exec` 85, `check` 63 | + +**Overturned 1: `check` is not the dominant call.** The ledger's 83% was read +as the shape of an mxcli loop, and the obvious follow-up — "why so many checks?" +— was queued as the next thing to attack. Two further projects put `check` at +15% and 22%. The ledger is the outlier, not the archetype. A distribution taken +from one project is an anecdote with a table around it, and the fix for reading +it that way is three, not a better argument about one. + +All three logs predate ako/mxcli#629, so their `-c` and `exec` counts are +**inflated by mxcli's own child processes** — `mxcli test` spawns three before a +single test runs, so demo-2's 11 test runs contributed ~33 phantom calls and +CapTrack's 5 contributed ~15. The direction of that error matters: it inflates +exactly the two commands that displaced `check` at the top. It does not overturn +the overturning — `check` is still nowhere near 83% in either — but any figure +in this table is a pre-#629 figure. + +**Overturned 2: the wall-time lever is `run`, and the report understates it.** + +| project | closed-run wall time | `run` calls | `run` closed | `run` median | +|---|---:|---:|---:|---:| +| CapTrackV6 | 1,775 s | 34 | 7 | 90.7 s | +| mxcli-demo-2 | 838 s | 30 | 4 | 70.5 s | + +Read naively, `run` is 27–29% of mxcli's wall time. That reading is wrong, and +wrong in the same direction both times: **a boot that is killed never writes a +summary record, so its duration is not counted at all.** 26 of 30 and 27 of 34 +`run` invocations are uncounted. The totals above are floors. + +Multiply the boot count by the per-boot median instead and the shape changes +completely: ~35 min of boots in demo-2, ~50 min in CapTrack — against sessions of +5 h and 2 h 45. Every other command in both tables is noise beside that. + +This also corrects a figure CapTrack reported about itself. Its write-up says "of +~2.5 h building, only ~30 min was spent inside mxcli" and ranks its levers on +that basis. The 30 min is the closed-run total (1,775 s), which already contains +the 7 boots that closed; the 27 it excludes add ~41 min at the measured median, +so the real figure is around 70 min — more than double, and enough to change +which levers are worth pulling. + ## What is actually asymmetric — and how little of it is irreducible An earlier draft said: "Vercel's agent writes a `.tsx` file and the work is @@ -288,6 +344,38 @@ Three consequences for this proposal: *test* loop is blocked on 11.14 as well, and the skills that recommend `--attach` need the same version caveat `--watch` already carries. +**But 11.14 is not the whole story, and a 11.13 project proves it.** When this +section was written the defect explained the restart-per-change, and that closed +the question. It should not have. mxcli-demo-2 ran **11.13.0** — the version this +proposal's own control measured hot-reloading in 3.4 s — and still took **30 full +restarts**, its findings saying plainly: + +> `run` without `--watch`: every model change meant a full restart (the 30 runs, +> 243 s). `--watch` would hot-apply most changes as one long-running invocation. + +So the two projects are a natural experiment, and they answer differently: + +| | Mendix | warm loop | restarts | +|---|---|---|---:| +| CapTrackV6 | 11.14.0 | blocked by the defect | 34 | +| mxcli-demo-2 | 11.13.0 | **available, ~3 s** | 30 | + +On 11.14 the restart is forced. On 11.13 it is chosen — by a loop that never +reached for `--watch`. An earlier draft of this proposal said exactly that ("the +fast paths exist — the session just didn't take them, which makes this a defaults +problem more than a capability one"), the 11.14 measurement contradicted it, and +the correction went one step too far: it replaced "defaults problem" with +"mxbuild problem" when both are true on different versions. A measurement that +explains a symptom on one version does not retire the hypothesis on the others — +and the tell was available, since the same control that proved the defect +(11.13 reloads in 3.4 s) also proved a working warm loop existed to be unused. + +**That makes the restart lever real on both versions, by different routes:** +fix the default *version* for 11.14, and fix the default *invocation* for +everything else. `run --local --watch` is not what a session reaches for, and +neither the generated CLAUDE.md gate list nor the skills tell it to. A one-line +default is the whole intervention on 11.13; it is worth ~30 boots. + **The version default makes this worse than it needs to be.** `bootstrap-app` chooses the newest version on the CDN when the environment has nothing cached — 11.14.0 at the time of writing — so a freshly bootstrapped project lands on @@ -479,11 +567,30 @@ mxcli already logs every invocation as JSON Lines under `~/.mxcli/logs/` (`diaglog`, wired at `newLoggedExecutor` in `cmd/mxcli/main.go` — so it covers all commands, not a curated subset). That is the instrument. -**`mxcli diag loop-report`** reads a session's log and prints: invocations by -verb, wall time by verb, `check`-immediately-before-`exec` pairs (pure waste), -restarts vs. hot reloads, and output bytes per command. That converts "the loop -feels expensive" into a ranked list of where the calls actually went — and, run -before and after each lever, into evidence that a lever worked. +**`mxcli diag loop-report`** reads a session's log and prints invocations by +verb, wall time by verb, and `check`-immediately-before-`exec` pairs. It shipped, +and it has now run against three complete builds — which is where the two +corrections above came from, so the instrument has already paid for itself by +falsifying its author. + +Testing it against real projects also found three defects in it, each of which +had been silently skewing the very numbers it exists to produce: it logged a +minority of invocations and presented it as all (ako/mxcli#617), its `failed` +field counted a population its name did not describe (#620), and it counted +mxcli's own child processes as calls the agent made while reporting their parent +as a failure (#629). A measurement tool is not exempt from needing its own +evidence, and none of the three was visible from inside the tool. + +**Two things it still cannot see, both of which matter to the lever above:** + +- **A killed `run` contributes no wall time.** mxcli writes its summary on a + normal exit, so the 26-of-30 boots stopped with a signal are counted as + invocations and as zero seconds. The largest cost in the session is the one + the report is least able to size — sequencing item 2d. +- **Hot reloads inside a long-running `run`.** One `--watch` invocation that + applies forty changes is one row. That is the right unit for counting + processes and the wrong one for showing that item 2c worked, so claiming 2c + needs the app's own reload count, not this report. **Then a benchmark.** One fixed app-brief, run end to end, recording model calls and tokens. Without it, every claim here is an argument; with it, each lever @@ -500,6 +607,8 @@ places once already, and a loop regression is exactly as invisible. | 1b | Measure the check↔build gap rate: how many builds in a real session caught something `check` did not | S | sizes the batching prize, and feeds the parity programme's queue | | 2 | Fix `projectGates` to teach `exec`, not `check`+`exec` (lever 1) | XS | ~1 call per change, every project, immediately | | 2b | Measure `test --attach` on 11.14; pin the bootstrap default off 11.14 | XS | removes a forced 35 s/change from new projects | +| 2c | **Make `--watch` the default invocation** in the skills and the generated gate list, wherever the version supports it | XS | the largest measured wall-time item: ~30 boots on a 11.13 project that had the warm loop and never used it | +| 2d | Count a killed `run` in `diag loop-report` rather than dropping it | S | the restart bill is invisible today — `run`'s reported wall time is a floor built from 4 of 30 invocations | | 3 | Publish the canonical `&&` chain in `projectGates` + skills (lever 1) | XS | the 5–8 → 1–2 collapse, with nothing built | | 4 | Terse/delta output for `exec` and the noisy listings (lever 2) | M | the token half of the chain win; helps every call | | 5 | Tiered verification rule in the skills (lever 3) | S | stops the default path at the cheapest sufficient gate | @@ -508,7 +617,14 @@ places once already, and a loop regression is exactly as invisible. Item 1 first is deliberate. Items 2, 3, 5 and 6 are all XS-to-S and can ship immediately after it — item 3 is now the one that changes the shape of the loop, -and it is a documentation change. Item 4 is the only substantial build, and it +and it is a documentation change. + +**Items 2c and 2d were added after three projects were measured, and 2c is the +largest single item in this table by wall time.** It is also the cheapest: a +default, in prose, in files that already exist. Note that 2c and 2d attack +different axes — 2c removes the boots, 2d makes the remaining ones visible — and +that 2d has to land for 2c to be claimable, because a lever that removes +uncounted time cannot be shown to have worked. Item 4 is the only substantial build, and it is what makes item 3 pay in tokens rather than only in call count. `mxcli apply` is deliberately **not** in this table. It is contingent on item 1 @@ -521,7 +637,9 @@ showing that the published chain is still being composed wrong. per-change. - The 11.14 serve-rebuild defect. It is mxbuild's, the controls are conclusive, and nothing mxcli does from outside repairs it. It should be reported upstream; - meanwhile the version default is the only lever we hold. + meanwhile the version default is the only lever we hold **on that version** — + on versions where the warm loop works, the restart is ours to remove (item 2c), + and a 11.13 project measured 30 restarts it did not have to take. - Scope. The reported sessions did not build the same thing. - MDL not being in training data (`PROPOSAL_llm_mdl_assistance.md` owns that). It is row 2 of the asymmetry table and the one property here that is not diff --git a/mdl-examples/bug-tests/1064-queued-microflow-must-return-nothing.fail.mdl b/mdl-examples/bug-tests/1064-queued-microflow-must-return-nothing.fail.mdl new file mode 100644 index 0000000000..0d4ccff207 --- /dev/null +++ b/mdl-examples/bug-tests/1064-queued-microflow-must-return-nothing.fail.mdl @@ -0,0 +1,33 @@ +-- NEGATIVE TEST — `mxcli check` must REFUSE this file. +-- +-- mendixlabs/mxcli#1064 — `CALL MICROFLOW M.F(…) IN QUEUE M.Q` where F returns +-- Boolean: `mxcli check --references` said "Check passed!" and the build failed: +-- +-- [CE7033] "A microflow used for background execution must have a Microflow +-- return type of 'Nothing'." at Call microflow activity 'F' +-- +-- The call and the queue both resolve; the constraint is on the flow the call +-- names. Measured on mxbuild 11.12.0: this shape → CE7033, the control beside +-- it → 0 errors. +-- +-- Verify: +-- mxcli check 1064-queued-microflow-must-return-nothing.fail.mdl +-- -- must report MDL088 naming Q1064.Work and CE7033, and exit non-zero +-- +-- The control lives beside this file as 1064-queued-microflow-must-return-nothing.mdl. + +create module Q1064; +/ +create queue Q1064.Work (Parallelism: 1); +/ +create microflow Q1064.ACT_Work () +returns boolean +begin + return true; +end; +/ +create microflow Q1064.ACT_Enqueue () +begin + call microflow Q1064.ACT_Work() in queue Q1064.Work; +end; +/ diff --git a/mdl-examples/bug-tests/1064-queued-microflow-must-return-nothing.mdl b/mdl-examples/bug-tests/1064-queued-microflow-must-return-nothing.mdl new file mode 100644 index 0000000000..2b0226be18 --- /dev/null +++ b/mdl-examples/bug-tests/1064-queued-microflow-must-return-nothing.mdl @@ -0,0 +1,25 @@ +-- CONTROL for 1064-queued-microflow-must-return-nothing.fail.mdl — must PASS. +-- +-- mendixlabs/mxcli#1064 — a microflow run on a task queue must return nothing +-- (CE7033 otherwise). The same call on a void microflow is what Mendix accepts: +-- measured on mxbuild 11.12.0, 0 errors. +-- +-- Verify: +-- mxcli check 1064-queued-microflow-must-return-nothing.mdl + +create module Q1064; +/ +create queue Q1064.Work (Parallelism: 1); +/ +create microflow Q1064.ACT_Work () +begin + log info node 'Q1064' 'working'; +end; +/ +create microflow Q1064.ACT_Enqueue () +begin + call microflow Q1064.ACT_Work() in queue Q1064.Work; + -- The same Boolean-free flow called without a queue is an ordinary call. + call microflow Q1064.ACT_Work(); +end; +/ diff --git a/mdl-examples/bug-tests/1176-import-mapping-object-describes-without-all.mdl b/mdl-examples/bug-tests/1176-import-mapping-object-describes-without-all.mdl new file mode 100644 index 0000000000..5b28f8a753 --- /dev/null +++ b/mdl-examples/bug-tests/1176-import-mapping-object-describes-without-all.mdl @@ -0,0 +1,54 @@ +-- Bug test for upstream issue #1176: DESCRIBE printed `all` on an import +-- activity that returns ONE object. +-- +-- Reported on v0.23.0 / Studio Pro 11.12.3: +-- +-- $objectResponse = import from mapping TestModule.MF_importmapping_rest_object($sCountryISOCode) all; +-- +-- The stored Range really is All — ConstantRange{SingleObject:false} is what +-- Studio Pro writes for an object-rooted mapping — so #881's `all` was +-- accurate, but it reads as "returns a list". DESCRIBE now leaves it off an +-- object result and keeps it on a list result. +-- +-- Dropping it is exact, not a shorthand: a missing keyword is written as All +-- explicitly (the runtime fix after #881), and the object is inferred from +-- the mapping's root. Both microflows below store the same activity. +-- +-- Expected: both describe as `import from mapping MyFirstModule.IMM_1176($P);` +-- with no trailing `all`, `mx check` reports 0 errors, and exec'ing the +-- described text changes nothing. + +create json structure MyFirstModule.JSON_1176 +snippet '{"code": "NL", "name": "Netherlands"}'; + +create non-persistent entity MyFirstModule.Country1176 ( Code: string, Name: string ); +/ + +-- Object-rooted: the activity binds ONE Country1176. +create import mapping MyFirstModule.IMM_1176 + with json structure MyFirstModule.JSON_1176 +{ + create MyFirstModule.Country1176 { + Code = code, + Name = name + } +}; +/ + +-- Written with `all`: still accepted, describes without it. +create microflow MyFirstModule.MF1176_All ( $P: string ) +returns MyFirstModule.Country1176 as $R +begin + $R = import from mapping MyFirstModule.IMM_1176($P) all; + return $R; +end; +/ + +-- The form DESCRIBE now emits for both. +create microflow MyFirstModule.MF1176_Bare ( $P: string ) +returns MyFirstModule.Country1176 as $R +begin + $R = import from mapping MyFirstModule.IMM_1176($P); + return $R; +end; +/ diff --git a/mdl-examples/bug-tests/attribute-guid-1119-alter-preserves-storage-guid.mdl b/mdl-examples/bug-tests/attribute-guid-1119-alter-preserves-storage-guid.mdl new file mode 100644 index 0000000000..dd233812da --- /dev/null +++ b/mdl-examples/bug-tests/attribute-guid-1119-alter-preserves-storage-guid.mdl @@ -0,0 +1,103 @@ +-- ============================================================================ +-- mendixlabs/mxcli#1119 — an ALTER must not re-mint an attribute's storage GUID +-- ============================================================================ +-- +-- Each attribute of a persistent entity carries TWO 16-byte identities: the +-- element `$ID`, and a separate `GUID` the runtime keys the database on +-- (`mendixsystem$attribute.id`). Every ALTER used to rebuild the entity from the +-- semantic model, which gave each attribute `GUID = $ID` and discarded the +-- stored one — so the next deploy against a database that already held data +-- dropped and recreated every column of that entity. Rows and associations +-- survived; all attribute values were gone. +-- +-- Nothing reported it. The model stays valid, `mxcli check`, `exec`, `mx check` +-- and the build are all clean, and `DESCRIBE ENTITY` is byte-identical before +-- and after, because the reader never surfaces the GUID. +-- +-- WHAT THIS FILE IS FOR, AND WHAT IT CANNOT DO +-- +-- MDL cannot read or assert a GUID, so this script cannot fail on the bug. It +-- exercises every ALTER form that routes through Backend.UpdateEntity, so the +-- fix can be checked against a REAL project — and, since the write path now +-- refuses a GUID it would otherwise have moved (canon.StorageGUIDError), a +-- regression shows up here as a REFUSED statement rather than as silence. +-- +-- Run it TWICE against a project whose entity Studio Pro authored: +-- +-- mxcli exec attribute-guid-1119-alter-preserves-storage-guid.mdl -p app.mpr +-- +-- Expected: every statement succeeds the first time; the second run reports the +-- entity unchanged. A "refusing to write unit ... would be written with a +-- different GUID" is the guard catching a regression in the carry. +-- +-- The assertions live where the GUID is visible — on the raw BSON, in +-- mdl/backend/modelsdk/issue1119_attribute_guid_test.go (and the index arm in +-- issue1119_index_guid_test.go), each with the carry stubbed as its control. +-- ============================================================================ + +CREATE MODULE BugAttrGuid1119; + +-- The subject: several attributes, so a rewrite that loses identities loses +-- many. One attribute would make the bug look like an edge case; it was not — +-- the reported project lost all 28 of an entity's attributes at once. +CREATE OR MODIFY PERSISTENT ENTITY BugAttrGuid1119.Organization ( + Name: String(200), + URLName: String(200), + Status: String(50), + Notes: String(500) +); + +CREATE OR MODIFY PERSISTENT ENTITY BugAttrGuid1119.Person ( + FullName: String(200) +); + +-- A sibling association, and an index: both carry a GUID of their own. The +-- association is the reporter's own negative result (it survived, because the +-- entities-list rebuild passes it through as stored bytes); the index is the +-- arm that would otherwise make every ALTER below get refused. +CREATE OR MODIFY ASSOCIATION BugAttrGuid1119.Person_Organization + FROM BugAttrGuid1119.Person TO BugAttrGuid1119.Organization TYPE Reference; + +ALTER ENTITY BugAttrGuid1119.Organization ADD INDEX (Name ASC); + +-- --------------------------------------------------------------------------- +-- The six forms the report tested, each of which destroyed every GUID. +-- --------------------------------------------------------------------------- + +-- The one that makes the mechanism plain: it touches no attribute at all. +ALTER ENTITY BugAttrGuid1119.Organization + SET DOCUMENTATION 'Storage GUIDs must survive a documentation edit.'; + +ALTER ENTITY BugAttrGuid1119.Organization ADD ATTRIBUTE Region: String(100); + +ALTER ENTITY BugAttrGuid1119.Organization MODIFY ATTRIBUTE Notes: String(1000); + +-- A rename must carry the GUID FORWARD, not mint a fresh one: Studio Pro +-- renames the column and keeps the data, so a fresh GUID here loses it just as +-- surely as changing an untouched attribute's. +ALTER ENTITY BugAttrGuid1119.Organization RENAME ATTRIBUTE URLName TO UrlSlug; + +ALTER ENTITY BugAttrGuid1119.Organization DROP ATTRIBUTE Region; + +-- CREATE OR MODIFY on an entity that already exists is the sixth form, and the +-- one most likely to be run repeatedly from a checked-in script. +CREATE OR MODIFY PERSISTENT ENTITY BugAttrGuid1119.Organization ( + Name: String(200), + UrlSlug: String(200), + Status: String(50), + Notes: String(1000) +); + +-- One more path that reaches the same rewrite and was not in the report: +-- CREATE VALIDATION RULE also goes through UpdateEntity (cmd_validationrules.go), +-- so it destroyed every GUID too. GRANT does NOT — it mutates the loaded gen +-- tree in place, which is why the reporter found association and GRANT writes +-- harmless and is the distinction worth remembering: the model -> gen rebuild is +-- the exposed path, in-place gen mutation is not. +CREATE OR MODIFY REGULAR EXPRESSION BugAttrGuid1119.SlugPattern ( + Expression: '^[a-z0-9-]+$' +); + +CREATE VALIDATION RULE FOR BugAttrGuid1119.Organization.UrlSlug + REGEX BugAttrGuid1119.SlugPattern + FEEDBACK 'Use lower-case letters, digits and hyphens only.'; diff --git a/mdl-examples/bug-tests/domainmodel-1169-whole-unit-storage-guids.mdl b/mdl-examples/bug-tests/domainmodel-1169-whole-unit-storage-guids.mdl new file mode 100644 index 0000000000..cc18588150 --- /dev/null +++ b/mdl-examples/bug-tests/domainmodel-1169-whole-unit-storage-guids.mdl @@ -0,0 +1,126 @@ +-- ============================================================================ +-- mendixlabs/mxcli#1169 — a whole-unit rewrite must not re-mint any storage GUID +-- ============================================================================ +-- +-- #1119 fixed the ALTER ENTITY path, which swaps ONE entity into an otherwise +-- untouched list. `Backend.UpdateDomainModel` is the other shape: it removes and +-- rebuilds the ENTIRE Entities and Associations lists, so every element arrived +-- with no stored bytes and the codec's EmitGUID default wrote `GUID = $ID` across +-- the whole module — entities, their attributes and indexes, and every +-- association, including the ones the statement never named. The reporter measured +-- 282 moved GUIDs in one module (37 entities, 224 attributes, 21 associations) and +-- a runtime crash: +-- +-- Cannot invoke "...Table.getTableName()" because "table" is null +-- +-- The title said ALTER ASSOCIATION. The path is shared, and that is the point of +-- this file: FIVE statements reach it, and the one that costs most is not an +-- association statement at all. +-- +-- RENAME ENTITY <- the expensive one +-- RENAME ASSOCIATION +-- ALTER ASSOCIATION ... SET COMMENT +-- ALTER ASSOCIATION ... SET OWNER +-- CREATE OR MODIFY ASSOCIATION (re-run against an unchanged project) +-- +-- WHY RENAME ENTITY IS THE WORST OF THEM +-- +-- An entity's name IS its table name, and the runtime resolves the entity by its +-- GUID, not by that name. Measured for #503 on 11.13.0 + PostgreSQL 16, same +-- 250-row starting state both ways: +-- +-- GUID preserved -> the table is RENAMED: 250 -> 250 rows +-- GUID re-minted -> the old table is DROPPED and an empty one created: 250 -> 0 +-- +-- So this loses a whole TABLE where an ALTER loses a column. `mxcli`'s own help +-- for CREATE OR MODIFY promised "preserves UUID. Safe to re-run" throughout — true +-- of the `$ID`, false of the `GUID`. +-- +-- WHAT THIS FILE IS FOR, AND WHAT IT CANNOT DO +-- +-- MDL cannot read or assert a GUID, so this script cannot fail on the bug itself. +-- What it does is exercise every statement that routes through the rewrite, so the +-- fix can be checked against a REAL project — and because the write path refuses a +-- GUID it would otherwise have moved (canon.StorageGUIDError), a regression in the +-- carry shows up here as a REFUSED statement rather than as silence: +-- +-- refusing to write unit ...: N element(s) kept their $ID but would be written +-- with a different GUID ... +-- +-- Run it against a project whose entities Studio Pro authored: +-- +-- mxcli exec domainmodel-1169-whole-unit-storage-guids.mdl -p app.mpr +-- +-- Expected: every statement succeeds. Note the trap this shares with #503's file — +-- everything CREATEd below is created by mxcli, so its GUID equals its $ID from +-- birth and a rewrite that re-mints `GUID = $ID` reproduces the very same value. +-- Against a project of only mxcli-created entities this file passes whether or not +-- the fix is present. The assertions therefore live where the GUID is visible — on +-- the raw BSON, over a Studio Pro-authored fixture, in +-- mdl/backend/modelsdk/issue1169_domainmodel_guid_test.go. +-- ============================================================================ + +CREATE MODULE BugUnitGuid1169; + +-- Several entities with several attributes each, because the defect's scale is the +-- whole unit rather than the statement's target: a rewrite driven by ONE +-- association below must leave all of these untouched. +CREATE OR MODIFY PERSISTENT ENTITY BugUnitGuid1169.Customer ( + Name: String(200), + Email: String(200), + Reference: String(50) +); + +CREATE OR MODIFY PERSISTENT ENTITY BugUnitGuid1169.Order ( + OrderNumber: String(50), + Total: Decimal, + Placed: DateTime +); + +CREATE OR MODIFY PERSISTENT ENTITY BugUnitGuid1169.OrderLine ( + Quantity: Integer, + UnitPrice: Decimal +); + +-- An index carries a GUID of its own, and is the arm that made every statement +-- below get refused once the entity carry was in place but the child carry was not. +ALTER ENTITY BugUnitGuid1169.Customer ADD INDEX (Reference ASC); + +CREATE OR MODIFY ASSOCIATION BugUnitGuid1169.Order_Customer + FROM BugUnitGuid1169.Order TO BugUnitGuid1169.Customer TYPE Reference; + +CREATE OR MODIFY ASSOCIATION BugUnitGuid1169.OrderLine_Order + FROM BugUnitGuid1169.OrderLine TO BugUnitGuid1169.Order TYPE Reference; + +-- --------------------------------------------------------------------------- +-- The five statements that reach the whole-unit rewrite. +-- --------------------------------------------------------------------------- + +-- 1. An association edit that names ONE association and rewrote every element in +-- the module. This is the reported statement. +ALTER ASSOCIATION BugUnitGuid1169.Order_Customer + SET COMMENT 'Storage GUIDs must survive an association comment edit.'; + +-- 2. The same, for a property Mendix actually stores on the association. +ALTER ASSOCIATION BugUnitGuid1169.OrderLine_Order SET OWNER BOTH; + +-- 3. A re-run of an unchanged CREATE OR MODIFY. Nothing should land at all +-- (ADR-0008 elides the write); before the fix this alone moved every GUID. +CREATE OR MODIFY ASSOCIATION BugUnitGuid1169.Order_Customer + FROM BugUnitGuid1169.Order TO BugUnitGuid1169.Customer TYPE Reference; + +-- 4. RENAME ASSOCIATION: the identity must travel with the new name. +RENAME ASSOCIATION BugUnitGuid1169.OrderLine_Order TO OrderRow_Order; + +-- 5. RENAME ENTITY — a table rename if the GUID is carried, a table DROP if it is +-- not. Renaming the entity that holds the index and is the TO side of an +-- association exercises the child carry and the pointer rewrite together. +RENAME ENTITY BugUnitGuid1169.Customer TO Client; + +-- And the control for the whole file: after all of the above, describing the +-- module must still show three entities with every attribute and index intact. An +-- element LOST to the rewrite would be a different defect wearing the same +-- symptom, and this is the one part MDL can check for itself. +DESCRIBE ENTITY BugUnitGuid1169.Client; +DESCRIBE ENTITY BugUnitGuid1169.Order; +DESCRIBE ENTITY BugUnitGuid1169.OrderLine; diff --git a/mdl-examples/bug-tests/javascript-action-1171-entity-argument-own-line.mdl b/mdl-examples/bug-tests/javascript-action-1171-entity-argument-own-line.mdl new file mode 100644 index 0000000000..02f8745a0c --- /dev/null +++ b/mdl-examples/bug-tests/javascript-action-1171-entity-argument-own-line.mdl @@ -0,0 +1,29 @@ +-- mendixlabs/mxcli#1171 +-- CALL JAVASCRIPT ACTION with an entity-type (`entity <>`) parameter, written +-- with the argument on its own line and `)` on the next. The visitor keeps an +-- argument's trailing whitespace so expressions round-trip as written, and the +-- entity-type path (added for #1137) stored that text as the entity name: +-- "$Type": "Microflows$EntityTypeCodeActionParameterValue" +-- "Entity": "Administration.Account\n" +-- +-- Measured on mxbuild 11.6.6 against testdata/expr-checker/minimal.mpr: +-- fixed -> "The app contains: 0 errors." +-- faulty -> CE1613 "The selected entity 'Administration.Account +-- ' no longer exists." at Call JavaScript action activity +-- `mxcli check --references` passes on both. The report's CE0115 was the +-- #1137 symptom (v0.23.0 predates that fix); on a build with #1137 and +-- without this fix, the same MDL fails with CE1613 instead. +-- +-- Needs an app with the NanoflowCommons module (testdata/expr-checker/minimal.mpr). +-- +-- Verify after exec: +-- ./bin/mxcli bson dump -p app.mpr --type nanoflow --object MyFirstModule.ACT_Refresh_Multiline +-- the parameter value's "Entity" must be exactly "Administration.Account". + +create or modify nanoflow MyFirstModule.ACT_Refresh_Multiline () +begin +$Var = call javascript action NanoflowCommons.RefreshEntity( +EntityToRefresh = Administration.Account +); +return; +end; diff --git a/mdl-examples/bug-tests/move-entity-503-preserves-storage-guids.mdl b/mdl-examples/bug-tests/move-entity-503-preserves-storage-guids.mdl new file mode 100644 index 0000000000..a8ca0b8d1c --- /dev/null +++ b/mdl-examples/bug-tests/move-entity-503-preserves-storage-guids.mdl @@ -0,0 +1,107 @@ +-- ============================================================================ +-- ako/mxcli#503 + #605 — what a cross-module MOVE ENTITY must preserve +-- ============================================================================ +-- +-- A moved entity, its attributes, and any association the move converts into a +-- cross-module association each carry a `GUID` the runtime keys the database on. +-- MoveEntity rebuilt all of them from the semantic model, so each came out with +-- `GUID = $ID`: #657 had fixed that for ALTER's entity and #1119 for its +-- children, and the move path got neither. +-- +-- WHY THE MOVE IS NOT A SPECIAL CASE, MEASURED +-- +-- A module move changes the table name (`myfirstmodule$moveprobe` -> +-- `administration$moveprobe`), so it was an open question whether the data could +-- survive at all. It can, and only if the GUID does. On Mendix 11.13.0 + +-- PostgreSQL 16, same 250-row starting state both ways: +-- +-- GUID re-minted (pre-fix) -> old table dropped, new one created: 250 -> 0 rows +-- GUID preserved (fixed) -> table RENAMED: 250 -> 250 rows, values intact +-- +-- The runtime resolves the entity by its GUID, not by its table name. So this is +-- the same defect as #1119, not a lesser one: it loses the whole table rather +-- than a column. +-- +-- WHAT THIS FILE CHECKS, AND WHAT IT CANNOT +-- +-- mxcli exec move-entity-503-preserves-storage-guids.mdl -p app.mpr +-- +-- Expected: every statement succeeds. It is a smoke test that both move +-- directions still execute — the conversion is asymmetric, since moving the +-- CHILD leaves the cross-association in the source module while moving the +-- PARENT sends it to the target. +-- +-- It CANNOT fail on the bug, and the reason is the trap worth carrying away: +-- everything this script creates is created by MXCLI, so its GUID equals its $ID +-- from birth, and a rewrite that "re-mints GUID = $ID" reproduces the very same +-- value. Measured — the pre-fix binary passes this file exactly as the fixed one +-- does. Only a STUDIO PRO-authored entity, whose GUID and $ID genuinely differ, +-- can detect the defect; run these statements against one and a regression in +-- the cross-association carry is refused outright by the #1119 write guard: +-- +-- refusing to write unit ...: 1 element(s) kept their $ID but would be written +-- with a different GUID ... (DomainModels$CrossAssociation) +-- +-- That refusal is how the bug was found. It covers only the association, because +-- that conversion happens in place in the source unit; the entity's and +-- attributes' GUIDs land in a DIFFERENT unit, where the guard has nothing to pair +-- against. The real assertions therefore live in +-- mdl/backend/modelsdk/issue503_move_guid_test.go — on raw BSON, over a Studio +-- Pro-authored fixture entity, with each of the three carries stubbed +-- independently as its own control. +-- +-- The #605 half below is DIFFERENT in this respect, and genuinely does fail against +-- the broken code: a stale qualified name has nothing to do with GUID divergence, +-- so an mxcli-created entity detects it perfectly well. Measured — pre-fix this +-- file leaves 3 CE1613s, fixed it is 0 errors. +-- ============================================================================ + +CREATE MODULE BugMoveGuid503A; +CREATE MODULE BugMoveGuid503B; + +-- Two INDEPENDENT pairs, because the two conversion directions must be exercised +-- without ever moving both endpoints of the SAME association. Doing that leaves a +-- pre-existing cross-association pointing at an element that is no longer in its +-- unit, and the project will not even open: +-- +-- System.AggregateException: The given key '' was not present in the dictionary. +-- +-- Measured with a binary predating both fixes here, so it is a third defect on this +-- command and not something these fixes introduced — but it is why this file keeps +-- one endpoint of each pair put. Filed as ako/mxcli#628: MoveEntity scans +-- AssociationsItems() and never CrossAssociationsItems(), so a cross-association +-- that already exists is invisible to the move. + +-- Pair 1 — the TO side moves, so the cross-association STAYS in the source module +-- and its qualified name does not change. +CREATE OR MODIFY PERSISTENT ENTITY BugMoveGuid503A.Parent1 ( Code: String(50) ); +CREATE OR MODIFY PERSISTENT ENTITY BugMoveGuid503A.Child1 ( Label: String(100) ); +CREATE OR MODIFY ASSOCIATION BugMoveGuid503A.Child1_Parent1 + FROM BugMoveGuid503A.Child1 TO BugMoveGuid503A.Parent1 TYPE Reference; + +-- Pair 2 — the FROM side moves, so the cross-association TRAVELS to the target and +-- its qualified name changes with it. Only this direction may be swept; sweeping +-- the other would rewrite references that are still correct. +CREATE OR MODIFY PERSISTENT ENTITY BugMoveGuid503A.Parent2 ( Code: String(50) ); +CREATE OR MODIFY PERSISTENT ENTITY BugMoveGuid503A.Child2 ( Label: String(100) ); +CREATE OR MODIFY ASSOCIATION BugMoveGuid503A.Child2_Parent2 + FROM BugMoveGuid503A.Child2 TO BugMoveGuid503A.Parent2 TYPE Reference; + +-- A consumer, so the #605 sweep has something to rewrite. Without a reference to +-- the moved entity, a move has nothing to sweep and the statements below pass +-- whether or not the sweep exists. Both the parameter and the retrieve name +-- Parent1, which is about to move. +CREATE OR MODIFY MICROFLOW BugMoveGuid503A.ACT_UsesParent1 ($P: BugMoveGuid503A.Parent1) +RETURNS list of BugMoveGuid503A.Parent1 +BEGIN + retrieve $all from BugMoveGuid503A.Parent1; + return $all; +END; + +-- Each of these now reports what it rewrote, e.g. +-- Updated references in 1 document(s): BugMoveGuid503A.Parent1 → BugMoveGuid503B.Parent1 +-- and `mxcli docker check -p app.mpr` is 0 errors afterwards. Before #605 the +-- microflow's parameter and retrieve were left naming a module the entity had left, +-- as CE1613 at build time with nothing said at move time. +MOVE ENTITY BugMoveGuid503A.Parent1 TO BugMoveGuid503B; +MOVE ENTITY BugMoveGuid503A.Child2 TO BugMoveGuid503B; diff --git a/mdl-examples/bug-tests/rename-self-references-clobbered-by-persist.mdl b/mdl-examples/bug-tests/rename-self-references-clobbered-by-persist.mdl new file mode 100644 index 0000000000..f2b4dedec5 --- /dev/null +++ b/mdl-examples/bug-tests/rename-self-references-clobbered-by-persist.mdl @@ -0,0 +1,111 @@ +-- ============================================================================ +-- RENAME ENTITY / RENAME ASSOCIATION left the renamed element's own member +-- references stale — the project-wide sweep was CLOBBERED by the persist +-- ============================================================================ +-- +-- An entity's access rules and validation rules name each member by QUALIFIED +-- name (`Module.Entity.Attr`), and an access rule names an association the same +-- way (`Module.Association`). Both of those strings contain the very name a +-- RENAME changes, and both live in the SAME unit as the element being renamed. +-- +-- THE MECHANISM IS A CLOBBER, NOT A MISSED SWEEP +-- +-- `execRenameEntity` / `execRenameAssociation` read the domain model, run the +-- project-wide `RenameReferences` pass — which DOES rewrite those names, in the +-- raw unit — and then persist the semantic model they read BEFORE the sweep, +-- putting the stale names straight back. The reference report and the build +-- errors line up exactly, which is the tell: measured on a Studio Pro-authored +-- 11.13 module, +-- +-- Renamed entity: Administration.AccountPasswordData → Administration.PasswordData +-- Updated 24 reference(s) in 11 document(s) +-- +-- and then four CE1613s naming three attributes and one association, every one of +-- them "at Access rule of entity" or "at Validation rule of entity" for the +-- element that had just been renamed. The sweep had done its job and the write +-- undid it. +-- +-- This is the RENAME sibling of ako/mxcli#605, where the same names went stale on +-- the MODULE prefix during a cross-module MOVE ENTITY. It is worse than a +-- dangling string: `entityToGen`'s `syncMemberAccesses` matches existing entries +-- BY qualified name, so a stale `Module.Old.Attr` never equals the rebuilt +-- `Module.New.Attr` and it appends the new entry while KEEPING the old — the +-- entity ends up carrying every member twice, half of the entries dangling. +-- `DESCRIBE` renders members bare, so the duplication is the only visible trace. +-- +-- The association half is the longest-lived of the four, because the stale name is +-- then re-read by every later statement that loads the unit: a RENAME ASSOCIATION +-- followed by an unrelated RENAME ENTITY carried it into the second write too. +-- +-- HOW TO RUN IT +-- +-- mxcli exec rename-self-references-clobbered-by-persist.mdl -p app.mpr +-- mxcli docker check -p app.mpr +-- +-- Expected: every statement succeeds AND the check is 0 errors. Unlike the GUID +-- scripts next to this one, THIS FILE GENUINELY FAILS against the broken code — +-- a stale qualified name has nothing to do with GUID divergence, so an +-- mxcli-created entity detects it perfectly well. Measured on 11.13.0: 3 CE1613s +-- before the fix (two access-rule attributes and one validation rule), 0 after. +-- +-- ONE ARM OF IT IS NOT CAUGHT BY `mx check` HERE, and the asymmetry is worth +-- knowing before trusting a green build. In this script the stale ASSOCIATION +-- reference raises NO error, because mxcli wrote the access rule and the +-- MemberAccess still carries a valid pointer to the association element — the +-- qualified name is what a reader shows, not what mxbuild resolves. Against a +-- STUDIO PRO-authored access rule the same rename does error: +-- +-- [CE1613] "The selected association 'Administration.AccountPasswordData_Account' +-- no longer exists." at Access rule of entity 'Administration.PasswordData' +-- +-- So the association half of the fix is proven on a real module, not by this file. +-- What this file shows for it is the visible trace: run it against the pre-fix +-- binary and the DESCRIBE below emits `read (..., Widget_Holder)` — the old name, +-- for an association that no longer has it — where the fixed binary emits +-- `Gadget_Holder`. +-- +-- The executor-level assertions, each with its re-point stubbed as its own +-- control, are in mdl/executor/rename_entity_self_refs_test.go. +-- ============================================================================ + +CREATE MODULE BugRenameRefs; +CREATE MODULE ROLE BugRenameRefs.User; + +-- The subject. `NOT NULL ERROR` is what puts a VALIDATION rule on the attribute; +-- without one, only the access-rule half of the defect is exercised and a fix +-- that covered access rules alone would look complete. +CREATE OR MODIFY PERSISTENT ENTITY BugRenameRefs.Widget ( + WidgetName: String(100) NOT NULL ERROR 'A name is required', + WidgetCode: String(50) +); + +CREATE OR MODIFY PERSISTENT ENTITY BugRenameRefs.Holder ( + HolderName: String(100) +); + +CREATE OR MODIFY ASSOCIATION BugRenameRefs.Widget_Holder + FROM BugRenameRefs.Widget TO BugRenameRefs.Holder TYPE Reference; + +-- The access rules: member entries naming both attributes AND the association. +-- Mendix stores an association's MemberAccess on the FROM entity only (putting one +-- on the TO entity is CE0066), so this has to be granted on Widget. +GRANT BugRenameRefs.User ON BugRenameRefs.Widget ( + CREATE, DELETE, + READ (WidgetName, WidgetCode, Widget_Holder), + WRITE (WidgetCode, Widget_Holder) +); + +-- --------------------------------------------------------------------------- +-- The two renames. Order matters: the association first, so the second rename +-- re-reads the unit and would carry a stale association name forward into its own +-- write — which is how the association half outlived a fix for the entity half. +-- --------------------------------------------------------------------------- + +RENAME ASSOCIATION BugRenameRefs.Widget_Holder TO Gadget_Holder; + +RENAME ENTITY BugRenameRefs.Widget TO Gadget; + +-- The visible trace, and the one part MDL can check for itself: each member must +-- appear ONCE. Two entries for the same member is the duplication +-- syncMemberAccesses introduces when the stale name fails to match the rebuilt one. +DESCRIBE ENTITY BugRenameRefs.Gadget; diff --git a/mdl-examples/doctype-tests/queued-calls.mdl b/mdl-examples/doctype-tests/queued-calls.mdl index 8a9ddf8bd9..8001809f2e 100644 --- a/mdl-examples/doctype-tests/queued-calls.mdl +++ b/mdl-examples/doctype-tests/queued-calls.mdl @@ -12,13 +12,17 @@ -- generated/metamodel does not) is NOT written: measured on 11.13, a call -- carrying only `Queue` is inert — mx check does not read it. -- --- Two traps, both verified on mxbuild 11.13.0: +-- Three traps, verified on mxbuild 11.13.0 (the second on 11.12.0): -- -- 1. A queued CALL JAVA ACTION must return Nothing. Anything else is -- CE7038 "A Java action used for background execution must have a return -- type of 'Nothing'." mxcli's default return type for CREATE JAVA ACTION -- is Boolean, so `returns void` is required here, not optional. --- 2. The queue must exist. A missing one is CE1613 on the CALL ACTIVITY — +-- 2. A queued CALL MICROFLOW must return Nothing too: CE7033 "A microflow +-- used for background execution must have a Microflow return type of +-- 'Nothing'." (measured on 11.12.0; `mxcli check` reports it as MDL088, +-- mendixlabs/mxcli#1064). Ops.ACT_Work below has no `returns` for that. +-- 3. The queue must exist. A missing one is CE1613 on the CALL ACTIVITY — -- it names the activity, not the script — so `mxcli check --references` -- resolves the name first and reports it against the statement. -- diff --git a/mdl/backend/domainmodel.go b/mdl/backend/domainmodel.go index 7ff7ef16bc..c383ee642a 100644 --- a/mdl/backend/domainmodel.go +++ b/mdl/backend/domainmodel.go @@ -3,6 +3,7 @@ package backend import ( + "github.com/mendixlabs/mxcli/mdl/types" "github.com/mendixlabs/mxcli/model" "github.com/mendixlabs/mxcli/sdk/domainmodel" ) @@ -30,7 +31,12 @@ type DomainModelBackend interface { CreateEntity(domainModelID model.ID, entity *domainmodel.Entity) error UpdateEntity(domainModelID model.ID, entity *domainmodel.Entity) error DeleteEntity(domainModelID model.ID, entityID model.ID) error - MoveEntity(entity *domainmodel.Entity, sourceDMID, targetDMID model.ID, sourceModuleName, targetModuleName string) ([]string, error) + // MoveEntity moves an entity between domain models, converting each association + // that touches it into a cross-module association. It reports one entry per + // conversion with the qualified name the association had and the one it has + // afterwards, because the two move directions differ and the caller must sweep + // references from the names rather than derive them (#605). + MoveEntity(entity *domainmodel.Entity, sourceDMID, targetDMID model.ID, sourceModuleName, targetModuleName string) ([]types.MovedAssociation, error) // Attributes AddAttribute(domainModelID model.ID, entityID model.ID, attr *domainmodel.Attribute) error diff --git a/mdl/backend/infrastructure.go b/mdl/backend/infrastructure.go index 1e96d116fc..a326855e62 100644 --- a/mdl/backend/infrastructure.go +++ b/mdl/backend/infrastructure.go @@ -48,6 +48,17 @@ type RawUnitBackend interface { // missing from its output is missing on purpose, and carrying it back would // undo a deliberate deletion. UpdateRawUnitOwningTranslations(unitID string, contents []byte) error + + // UpdateRawUnitOwningStorageGUIDs is UpdateRawUnit for a write that + // deliberately transplants storage GUIDs onto elements that keep their $ID. + // + // The write path refuses that pattern by default, because for every ordinary + // write it means the database's identity for an element was dropped and the + // next deploy will drop its column (#1119). The marketplace module update is + // the exception: it captures a module's GUIDs and puts them back onto the + // documents replacing it, which is what stops an update destroying that + // module's data. Only a caller doing that may use this. + UpdateRawUnitOwningStorageGUIDs(unitID string, contents []byte) error } // MetadataBackend provides project-level metadata and introspection. diff --git a/mdl/backend/mcp/unsupported_gen.go b/mdl/backend/mcp/unsupported_gen.go index 5cbd5c6d3c..a1dc96a9c1 100644 --- a/mdl/backend/mcp/unsupported_gen.go +++ b/mdl/backend/mcp/unsupported_gen.go @@ -927,7 +927,7 @@ func (unsupportedBackend) MoveDocument(_ model.ID, _ model.ID) (err0 error) { return } -func (unsupportedBackend) MoveEntity(_ *domainmodel.Entity, _ model.ID, _ model.ID, _ string, _ string) (r0 []string, err1 error) { +func (unsupportedBackend) MoveEntity(_ *domainmodel.Entity, _ model.ID, _ model.ID, _ string, _ string) (r0 []types.MovedAssociation, err1 error) { err1 = errUnsupported("MoveEntity") return } @@ -1332,6 +1332,11 @@ func (unsupportedBackend) UpdateRawUnit(_ string, _ []uint8) (err0 error) { return } +func (unsupportedBackend) UpdateRawUnitOwningStorageGUIDs(_ string, _ []uint8) (err0 error) { + err0 = errUnsupported("UpdateRawUnitOwningStorageGUIDs") + return +} + func (unsupportedBackend) UpdateRawUnitOwningTranslations(_ string, _ []uint8) (err0 error) { err0 = errUnsupported("UpdateRawUnitOwningTranslations") return diff --git a/mdl/backend/mock/backend.go b/mdl/backend/mock/backend.go index da89235572..b5fa2477a4 100644 --- a/mdl/backend/mock/backend.go +++ b/mdl/backend/mock/backend.go @@ -68,7 +68,7 @@ type MockBackend struct { CreateEntityFunc func(domainModelID model.ID, entity *domainmodel.Entity) error UpdateEntityFunc func(domainModelID model.ID, entity *domainmodel.Entity) error DeleteEntityFunc func(domainModelID model.ID, entityID model.ID) error - MoveEntityFunc func(entity *domainmodel.Entity, sourceDMID, targetDMID model.ID, sourceModuleName, targetModuleName string) ([]string, error) + MoveEntityFunc func(entity *domainmodel.Entity, sourceDMID, targetDMID model.ID, sourceModuleName, targetModuleName string) ([]types.MovedAssociation, error) AddAttributeFunc func(domainModelID model.ID, entityID model.ID, attr *domainmodel.Attribute) error UpdateAttributeFunc func(domainModelID model.ID, entityID model.ID, attr *domainmodel.Attribute) error DeleteAttributeFunc func(domainModelID model.ID, entityID model.ID, attrID model.ID) error @@ -317,6 +317,12 @@ type MockBackend struct { // when unset, because most tests do not care about the distinction. UpdateRawUnitOwningTranslationsFunc func(unitID string, contents []byte) error + // UpdateRawUnitOwningStorageGUIDsFunc stubs the write path that deliberately + // transplants storage GUIDs (the marketplace module update). Separate from + // UpdateRawUnitFunc so a test cannot satisfy it by accident: a write that + // moves a GUID without meaning to is the #1119 data-loss defect. + UpdateRawUnitOwningStorageGUIDsFunc func(unitID string, contents []byte) error + // MetadataBackend ListAllUnitIDsFunc func() ([]string, error) ListUnitsFunc func() ([]*types.UnitInfo, error) diff --git a/mdl/backend/mock/mock_domainmodel.go b/mdl/backend/mock/mock_domainmodel.go index eb2c47681c..199df21bb8 100644 --- a/mdl/backend/mock/mock_domainmodel.go +++ b/mdl/backend/mock/mock_domainmodel.go @@ -4,6 +4,7 @@ package mock import ( "fmt" + "github.com/mendixlabs/mxcli/mdl/types" "github.com/mendixlabs/mxcli/model" "github.com/mendixlabs/mxcli/sdk/domainmodel" @@ -65,7 +66,7 @@ func (m *MockBackend) DeleteEntity(domainModelID model.ID, entityID model.ID) er return nil } -func (m *MockBackend) MoveEntity(entity *domainmodel.Entity, sourceDMID, targetDMID model.ID, sourceModuleName, targetModuleName string) ([]string, error) { +func (m *MockBackend) MoveEntity(entity *domainmodel.Entity, sourceDMID, targetDMID model.ID, sourceModuleName, targetModuleName string) ([]types.MovedAssociation, error) { if m.MoveEntityFunc != nil { return m.MoveEntityFunc(entity, sourceDMID, targetDMID, sourceModuleName, targetModuleName) } diff --git a/mdl/backend/mock/mock_infrastructure.go b/mdl/backend/mock/mock_infrastructure.go index 33db30ee4d..96105aa307 100644 --- a/mdl/backend/mock/mock_infrastructure.go +++ b/mdl/backend/mock/mock_infrastructure.go @@ -98,6 +98,13 @@ func (m *MockBackend) UpdateRawUnitOwningTranslations(unitID string, contents [] return fmt.Errorf("MockBackend.UpdateRawUnitOwningTranslations not configured") } +func (m *MockBackend) UpdateRawUnitOwningStorageGUIDs(unitID string, contents []byte) error { + if m.UpdateRawUnitOwningStorageGUIDsFunc != nil { + return m.UpdateRawUnitOwningStorageGUIDsFunc(unitID, contents) + } + return fmt.Errorf("MockBackend.UpdateRawUnitOwningStorageGUIDs not configured") +} + // --------------------------------------------------------------------------- // MetadataBackend // --------------------------------------------------------------------------- diff --git a/mdl/backend/modelsdk/association_move_write.go b/mdl/backend/modelsdk/association_move_write.go index 322a8d87e0..30782193c4 100644 --- a/mdl/backend/modelsdk/association_move_write.go +++ b/mdl/backend/modelsdk/association_move_write.go @@ -6,6 +6,10 @@ import ( "fmt" "strings" + "go.mongodb.org/mongo-driver/v2/bson" + "go.mongodb.org/mongo-driver/v2/x/bsonx/bsoncore" + + "github.com/mendixlabs/mxcli/mdl/types" "github.com/mendixlabs/mxcli/model" "github.com/mendixlabs/mxcli/modelsdk/codec" "github.com/mendixlabs/mxcli/modelsdk/element" @@ -68,10 +72,35 @@ func crossAssocToGen(ca *domainmodel.CrossModuleAssociation) *genDm.CrossAssocia return out } -// crossAssocFromGenAssoc builds a gen CrossAssociation from a (regular) gen -// Association being converted during a cross-module move. parentID is the local -// FROM entity; childRef is the remote TO entity's qualified name. +// crossAssocFromGenAssoc converts a (regular) gen Association into the +// CrossAssociation a cross-module move turns it into. parentID is the local FROM +// entity; childRef is the remote TO entity's qualified name. +// +// It prefers a RAW transform of the stored document (crossAssocRawFromAssoc), +// because the conversion is a re-addressing of the same association and every +// property it does not touch should survive byte-for-byte — the GUID above all. +// The runtime keys the database on that GUID, and re-minting it here is what made +// the #1119 write guard refuse MOVE ENTITY for any entity in an association +// (ako/mxcli#503). +// +// The property-by-property build below is the fallback for an association with no +// stored bytes — one created earlier in the same session and moved before it was +// ever persisted. It is also the cautionary case: a hand-maintained copy list +// silently drops whatever nobody thought to add, which is how it lost the view +// source once (see the Source arm) and the GUID until #503. func crossAssocFromGenAssoc(a *genDm.Association, parentID, childRef string) *genDm.CrossAssociation { + if raw, ok := crossAssocRawFromAssoc(a, childRef); ok { + out := genDm.NewCrossAssociation() + // Clean element: SetRaw + InitFromRaw bind the properties without dirtying + // any, so the encoder's existing-element path passes the whole document + // through verbatim. The registered EmitGUID/NullFields defaults apply only + // to a fresh (raw == nil) element, so nothing is appended on top. + out.SetRaw(raw) + out.InitFromRaw(raw) + out.SetID(a.ID()) + return out + } + out := genDm.NewCrossAssociation() out.SetID(a.ID()) // preserve the original association's ID out.SetName(a.Name()) @@ -106,6 +135,82 @@ func crossAssocFromGenAssoc(a *genDm.Association, parentID, childRef string) *ge return out } +// crossAssocRawFromAssoc rewrites a stored DomainModels$Association document as a +// DomainModels$CrossAssociation one. Three edits, and everything else passes +// through untouched: +// +// - $Type becomes DomainModels$CrossAssociation. +// - ChildPointer (a 16-byte element id, only resolvable inside one unit) becomes +// Child, the target entity's qualified name. +// - ChildConnection and ParentConnection are dropped. They are the association +// line's on-canvas waypoints, and a cross-module association has no line to +// draw to the other module. +// +// That difference is not guessed: it is the whole difference between the two types +// in `generated/metamodel` — the arbiter when the two generated sources disagree — +// whose DomainModelsCrossAssociation declares Child, GUID, DeleteBehavior, +// Documentation, ExportLevel, Name, Owner, ParentPointer, Source, StorageFormat +// and Type, and DomainModelsAssociation the same set with ChildPointer in place of +// Child plus the two connection points. Verified against the stored key set of a +// real association: exactly those thirteen keys plus $ID and $Type. +// +// ParentPointer is kept verbatim in both move directions. When the CHILD moves the +// cross-association stays in the source unit beside its unchanged parent; when the +// PARENT moves it travels to the target unit with it. Either way the id it holds +// still resolves in the unit the document ends up in. +// +// Key ORDER follows the stored association rather than any reference +// cross-association, because there is no Studio Pro-authored one to pin against +// here. That is sound rather than a gap: mxcli already writes cross-associations +// in gen-property order today and they load, so the order of properties is not +// something Mendix's reader depends on. +// +// Returns false when the association has no stored bytes, which is the one case +// the caller must build from properties instead. +func crossAssocRawFromAssoc(a *genDm.Association, childRef string) (bson.Raw, bool) { + raw := a.Raw() + if raw == nil { + return nil, false + } + elems, err := bsoncore.Document(raw).Elements() + if err != nil { + return nil, false + } + + out := make(bson.D, 0, len(elems)) + child := false + for _, e := range elems { + switch e.Key() { + case "$Type": + out = append(out, bson.E{Key: "$Type", Value: "DomainModels$CrossAssociation"}) + case "ChildPointer": + out = append(out, bson.E{Key: "Child", Value: childRef}) + child = true + case "ChildConnection", "ParentConnection": + // No line to the other module; the type declares neither. + default: + v := e.Value() + out = append(out, bson.E{ + Key: e.Key(), + Value: bson.RawValue{Type: bson.Type(v.Type), Value: v.Data}, + }) + } + } + + // Child is mandatory on the target type (no omitempty in the metamodel), so a + // source document without a ChildPointer to rename would produce a document + // missing it. Hand that case to the property build rather than emit one. + if !child { + return nil, false + } + + b, err := bson.Marshal(out) + if err != nil { + return nil, false + } + return bson.Raw(b), true +} + // deleteBehaviorToGen builds an AssociationDeleteBehavior with the given parent/ // child behaviors (its null error-message slots come from the registered default). func deleteBehaviorToGen(parent, child string) *genDm.AssociationDeleteBehavior { @@ -181,10 +286,13 @@ func (b *Backend) UpdateEnumerationRefsInAllDomainModels(oldQualifiedName, newQu // MoveEntity moves an entity from a source domain model to a target one, // converting any same-DM associations that reference it into cross-module // associations (FROM-child stays in source, FROM-parent goes to target), and -// rewriting the entity's view source / validation-rule attribute refs to the new -// module. Mirrors the legacy MoveEntity. Returns the names of converted -// associations as warnings. -func (b *Backend) MoveEntity(entity *domainmodel.Entity, sourceDMID, targetDMID model.ID, sourceModuleName, targetModuleName string) ([]string, error) { +// rewriting the entity's view source, validation-rule attribute refs and +// access-rule member refs to the new module. +// +// Returns one entry per converted association, carrying the qualified name it had +// and the one it has afterwards, so the caller can sweep references from the names +// instead of deriving them — the two move directions differ (see MovedAssociation). +func (b *Backend) MoveEntity(entity *domainmodel.Entity, sourceDMID, targetDMID model.ID, sourceModuleName, targetModuleName string) ([]types.MovedAssociation, error) { if entity == nil { return nil, fmt.Errorf("MoveEntity: nil entity") } @@ -208,21 +316,30 @@ func (b *Backend) MoveEntity(entity *domainmodel.Entity, sourceDMID, targetDMID } } - // Remove the moved entity from the source DM. - removed := false + // Remove the moved entity from the source DM, keeping the stored element so + // its identities can be carried onto the rebuild that lands in the target. + var orig *genDm.Entity for i, el := range sourceDM.EntitiesItems() { - if string(el.ID()) == string(entity.ID) { - sourceDM.RemoveEntities(i) - removed = true - break + if string(el.ID()) != string(entity.ID) { + continue } + orig, _ = el.(*genDm.Entity) + sourceDM.RemoveEntities(i) + break } - if !removed { + if orig == nil { return nil, fmt.Errorf("entity not found in source domain model: %s", entity.ID) } // Convert associations referencing the moved entity to cross-associations. - var converted []string + // + // Each conversion records the qualified name the association had and the one it + // has afterwards, because Mendix stores an association in the module of its FROM + // entity and the two directions therefore differ: the parent moving takes the + // cross-association to the target module, the child moving leaves it in the + // source. The caller sweeps references from these names, so the case that does + // not move is a no-op rather than a branch it has to know about (#605). + var converted []types.MovedAssociation var removeIdx []int for i, el := range sourceDM.AssociationsItems() { a, ok := el.(*genDm.Association) @@ -230,15 +347,21 @@ func (b *Backend) MoveEntity(entity *domainmodel.Entity, sourceDMID, targetDMID continue } parentID, childID := string(a.ParentRefID()), string(a.ChildRefID()) + moved := types.MovedAssociation{ + Name: a.Name(), + OldQualifiedName: sourceModuleName + "." + a.Name(), + NewQualifiedName: sourceModuleName + "." + a.Name(), + } switch { case childID == string(entity.ID): // child moved → cross-assoc stays in source sourceDM.AddCrossAssociations(crossAssocFromGenAssoc(a, parentID, targetModuleName+"."+entity.Name)) removeIdx = append(removeIdx, i) - converted = append(converted, a.Name()) + converted = append(converted, moved) case parentID == string(entity.ID): // parent moved → cross-assoc goes to target targetDM.AddCrossAssociations(crossAssocFromGenAssoc(a, parentID, sourceModuleName+"."+nameByID[childID])) removeIdx = append(removeIdx, i) - converted = append(converted, a.Name()) + moved.NewQualifiedName = targetModuleName + "." + a.Name() + converted = append(converted, moved) } } for i := len(removeIdx) - 1; i >= 0; i-- { @@ -256,14 +379,62 @@ func (b *Backend) MoveEntity(entity *domainmodel.Entity, sourceDMID, targetDMID } } + // The same re-pointing for the entity's own ACCESS RULES, which name each member + // by qualified name and were the missed sibling of the validation rules above + // (#605). Leaving them stale is worse than a dangling string: entityToGen's + // syncMemberAccesses matches existing entries by qualified name, so a stale + // `Source.Entity.Attr` never equals the rebuilt `Target.Entity.Attr` and it + // appends the new one while keeping the old — the moved entity ends up carrying + // every member twice, half of the entries dangling. DESCRIBE renders members + // bare, so the duplication is the only thing that shows. + // + // An ATTRIBUTE reference always follows the entity, so its module prefix is + // rewritten unconditionally. An ASSOCIATION reference is rewritten only for the + // associations this move actually sent to the target module: a pre-existing + // cross-association whose parent is the moved entity is not in the conversion + // list and does not travel, so a blanket prefix swap would break it. + assocRenames := make(map[string]string, len(converted)) + for _, m := range converted { + if m.Moved() { + assocRenames[m.OldQualifiedName] = m.NewQualifiedName + } + } + for _, ar := range entity.AccessRules { + for _, ma := range ar.MemberAccesses { + if strings.HasPrefix(ma.AttributeName, oldPrefix) { + ma.AttributeName = newPrefix + ma.AttributeName[len(oldPrefix):] + } + if to, ok := assocRenames[ma.AssociationName]; ok { + ma.AssociationName = to + } + } + } + if err := b.persistDM(sourceDMID, sourceDM); err != nil { return nil, fmt.Errorf("MoveEntity: persist source: %w", err) } - // Add the (rebuilt) entity to the target DM. + // Add the (rebuilt) entity to the target DM, carrying the identities the + // rebuild has no business re-minting — the same two carries UpdateEntity does, + // which MoveEntity never got (#657 for the entity, #1119 for its children). + // + // Nothing in the entity's unmodeled properties goes stale when the module + // changes, which is what makes a blanket raw carry safe here as well as there. + // `Image` is a qualified name pointing at an image document that does NOT move + // with the entity, so keeping it is correct rather than stale — and it is + // unmodeled, so without this carry a move silently drops an entity's + // domain-model image as well as its GUID. + // Everything that does need rewriting is modeled and therefore dirty: + // `MaybeGeneralization`, `Location`, `ExportLevel` and the member lists are all + // re-encoded from the rebuild, and `Source`/`ValidationRules` were re-pointed on + // `entity` just above. ge := entityToGen(entity, targetModuleName, b.majorVersion()) ge.SetID(element.ID(entity.ID)) assignEntityIDs(ge) + if raw := orig.Raw(); raw != nil { + ge.SetRaw(raw) + } + carryChildIdentity(ge, orig, entity) targetDM.AddEntities(ge) if err := b.persistDM(targetDMID, targetDM); err != nil { return nil, fmt.Errorf("MoveEntity: persist target: %w", err) diff --git a/mdl/backend/modelsdk/create_or_modify_entity_guid_test.go b/mdl/backend/modelsdk/create_or_modify_entity_guid_test.go new file mode 100644 index 0000000000..e0802c0789 --- /dev/null +++ b/mdl/backend/modelsdk/create_or_modify_entity_guid_test.go @@ -0,0 +1,238 @@ +// SPDX-License-Identifier: Apache-2.0 + +package modelsdkbackend + +import ( + "testing" + + "github.com/mendixlabs/mxcli/sdk/domainmodel" +) + +// The shape `CREATE OR MODIFY PERSISTENT ENTITY` hands to UpdateEntity is not the +// shape every ALTER hands it, and the difference is the whole defect. +// +// `mergeDeclaredOntoStoredEntity` sets `merged.Attributes = declared.Attributes` and +// `merged.Indexes = declared.Indexes` — the members the STATEMENT declares, built from +// text by the visitor, carrying **no element ID at all**. carryChildIdentity keyed +// entirely on that ID and read an empty one as "a genuinely new member, so a fresh +// GUID is right", so re-declaring an existing entity re-minted every attribute GUID +// and dropped every column on the next deploy. +// +// Same data loss as #1119 through a different executor path. The write guard is what +// surfaced it, as a REFUSAL on a doctype script that had been passing for months: +// +// failed to update entity: refusing to write unit d82b0484-…: 1 element(s) kept +// their $ID but would be written with a different GUID +// dff2ced1-… (DomainModels$Attribute): stored 4b52b36b-…, would write dff2ced1-… +// +// Note WHY the refused element still had the stored `$ID`, because that is the part +// that makes the report look impossible: the fresh element is encoded with +// `GUID = $ID` (EmitGUID), and `canon.TransplantIDs` then substitutes the stored `$ID` +// over *every* 16-byte binary in the document — the GUID field included. The +// corruption therefore arrives wearing the correct `$ID`, which is exactly the +// pairing the guard uses, and is why the guard could see it at all. +// +// The fix is to fall back to the attribute NAME when there is no ID. A name is unique +// within an entity, so a stored attribute of that name IS the same member — which is +// what Studio Pro assumes when a re-declared attribute keeps its column. + +// TestCreateOrModifyEntity_PreservesAttributeGUIDsWithoutSemanticIDs is the arm the CI +// failure measured: attributes declared by the statement, so without IDs. +func TestCreateOrModifyEntity_PreservesAttributeGUIDsWithoutSemanticIDs(t *testing.T) { + proj := copyFixture(t) + b := New() + if err := b.Connect(proj); err != nil { + t.Fatalf("connect: %v", err) + } + + dmID, ent := richestEntity(t, b) + before := attributeGUIDs(t, b, dmID, ent.ID) + if len(before) < 3 { + t.Fatalf("fixture entity %s has %d attributes; need >= 3", ent.Name, len(before)) + } + // Without GUID != $ID this cannot fail against the broken code: an element mxcli + // created has them equal from birth, so re-minting reproduces the same value. + ids := attributeIDs(t, b, dmID, ent.ID) + for name, guid := range before { + if guid == "" || guid == ids[name] { + t.Fatalf("fixture attribute %s has GUID %q and $ID %q; need them to differ", name, guid, ids[name]) + } + } + + // What mergeDeclaredOntoStoredEntity produces: the stored entity with the + // statement's member list swapped in, and that list carries no IDs. + for _, a := range ent.Attributes { + a.ID = "" + } + + if err := b.UpdateEntity(dmID, ent); err != nil { + // A refusal here IS the failure: the guard firing because the carry did not + // happen. It is not an unrelated error. + t.Fatalf("UpdateEntity with statement-declared attributes: %v", err) + } + if err := b.Disconnect(); err != nil { + t.Fatalf("disconnect: %v", err) + } + + // Reopen: preservation only counts if it reached disk. + b2 := New() + if err := b2.Connect(proj); err != nil { + t.Fatalf("reconnect: %v", err) + } + t.Cleanup(func() { _ = b2.Disconnect() }) + + after := attributeGUIDs(t, b2, dmID, ent.ID) + moved := 0 + for name, want := range before { + got, ok := after[name] + if !ok { + t.Errorf("attribute %s is gone after the re-declaration", name) + continue + } + if got != want { + moved++ + if moved <= 3 { + t.Errorf("attribute %s: storage GUID changed %s -> %s", name, want, got) + } + } + } + if moved > 3 { + t.Errorf("... and %d more attributes whose storage GUID changed", moved-3) + } +} + +// TestCreateOrModifyEntity_NewAttributeStillGetsAFreshGUID is the control for the name +// fallback, and the reason the fallback claims each stored attribute at most once: an +// attribute the statement genuinely introduces must NOT inherit an existing or removed +// member's GUID, or the runtime hands it that column's data. +func TestCreateOrModifyEntity_NewAttributeStillGetsAFreshGUID(t *testing.T) { + proj := copyFixture(t) + b := New() + if err := b.Connect(proj); err != nil { + t.Fatalf("connect: %v", err) + } + + dmID, ent := richestEntity(t, b) + before := attributeGUIDs(t, b, dmID, ent.ID) + + const added = "CreateOrModifyAdded" + for _, a := range ent.Attributes { + a.ID = "" + } + ent.Attributes = append(ent.Attributes, &domainmodel.Attribute{ + Name: added, + Type: &domainmodel.StringAttributeType{Length: 20}, + }) + + if err := b.UpdateEntity(dmID, ent); err != nil { + t.Fatalf("UpdateEntity: %v", err) + } + if err := b.Disconnect(); err != nil { + t.Fatalf("disconnect: %v", err) + } + + b2 := New() + if err := b2.Connect(proj); err != nil { + t.Fatalf("reconnect: %v", err) + } + t.Cleanup(func() { _ = b2.Disconnect() }) + + after := attributeGUIDs(t, b2, dmID, ent.ID) + got := after[added] + if got == "" { + t.Fatalf("new attribute %s has no storage GUID", added) + } + for name, old := range before { + if old == got { + t.Errorf("new attribute %s inherited %s's storage GUID %s — the runtime would hand it that column's data", + added, name, got) + } + } + // And the pre-existing ones still kept theirs, so the fallback did not spend its + // match on the newcomer. + for name, want := range before { + if after[name] != want { + t.Errorf("attribute %s: storage GUID changed %s -> %s", name, want, after[name]) + } + } +} + +// TestCreateOrModifyEntity_PreservesIndexGUIDsWithoutSemanticIDs is the index arm. +// `merged.Indexes = declared.Indexes` strips index IDs the same way, and an index has +// no name to fall back to — it is addressed by position, which is the correspondence +// the ID-bearing path already trusts (entityToGen appends one gen index per semantic +// index, in order). +// +// No data rides on an index GUID — the platform rebuilds the index — so the cost of a +// wrong pairing here is a rebuild, not a lost column. What a missing carry costs is +// the write itself: the guard refuses it, and `CREATE OR MODIFY` on an indexed entity +// stops working. +func TestCreateOrModifyEntity_PreservesIndexGUIDsWithoutSemanticIDs(t *testing.T) { + proj := copyFixture(t) + + b := New() + if err := b.Connect(proj); err != nil { + t.Fatalf("connect: %v", err) + } + dmID, _ := richestEntity(t, b) + const entName = "CreateOrModifyIndexed" + if err := b.CreateEntity(dmID, &domainmodel.Entity{ + Name: entName, + Persistable: true, + Attributes: []*domainmodel.Attribute{ + {Name: "Code", Type: &domainmodel.StringAttributeType{Length: 50}}, + {Name: "Descr", Type: &domainmodel.StringAttributeType{Length: 50}}, + }, + }); err != nil { + t.Fatalf("CreateEntity: %v", err) + } + stored := mustFindEntity(t, b, dmID, entName) + stored.Indexes = []*domainmodel.Index{{ + Attributes: []*domainmodel.IndexAttribute{ + {AttributeID: stored.Attributes[0].ID, Ascending: true}, + }, + }} + if err := b.UpdateEntity(dmID, stored); err != nil { + t.Fatalf("UpdateEntity (add index): %v", err) + } + if err := b.Disconnect(); err != nil { + t.Fatalf("disconnect: %v", err) + } + + // An index mxcli created has GUID == $ID legitimately and cannot detect the + // conflation, so seed the divergence the way Studio Pro leaves it, below the + // writer (which would rightly refuse a write that moves a GUID). + seeded := patchIndexGUID(t, proj, dmID, entName) + + b2 := New() + if err := b2.Connect(proj); err != nil { + t.Fatalf("reconnect: %v", err) + } + target := mustFindEntity(t, b2, dmID, entName) + for _, a := range target.Attributes { + a.ID = "" + } + for _, i := range target.Indexes { + i.ID = "" + } + if err := b2.UpdateEntity(dmID, target); err != nil { + t.Fatalf("UpdateEntity with statement-declared members: %v", err) + } + if err := b2.Disconnect(); err != nil { + t.Fatalf("disconnect: %v", err) + } + + b3 := New() + if err := b3.Connect(proj); err != nil { + t.Fatalf("reconnect: %v", err) + } + t.Cleanup(func() { _ = b3.Disconnect() }) + + got := indexGUIDs(t, b3, dmID, entName) + if len(got) != 1 { + t.Fatalf("expected 1 index after the re-declaration, got %d", len(got)) + } + if got[0] != seeded { + t.Errorf("index storage GUID changed %s -> %s", seeded, got[0]) + } +} diff --git a/mdl/backend/modelsdk/domainmodel_alter.go b/mdl/backend/modelsdk/domainmodel_alter.go index ccd4851f06..fd7c75c014 100644 --- a/mdl/backend/modelsdk/domainmodel_alter.go +++ b/mdl/backend/modelsdk/domainmodel_alter.go @@ -183,6 +183,22 @@ func (b *Backend) UpdateEntity(domainModelID model.ID, entity *domainmodel.Entit ge.SetRaw(raw) } + // The same carry, one level down, for the entity's GUID-bearing CHILDREN. + // #657 closed this for the entity element and noted that siblings survive via + // the list-rebuild raw passthrough — but the target's own children do not: + // entityToGen rebuilds every attribute and index from the semantic model, so + // each arrives raw==nil and the codec's EmitGUID default writes GUID = $ID. + // + // For an attribute that GUID is the database's identity, not a cross-reference: + // the runtime keys mendixsystem$attribute.id on it, so re-minting it makes the + // synchroniser treat every column as deleted-and-re-added and DROP it on the + // next deploy. Measured on a real project: one ALTER, 0 of 28 GUIDs surviving, + // all 607 rows' attribute values gone (issue #1119). Nothing caught it — the + // model stays valid, mx check is clean, and because the new GUID is derived + // from a now-stable $ID the damage is idempotent, so a second run is elided + // and reports "Unchanged". + carryChildIdentity(ge, orig, entity) + // When an update empties a child list, the fresh (empty) list on ge is "clean" // — entityToGen appended nothing to it — so the codec passes the STORED raw // bytes through unchanged and the removal silently does not happen. Touching @@ -249,10 +265,22 @@ func (b *Backend) UpdateEntity(domainModelID model.ID, entity *domainmodel.Entit // UpdateDomainModel persists a whole mutated domain model (the executor's // read-modify-write path for ALTER ASSOCIATION, CREATE OR MODIFY ASSOCIATION, // and RENAME). It rebuilds the Entities and Associations lists from the semantic -// model via the byte-faithful converters, preserving each element's identity. +// model via the byte-faithful converters, carrying each element's stored identity +// — both its $ID and its storage GUID — onto the rebuild. // CrossAssociations and Annotations are NOT represented in domainmodel.DomainModel, // so they are left as gen passthrough rather than dropped (ADR-0005: guard // fidelity — the existing raw bytes carry forward unchanged). +// +// The GUID half of that was missing until ako/mxcli#1169. Unlike UpdateEntity, +// which swaps one entity into an otherwise raw-passthrough list, this rebuilds +// EVERY entity and EVERY association — so every element arrived raw==nil and the +// codec's EmitGUID default wrote GUID = $ID across the whole unit, including +// elements the statement never named. The reporter measured 282 moved GUIDs in one +// module. What that costs is in the carry comments below; the short version is +// that the runtime keys the database on the GUID, so re-minting one drops a column +// (an attribute) or a whole table (an entity, whose name is the table name — which +// is why RENAME ENTITY routing through here was the worst of the affected +// statements). func (b *Backend) UpdateDomainModel(dm *domainmodel.DomainModel) error { if dm == nil { return fmt.Errorf("UpdateDomainModel: nil domain model") @@ -267,6 +295,25 @@ func (b *Backend) UpdateDomainModel(dm *domainmodel.DomainModel) error { moduleName := b.moduleNameFor(dm.ID) major := b.majorVersion() + // Index the stored elements by $ID BEFORE the removal loops empty the lists. + // The $ID is the right key because the read path round-trips it into the + // semantic model, so it is exact rather than structural: a RENAME keeps the ID + // and carries the identity forward (Studio Pro renames the table or column and + // keeps the data), while a genuinely new element arrives with an empty ID, + // matches nothing, and correctly gets a fresh GUID. + storedEntities := make(map[string]*genDm.Entity, len(gdm.EntitiesItems())) + for _, el := range gdm.EntitiesItems() { + if ge, ok := el.(*genDm.Entity); ok { + storedEntities[string(ge.ID())] = ge + } + } + storedAssociations := make(map[string]*genDm.Association, len(gdm.AssociationsItems())) + for _, el := range gdm.AssociationsItems() { + if ga, ok := el.(*genDm.Association); ok { + storedAssociations[string(ga.ID())] = ga + } + } + for i := len(gdm.EntitiesItems()) - 1; i >= 0; i-- { gdm.RemoveEntities(i) } @@ -274,6 +321,18 @@ func (b *Backend) UpdateDomainModel(dm *domainmodel.DomainModel) error { ge := entityToGen(e, moduleName, major) ge.SetID(element.ID(e.ID)) assignEntityIDs(ge) + // Carry the stored raw bytes so the codec treats the rebuild as an EXISTING + // element: the properties entityToGen set re-encode, while the ones the + // semantic model does not carry — the GUID above all — pass through verbatim. + // This is #657's carry (for the entity) and #1119's (for its attributes and + // indexes) applied to a path that had neither, because it rebuilds the whole + // list instead of swapping one member into it (#1169). + if orig := storedEntities[string(e.ID)]; orig != nil { + if raw := orig.Raw(); raw != nil { + ge.SetRaw(raw) + } + carryChildIdentity(ge, orig, e) + } gdm.AddEntities(ge) } @@ -286,6 +345,20 @@ func (b *Backend) UpdateDomainModel(dm *domainmodel.DomainModel) error { ga.SetID(element.ID(a.ID)) } assignAssociationIDs(ga) + // The same carry for associations. A direct SetRaw is enough here, where the + // cross-module MOVE needed a raw transform (crossAssocRawFromAssoc, #503): + // assocToGen is Association -> Association, so the stored $Type and the + // rebuilt one agree and nothing has to be rewritten on the way. + // + // It also subsumes the property-by-property patch #872 made for the line + // anchors: those were lost to this same "the rebuild only carries what the + // semantic model models" mechanism, and raw passthrough covers the whole + // class rather than one property of it. + if orig := storedAssociations[string(a.ID)]; orig != nil { + if raw := orig.Raw(); raw != nil { + ga.SetRaw(raw) + } + } gdm.AddAssociations(ga) } diff --git a/mdl/backend/modelsdk/domainmodel_child_identity.go b/mdl/backend/modelsdk/domainmodel_child_identity.go new file mode 100644 index 0000000000..51fdef46c7 --- /dev/null +++ b/mdl/backend/modelsdk/domainmodel_child_identity.go @@ -0,0 +1,165 @@ +// SPDX-License-Identifier: Apache-2.0 + +package modelsdkbackend + +import ( + genDm "github.com/mendixlabs/mxcli/modelsdk/gen/domainmodels" + "github.com/mendixlabs/mxcli/sdk/domainmodel" +) + +// carryChildIdentity copies each stored child's raw bytes onto its rebuilt +// counterpart, so the codec treats the child as an EXISTING element: the +// properties entityToGen set re-encode from the rebuild, while the ones the +// semantic model does not carry — the GUID above all — pass through verbatim. +// +// Only the two children that carry a GUID are covered. Attributes and indexes +// are the entity's GUID-bearing children (`RegisterTypeDefaults` in +// domainmodel_write.go registers EmitGUID for DomainModels$Attribute and +// DomainModels$EntityIndex); access rules, validation rules and event handlers +// have no GUID, so there is no identity to carry and raw-carrying them would +// only risk resurrecting a property a rewrite means to clear. +// +// The correspondence is the stored $ID, which the read path round-trips into the +// semantic model (attributeFromGen / indexFromGen set .ID). That is what makes +// this exact rather than structural: the executor preserves an attribute's ID +// across a RENAME, so a rename carries the GUID forward — Studio Pro renames the +// column and keeps the data — while a genuinely new attribute arrives with an +// empty ID, matches nothing, and correctly gets a fresh GUID. +// +// Pairing by name (not by list position) for attributes is deliberate: names are +// unique within an entity, so the lookup does not depend on entityToGen and the +// semantic list staying in lockstep. Indexes have no name, so they pair by +// position — which holds because entityToGen appends one gen index per semantic +// index in the same loop. +// +// THE SEMANTIC ID IS NOT ALWAYS THERE, and reading its absence as "a new member" +// was a second instance of #1119 rather than a safe default. +// `CREATE OR MODIFY ENTITY` reaches UpdateEntity through +// mergeDeclaredOntoStoredEntity, which sets `Attributes` and `Indexes` to the lists +// the STATEMENT declares — built from text by the visitor, carrying no ID at all — +// so an ID-only pairing carried nothing and every attribute of a re-declared entity +// was re-minted. The write guard caught it as a refusal on a doctype script that had +// been passing for months; a stored attribute of the same NAME is the same member, +// which is what Studio Pro assumes when a re-declared attribute keeps its column. +// +// So each list is paired in two passes: the exact key first, the weaker one only for +// what it left over. A stored element is claimed at most once, which is what keeps a +// rename-plus-re-add from handing the newcomer the renamed member's data — the ID +// match takes the stored element, and the name fallback then finds it claimed. +func carryChildIdentity(ge, orig *genDm.Entity, entity *domainmodel.Entity) { + if ge == nil || orig == nil || entity == nil { + return + } + carryAttributeIdentity(ge, orig, entity) + carryIndexIdentity(ge, orig, entity) +} + +func carryAttributeIdentity(ge, orig *genDm.Entity, entity *domainmodel.Entity) { + storedByID := map[string]*genDm.Attribute{} + storedByName := map[string]*genDm.Attribute{} + for _, el := range orig.AttributesItems() { + if a, ok := el.(*genDm.Attribute); ok && a.Raw() != nil { + storedByID[string(a.ID())] = a + storedByName[a.Name()] = a + } + } + if len(storedByID) == 0 { + return + } + + // The semantic attribute the rebuild came from, by its (possibly new) name. + semantic := make(map[string]*domainmodel.Attribute, len(entity.Attributes)) + for _, a := range entity.Attributes { + semantic[a.Name] = a + } + + var rebuilt []*genDm.Attribute + for _, el := range ge.AttributesItems() { + if ga, ok := el.(*genDm.Attribute); ok { + rebuilt = append(rebuilt, ga) + } + } + + claimed := make(map[string]bool, len(storedByID)) + carry := func(ga, sa *genDm.Attribute) { + ga.SetID(sa.ID()) + ga.SetRaw(sa.Raw()) + claimed[string(sa.ID())] = true + } + + // Pass 1 — the exact key. The executor preserves an attribute's ID across a + // RENAME, so this is what carries the GUID onto the new name. + paired := make(map[*genDm.Attribute]bool, len(rebuilt)) + for _, ga := range rebuilt { + sem := semantic[ga.Name()] + if sem == nil || sem.ID == "" { + continue + } + sa := storedByID[string(sem.ID)] + if sa == nil { + continue + } + carry(ga, sa) + paired[ga] = true + } + + // Pass 2 — by name, for what pass 1 left. This is the statement-declared case, + // where there is no ID to pair on. A stored attribute pass 1 already claimed is + // skipped, so an attribute genuinely introduced under a name another member has + // just vacated still gets a fresh GUID rather than that member's data. + for _, ga := range rebuilt { + if paired[ga] { + continue + } + sa := storedByName[ga.Name()] + if sa == nil || claimed[string(sa.ID())] { + continue // genuinely new: nothing to carry, a fresh GUID is right + } + carry(ga, sa) + } +} + +func carryIndexIdentity(ge, orig *genDm.Entity, entity *domainmodel.Entity) { + storedByID := map[string]*genDm.Index{} + var storedInOrder []*genDm.Index + for _, el := range orig.IndexesItems() { + if idx, ok := el.(*genDm.Index); ok && idx.Raw() != nil { + storedByID[string(idx.ID())] = idx + storedInOrder = append(storedInOrder, idx) + } + } + if len(storedByID) == 0 { + return + } + + rebuilt := ge.IndexesItems() + claimed := make(map[string]bool, len(storedByID)) + for i, sem := range entity.Indexes { + if i >= len(rebuilt) { + break + } + gi, ok := rebuilt[i].(*genDm.Index) + if !ok || sem == nil { + continue + } + var si *genDm.Index + switch { + case sem.ID != "": + si = storedByID[string(sem.ID)] + case i < len(storedInOrder): + // The statement-declared case: no ID to pair on, and an index has no name + // either, so position is what is left. It is the same correspondence the + // ID-bearing branch already trusts, and it is the weaker one — a reordered + // or dropped index pairs the wrong way round. That costs an index rebuild + // and no data, because nothing the platform keys on rides on an index + // GUID; what a MISSING carry costs is the write, which the guard refuses. + si = storedInOrder[i] + } + if si == nil || claimed[string(si.ID())] { + continue + } + gi.SetID(si.ID()) + gi.SetRaw(si.Raw()) + claimed[string(si.ID())] = true + } +} diff --git a/mdl/backend/modelsdk/issue1119_attribute_guid_test.go b/mdl/backend/modelsdk/issue1119_attribute_guid_test.go new file mode 100644 index 0000000000..ec751ac5c9 --- /dev/null +++ b/mdl/backend/modelsdk/issue1119_attribute_guid_test.go @@ -0,0 +1,301 @@ +// SPDX-License-Identifier: Apache-2.0 + +package modelsdkbackend + +import ( + "testing" + + "github.com/mendixlabs/mxcli/model" + genDm "github.com/mendixlabs/mxcli/modelsdk/gen/domainmodels" + "github.com/mendixlabs/mxcli/sdk/domainmodel" +) + +// TestIssue1119_AlterPreservesAttributeGUIDs guards GitHub issue #1119: an ALTER +// of an existing entity must not change any attribute's storage GUID. +// +// #657 closed this for the entity element. Its children were left behind: +// entityToGen rebuilds every attribute from the semantic model, so each arrived +// raw==nil and the codec's EmitGUID default wrote GUID = $ID. The runtime keys +// mendixsystem$attribute.id on that GUID, so on the next deploy the synchroniser +// treats every attribute as deleted-and-re-added and drops its column — reported +// from production as 28 attributes of 607 rows emptied by a single MDL edit. +// +// Every ALTER ENTITY form routes through Backend.UpdateEntity (cmd_entities.go), +// so exercising that covers all of them — including SET DOCUMENTATION, which +// touches no attribute at all and destroyed every GUID anyway. +func TestIssue1119_AlterPreservesAttributeGUIDs(t *testing.T) { + cases := []struct { + name string + // mutate applies one ALTER form. newAttr names an attribute the mutation + // creates, and renamedFrom/renamedTo the pair a rename produces. + mutate func(*domainmodel.Entity) + newAttr string + renamedTo string + renamedFrom int // index into the entity's attributes, before mutation + }{ + { + name: "SetDocumentation", + mutate: func(e *domainmodel.Entity) { e.Documentation = "issue 1119" }, + }, + { + name: "AddAttribute", + mutate: func(e *domainmodel.Entity) { + e.Attributes = append(e.Attributes, &domainmodel.Attribute{ + Name: "Issue1119Added", + Type: &domainmodel.StringAttributeType{Length: 20}, + }) + }, + newAttr: "Issue1119Added", + }, + { + name: "RenameAttribute", + mutate: func(e *domainmodel.Entity) { e.Attributes[0].Name = "Issue1119Renamed" }, + renamedTo: "Issue1119Renamed", + renamedFrom: 0, + }, + { + name: "DropAttribute", + mutate: func(e *domainmodel.Entity) { e.Attributes = e.Attributes[1:] }, + }, + { + name: "ModifyAttribute", + mutate: func(e *domainmodel.Entity) { + e.Attributes[1].Type = &domainmodel.StringAttributeType{Length: 999} + }, + }, + { + name: "RenameEntity", + mutate: func(e *domainmodel.Entity) { e.Name = "Issue1119Renamed" + e.Name }, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + proj := copyFixture(t) + b := New() + if err := b.Connect(proj); err != nil { + t.Fatalf("connect: %v", err) + } + + dmID, ent := richestEntity(t, b) + before := attributeGUIDs(t, b, dmID, ent.ID) + if len(before) < 3 { + t.Fatalf("fixture entity %s has %d attributes; need >= 3", ent.Name, len(before)) + } + // Without GUID != $ID the test cannot detect the conflation at all — + // it would pass against the broken code. Same precondition as #657's. + ids := attributeIDs(t, b, dmID, ent.ID) + for name, guid := range before { + if guid == "" { + t.Fatalf("fixture attribute %s has no GUID; cannot verify preservation", name) + } + if guid == ids[name] { + t.Fatalf("fixture attribute %s has GUID == $ID; need them to differ to detect #1119", name) + } + } + originalName := ent.Attributes[tc.renamedFrom].Name + + tc.mutate(ent) + if err := b.UpdateEntity(dmID, ent); err != nil { + t.Fatalf("UpdateEntity: %v", err) + } + if err := b.Disconnect(); err != nil { + t.Fatalf("disconnect: %v", err) + } + + // Reopen: GUID preservation only counts if it reached disk. + b2 := New() + if err := b2.Connect(proj); err != nil { + t.Fatalf("reconnect: %v", err) + } + t.Cleanup(func() { _ = b2.Disconnect() }) + after := attributeGUIDs(t, b2, dmID, ent.ID) + + // Every attribute that survived under its own name keeps its GUID. + changed := 0 + for name, want := range before { + got, ok := after[name] + if !ok { + continue // dropped or renamed; handled below + } + if got != want { + changed++ + if changed <= 3 { + t.Errorf("attribute %s: storage GUID changed %s -> %s", name, want, got) + } + } + } + if changed > 3 { + t.Errorf("... and %d more attributes whose storage GUID changed", changed-3) + } + + // A rename must carry the GUID FORWARD: Studio Pro renames the column + // and keeps the data. Minting a fresh one here loses it just as surely + // as changing an untouched attribute's. + if tc.renamedTo != "" { + if got, want := after[tc.renamedTo], before[originalName]; got != want { + t.Errorf("rename %s -> %s did not carry the storage GUID: %s -> %s", + originalName, tc.renamedTo, want, got) + } + } + + // And a genuinely new attribute must get a GUID of its own — not an + // empty one, and not one inherited from an existing or dropped + // attribute, which would make the runtime adopt that column. + if tc.newAttr != "" { + got := after[tc.newAttr] + if got == "" { + t.Errorf("new attribute %s has no storage GUID", tc.newAttr) + } + for name, old := range before { + if old == got { + t.Errorf("new attribute %s inherited %s's storage GUID %s", tc.newAttr, name, got) + } + } + } + }) + } +} + +// TestIssue1119_AlterPreservesAssociationGUIDs is the control for the reporter's +// own negative result: a sibling association in the same unit was already safe, +// because the entities-list rebuild passes it through as raw. It is here so a +// future change that starts rebuilding DM-level children is caught with the +// attribute case rather than after it. +func TestIssue1119_AlterPreservesAssociationGUIDs(t *testing.T) { + proj := copyFixture(t) + b := New() + if err := b.Connect(proj); err != nil { + t.Fatalf("connect: %v", err) + } + + // Find a domain model that holds both an association and an entity to alter. + dms, err := b.ListDomainModels() + if err != nil { + t.Fatalf("ListDomainModels: %v", err) + } + var ( + dmID model.ID + ent *domainmodel.Entity + ) + for _, d := range dms { + gdm, err := b.loadDomainModelGen(d.ID) + if err != nil || len(gdm.AssociationsItems()) == 0 { + continue + } + for _, e := range d.Entities { + if len(e.Attributes) > 0 { + dmID, ent = d.ID, e + break + } + } + if ent != nil { + break + } + } + if ent == nil { + t.Skip("no domain model in the fixture holds both an association and an entity with attributes") + } + + before := associationGUIDs(t, b, dmID) + if len(before) == 0 { + t.Fatal("no associations read back; the control cannot prove anything") + } + + ent.Documentation = "issue 1119 association control" + if err := b.UpdateEntity(dmID, ent); err != nil { + t.Fatalf("UpdateEntity: %v", err) + } + if err := b.Disconnect(); err != nil { + t.Fatalf("disconnect: %v", err) + } + + b2 := New() + if err := b2.Connect(proj); err != nil { + t.Fatalf("reconnect: %v", err) + } + t.Cleanup(func() { _ = b2.Disconnect() }) + + after := associationGUIDs(t, b2, dmID) + for name, want := range before { + if got := after[name]; got != want { + t.Errorf("association %s: storage GUID changed %s -> %s", name, want, got) + } + } +} + +// richestEntity returns the loadable domain model and the entity with the most +// attributes in the fixture — the most sensitive target for a GUID check. +func richestEntity(t *testing.T, b *Backend) (model.ID, *domainmodel.Entity) { + t.Helper() + dms, err := b.ListDomainModels() + if err != nil { + t.Fatalf("ListDomainModels: %v", err) + } + var ( + dmID model.ID + best *domainmodel.Entity + ) + for _, d := range dms { + // The System module's unit is not on disk in the fixture; skip what we + // cannot read rather than failing on it. + if _, err := b.loadDomainModelGen(d.ID); err != nil { + continue + } + for _, e := range d.Entities { + if best == nil || len(e.Attributes) > len(best.Attributes) { + dmID, best = d.ID, e + } + } + } + if best == nil { + t.Fatal("no loadable domain model with entities in the fixture") + } + return dmID, best +} + +// attributeGUIDs maps attribute name -> storage GUID, read from the raw BSON. +// The GUID is not surfaced by the semantic reader, which is why #657's finding +// says to assert on raw bytes rather than on DESCRIBE. +func attributeGUIDs(t *testing.T, b *Backend, dmID, entID model.ID) map[string]string { + return attributeRawKey(t, b, dmID, entID, "GUID") +} + +// attributeIDs maps attribute name -> element $ID, for the GUID != $ID precondition. +func attributeIDs(t *testing.T, b *Backend, dmID, entID model.ID) map[string]string { + return attributeRawKey(t, b, dmID, entID, "$ID") +} + +func attributeRawKey(t *testing.T, b *Backend, dmID, entID model.ID, key string) map[string]string { + t.Helper() + gdm, err := b.loadDomainModelGen(dmID) + if err != nil { + t.Fatalf("loadDomainModelGen: %v", err) + } + ge := findGenEntity(gdm, entID) + if ge == nil { + t.Fatalf("gen entity %s not found", entID) + } + out := map[string]string{} + for _, el := range ge.AttributesItems() { + if a, ok := el.(*genDm.Attribute); ok { + out[a.Name()] = rawKeyHex(t, a.Raw(), key) + } + } + return out +} + +func associationGUIDs(t *testing.T, b *Backend, dmID model.ID) map[string]string { + t.Helper() + gdm, err := b.loadDomainModelGen(dmID) + if err != nil { + t.Fatalf("loadDomainModelGen: %v", err) + } + out := map[string]string{} + for _, el := range gdm.AssociationsItems() { + if a, ok := el.(*genDm.Association); ok { + out[a.Name()] = rawKeyHex(t, a.Raw(), "GUID") + } + } + return out +} diff --git a/mdl/backend/modelsdk/issue1119_index_guid_test.go b/mdl/backend/modelsdk/issue1119_index_guid_test.go new file mode 100644 index 0000000000..9ad7de6741 --- /dev/null +++ b/mdl/backend/modelsdk/issue1119_index_guid_test.go @@ -0,0 +1,286 @@ +// SPDX-License-Identifier: Apache-2.0 + +package modelsdkbackend + +import ( + "bytes" + "crypto/sha256" + "database/sql" + "encoding/base64" + "encoding/hex" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/mendixlabs/mxcli/model" + genDm "github.com/mendixlabs/mxcli/modelsdk/gen/domainmodels" + "github.com/mendixlabs/mxcli/sdk/domainmodel" + "go.mongodb.org/mongo-driver/v2/bson" +) + +// An entity INDEX carries a GUID of its own (RegisterTypeDefaults registers +// EmitGUID for DomainModels$EntityIndex), and entityToGen rebuilds it from the +// semantic model exactly as it rebuilds an attribute — so it has the same #1119 +// exposure, and carrying it is load-bearing for a second reason: without the +// carry, the storage-GUID guard would refuse every ALTER on an indexed entity. +// +// The fixture holds no Studio Pro-authored index, and one mxcli creates has +// GUID == $ID legitimately, which cannot detect the conflation. So the stored +// state is seeded the way Studio Pro leaves it — the index's GUID made to differ +// from its $ID — by patching the unit on disk, below the writer. +func TestIssue1119_AlterPreservesIndexGUIDs(t *testing.T) { + proj := copyFixture(t) + + // 1. Create an entity with an index through the ordinary path. + b := New() + if err := b.Connect(proj); err != nil { + t.Fatalf("connect: %v", err) + } + dmID, _ := richestEntity(t, b) + const entName = "Issue1119Indexed" + ent := &domainmodel.Entity{ + Name: entName, + Persistable: true, + Attributes: []*domainmodel.Attribute{ + {Name: "Code", Type: &domainmodel.StringAttributeType{Length: 50}}, + {Name: "Label", Type: &domainmodel.StringAttributeType{Length: 50}}, + }, + } + if err := b.CreateEntity(dmID, ent); err != nil { + t.Fatalf("CreateEntity: %v", err) + } + // Re-read to learn the attribute IDs the index must point at. + stored := mustFindEntity(t, b, dmID, entName) + stored.Indexes = []*domainmodel.Index{{ + Attributes: []*domainmodel.IndexAttribute{ + {AttributeID: stored.Attributes[0].ID, Ascending: true}, + }, + }} + if err := b.UpdateEntity(dmID, stored); err != nil { + t.Fatalf("UpdateEntity (add index): %v", err) + } + if err := b.Disconnect(); err != nil { + t.Fatalf("disconnect: %v", err) + } + + // 2. Make the index's GUID differ from its $ID, as a Studio Pro-authored one + // does. This goes straight at the stored bytes: the writer would (rightly) + // refuse a write that moves a GUID. + seededGUID := patchIndexGUID(t, proj, dmID, entName) + + // 3. ALTER the entity. Nothing about the index is mentioned. + b2 := New() + if err := b2.Connect(proj); err != nil { + t.Fatalf("reconnect: %v", err) + } + target := mustFindEntity(t, b2, dmID, entName) + target.Documentation = "issue 1119 index arm" + if err := b2.UpdateEntity(dmID, target); err != nil { + // A refusal here is the guard firing because the carry did not happen — + // which is the failure this test exists to catch, not an unrelated error. + t.Fatalf("UpdateEntity on an indexed entity: %v", err) + } + if err := b2.Disconnect(); err != nil { + t.Fatalf("disconnect: %v", err) + } + + // 4. The index kept the GUID it was seeded with. + b3 := New() + if err := b3.Connect(proj); err != nil { + t.Fatalf("reconnect: %v", err) + } + t.Cleanup(func() { _ = b3.Disconnect() }) + + got := indexGUIDs(t, b3, dmID, entName) + if len(got) != 1 { + t.Fatalf("expected 1 index after the ALTER, got %d", len(got)) + } + if got[0] != seededGUID { + t.Errorf("index storage GUID changed on ALTER: seeded %s, now %s", seededGUID, got[0]) + } +} + +func mustFindEntity(t *testing.T, b *Backend, dmID model.ID, name string) *domainmodel.Entity { + t.Helper() + dms, err := b.ListDomainModels() + if err != nil { + t.Fatalf("ListDomainModels: %v", err) + } + for _, d := range dms { + if d.ID != dmID { + continue + } + for _, e := range d.Entities { + if e.Name == name { + return e + } + } + } + t.Fatalf("entity %s not found in domain model %s", name, dmID) + return nil +} + +func indexGUIDs(t *testing.T, b *Backend, dmID model.ID, entityName string) []string { + t.Helper() + gdm, err := b.loadDomainModelGen(dmID) + if err != nil { + t.Fatalf("loadDomainModelGen: %v", err) + } + var out []string + for _, el := range gdm.EntitiesItems() { + ge, ok := el.(*genDm.Entity) + if !ok || ge.Name() != entityName { + continue + } + for _, iel := range ge.IndexesItems() { + if idx, ok := iel.(*genDm.Index); ok { + out = append(out, rawKeyHex(t, idx.Raw(), "GUID")) + } + } + } + return out +} + +// patchIndexGUID rewrites the stored unit so every DomainModels$EntityIndex +// carries a GUID that differs from its $ID, and returns the hex it wrote. It +// edits the file (and the .mpr's ContentsHash) directly, below the writer. +func patchIndexGUID(t *testing.T, proj string, dmID model.ID, entityName string) string { + t.Helper() + path := unitFilePath(t, proj, entityName) + raw, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read unit: %v", err) + } + var doc bson.D + if err := bson.Unmarshal(raw, &doc); err != nil { + t.Fatalf("unmarshal unit: %v", err) + } + + seeded := make([]byte, 16) + for i := range seeded { + seeded[i] = 0xAB + } + patched := 0 + var walk func(any) + walk = func(v any) { + switch t2 := v.(type) { + case bson.D: + isIndex := false + for _, e := range t2 { + if e.Key == "$Type" { + if s, ok := e.Value.(string); ok && s == "DomainModels$EntityIndex" { + isIndex = true + } + } + } + for i, e := range t2 { + if isIndex && e.Key == "GUID" { + if bin, ok := e.Value.(bson.Binary); ok { + t2[i].Value = bson.Binary{Subtype: bin.Subtype, Data: append([]byte{}, seeded...)} + patched++ + } + } + walk(e.Value) + } + case bson.A: + for _, e := range t2 { + walk(e) + } + } + } + walk(doc) + if patched == 0 { + t.Fatal("no DomainModels$EntityIndex GUID found to seed; the index was not written") + } + + out, err := bson.Marshal(doc) + if err != nil { + t.Fatalf("marshal patched unit: %v", err) + } + if err := os.WriteFile(path, out, 0644); err != nil { + t.Fatalf("write patched unit: %v", err) + } + + // The .mpr keeps a hash of each v2 unit file. Leave it consistent, so the + // reader is not handed a unit the project believes is something else — and so + // the seeding is not itself what the ALTER reacts to. + sum := sha256.Sum256(out) + db, err := sql.Open("sqlite", proj) + if err != nil { + t.Fatalf("open mpr: %v", err) + } + defer db.Close() + res, err := db.Exec( + "UPDATE Unit SET ContentsHash = ? WHERE lower(hex(UnitID)) = ?", + base64.StdEncoding.EncodeToString(sum[:]), storedUnitIDHex(t, string(dmID)), + ) + if err != nil { + t.Fatalf("update ContentsHash: %v", err) + } + n, err := res.RowsAffected() + if err != nil { + t.Fatalf("RowsAffected: %v", err) + } + if n != 1 { + t.Fatalf("ContentsHash update matched %d rows, want 1", n) + } + return "abababababababababababababababab" +} + +// storedUnitIDHex renders a unit id the way the .mpr's Unit.UnitID blob holds +// it: .NET GUID layout, so the first three fields are little-endian. Both +// spellings of the id are in play around a v2 project — neither keying on the +// plain id nor on the file's base name matches this column, and both fail +// SILENTLY as a zero-row UPDATE. +func storedUnitIDHex(t *testing.T, id string) string { + t.Helper() + raw, err := hex.DecodeString(strings.ReplaceAll(id, "-", "")) + if err != nil || len(raw) != 16 { + t.Fatalf("not a uuid: %q (%v)", id, err) + } + out := make([]byte, 16) + copy(out, raw) + for _, f := range [][2]int{{0, 4}, {4, 6}, {6, 8}} { + lo, hi := f[0], f[1] + for i, j := lo, hi-1; i < j; i, j = i+1, j-1 { + out[i], out[j] = out[j], out[i] + } + } + return hex.EncodeToString(out) +} + +// unitFilePath finds the domain-model unit's file by its CONTENT rather than by +// deriving the on-disk name, which uses the .NET-swapped byte order: the unit +// wanted is the one whose $Type is DomainModels$DomainModel and which holds the +// entity just created. +func unitFilePath(t *testing.T, proj, entityName string) string { + t.Helper() + matches, _ := filepath.Glob(filepath.Join(filepath.Dir(proj), "mprcontents", "*", "*", "*.mxunit")) + var found []string + for _, m := range matches { + raw, err := os.ReadFile(m) + if err != nil { + continue + } + var doc bson.D + if bson.Unmarshal(raw, &doc) != nil { + continue + } + isDM := false + for _, e := range doc { + if e.Key == "$Type" { + if s, ok := e.Value.(string); ok && s == "DomainModels$DomainModel" { + isDM = true + } + } + } + if isDM && bytes.Contains(raw, []byte(entityName)) { + found = append(found, m) + } + } + if len(found) != 1 { + t.Fatalf("expected exactly one domain-model unit holding %s, found %v", entityName, found) + } + return found[0] +} diff --git a/mdl/backend/modelsdk/issue1169_domainmodel_guid_test.go b/mdl/backend/modelsdk/issue1169_domainmodel_guid_test.go new file mode 100644 index 0000000000..f05b44d0bb --- /dev/null +++ b/mdl/backend/modelsdk/issue1169_domainmodel_guid_test.go @@ -0,0 +1,321 @@ +// SPDX-License-Identifier: Apache-2.0 + +package modelsdkbackend + +import ( + "fmt" + "testing" + + "github.com/mendixlabs/mxcli/model" + genDm "github.com/mendixlabs/mxcli/modelsdk/gen/domainmodels" + "github.com/mendixlabs/mxcli/sdk/domainmodel" +) + +// TestIssue1169_UpdateDomainModelPreservesStorageGUIDs guards ako/mxcli#1169: +// UpdateDomainModel must not re-mint any storage GUID in the unit it rewrites. +// +// It removes and rebuilds the WHOLE Entities and Associations lists, so every +// element arrived raw==nil and the codec's EmitGUID default wrote GUID = $ID — +// entities, their attributes and indexes, and every association, including the +// ones the statement never named. #657 fixed this for the ALTER target entity and +// #1119 for that entity's children; this path got neither, and it is the wider of +// the two, because the blast radius is the unit rather than one element. +// +// Its doc comment claimed it preserved "each element's identity". That is true of +// the $ID and false of the GUID, which is the identity that matters to the +// database: the runtime keys mendixsystem$entity.id and mendixsystem$attribute.id +// on it. The reporter measured 282 moved GUIDs in one module and a runtime crash +// (`getTableName() because "table" is null`). +// +// EVERY CASE HERE IS A DIFFERENT MDL STATEMENT, and that is the point — the title +// says ALTER ASSOCIATION, but the path is shared: +// +// RenameEntity RENAME ENTITY +// RenameAssociation RENAME ASSOCIATION +// SetAssociationComment ALTER ASSOCIATION ... SET COMMENT +// SetAssociationOwner ALTER ASSOCIATION ... SET OWNER +// NoChange CREATE OR MODIFY ASSOCIATION re-run +// +// RENAME ENTITY is the most expensive of them: the entity name is the table name, +// and per the #503 measurement a re-minted GUID makes the runtime DROP the old +// table and create an empty one instead of renaming it — a whole table, where an +// ALTER loses a column. +// +// CONTROL. Against the unfixed code every case fails, but on the refusal rather +// than the GUID assertion: the #1119 write guard (canon.StorageGUIDError) pairs by +// $ID within the unit, which is exactly this shape, so it catches the corruption +// before it reaches disk and UpdateDomainModel returns an error. That is the +// symptom on this branch; upstream, with no guard, the GUIDs move. The GUID +// assertions below are therefore both the primary statement of intent and the +// backstop for anyone who opts a write path out of the guard. +func TestIssue1169_UpdateDomainModelPreservesStorageGUIDs(t *testing.T) { + for _, tc := range []struct { + name string + // mutate applies one statement's worth of change to the semantic model and + // returns a check that the change actually reached disk. A fix that simply + // stopped writing would pass every GUID assertion here. + mutate func(t *testing.T, dm *domainmodel.DomainModel) func(t *testing.T, after *domainmodel.DomainModel) + }{ + { + name: "RenameEntity", + mutate: func(t *testing.T, dm *domainmodel.DomainModel) func(*testing.T, *domainmodel.DomainModel) { + ent := firstEntityWithAttributes(t, dm) + id, want := ent.ID, "Issue1169Renamed"+ent.Name + ent.Name = want + return func(t *testing.T, after *domainmodel.DomainModel) { + for _, e := range after.Entities { + if e.ID == id && e.Name != want { + t.Errorf("entity name not persisted: got %q, want %q", e.Name, want) + } + } + } + }, + }, + { + name: "RenameAssociation", + mutate: func(t *testing.T, dm *domainmodel.DomainModel) func(*testing.T, *domainmodel.DomainModel) { + a := dm.Associations[0] + id, want := a.ID, "Issue1169Renamed_"+a.Name + a.Name = want + return func(t *testing.T, after *domainmodel.DomainModel) { + for _, x := range after.Associations { + if x.ID == id && x.Name != want { + t.Errorf("association name not persisted: got %q, want %q", x.Name, want) + } + } + } + }, + }, + { + name: "SetAssociationComment", + mutate: func(t *testing.T, dm *domainmodel.DomainModel) func(*testing.T, *domainmodel.DomainModel) { + a := dm.Associations[0] + id, want := a.ID, "issue 1169" + a.Documentation = want + return func(t *testing.T, after *domainmodel.DomainModel) { + for _, x := range after.Associations { + if x.ID == id && x.Documentation != want { + t.Errorf("association documentation not persisted: got %q, want %q", x.Documentation, want) + } + } + } + }, + }, + { + name: "SetAssociationOwner", + mutate: func(t *testing.T, dm *domainmodel.DomainModel) func(*testing.T, *domainmodel.DomainModel) { + a := dm.Associations[0] + want := domainmodel.AssociationOwnerBoth + if a.Owner == domainmodel.AssociationOwnerBoth { + want = domainmodel.AssociationOwnerDefault + } + id := a.ID + a.Owner = want + return func(t *testing.T, after *domainmodel.DomainModel) { + for _, x := range after.Associations { + if x.ID == id && x.Owner != want { + t.Errorf("association owner not persisted: got %q, want %q", x.Owner, want) + } + } + } + }, + }, + { + // CREATE OR MODIFY ASSOCIATION re-run against an unchanged project. The + // CLI's own help promises "preserves UUID. Safe to re-run" — true of the + // $ID and, before this fix, false of the GUID. Nothing lands (ADR-0008 + // elides the write), so there is nothing to assert as persisted; what + // matters is that it is neither refused nor destructive. + name: "NoChange", + mutate: func(t *testing.T, dm *domainmodel.DomainModel) func(*testing.T, *domainmodel.DomainModel) { + return nil + }, + }, + } { + t.Run(tc.name, func(t *testing.T) { + proj := copyFixture(t) + b := New() + if err := b.Connect(proj); err != nil { + t.Fatalf("connect: %v", err) + } + + dm := domainModelWithAssociation(t, b) + before := domainModelGUIDs(t, b, dm.ID) + + // Preconditions. Without GUID != $ID this cannot fail against the broken + // code: an element mxcli created has GUID == $ID from birth, so a rebuild + // that re-mints GUID = $ID reproduces the very same value. That is the + // trap that voided a live-database control during #503. + divergent := 0 + for id, rec := range before { + if rec.guid == "" { + t.Fatalf("%s has no GUID; cannot verify preservation", rec.label) + } + if rec.guid != id { + divergent++ + } + } + if divergent < len(before) { + t.Fatalf("%d of %d fixture elements have GUID == $ID; all must differ to detect #1169", + len(before)-divergent, len(before)) + } + if divergent < 3 { + t.Fatalf("only %d GUID-bearing elements in the fixture unit; need >= 3", divergent) + } + + persisted := tc.mutate(t, dm) + if err := b.UpdateDomainModel(dm); err != nil { + t.Fatalf("UpdateDomainModel: %v", err) + } + if err := b.Disconnect(); err != nil { + t.Fatalf("disconnect: %v", err) + } + + // Reopen: preservation only counts if it reached disk. + b2 := New() + if err := b2.Connect(proj); err != nil { + t.Fatalf("reconnect: %v", err) + } + t.Cleanup(func() { _ = b2.Disconnect() }) + + after := domainModelGUIDs(t, b2, dm.ID) + + moved, gone := 0, 0 + for id, want := range before { + got, ok := after[id] + if !ok { + gone++ + if gone <= 3 { + t.Errorf("%s is gone from the unit after the write", want.label) + } + continue + } + if got.guid != want.guid { + moved++ + if moved <= 5 { + t.Errorf("%s: storage GUID changed %s -> %s", want.label, want.guid, got.guid) + } + } + } + if moved > 5 { + t.Errorf("... and %d more elements whose storage GUID changed", moved-5) + } + if gone > 3 { + t.Errorf("... and %d more elements gone from the unit", gone-3) + } + + if persisted != nil { + dmAfter := reloadDomainModel(t, b2, dm.ID) + persisted(t, dmAfter) + } + }) + } +} + +// guidRec is one GUID-bearing element: its human label, for a failure message +// that says WHICH element moved, and its stored GUID. +type guidRec struct { + label string + guid string +} + +// domainModelGUIDs censuses every GUID-bearing element in a domain model unit, +// keyed by element $ID. +// +// The key is the $ID and not the name, deliberately: a RENAME case keyed on name +// reads as "the old element vanished and a new one appeared", which looks like +// four changes and one addition rather than the zero it is. That false reading +// cost a measurement during this investigation. +func domainModelGUIDs(t *testing.T, b *Backend, dmID model.ID) map[string]guidRec { + t.Helper() + gdm, err := b.loadDomainModelGen(dmID) + if err != nil { + t.Fatalf("loadDomainModelGen: %v", err) + } + out := map[string]guidRec{} + for _, el := range gdm.EntitiesItems() { + ge, ok := el.(*genDm.Entity) + if !ok { + continue + } + out[string(ge.ID())] = guidRec{fmt.Sprintf("entity %s", ge.Name()), rawKeyHex(t, ge.Raw(), "GUID")} + for _, ael := range ge.AttributesItems() { + if a, ok := ael.(*genDm.Attribute); ok { + out[string(a.ID())] = guidRec{ + fmt.Sprintf("attribute %s.%s", ge.Name(), a.Name()), + rawKeyHex(t, a.Raw(), "GUID"), + } + } + } + for i, iel := range ge.IndexesItems() { + if idx, ok := iel.(*genDm.Index); ok { + out[string(idx.ID())] = guidRec{ + fmt.Sprintf("index %s[%d]", ge.Name(), i), + rawKeyHex(t, idx.Raw(), "GUID"), + } + } + } + } + for _, el := range gdm.AssociationsItems() { + if a, ok := el.(*genDm.Association); ok { + out[string(a.ID())] = guidRec{fmt.Sprintf("association %s", a.Name()), rawKeyHex(t, a.Raw(), "GUID")} + } + } + for _, el := range gdm.CrossAssociationsItems() { + if a, ok := el.(*genDm.CrossAssociation); ok { + out[string(a.ID())] = guidRec{fmt.Sprintf("cross-association %s", a.Name()), rawKeyHex(t, a.Raw(), "GUID")} + } + } + return out +} + +// domainModelWithAssociation returns the first loadable domain model that holds +// both an association and an entity with attributes. +func domainModelWithAssociation(t *testing.T, b *Backend) *domainmodel.DomainModel { + t.Helper() + dms, err := b.ListDomainModels() + if err != nil { + t.Fatalf("ListDomainModels: %v", err) + } + for _, d := range dms { + if _, err := b.loadDomainModelGen(d.ID); err != nil { + continue // the System module's unit is not on disk in the fixture + } + if len(d.Associations) == 0 { + continue + } + for _, e := range d.Entities { + if len(e.Attributes) > 0 { + return d + } + } + } + t.Fatal("no loadable domain model in the fixture holds an association and an entity with attributes") + return nil +} + +func firstEntityWithAttributes(t *testing.T, dm *domainmodel.DomainModel) *domainmodel.Entity { + t.Helper() + for _, e := range dm.Entities { + if len(e.Attributes) > 0 { + return e + } + } + t.Fatal("domain model has no entity with attributes") + return nil +} + +func reloadDomainModel(t *testing.T, b *Backend, dmID model.ID) *domainmodel.DomainModel { + t.Helper() + dms, err := b.ListDomainModels() + if err != nil { + t.Fatalf("ListDomainModels: %v", err) + } + for _, d := range dms { + if d.ID == dmID { + return d + } + } + t.Fatalf("domain model %s not found after the write", dmID) + return nil +} diff --git a/mdl/backend/modelsdk/issue503_move_guid_test.go b/mdl/backend/modelsdk/issue503_move_guid_test.go new file mode 100644 index 0000000000..04e9873cb3 --- /dev/null +++ b/mdl/backend/modelsdk/issue503_move_guid_test.go @@ -0,0 +1,264 @@ +// SPDX-License-Identifier: Apache-2.0 + +package modelsdkbackend + +import ( + "testing" + + "github.com/mendixlabs/mxcli/model" + genDm "github.com/mendixlabs/mxcli/modelsdk/gen/domainmodels" + "github.com/mendixlabs/mxcli/sdk/domainmodel" +) + +// TestIssue503_MovePreservesStorageGUIDs guards ako/mxcli#503: MOVE ENTITY must +// not re-mint a storage GUID — not the moved entity's, not its attributes', and +// not that of an association the move converts to a cross-association. +// +// #657 fixed the entity GUID for UpdateEntity and #1119 its children; MoveEntity +// got neither, and additionally dropped the association's GUID when converting +// it. That last one is why the #1119 write guard began refusing MOVE ENTITY for +// any entity in an association: the conversion happens in place in the source +// unit, keeping the element's $ID, so the guard could see the GUID move. The +// other losses are invisible to it — the entity lands in a DIFFERENT unit, where +// its $ID matches nothing stored, and cross-unit moves are outside the guard's +// reach by construction. +// +// Both endpoints are covered because the conversion is asymmetric: moving the +// CHILD leaves the cross-association in the source unit, moving the PARENT sends +// it to the target. +func TestIssue503_MovePreservesStorageGUIDs(t *testing.T) { + for _, tc := range []struct { + name string + // endpoint picks which side of the association to move: the child (TO) + // entity or the parent (FROM) entity. + moveChild bool + // assocLandsIn says which domain model should hold the cross-association + // afterwards, so the assertion looks in the right unit. + assocInSource bool + }{ + {name: "MoveChild_CrossAssocStaysInSource", moveChild: true, assocInSource: true}, + {name: "MoveParent_CrossAssocGoesToTarget", moveChild: false, assocInSource: false}, + } { + t.Run(tc.name, func(t *testing.T) { + proj := copyFixture(t) + b := New() + if err := b.Connect(proj); err != nil { + t.Fatalf("connect: %v", err) + } + + srcID, srcMod, assocName, assocGUIDBefore, childID, parentID := associationSubject(t, b) + dstID, dstMod := otherDomainModel(t, b, srcID) + if dstID == "" { + t.Skip("fixture has no second loadable domain model to move into") + } + + entID := childID + if !tc.moveChild { + entID = parentID + } + ent := entityByID(t, b, srcID, entID) + if len(ent.Attributes) == 0 { + t.Skipf("subject entity %s has no attributes; nothing to detect the conflation with", ent.Name) + } + + // Preconditions: without GUID != $ID on both the association and the + // attributes, this test cannot fail against the broken code. + attrsBefore := attributeGUIDs(t, b, srcID, entID) + attrIDs := attributeIDs(t, b, srcID, entID) + for name, guid := range attrsBefore { + if guid == "" || guid == attrIDs[name] { + t.Fatalf("fixture attribute %s has GUID %q and $ID %q; need them to differ", name, guid, attrIDs[name]) + } + } + entGUIDBefore := entityGUID(t, b, srcID, entID) + entIDHex := entityRawKey(t, b, srcID, entID, "$ID") + if entGUIDBefore == "" || entGUIDBefore == entIDHex { + t.Fatalf("fixture entity %s has GUID %q and $ID %q; need them to differ", ent.Name, entGUIDBefore, entIDHex) + } + if assocGUIDBefore == "" { + t.Fatalf("association %s has no GUID; cannot verify preservation", assocName) + } + + converted, err := b.MoveEntity(ent, srcID, dstID, srcMod, dstMod) + if err != nil { + t.Fatalf("MoveEntity: %v", err) + } + if len(converted) == 0 { + t.Fatalf("expected %s to be converted to a cross-association, got %v", assocName, converted) + } + if err := b.Disconnect(); err != nil { + t.Fatalf("disconnect: %v", err) + } + + // Reopen: preservation only counts if it reached disk. + b2 := New() + if err := b2.Connect(proj); err != nil { + t.Fatalf("reconnect: %v", err) + } + t.Cleanup(func() { _ = b2.Disconnect() }) + + // The move must actually have happened — preserving GUIDs on an entity + // that never moved proves nothing. + if e := findEntityByName(t, b2, dstID, ent.Name); e == nil { + t.Fatalf("entity %s is not in the target domain model after the move", ent.Name) + } + if e := findEntityByName(t, b2, srcID, ent.Name); e != nil { + t.Errorf("entity %s is still in the source domain model after the move", ent.Name) + } + + if got := entityGUID(t, b2, dstID, entID); got != entGUIDBefore { + t.Errorf("entity storage GUID changed on MOVE: before=%s after=%s", entGUIDBefore, got) + } + attrsAfter := attributeGUIDs(t, b2, dstID, entID) + for name, want := range attrsBefore { + got, ok := attrsAfter[name] + if !ok { + t.Errorf("attribute %s is missing after the move", name) + continue + } + if got != want { + t.Errorf("attribute %s storage GUID changed on MOVE: before=%s after=%s", name, want, got) + } + } + + assocDM := dstID + if tc.assocInSource { + assocDM = srcID + } + got := crossAssocGUID(t, b2, assocDM, assocName) + if got == "" { + t.Fatalf("cross-association %s not found in the expected domain model", assocName) + } + if got != assocGUIDBefore { + t.Errorf("cross-association %s storage GUID changed on MOVE: before=%s after=%s", + assocName, assocGUIDBefore, got) + } + }) + } +} + +// associationSubject returns the first loadable domain model holding a regular +// association, with that association's name, stored GUID, and its child (TO) and +// parent (FROM) entity ids. +func associationSubject(t *testing.T, b *Backend) (dmID model.ID, moduleName, assocName, assocGUID string, childID, parentID model.ID) { + t.Helper() + dms, err := b.ListDomainModels() + if err != nil { + t.Fatalf("ListDomainModels: %v", err) + } + for _, d := range dms { + gdm, err := b.loadDomainModelGen(d.ID) + if err != nil { + continue + } + mod, err := b.GetModule(d.ContainerID) + if err != nil || mod == nil { + continue + } + for _, el := range gdm.AssociationsItems() { + a, ok := el.(*genDm.Association) + if !ok || a.Raw() == nil { + continue + } + return d.ID, mod.Name, a.Name(), rawKeyHex(t, a.Raw(), "GUID"), + model.ID(a.ChildRefID()), model.ID(a.ParentRefID()) + } + } + t.Fatal("no loadable domain model with a regular association in the fixture") + return +} + +// otherDomainModel returns a loadable domain model that is not exclude. +func otherDomainModel(t *testing.T, b *Backend, exclude model.ID) (model.ID, string) { + t.Helper() + dms, err := b.ListDomainModels() + if err != nil { + t.Fatalf("ListDomainModels: %v", err) + } + for _, d := range dms { + if d.ID == exclude { + continue + } + if _, err := b.loadDomainModelGen(d.ID); err != nil { + continue + } + mod, err := b.GetModule(d.ContainerID) + if err != nil || mod == nil { + continue + } + return d.ID, mod.Name + } + return "", "" +} + +func entityByID(t *testing.T, b *Backend, dmID, entID model.ID) *domainmodel.Entity { + t.Helper() + dms, err := b.ListDomainModels() + if err != nil { + t.Fatalf("ListDomainModels: %v", err) + } + for _, d := range dms { + if d.ID != dmID { + continue + } + for _, e := range d.Entities { + if e.ID == entID { + return e + } + } + } + t.Fatalf("entity %s not found in domain model %s", entID, dmID) + return nil +} + +func findEntityByName(t *testing.T, b *Backend, dmID model.ID, name string) *domainmodel.Entity { + t.Helper() + dms, err := b.ListDomainModels() + if err != nil { + t.Fatalf("ListDomainModels: %v", err) + } + for _, d := range dms { + if d.ID != dmID { + continue + } + for _, e := range d.Entities { + if e.Name == name { + return e + } + } + } + return nil +} + +func entityGUID(t *testing.T, b *Backend, dmID, entID model.ID) string { + return entityRawKey(t, b, dmID, entID, "GUID") +} + +func entityRawKey(t *testing.T, b *Backend, dmID, entID model.ID, key string) string { + t.Helper() + gdm, err := b.loadDomainModelGen(dmID) + if err != nil { + t.Fatalf("loadDomainModelGen: %v", err) + } + ge := findGenEntity(gdm, entID) + if ge == nil { + return "" + } + return rawKeyHex(t, ge.Raw(), key) +} + +func crossAssocGUID(t *testing.T, b *Backend, dmID model.ID, name string) string { + t.Helper() + gdm, err := b.loadDomainModelGen(dmID) + if err != nil { + t.Fatalf("loadDomainModelGen: %v", err) + } + for _, el := range gdm.CrossAssociationsItems() { + ca, ok := el.(*genDm.CrossAssociation) + if !ok || ca.Name() != name { + continue + } + return rawKeyHex(t, ca.Raw(), "GUID") + } + return "" +} diff --git a/mdl/backend/modelsdk/issue605_move_refs_test.go b/mdl/backend/modelsdk/issue605_move_refs_test.go new file mode 100644 index 0000000000..4ce53d4df6 --- /dev/null +++ b/mdl/backend/modelsdk/issue605_move_refs_test.go @@ -0,0 +1,230 @@ +// SPDX-License-Identifier: Apache-2.0 + +package modelsdkbackend + +import ( + "strings" + "testing" + + "github.com/mendixlabs/mxcli/model" + genDm "github.com/mendixlabs/mxcli/modelsdk/gen/domainmodels" +) + +// TestIssue605_MovePreservesOwnAccessRuleRefs guards the half of ako/mxcli#605 +// that lives inside the moved document: a moved entity's OWN access rules kept +// the source module's prefix on every member they name. +// +// MoveEntity re-points the view source and each validation rule's attribute (see +// the rewrites just before the rebuild), and access rules were the missed sibling. +// The consequence is worse than a stale string, because entityToGen's +// syncMemberAccesses matches existing entries BY QUALIFIED NAME: the stale +// `Source.Entity.Attr` never equals the freshly built `Target.Entity.Attr`, so it +// appends the new one and keeps the old — the entity ends up with duplicated +// MemberAccess entries, half of them dangling. Measured on a blank 11.13 app, 9 of +// 33 CE1613s were "at Access rule of entity" for the entity that had just moved. +// +// DESCRIBE cannot show this: it renders members bare, so the only visible trace is +// each member appearing twice. The assertion is therefore on the stored qualified +// names. +func TestIssue605_MovePreservesOwnAccessRuleRefs(t *testing.T) { + proj := copyFixture(t) + b := New() + if err := b.Connect(proj); err != nil { + t.Fatalf("connect: %v", err) + } + + srcID, srcMod, entID := entityWithAccessRules(t, b) + dstID, dstMod := otherDomainModel(t, b, srcID) + if dstID == "" { + t.Skip("fixture has no second loadable domain model to move into") + } + ent := entityByID(t, b, srcID, entID) + + // Precondition: the stored rules must actually name members under the source + // module, or nothing here can fail. + before := accessRuleMemberRefs(t, b, srcID, entID) + if len(before) == 0 { + t.Skipf("entity %s has no access-rule member references", ent.Name) + } + srcPrefix := srcMod + "." + found := false + for _, r := range before { + if strings.HasPrefix(r, srcPrefix) { + found = true + } + } + if !found { + t.Fatalf("no member reference under %q to go stale; refs=%v", srcPrefix, before) + } + + if _, err := b.MoveEntity(ent, srcID, dstID, srcMod, dstMod); err != nil { + t.Fatalf("MoveEntity: %v", err) + } + if err := b.Disconnect(); err != nil { + t.Fatalf("disconnect: %v", err) + } + + b2 := New() + if err := b2.Connect(proj); err != nil { + t.Fatalf("reconnect: %v", err) + } + t.Cleanup(func() { _ = b2.Disconnect() }) + + after := accessRuleMemberRefs(t, b2, dstID, entID) + + // 1. Nothing may still point at the source module. + for _, r := range after { + if strings.HasPrefix(r, srcPrefix) { + t.Errorf("access rule still references %s after the move to %s", r, dstMod) + } + } + // 2. No member may be named twice — the duplication syncMemberAccesses + // introduces when the stale entry fails to match the rebuilt one. + seen := map[string]int{} + for _, r := range after { + seen[r]++ + } + for r, n := range seen { + if n > 1 { + t.Errorf("member %s appears %d times in the moved entity's access rules", r, n) + } + } + // 3. And the rules must still cover what they covered — a "fix" that dropped + // every member would pass 1 and 2. + if len(seen) < len(uniq(before)) { + t.Errorf("member count shrank across the move: %d distinct before, %d after (before=%v after=%v)", + len(uniq(before)), len(seen), before, after) + } +} + +// TestIssue605_MoveReportsAssociationRenamesInBothDirections pins the contract the +// project-wide sweep is driven from: MoveEntity must report, per converted +// association, the qualified name it had and the one it now has. +// +// The two directions differ, and a sweep that assumed either one would be wrong in +// the other — the parent moving takes the cross-association to the target module, +// the child moving leaves it where it was. +func TestIssue605_MoveReportsAssociationRenamesInBothDirections(t *testing.T) { + for _, tc := range []struct { + name string + moveChild bool + // wantMoved says whether the association's qualified name should change. + wantMoved bool + }{ + {name: "ParentMoved_AssociationFollows", moveChild: false, wantMoved: true}, + {name: "ChildMoved_AssociationStays", moveChild: true, wantMoved: false}, + } { + t.Run(tc.name, func(t *testing.T) { + proj := copyFixture(t) + b := New() + if err := b.Connect(proj); err != nil { + t.Fatalf("connect: %v", err) + } + t.Cleanup(func() { _ = b.Disconnect() }) + + srcID, srcMod, assocName, _, childID, parentID := associationSubject(t, b) + dstID, dstMod := otherDomainModel(t, b, srcID) + if dstID == "" { + t.Skip("fixture has no second loadable domain model to move into") + } + entID := parentID + if tc.moveChild { + entID = childID + } + ent := entityByID(t, b, srcID, entID) + + moved, err := b.MoveEntity(ent, srcID, dstID, srcMod, dstMod) + if err != nil { + t.Fatalf("MoveEntity: %v", err) + } + if len(moved) != 1 { + t.Fatalf("expected 1 converted association, got %d: %+v", len(moved), moved) + } + m := moved[0] + if m.Name != assocName { + t.Errorf("reported name %q, want %q", m.Name, assocName) + } + if got, want := m.OldQualifiedName, srcMod+"."+assocName; got != want { + t.Errorf("OldQualifiedName = %q, want %q", got, want) + } + wantNew := srcMod + "." + assocName + if tc.wantMoved { + wantNew = dstMod + "." + assocName + } + if got := m.NewQualifiedName; got != wantNew { + t.Errorf("NewQualifiedName = %q, want %q", got, wantNew) + } + if m.Moved() != tc.wantMoved { + t.Errorf("Moved() = %v, want %v", m.Moved(), tc.wantMoved) + } + }) + } +} + +// entityWithAccessRules returns a loadable domain model and an entity in it whose +// access rules name at least one member. +func entityWithAccessRules(t *testing.T, b *Backend) (dmID model.ID, moduleName string, entID model.ID) { + t.Helper() + dms, err := b.ListDomainModels() + if err != nil { + t.Fatalf("ListDomainModels: %v", err) + } + for _, d := range dms { + if _, err := b.loadDomainModelGen(d.ID); err != nil { + continue + } + mod, err := b.GetModule(d.ContainerID) + if err != nil || mod == nil { + continue + } + for _, e := range d.Entities { + if len(accessRuleMemberRefs(t, b, d.ID, e.ID)) > 0 { + return d.ID, mod.Name, e.ID + } + } + } + t.Fatal("no entity with access-rule member references in the fixture") + return +} + +// accessRuleMemberRefs returns every qualified name an entity's access rules name, read +// from the stored gen elements (attributes and associations alike). +func accessRuleMemberRefs(t *testing.T, b *Backend, dmID, entID model.ID) []string { + t.Helper() + gdm, err := b.loadDomainModelGen(dmID) + if err != nil { + t.Fatalf("loadDomainModelGen: %v", err) + } + ge := findGenEntity(gdm, entID) + if ge == nil { + return nil + } + var out []string + for _, el := range ge.AccessRulesItems() { + ar, ok := el.(*genDm.AccessRule) + if !ok { + continue + } + for _, mel := range ar.MemberAccessesItems() { + ma, ok := mel.(*genDm.MemberAccess) + if !ok { + continue + } + if qn := ma.AttributeQualifiedName(); qn != "" { + out = append(out, qn) + } + if qn := ma.AssociationQualifiedName(); qn != "" { + out = append(out, qn) + } + } + } + return out +} + +func uniq(in []string) map[string]bool { + out := map[string]bool{} + for _, s := range in { + out[s] = true + } + return out +} diff --git a/mdl/backend/modelsdk/unimplemented_gen.go b/mdl/backend/modelsdk/unimplemented_gen.go index 4fbbbd04f9..232eed4a25 100644 --- a/mdl/backend/modelsdk/unimplemented_gen.go +++ b/mdl/backend/modelsdk/unimplemented_gen.go @@ -838,8 +838,8 @@ func (unimplemented) MoveDocument(_ model.ID, _ model.ID) error { return errUnimplemented("MoveDocument") } -func (unimplemented) MoveEntity(_ *domainmodel.Entity, _ model.ID, _ model.ID, _ string, _ string) ([]string, error) { - var r0 []string +func (unimplemented) MoveEntity(_ *domainmodel.Entity, _ model.ID, _ model.ID, _ string, _ string) ([]types.MovedAssociation, error) { + var r0 []types.MovedAssociation return r0, errUnimplemented("MoveEntity") } @@ -1184,6 +1184,10 @@ func (unimplemented) UpdateRawUnit(_ string, _ []uint8) error { return errUnimplemented("UpdateRawUnit") } +func (unimplemented) UpdateRawUnitOwningStorageGUIDs(_ string, _ []uint8) error { + return errUnimplemented("UpdateRawUnitOwningStorageGUIDs") +} + func (unimplemented) UpdateRawUnitOwningTranslations(_ string, _ []uint8) error { return errUnimplemented("UpdateRawUnitOwningTranslations") } diff --git a/mdl/backend/modelsdk/units.go b/mdl/backend/modelsdk/units.go index ca1fddf5a6..b494c3887c 100644 --- a/mdl/backend/modelsdk/units.go +++ b/mdl/backend/modelsdk/units.go @@ -45,6 +45,12 @@ func (b *Backend) UpdateRawUnitOwningTranslations(unitID string, contents []byte return b.writer.UpdateRawUnitOwningTranslations(unitID, contents) } +// UpdateRawUnitOwningStorageGUIDs writes a unit whose contents deliberately move +// storage GUIDs — see the interface for why that needs saying out loud. +func (b *Backend) UpdateRawUnitOwningStorageGUIDs(unitID string, contents []byte) error { + return b.writer.UpdateRawUnitOwningStorageGUIDs(unitID, contents) +} + // ListRawUnitsByType returns every unit whose $Type has the given prefix, with // resolved raw contents — the catalog uses this for document types that have no // dedicated typed reader (e.g. JavaScript actions, data transformers). Delegates diff --git a/mdl/catalog/lint_rule_vocabulary_test.go b/mdl/catalog/lint_rule_vocabulary_test.go index 7b7e6708f1..61b6445f39 100644 --- a/mdl/catalog/lint_rule_vocabulary_test.go +++ b/mdl/catalog/lint_rule_vocabulary_test.go @@ -55,10 +55,18 @@ func TestCONV010AllowsWhatTheCatalogCallsUIActions(t *testing.T) { src := readRule(t, "conv010_act_microflow_content.star") // The activities CONV010 documents as permitted in an ACT_ microflow. + // + // NanoflowCallAction is here because microflows() yields NANOFLOWS too, so + // CONV010 lints an ACT_ nanoflow — which delegates with a nanoflow call, not + // a microflow call. Without it the rule flagged the delegation it demands and + // an ACT_ nanoflow could satisfy it in no way at all (ako/mxcli#644). That is + // the third short allowlist here: the wrong vocabulary once, a missing + // ExclusiveMerge once, and now the client-side half of "call a sub-flow". permitted := []microflows.MicroflowAction{ µflows.ShowPageAction{}, µflows.ClosePageAction{}, µflows.MicroflowCallAction{}, + µflows.NanoflowCallAction{}, } for _, action := range permitted { diff --git a/mdl/executor/cmd_microflows_builder_calls.go b/mdl/executor/cmd_microflows_builder_calls.go index d5a0aae07b..d5c556063c 100644 --- a/mdl/executor/cmd_microflows_builder_calls.go +++ b/mdl/executor/cmd_microflows_builder_calls.go @@ -339,17 +339,9 @@ func (fb *flowBuilder) addCallJavaActionAction(s *ast.CallJavaActionStmt) model. if entityTypeParams[arg.Name] { // Entity type parameter: value is the entity qualified name, not the variable reference. // When the argument is a variable like $Email, resolve its entity type from varTypes. - valueExpr := fb.exprToString(arg.Value) - entityName := strings.Trim(valueExpr, "'") - if strings.HasPrefix(entityName, "$") { - varName := strings.TrimPrefix(entityName, "$") - if resolvedType, ok := fb.varTypes[varName]; ok { - entityName = resolvedType - } - } value = µflows.EntityTypeCodeActionParameterValue{ BaseElement: model.BaseElement{ID: model.ID(types.GenerateID())}, - Entity: entityName, + Entity: fb.entityTypeArgument(arg.Value), } } else if isEmptyJavaActionArgument(arg.Value) { if microflowTypeParams[arg.Name] { @@ -531,19 +523,9 @@ func (fb *flowBuilder) addCallJavaScriptActionAction(s *ast.CallJavaScriptAction var value microflows.CodeActionParameterValue valueExpr := fb.exprToString(arg.Value) if entityTypeParams[arg.Name] { - // Entity-type parameter: the value is an entity qualified name. - // When the argument is a variable like $Order, resolve the entity - // it holds from varTypes, mirroring the Java-action builder. - entityName := strings.Trim(valueExpr, "'") - if strings.HasPrefix(entityName, "$") { - varName := strings.TrimPrefix(entityName, "$") - if resolvedType, ok := fb.varTypes[varName]; ok { - entityName = resolvedType - } - } value = µflows.EntityTypeCodeActionParameterValue{ BaseElement: model.BaseElement{ID: model.ID(types.GenerateID())}, - Entity: entityName, + Entity: fb.entityTypeArgument(arg.Value), } } else { value = µflows.BasicCodeActionParameterValue{ @@ -590,6 +572,25 @@ func (fb *flowBuilder) addCallJavaScriptActionAction(s *ast.CallJavaScriptAction return activity.ID } +// entityTypeArgument returns the entity qualified name an entity-type +// (`entity <>`) code-action argument names. The value is stored as a bare +// name under EntityTypeCodeActionParameterValue.Entity, so it is NOT the +// argument's expression text: the visitor keeps an argument's trailing +// whitespace so expressions round-trip as written, and an argument on its own +// line therefore arrives as "Mod.Entity\n". Stored verbatim that names an +// entity that does not exist — CE0115 at build, with `mxcli check` clean +// (mendixlabs/mxcli#1171). A variable argument such as $Order resolves to the +// entity it holds. +func (fb *flowBuilder) entityTypeArgument(expr ast.Expression) string { + entityName := strings.Trim(strings.TrimSpace(fb.exprToString(expr)), "'") + if varName, ok := strings.CutPrefix(entityName, "$"); ok { + if resolvedType, ok := fb.varTypes[varName]; ok { + entityName = resolvedType + } + } + return entityName +} + func isEmptyJavaActionArgument(expr ast.Expression) bool { lit, ok := expr.(*ast.LiteralExpr) return ok && (lit.Kind == ast.LiteralEmpty || lit.Kind == ast.LiteralNull) diff --git a/mdl/executor/cmd_microflows_builder_control.go b/mdl/executor/cmd_microflows_builder_control.go index bd26a1bf42..f7203efcba 100644 --- a/mdl/executor/cmd_microflows_builder_control.go +++ b/mdl/executor/cmd_microflows_builder_control.go @@ -963,7 +963,17 @@ func (fb *flowBuilder) addWhileStatement(s *ast.WhileStmt) model.ID { loopWidth := max(bodyBounds.Width+2*LoopPadding, MinLoopWidth) loopHeight := max(bodyBounds.Height+2*LoopPadding, MinLoopHeight) - innerStartX := LoopPadding + // A child's Position is its CENTRE, so the first one's centre must sit half an + // activity in from the padding or its left edge hangs outside the box. This + // read `LoopPadding` alone and put every while loop's first activity at x=50 + // with its left edge at -10 — MPR011 on every while loop mxcli wrote, single + // level included (ako/mxcli#645). + // + // The doc comment above says the layout "matches addLoopStatement but without + // iterator icon space", and dropping the iterator space is right; taking + // ActivityWidth/2 with it was not, because that term is not iterator space — + // it is what converts a centre to a left edge. The Y line below always had it. + innerStartX := LoopPadding + ActivityWidth/2 innerStartY := LoopPadding + ActivityHeight/2 // posX is where the builder would CENTRE the next element. A loop box placed diff --git a/mdl/executor/cmd_microflows_builder_entity_type_arg_test.go b/mdl/executor/cmd_microflows_builder_entity_type_arg_test.go new file mode 100644 index 0000000000..e649051bc9 --- /dev/null +++ b/mdl/executor/cmd_microflows_builder_entity_type_arg_test.go @@ -0,0 +1,178 @@ +// SPDX-License-Identifier: Apache-2.0 + +package executor + +import ( + "testing" + + "github.com/mendixlabs/mxcli/mdl/ast" + "github.com/mendixlabs/mxcli/mdl/backend/mock" + "github.com/mendixlabs/mxcli/mdl/types" + "github.com/mendixlabs/mxcli/mdl/visitor" + "github.com/mendixlabs/mxcli/sdk/javaactions" + "github.com/mendixlabs/mxcli/sdk/microflows" +) + +// parsedFlowBody parses src (one CREATE MICROFLOW/NANOFLOW statement) and +// returns its body. These tests go through the parser on purpose: the defect +// lives in the text the visitor preserves for an argument, and a hand-built +// ast.IdentifierExpr — which is what the #1137 tests use — carries none of it. +func parsedFlowBody(t *testing.T, src string) []ast.MicroflowStatement { + t.Helper() + prog, errs := visitor.Build(src) + if len(errs) > 0 { + t.Fatalf("parse errors: %v", errs) + } + for _, stmt := range prog.Statements { + switch s := stmt.(type) { + case *ast.CreateMicroflowStmt: + return s.Body + case *ast.CreateNanoflowStmt: + return s.Body + } + } + t.Fatalf("no microflow or nanoflow in %q", src) + return nil +} + +func refreshEntityBackend() *mock.MockBackend { + return &mock.MockBackend{ + ReadJavaScriptActionByNameFunc: func(string) (*types.JavaScriptAction, error) { + return &types.JavaScriptAction{ + Name: "RefreshEntity", + Parameters: []*types.JavaActionParameter{ + {Name: "EntityToRefresh", ParameterType: &types.EntityTypeParameterType{}}, + }, + }, nil + }, + } +} + +func jsEntityArgument(t *testing.T, fb *flowBuilder, src string) string { + t.Helper() + for _, stmt := range parsedFlowBody(t, src) { + call, ok := stmt.(*ast.CallJavaScriptActionStmt) + if !ok { + continue + } + action := jsActionActivity(t, fb, call) + value, ok := action.ParameterMappings[0].Value.(*microflows.EntityTypeCodeActionParameterValue) + if !ok { + t.Fatalf("value = %T, want *EntityTypeCodeActionParameterValue", action.ParameterMappings[0].Value) + } + return value.Entity + } + t.Fatal("no call javascript action in body") + return "" +} + +// TestBuildJavaScriptAction_EntityTypeArgumentOnItsOwnLine reproduces +// mendixlabs/mxcli#1171, verbatim in shape: the argument on its own line and +// the closing parenthesis on the next. +// +// $Var = call javascript action NanoflowCommons.RefreshEntity( +// EntityToRefresh = ReferenceData.Dimension +// ); +// +// The visitor keeps an argument's trailing whitespace so an *expression* +// round-trips as written. The #1137 fix then took that same text as the +// entity's qualified name and stored `Entity: "ReferenceData.Dimension\n"` — +// the right $Type, naming an entity that does not exist. mxbuild reports it +// as the same CE0115 #1137 fixed ("The arguments that are passed to +// JavaScript action 'NanoflowCommons.RefreshEntity' do not match the expected +// parameters and need to be refreshed"), while `mxcli check --references` +// passes and DESCRIBE prints the stray newline as harmless layout. +func TestBuildJavaScriptAction_EntityTypeArgumentOnItsOwnLine(t *testing.T) { + fb := &flowBuilder{posX: 100, posY: 100, spacing: HorizontalSpacing, backend: refreshEntityBackend()} + got := jsEntityArgument(t, fb, `create or modify nanoflow CustomModule.ACT_Example () +begin +$Var = call javascript action NanoflowCommons.RefreshEntity( +EntityToRefresh = ReferenceData.Dimension +); +return; +end;`) + if got != "ReferenceData.Dimension" { + t.Errorf("Entity = %q, want %q (issue #1171: a stored name with trailing whitespace is CE0115)", got, "ReferenceData.Dimension") + } +} + +// TestBuildJavaScriptAction_EntityTypeArgumentOneLine is the control: the +// single-line spelling #1137's example uses. It passed before the #1171 fix, +// which is what shows the failure above is the layout and not the value. +func TestBuildJavaScriptAction_EntityTypeArgumentOneLine(t *testing.T) { + fb := &flowBuilder{posX: 100, posY: 100, spacing: HorizontalSpacing, backend: refreshEntityBackend()} + got := jsEntityArgument(t, fb, `create nanoflow CustomModule.ACT_Example () +begin + call javascript action NanoflowCommons.RefreshEntity(EntityToRefresh = ReferenceData.Dimension); +end;`) + if got != "ReferenceData.Dimension" { + t.Errorf("Entity = %q, want ReferenceData.Dimension", got) + } +} + +// TestBuildJavaScriptAction_EntityTypeVariableArgumentOnItsOwnLine: a variable +// argument is resolved to the entity it holds through varTypes, keyed by the +// bare name. Trailing whitespace made that lookup miss too, so the stored +// value became the literal text "$Dim\n". +func TestBuildJavaScriptAction_EntityTypeVariableArgumentOnItsOwnLine(t *testing.T) { + fb := &flowBuilder{ + posX: 100, posY: 100, spacing: HorizontalSpacing, + backend: refreshEntityBackend(), + varTypes: map[string]string{"Dim": "ReferenceData.Dimension"}, + } + got := jsEntityArgument(t, fb, `create nanoflow CustomModule.ACT_Example ($Dim: ReferenceData.Dimension) +begin + call javascript action NanoflowCommons.RefreshEntity( + EntityToRefresh = $Dim + ); +end;`) + if got != "ReferenceData.Dimension" { + t.Errorf("Entity = %q, want ReferenceData.Dimension", got) + } +} + +// TestBuildJavaAction_EntityTypeArgumentOnItsOwnLine: the Java-action builder +// is the twin #1137 copied, and has the same hole. +func TestBuildJavaAction_EntityTypeArgumentOnItsOwnLine(t *testing.T) { + fb := &flowBuilder{ + posX: 100, posY: 100, spacing: HorizontalSpacing, + backend: &mock.MockBackend{ + ReadJavaActionByNameFunc: func(string) (*javaactions.JavaAction, error) { + return &javaactions.JavaAction{ + Name: "Export", + Parameters: []*javaactions.JavaActionParameter{ + {Name: "EntityType", ParameterType: &javaactions.EntityTypeParameterType{}}, + }, + }, nil + }, + }, + } + body := parsedFlowBody(t, `create microflow CustomModule.ACT_Export () +begin + call java action CustomModule.Export( + EntityType = ReferenceData.Dimension + ); +end;`) + for _, stmt := range body { + call, ok := stmt.(*ast.CallJavaActionStmt) + if !ok { + continue + } + id := fb.addCallJavaActionAction(call) + for _, obj := range fb.objects { + if obj.GetID() != id { + continue + } + action := obj.(*microflows.ActionActivity).Action.(*microflows.JavaActionCallAction) + value, ok := action.ParameterMappings[0].Value.(*microflows.EntityTypeCodeActionParameterValue) + if !ok { + t.Fatalf("value = %T, want *EntityTypeCodeActionParameterValue", action.ParameterMappings[0].Value) + } + if value.Entity != "ReferenceData.Dimension" { + t.Errorf("Entity = %q, want ReferenceData.Dimension", value.Entity) + } + return + } + } + t.Fatal("no call java action in body") +} diff --git a/mdl/executor/cmd_microflows_format_action.go b/mdl/executor/cmd_microflows_format_action.go index 8ad527800f..63ddb44be2 100644 --- a/mdl/executor/cmd_microflows_format_action.go +++ b/mdl/executor/cmd_microflows_format_action.go @@ -1573,13 +1573,17 @@ func formatImportXmlAction(ctx *ExecContext, a *microflows.ImportXmlAction, enti // formatImportMappingRange renders the activity's Range — Studio Pro's // All / First / Custom setting. // -// ALWAYS emits one of the three, never nothing. Omitting it would leave the -// builder inferring cardinality from the mapping's root shape, and an -// object-rooted mapping set to All is a real state that inference turns into -// First — Studio Pro's own default, shipped in the blank app's -// FeedbackModule.IMM_PostResponse. Before this, all three settings described -// identically, so the describe→edit→exec cycle silently rewrote the activity. -// (issue #881) +// Before #881 all three settings described identically, so the +// describe→edit→exec cycle silently rewrote the activity; each is now distinct. +// +// The one form left bare is All against an OBJECT variable — Studio Pro's +// default for an object-rooted mapping (the blank app's +// FeedbackModule.IMM_PostResponse). There `all` read as "returns a list" +// (upstream #1176). Bare is exact, not a shorthand: the builder writes a +// missing keyword as All explicitly and infers the object from the mapping. +// That equivalence is what makes omitting it safe — before the builder did +// so, silence stored First and the runtime threw on import. A list result +// keeps its `all`. func formatImportMappingRange(h *microflows.ResultHandlingMapping) string { if h == nil { return "" @@ -1603,6 +1607,9 @@ func formatImportMappingRange(h *microflows.ResultHandlingMapping) string { if microflows.RangeSingleObjectOf(h) { return " first" } + if h.SingleObject { + return "" + } return " all" } diff --git a/mdl/executor/cmd_microflows_import_range_test.go b/mdl/executor/cmd_microflows_import_range_test.go index 9f1a396cd7..09aa13c89e 100644 --- a/mdl/executor/cmd_microflows_import_range_test.go +++ b/mdl/executor/cmd_microflows_import_range_test.go @@ -135,10 +135,12 @@ func TestImportRange_UnauthoredKeepsTheCardinalityInference(t *testing.T) { } } -// DESCRIBE always emits one of the three forms — never nothing. Omitting it -// would leave the builder inferring on re-exec, and an object-rooted mapping set -// to All (Studio Pro's default, shipped in the blank app) would come back as -// First. That silent rewrite is what #881 reported. +// DESCRIBE emits the range whenever it carries information. The one case it +// omits is All against an OBJECT variable — Studio Pro's default for an +// object-rooted mapping — where `all` read as "returns a list" (upstream #1176). +// Omitting it is safe only because the builder writes a missing keyword as All +// explicitly; before that fix, silence re-entered the inference and came back +// as First, which is the silent rewrite #881 reported. func TestFormatImportMappingRange(t *testing.T) { first, no := true, false for _, tc := range []struct { @@ -151,10 +153,12 @@ func TestFormatImportMappingRange(t *testing.T) { { // Mendix's own SUB_Feedback_PostToAppInsights: range All, variable an // object. Describing this as `first` — which reading SingleObject does — - // changes the activity on re-exec. + // changes the activity on re-exec; describing it as `all` reads as a + // list (upstream #1176). The bare form is what the builder writes back + // as exactly this. "all against an object variable", µflows.ResultHandlingMapping{SingleObject: true, RangeSingleObject: &no}, - " all", + "", }, {"limit", µflows.ResultHandlingMapping{LimitExpression: "10"}, " limit 10"}, {"limit+offset", µflows.ResultHandlingMapping{LimitExpression: "10", OffsetExpression: "5"}, " limit 10 offset 5"}, @@ -207,3 +211,34 @@ func TestImportRange_UnauthoredObjectRootedWritesAllNotFirst(t *testing.T) { "binds an object; only the RANGE changed") } } + +// upstream #1176: an object-returning import described as +// +// $objectResponse = import from mapping M.IMM($s) all; +// +// — `all` on an activity that binds one object reads as "returns a list". The +// bare form must describe it, and it must round-trip: re-executing the bare form +// has to store the same activity that `all` stores, or dropping the keyword +// from DESCRIBE silently rewrites the model (the #881 defect, in reverse). +func TestImportRange_ObjectResultDescribesWithoutAll(t *testing.T) { + bare := buildImportRange(t, false, &ast.ImportFromMappingStmt{}) + if got := formatImportMappingRange(bare); got != "" { + t.Errorf("object-rooted, unauthored range: describe emits %q, want \"\" — "+ + "`all` on an object result reads as a list", got) + } + + all := buildImportRange(t, false, &ast.ImportFromMappingStmt{All: true}) + if bare.SingleObject != all.SingleObject || + microflows.RangeSingleObjectOf(bare) != microflows.RangeSingleObjectOf(all) || + (bare.ForceSingleOccurrence == nil) != (all.ForceSingleOccurrence == nil) || + (bare.ForceSingleOccurrence != nil && *bare.ForceSingleOccurrence != *all.ForceSingleOccurrence) { + t.Errorf("bare and `all` build different activities (bare %+v, all %+v) — "+ + "omitting `all` from DESCRIBE would rewrite the model on re-exec", bare, all) + } + + // A list result keeps its `all`: there it is the Range the reader expects. + list := buildImportRange(t, true, &ast.ImportFromMappingStmt{}) + if got := formatImportMappingRange(list); got != " all" { + t.Errorf("list-rooted, unauthored range: describe emits %q, want \" all\"", got) + } +} diff --git a/mdl/executor/cmd_move.go b/mdl/executor/cmd_move.go index 57c970799e..e47b9f5731 100644 --- a/mdl/executor/cmd_move.go +++ b/mdl/executor/cmd_move.go @@ -8,6 +8,7 @@ import ( "github.com/mendixlabs/mxcli/mdl/ast" mdlerrors "github.com/mendixlabs/mxcli/mdl/errors" + "github.com/mendixlabs/mxcli/mdl/types" "github.com/mendixlabs/mxcli/model" "github.com/mendixlabs/mxcli/sdk/domainmodel" ) @@ -319,7 +320,8 @@ func moveEntity(ctx *ExecContext, name ast.QualifiedName, sourceModule, targetMo return mdlerrors.NewBackend("get target domain model", err) } - // Move entity via writer (converts associations to CrossAssociations, updates validation rule refs) + // Move entity via writer (converts associations to CrossAssociations, updates + // validation rule and access rule refs) convertedAssocs, err := ctx.Backend.MoveEntity(entity, sourceDM.ID, targetDM.ID, sourceModule.Name, targetModule.Name) if err != nil { return mdlerrors.NewBackend("move entity", err) @@ -344,10 +346,46 @@ func moveEntity(ctx *ExecContext, name ast.QualifiedName, sourceModule, targetMo fmt.Fprintf(ctx.Output, "Updated %d OQL query(ies) referencing %s\n", oqlUpdated, oldQualifiedName) } + // Every OTHER cross-module move sweeps the project for BY_NAME references (the + // isCrossModuleMove branch in execMove). An entity move returns early, before + // both the doctype switch and that sweep — correctly, because an entity is not a + // top-level unit, but the sweep is purely name-based and applies just the same. + // Without it a move reported success and left every reference to the entity, its + // attribute paths and the converted association naming the old module: 33 CE1613s + // in a blank 11.13 app, in microflow activities, page widgets, page parameters and + // access rules (#605). + // + // The association renames come from the backend rather than from the module names, + // because only the direction that carried the cross-association to the target + // changed its qualified name; the other is a no-op here by construction. + sweep := func(oldQN, newQN string) error { + if oldQN == newQN { + return nil + } + updated, err := ctx.Backend.UpdateQualifiedNameInAllUnits(oldQN, newQN) + if err != nil { + return mdlerrors.NewBackend("update references", err) + } + if updated > 0 { + fmt.Fprintf(ctx.Output, "Updated references in %d document(s): %s → %s\n", updated, oldQN, newQN) + } + return nil + } + // The entity itself, which also covers every `Module.Entity.Attribute` path + // because the sweep matches a name that equals the old one or is prefixed by it. + if err := sweep(oldQualifiedName, newQualifiedName); err != nil { + return err + } + for _, assoc := range convertedAssocs { + if err := sweep(assoc.OldQualifiedName, assoc.NewQualifiedName); err != nil { + return err + } + } + fmt.Fprintf(ctx.Output, "Moved entity %s to %s\n", name.String(), targetModule.Name) if len(convertedAssocs) > 0 { fmt.Fprintf(ctx.Output, "Converted %d association(s) to cross-module associations:\n", len(convertedAssocs)) - for _, assocName := range convertedAssocs { + for _, assocName := range types.MovedAssociationNames(convertedAssocs) { fmt.Fprintf(ctx.Output, " - %s\n", assocName) } } diff --git a/mdl/executor/cmd_rename.go b/mdl/executor/cmd_rename.go index 863e9b32d4..ed5f5261f3 100644 --- a/mdl/executor/cmd_rename.go +++ b/mdl/executor/cmd_rename.go @@ -10,6 +10,8 @@ import ( "github.com/mendixlabs/mxcli/mdl/ast" mdlerrors "github.com/mendixlabs/mxcli/mdl/errors" "github.com/mendixlabs/mxcli/mdl/types" + "github.com/mendixlabs/mxcli/model" + "github.com/mendixlabs/mxcli/sdk/domainmodel" ) // execRename handles RENAME statements for all document types. @@ -91,6 +93,7 @@ func execRenameEntity(ctx *ExecContext, s *ast.RenameStmt) error { for _, ent := range dm.Entities { if ent.Name == s.Name.Name { ent.Name = s.NewName + repointEntitySelfRefs(ent, oldQualifiedName, newQualifiedName) break } } @@ -108,6 +111,47 @@ func execRenameEntity(ctx *ExecContext, s *ast.RenameStmt) error { return nil } +// repointEntitySelfRefs rewrites the qualified names the entity uses to refer to +// its OWN members, which embed the entity name that has just changed. +// +// It is needed because of a clobber, not because the sweep missed them: the +// project-wide RenameReferences pass above does rewrite these names in the raw +// unit, and then UpdateDomainModel persists the semantic model read BEFORE it and +// puts the stale ones back. Measured on a real 11.13 app, the report and the errors +// line up exactly — "Updated 3 reference(s) in 1 document(s)" followed by three +// CE1613s at "Access rule of entity" and "Validation rule of entity" for the entity +// just renamed. Re-pointing here makes the persist agree with the sweep instead of +// undoing it. +// +// Leaving them stale is worse than a dangling string: entityToGen's +// syncMemberAccesses matches existing entries BY qualified name, so a stale +// `Module.Old.Attr` never equals the rebuilt `Module.New.Attr` and it appends the +// new entry while keeping the old — the entity ends up carrying every member twice, +// half of the entries dangling. DESCRIBE renders members bare, so the duplication is +// the only visible trace. This is the RENAME sibling of the MOVE ENTITY defect in +// ako/mxcli#605, where the same names went stale on the module prefix instead. +// +// Only names qualified by the ENTITY are rewritten. An association member is named +// `Module.Association` — it carries no entity name — so a blanket prefix swap would +// corrupt a reference that is still correct. +func repointEntitySelfRefs(ent *domainmodel.Entity, oldQualifiedName, newQualifiedName string) { + oldPrefix, newPrefix := oldQualifiedName+".", newQualifiedName+"." + repoint := func(name string) string { + if strings.HasPrefix(name, oldPrefix) { + return newPrefix + name[len(oldPrefix):] + } + return name + } + for _, ar := range ent.AccessRules { + for _, ma := range ar.MemberAccesses { + ma.AttributeName = repoint(ma.AttributeName) + } + } + for _, vr := range ent.ValidationRules { + vr.AttributeID = model.ID(repoint(string(vr.AttributeID))) + } +} + // execRenameModule renames a module and updates all BY_NAME references with the module prefix. func execRenameModule(ctx *ExecContext, s *ast.RenameStmt) error { oldModuleName := s.Name.Module @@ -392,6 +436,7 @@ func execRenameAssociation(ctx *ExecContext, s *ast.RenameStmt) error { break } } + repointAssociationMemberRefs(dm, oldQualifiedName, newQualifiedName) if err := ctx.Backend.UpdateDomainModel(dm); err != nil { return mdlerrors.NewBackend("update association name", err) } @@ -406,6 +451,39 @@ func execRenameAssociation(ctx *ExecContext, s *ast.RenameStmt) error { return nil } +// repointAssociationMemberRefs rewrites the entity access rules in this unit that +// name the renamed association, which they do by qualified name. +// +// Same clobber as repointEntitySelfRefs, one document type over: RenameReferences +// rewrites these in the raw unit and the UpdateDomainModel that follows puts the +// stale name back. It was the last of the four CE1613s a RENAME ASSOCIATION followed +// by a RENAME ENTITY left on a real 11.13 app, and the one that survives longest, +// because the stale name is re-read by every later statement that loads the unit. +// +// The match is EXACT rather than a prefix: an association member is named +// `Module.Association` with nothing after it, and a prefix match would also rewrite +// a differently-named association that happens to start with the same text. +// +// Every entity is walked, not just the association's FROM side. Mendix stores a +// MemberAccess for an association only on the FROM entity (adding one to the TO +// entity is CE0066), so in a well-formed model this finds them all on one entity — +// but a sweep that assumed the storage rule would silently skip anything a model +// carries in spite of it. +func repointAssociationMemberRefs(dm *domainmodel.DomainModel, oldQualifiedName, newQualifiedName string) { + if dm == nil { + return + } + for _, ent := range dm.Entities { + for _, ar := range ent.AccessRules { + for _, ma := range ar.MemberAccesses { + if ma.AssociationName == oldQualifiedName { + ma.AssociationName = newQualifiedName + } + } + } + } +} + // execRenameJavaAction renames a Java action and its .java source file. func execRenameJavaAction(ctx *ExecContext, s *ast.RenameStmt) error { oldQualifiedName := s.Name.Module + "." + s.Name.Name diff --git a/mdl/executor/issue605_move_entity_sweep_test.go b/mdl/executor/issue605_move_entity_sweep_test.go new file mode 100644 index 0000000000..647f80920c --- /dev/null +++ b/mdl/executor/issue605_move_entity_sweep_test.go @@ -0,0 +1,136 @@ +// SPDX-License-Identifier: Apache-2.0 + +package executor + +import ( + "testing" + + "github.com/mendixlabs/mxcli/mdl/ast" + "github.com/mendixlabs/mxcli/mdl/backend/mock" + "github.com/mendixlabs/mxcli/mdl/types" + "github.com/mendixlabs/mxcli/model" + "github.com/mendixlabs/mxcli/sdk/domainmodel" +) + +// TestIssue605_MoveEntitySweepsQualifiedNames guards the call site, because the +// call site is what was missing. +// +// execMove returns early for ENTITY — before the doctype switch and before the +// isCrossModuleMove sweep every other doctype gets — so a cross-module entity move +// rewrote no references at all: 33 CE1613s in a blank 11.13 app, reported as +// success. The backend contract (which association changed module, and to what) is +// covered in mdl/backend/modelsdk/issue605_move_refs_test.go; what this asserts is +// that moveEntity actually drives the sweep from it. +// +// A mock is the right instrument here: the real sweep is already exercised by the +// doctypes that never lost it, and what needs pinning is which (old, new) pairs +// reach it — in particular that an association whose module did NOT change is not +// swept, since renaming it would corrupt references that are still correct. +func TestIssue605_MoveEntitySweepsQualifiedNames(t *testing.T) { + for _, tc := range []struct { + name string + // converted is what the backend reports for the move. + converted []types.MovedAssociation + // wantSweeps lists the (old, new) pairs the sweep must be called with, in + // any order. + wantSweeps map[string]string + }{ + { + name: "ParentMoved_EntityAndAssociationSwept", + converted: []types.MovedAssociation{{ + Name: "Child_Parent", + OldQualifiedName: "Source.Child_Parent", + NewQualifiedName: "Target.Child_Parent", + }}, + wantSweeps: map[string]string{ + "Source.Subject": "Target.Subject", + "Source.Child_Parent": "Target.Child_Parent", + }, + }, + { + // The cross-association stayed in the source module, so its qualified + // name is unchanged and must NOT be swept. + name: "ChildMoved_OnlyEntitySwept", + converted: []types.MovedAssociation{{ + Name: "Child_Parent", + OldQualifiedName: "Source.Child_Parent", + NewQualifiedName: "Source.Child_Parent", + }}, + wantSweeps: map[string]string{ + "Source.Subject": "Target.Subject", + }, + }, + { + name: "NoAssociations_EntityStillSwept", + converted: nil, + wantSweeps: map[string]string{"Source.Subject": "Target.Subject"}, + }, + } { + t.Run(tc.name, func(t *testing.T) { + src := mkModule("Source") + dst := mkModule("Target") + subject := &domainmodel.Entity{Name: "Subject", Persistable: true} + subject.ID = nextID("entity") + + gotSweeps := map[string]string{} + mb := &mock.MockBackend{ + IsConnectedFunc: func() bool { return true }, + ListModulesFunc: func() ([]*model.Module, error) { return []*model.Module{src, dst}, nil }, + GetModuleByNameFunc: func(name string) (*model.Module, error) { + switch name { + case src.Name: + return src, nil + case dst.Name: + return dst, nil + } + return nil, nil + }, + GetDomainModelFunc: func(moduleID model.ID) (*domainmodel.DomainModel, error) { + dm := &domainmodel.DomainModel{ContainerID: moduleID} + dm.ID = nextID("dm") + if moduleID == src.ID { + dm.Entities = []*domainmodel.Entity{subject} + } + return dm, nil + }, + MoveEntityFunc: func(_ *domainmodel.Entity, _, _ model.ID, _, _ string) ([]types.MovedAssociation, error) { + return tc.converted, nil + }, + UpdateOqlQueriesForMovedEntityFunc: func(_, _ string) (int, error) { return 0, nil }, + UpdateQualifiedNameInAllUnitsFunc: func(oldName, newName string) (int, error) { + if prev, dup := gotSweeps[oldName]; dup { + t.Errorf("swept %s twice (%s then %s)", oldName, prev, newName) + } + gotSweeps[oldName] = newName + return 1, nil + }, + } + + ctx, _ := newMockCtx(t, withBackend(mb), withHierarchy(mkHierarchy(src, dst))) + err := execMove(ctx, &ast.MoveStmt{ + DocumentType: ast.DocumentTypeEntity, + Name: ast.QualifiedName{Module: src.Name, Name: subject.Name}, + TargetModule: dst.Name, + }) + if err != nil { + t.Fatalf("execMove: %v", err) + } + + for old, want := range tc.wantSweeps { + got, ok := gotSweeps[old] + if !ok { + t.Errorf("no reference sweep for %s → %s", old, want) + continue + } + if got != want { + t.Errorf("sweep for %s went to %s, want %s", old, got, want) + } + } + for old := range gotSweeps { + if _, want := tc.wantSweeps[old]; !want { + t.Errorf("unexpected reference sweep for %s → %s; renaming a name that did not change corrupts references that were still correct", old, gotSweeps[old]) + } + } + }) + } +} diff --git a/mdl/executor/loop_containment_test.go b/mdl/executor/loop_containment_test.go index 27710d7de2..2bfa385aa2 100644 --- a/mdl/executor/loop_containment_test.go +++ b/mdl/executor/loop_containment_test.go @@ -153,3 +153,77 @@ func TestLoopBox_NestedLoopsEachContainTheirChildren(t *testing.T) { logStmt("before"), inner, logStmt("after"), })) } + +// buildWhileFlow is the same fixture for a WHILE loop, which has its own builder +// (addWhileStatement) and therefore its own layout code. +func buildWhileFlow(t *testing.T, body []ast.MicroflowStatement) []microflows.MicroflowObject { + t.Helper() + fb := &flowBuilder{ + posX: 100, + posY: 200, + spacing: HorizontalSpacing, + varTypes: map[string]string{"N": "Integer"}, + declaredVars: map[string]string{}, + measurer: &layoutMeasurer{varTypes: map[string]string{"N": "Integer"}}, + } + fb.addWhileStatement(&ast.WhileStmt{ + Condition: &ast.LiteralExpr{Kind: ast.LiteralBoolean, Value: true}, + Body: body, + }) + return fb.objects +} + +// A WHILE loop's first activity escaped its own box, on every while loop mxcli +// wrote — reported from a real project as "MPR011 on every `while` loop … first +// activity at (50,80) lies outside the loop box … even single-level loops" +// (ako/mxcli#645). +// +// The cause is one missing term. addWhileStatement's comment says its "layout +// matches addLoopStatement but without iterator icon space", and it dropped the +// iterator space correctly — but took ActivityWidth/2 with it: +// +// foreach: innerStartX = LoopPadding + iteratorSpace + ActivityWidth/2 = 210 +// while: innerStartX = LoopPadding = 50 +// +// A child's Position is its CENTRE ("RelativeMiddlePoint in Mendix"), so a +// centre at x=50 with ActivityWidth=120 puts the left edge at -10. The half-width +// is not iterator space; it is what converts a centre to a left edge. The very +// next line proves the omission was accidental rather than a choice — it adds +// ActivityHeight/2 for exactly this reason on the Y axis. +// +// This survived because the containment invariant WAS tested, on the foreach +// builder only. An invariant is worth what its coverage is: two builders, one +// tested, and the untested one shipped the violation to every project. +func TestWhileLoopBox_ContainsDefaultLaidOutChildren(t *testing.T) { + for _, n := range []int{1, 2, 4, 7} { + body := make([]ast.MicroflowStatement, 0, n) + for i := 0; i < n; i++ { + body = append(body, logStmt("x")) + } + assertLoopContainsItsChildren(t, buildWhileFlow(t, body)) + } +} + +// The X axis specifically, so a regression cannot hide behind a box that grew +// tall enough. A box that merely got wider on the right does not fix a child +// hanging off the left edge. +func TestWhileLoopFirstChildLeftEdgeIsInsideTheBox(t *testing.T) { + objects := buildWhileFlow(t, []ast.MicroflowStatement{logStmt("only")}) + + for _, o := range objects { + loop, ok := o.(*microflows.LoopedActivity) + if !ok { + continue + } + minX, _, _, _, n := loopChildBounds(loop) + if n == 0 { + t.Fatal("the while loop has no children — fixture is wrong") + } + if minX < 0 { + t.Errorf("the first activity's left edge is at x=%d, outside its own box: "+ + "a centre at LoopPadding is half an activity too far left", minX) + } + return + } + t.Fatal("no LoopedActivity in the built flow — fixture is wrong") +} diff --git a/mdl/executor/rename_entity_self_refs_test.go b/mdl/executor/rename_entity_self_refs_test.go new file mode 100644 index 0000000000..c04e44233a --- /dev/null +++ b/mdl/executor/rename_entity_self_refs_test.go @@ -0,0 +1,289 @@ +// SPDX-License-Identifier: Apache-2.0 + +package executor + +import ( + "testing" + + "github.com/mendixlabs/mxcli/mdl/ast" + "github.com/mendixlabs/mxcli/mdl/backend/mock" + "github.com/mendixlabs/mxcli/mdl/types" + "github.com/mendixlabs/mxcli/model" + "github.com/mendixlabs/mxcli/sdk/domainmodel" +) + +// TestRenameEntity_RepointsOwnMemberReferences guards the half of a RENAME ENTITY +// that lives INSIDE the renamed entity: its own access rules and validation rules +// name each member by qualified name (`Module.Entity.Attr`), and that name contains +// the entity name being changed. +// +// The mechanism is a clobber, and the reference report is what makes it visible. +// execRenameEntity reads the domain model, runs the project-wide RenameReferences +// sweep — which DOES rewrite those names, in the raw unit — and then persists the +// semantic model it read BEFORE the sweep, overwriting the unit with the stale +// names. Measured on a real 11.13 app: "Updated 3 reference(s) in 1 document(s)" +// followed by exactly three CE1613s naming those three members, at "Validation rule +// of entity" and "Access rule of entity" for the entity that had just been renamed. +// +// It is the sibling of the MOVE ENTITY defect in ako/mxcli#605 — same stale +// qualified names inside the moved/renamed entity, different command — and worse +// than a dangling string for the same reason: entityToGen's syncMemberAccesses +// matches existing entries BY qualified name, so a stale `Module.Old.Attr` never +// equals the rebuilt `Module.New.Attr` and it appends the new entry while keeping +// the old one. +// +// A mock is the right instrument: the defect is the executor handing +// UpdateDomainModel a semantic model whose self-references are stale, so what needs +// pinning is the model it passes. The end-to-end proof that mxbuild then accepts the +// project is mdl-examples/bug-tests/domainmodel-1169-whole-unit-storage-guids.mdl. +func TestRenameEntity_RepointsOwnMemberReferences(t *testing.T) { + const ( + modName = "Ren" + oldName = "Widget" + newName = "Gadget" + ) + + mod := mkModule(modName) + attr := &domainmodel.Attribute{Name: "WidgetName", Type: &domainmodel.StringAttributeType{Length: 100}} + attr.ID = nextID("attr") + + ent := &domainmodel.Entity{Name: oldName, Persistable: true, Attributes: []*domainmodel.Attribute{attr}} + ent.ID = nextID("entity") + // An access rule naming the attribute, and — the case a fix that only covered + // access rules would miss — a validation rule naming it too. + ma := &domainmodel.MemberAccess{ + AttributeID: attr.ID, + AttributeName: modName + "." + oldName + "." + attr.Name, + AccessRights: domainmodel.MemberAccessRightsReadWrite, + } + ma.ID = nextID("ma") + ar := &domainmodel.AccessRule{ContainerID: ent.ID, AllowRead: true, MemberAccesses: []*domainmodel.MemberAccess{ma}} + ar.ID = nextID("ar") + ent.AccessRules = []*domainmodel.AccessRule{ar} + + vr := &domainmodel.ValidationRule{ + ContainerID: ent.ID, + AttributeID: model.ID(modName + "." + oldName + "." + attr.Name), + Type: "Required", + } + vr.ID = nextID("vr") + ent.ValidationRules = []*domainmodel.ValidationRule{vr} + + // An association reference is NOT qualified by the entity name (it is + // `Module.Association`), so renaming the entity must leave it exactly as it is. + // A blanket prefix swap would corrupt it. + assocRef := modName + ".Widget_Other" + maAssoc := &domainmodel.MemberAccess{ + AssociationName: assocRef, + AccessRights: domainmodel.MemberAccessRightsReadWrite, + } + maAssoc.ID = nextID("ma") + ar.MemberAccesses = append(ar.MemberAccesses, maAssoc) + + // A sibling entity whose own members must not be touched: the rename rewrites + // what the RENAMED entity says about itself, nothing else in the unit. + sibAttr := &domainmodel.Attribute{Name: "SibName", Type: &domainmodel.StringAttributeType{Length: 50}} + sibAttr.ID = nextID("attr") + sib := &domainmodel.Entity{Name: "Sibling", Persistable: true, Attributes: []*domainmodel.Attribute{sibAttr}} + sib.ID = nextID("entity") + sibMA := &domainmodel.MemberAccess{ + AttributeID: sibAttr.ID, + AttributeName: modName + ".Sibling." + sibAttr.Name, + AccessRights: domainmodel.MemberAccessRightsReadOnly, + } + sibMA.ID = nextID("ma") + sibAR := &domainmodel.AccessRule{ContainerID: sib.ID, AllowRead: true, MemberAccesses: []*domainmodel.MemberAccess{sibMA}} + sibAR.ID = nextID("ar") + sib.AccessRules = []*domainmodel.AccessRule{sibAR} + + dm := &domainmodel.DomainModel{ContainerID: mod.ID, Entities: []*domainmodel.Entity{ent, sib}} + dm.ID = nextID("dm") + + var persisted *domainmodel.DomainModel + mb := &mock.MockBackend{ + IsConnectedFunc: func() bool { return true }, + ListModulesFunc: func() ([]*model.Module, error) { return []*model.Module{mod}, nil }, + GetModuleByNameFunc: func(name string) (*model.Module, error) { return mod, nil }, + GetDomainModelFunc: func(model.ID) (*domainmodel.DomainModel, error) { + return dm, nil + }, + RenameReferencesFunc: func(_, _ string, _ bool) ([]types.RenameHit, error) { return nil, nil }, + UpdateDomainModelFunc: func(d *domainmodel.DomainModel) error { + persisted = d + return nil + }, + } + + ctx, _ := newMockCtx(t, withBackend(mb), withHierarchy(mkHierarchy(mod))) + if err := execRename(ctx, &ast.RenameStmt{ + ObjectType: "entity", + Name: ast.QualifiedName{Module: modName, Name: oldName}, + NewName: newName, + }); err != nil { + t.Fatalf("execRename: %v", err) + } + if persisted == nil { + t.Fatal("UpdateDomainModel was never called; the rename did not persist") + } + + var renamed, sibling *domainmodel.Entity + for _, e := range persisted.Entities { + switch e.ID { + case ent.ID: + renamed = e + case sib.ID: + sibling = e + } + } + if renamed == nil || sibling == nil { + t.Fatalf("persisted model lost an entity: renamed=%v sibling=%v", renamed != nil, sibling != nil) + } + if renamed.Name != newName { + t.Fatalf("entity not renamed: got %q, want %q", renamed.Name, newName) + } + + wantAttrRef := modName + "." + newName + "." + attr.Name + gotAttrRefs := 0 + for _, r := range renamed.AccessRules { + for _, m := range r.MemberAccesses { + if m.AttributeName != "" { + gotAttrRefs++ + if m.AttributeName != wantAttrRef { + t.Errorf("access rule still names %q, want %q", m.AttributeName, wantAttrRef) + } + } + if m.AssociationName != "" && m.AssociationName != assocRef { + t.Errorf("association reference was rewritten to %q; it is not qualified by the entity name and must stay %q", + m.AssociationName, assocRef) + } + } + } + if gotAttrRefs != 1 { + t.Errorf("expected 1 attribute member reference, got %d", gotAttrRefs) + } + + for _, r := range renamed.ValidationRules { + if got := string(r.AttributeID); got != wantAttrRef { + t.Errorf("validation rule still names %q, want %q", got, wantAttrRef) + } + } + + // The sibling is the control: a fix that swapped the module prefix, or every + // occurrence of the old name anywhere in the unit, would have moved this too. + for _, r := range sibling.AccessRules { + for _, m := range r.MemberAccesses { + if want := modName + ".Sibling." + sibAttr.Name; m.AttributeName != want { + t.Errorf("sibling entity's member reference changed to %q, want %q", m.AttributeName, want) + } + } + } +} + +// TestRenameAssociation_RepointsEntityMemberReferences is the same defect one +// document type over: an entity access rule names an association by qualified name, +// and a RENAME ASSOCIATION left that name stale for exactly the same reason — the +// sweep fixes the raw unit, the UpdateDomainModel that follows puts the old name back. +// +// It is the longest-lived of the four: the stale name is then re-read by every later +// statement that loads the unit, so a RENAME ASSOCIATION followed by an unrelated +// RENAME ENTITY carried it forward into the second write too. On the real 11.13 app +// it was the one CE1613 that survived fixing the entity half. +func TestRenameAssociation_RepointsEntityMemberReferences(t *testing.T) { + const ( + modName = "Ren" + oldName = "Child_Parent" + newName = "Kid_Parent" + ) + mod := mkModule(modName) + + child := &domainmodel.Entity{Name: "Child", Persistable: true} + child.ID = nextID("entity") + parent := &domainmodel.Entity{Name: "Parent", Persistable: true} + parent.ID = nextID("entity") + + // The MemberAccess sits on the FROM (child) entity — Mendix stores it there and + // only there (a MemberAccess for an association on the TO entity is CE0066). + ma := &domainmodel.MemberAccess{ + AssociationName: modName + "." + oldName, + AccessRights: domainmodel.MemberAccessRightsReadWrite, + } + ma.ID = nextID("ma") + // A similarly-named association that is NOT the one being renamed: the control + // for a prefix match, which would rewrite this too. + other := &domainmodel.MemberAccess{ + AssociationName: modName + "." + oldName + "_Extra", + AccessRights: domainmodel.MemberAccessRightsReadOnly, + } + other.ID = nextID("ma") + ar := &domainmodel.AccessRule{ + ContainerID: child.ID, + AllowRead: true, + MemberAccesses: []*domainmodel.MemberAccess{ma, other}, + } + ar.ID = nextID("ar") + child.AccessRules = []*domainmodel.AccessRule{ar} + + assoc := &domainmodel.Association{ + Name: oldName, + ParentID: child.ID, + ChildID: parent.ID, + Type: domainmodel.AssociationTypeReference, + Owner: domainmodel.AssociationOwnerDefault, + } + assoc.ID = nextID("assoc") + + dm := &domainmodel.DomainModel{ + ContainerID: mod.ID, + Entities: []*domainmodel.Entity{child, parent}, + Associations: []*domainmodel.Association{assoc}, + } + dm.ID = nextID("dm") + + var persisted *domainmodel.DomainModel + mb := &mock.MockBackend{ + IsConnectedFunc: func() bool { return true }, + ListModulesFunc: func() ([]*model.Module, error) { return []*model.Module{mod}, nil }, + GetModuleByNameFunc: func(string) (*model.Module, error) { return mod, nil }, + GetDomainModelFunc: func(model.ID) (*domainmodel.DomainModel, error) { return dm, nil }, + RenameReferencesFunc: func(_, _ string, _ bool) ([]types.RenameHit, error) { return nil, nil }, + UpdateDomainModelFunc: func(d *domainmodel.DomainModel) error { + persisted = d + return nil + }, + } + + ctx, _ := newMockCtx(t, withBackend(mb), withHierarchy(mkHierarchy(mod))) + if err := execRename(ctx, &ast.RenameStmt{ + ObjectType: "association", + Name: ast.QualifiedName{Module: modName, Name: oldName}, + NewName: newName, + }); err != nil { + t.Fatalf("execRename: %v", err) + } + if persisted == nil { + t.Fatal("UpdateDomainModel was never called; the rename did not persist") + } + + if got := persisted.Associations[0].Name; got != newName { + t.Fatalf("association not renamed: got %q, want %q", got, newName) + } + want := modName + "." + newName + wantOther := modName + "." + oldName + "_Extra" + for _, e := range persisted.Entities { + for _, r := range e.AccessRules { + for _, m := range r.MemberAccesses { + switch m.ID { + case ma.ID: + if m.AssociationName != want { + t.Errorf("access rule still names %q, want %q", m.AssociationName, want) + } + case other.ID: + if m.AssociationName != wantOther { + t.Errorf("a differently-named association was rewritten to %q, want %q "+ + "(the match must be exact, not a prefix)", m.AssociationName, wantOther) + } + } + } + } + } +} diff --git a/mdl/executor/validate.go b/mdl/executor/validate.go index 2b87a26bdc..bac143cac5 100644 --- a/mdl/executor/validate.go +++ b/mdl/executor/validate.go @@ -847,6 +847,12 @@ func validateFlowBodyReferences(ctx *ExecContext, body []ast.MicroflowStatement, } } + // A queued call's target must return nothing (CE7033) — resolving the name + // says nothing about that (mendixlabs/mxcli#1064). + if len(refs.queues) > 0 && len(refs.microflows) > 0 { + errors = append(errors, validateQueuedMicroflowTargets(body, buildMicroflowReturnTypes(ctx), sc)...) + } + if len(refs.nanoflows) > 0 { known := buildNanoflowQualifiedNames(ctx) for _, ref := range refs.nanoflows { diff --git a/mdl/executor/validate_program.go b/mdl/executor/validate_program.go index d0ca4fba75..3c98718616 100644 --- a/mdl/executor/validate_program.go +++ b/mdl/executor/validate_program.go @@ -230,6 +230,12 @@ func ValidateProgram(prog *ast.Program, projectPath string) []linter.Violation { // is in the script and needs no project (CapTrackV2 FINDINGS §6). violations = append(violations, ValidateAfterStartupReturnType(prog)...) + // Flag `CALL MICROFLOW … IN QUEUE …` on a microflow that returns a value — + // Mendix requires a background microflow to return nothing (CE7033). Same + // shape as the after-startup check: the call resolves, and the constraint is + // on the flow it names (mendixlabs/mxcli#1064). + violations = append(violations, ValidateQueuedCallReturnType(prog)...) + // Flag an export mapping value whose member is a nested path — an export has // to produce the intermediate node, so Mendix rejects it with CE5015. The // answer is in the statement, so it runs here rather than under --references diff --git a/mdl/executor/validate_queued_call_return.go b/mdl/executor/validate_queued_call_return.go new file mode 100644 index 0000000000..e73ce2a81d --- /dev/null +++ b/mdl/executor/validate_queued_call_return.go @@ -0,0 +1,171 @@ +// SPDX-License-Identifier: Apache-2.0 + +// CE7033: "A microflow used for background execution must have a Microflow +// return type of 'Nothing'." +// +// `CALL MICROFLOW M.F(…) IN QUEUE M.Q` runs F in the background, and Mendix +// refuses the call unless F returns nothing. Both halves are in the model — the +// binding mxcli writes and the signature of the flow it names — and nothing +// compared them, so `mxcli check --references` said "Check passed!" and the build +// failed (mendixlabs/mxcli#1064). It is the queued-call sibling of MDL073's +// after-startup check (CE0142): the reference RESOLVES, and the constraint is on +// the thing it names. +// +// Only CALL MICROFLOW is checked. A queued CALL JAVA ACTION has its own rule +// (CE7038, documented in the queues skill) and is not part of this report. +package executor + +import ( + "fmt" + "reflect" + "strings" + + "github.com/mendixlabs/mxcli/mdl/ast" + mdlerrors "github.com/mendixlabs/mxcli/mdl/errors" + "github.com/mendixlabs/mxcli/mdl/linter" +) + +// queuedMicroflowCall is one `CALL MICROFLOW … IN QUEUE …` found in a body. +type queuedMicroflowCall struct { + target string // qualified name of the called microflow + queue string // qualified name of the queue +} + +// queuedMicroflowCalls returns every queued microflow call under root, found by +// walking the whole statement tree — calls nest inside IF / LOOP / WHILE / error +// handlers, and a hand-written switch over statement kinds silently misses +// whichever nesting was added last (the same reason authoredQueueTargets walks +// by reflection). +func queuedMicroflowCalls(root any) []queuedMicroflowCall { + var out []queuedMicroflowCall + var walk func(v reflect.Value) + seen := map[uintptr]bool{} + walk = func(v reflect.Value) { + switch v.Kind() { + case reflect.Ptr, reflect.Interface: + if v.IsNil() { + return + } + if v.Kind() == reflect.Ptr { + if seen[v.Pointer()] { + return + } + seen[v.Pointer()] = true + if s, ok := v.Interface().(*ast.CallMicroflowStmt); ok && s.Queue != nil && s.MicroflowName.Module != "" { + out = append(out, queuedMicroflowCall{ + target: s.MicroflowName.String(), + queue: s.Queue.Module + "." + s.Queue.Name, + }) + } + } + walk(v.Elem()) + case reflect.Slice, reflect.Array: + for i := 0; i < v.Len(); i++ { + walk(v.Index(i)) + } + case reflect.Struct: + for i := 0; i < v.NumField(); i++ { + if v.Type().Field(i).PkgPath != "" { + continue // unexported + } + walk(v.Field(i)) + } + } + } + walk(reflect.ValueOf(root)) + return out +} + +// checkQueuedMicroflowReturnsNothing reports a queued call whose target returns +// a value. returnType is the target's return type name as the backend or the +// script declares it; "" and "Void" both mean Nothing. +func checkQueuedMicroflowReturnsNothing(call queuedMicroflowCall, returnType string) error { + if returnType == "" || strings.EqualFold(returnType, "Void") { + return nil + } + return mdlerrors.NewValidationf( + "microflow %s is called in queue %s but returns %s — a microflow run in the background must "+ + "return nothing, and the build reports CE7033 \"A microflow used for background execution "+ + "must have a Microflow return type of 'Nothing'.\"", + call.target, call.queue, returnType) +} + +// validateQueuedMicroflowTargets is the --references half: queued calls whose +// target is already stored in the project, typed by returnTypes (from +// buildMicroflowReturnTypes). +// +// A target the script itself (re)creates is skipped — its declared signature +// is what will be written, and ValidateQueuedCallReturnType reports it without +// a project; checking it here too would print the same fault twice. A target +// whose return type is unavailable is left alone: nothing is KNOWN to be wrong, +// and a refusal would block a script that builds. +func validateQueuedMicroflowTargets(body []ast.MicroflowStatement, returnTypes map[string]string, sc *scriptContext) []string { + var errs []string + for _, call := range queuedMicroflowCalls(body) { + if sc != nil && sc.microflows[call.target] { + continue + } + ret, ok := returnTypes[call.target] + if !ok { + continue + } + if err := checkQueuedMicroflowReturnsNothing(call, ret); err != nil { + errs = append(errs, err.Error()) + } + } + return errs +} + +// ValidateQueuedCallReturnType (MDL088) is the project-less half: a queued call +// whose target the same script creates. That is the common shape — write the +// worker microflow, then enqueue it — so the answer is in the script and +// `mxcli check` with no project can give it. +func ValidateQueuedCallReturnType(prog *ast.Program) []linter.Violation { + if prog == nil { + return nil + } + + returnTypes := map[string]string{} + for _, stmt := range prog.Statements { + mf, ok := stmt.(*ast.CreateMicroflowStmt) + if !ok { + continue + } + if mf.ReturnType == nil || mf.ReturnType.Type.Kind == ast.TypeVoid { + returnTypes[mf.Name.String()] = "" + continue + } + if kind := mf.ReturnType.Type.Kind.String(); kind != "Unknown" { + returnTypes[mf.Name.String()] = kind + } + } + if len(returnTypes) == 0 { + return nil + } + + var out []linter.Violation + for _, stmt := range prog.Statements { + mf, ok := stmt.(*ast.CreateMicroflowStmt) + if !ok { + continue + } + for _, call := range queuedMicroflowCalls(mf.Body) { + // Only a microflow this script defines: anything else needs the + // project, and reporting it from here would guess. + ret, defined := returnTypes[call.target] + if !defined { + continue + } + if err := checkQueuedMicroflowReturnsNothing(call, ret); err != nil { + out = append(out, linter.Violation{ + RuleID: "MDL088", + Severity: linter.SeverityError, + Message: fmt.Sprintf("microflow %s: %s", mf.Name.String(), err.Error()), + Suggestion: "Drop the `returns` clause from " + call.target + + " (and its `return` value), or call it without `in queue`", + }) + } + } + } + return out +} diff --git a/mdl/executor/validate_queued_call_return_test.go b/mdl/executor/validate_queued_call_return_test.go new file mode 100644 index 0000000000..80c262e986 --- /dev/null +++ b/mdl/executor/validate_queued_call_return_test.go @@ -0,0 +1,175 @@ +// SPDX-License-Identifier: Apache-2.0 + +package executor + +import ( + "strings" + "testing" + + "github.com/mendixlabs/mxcli/mdl/ast" + "github.com/mendixlabs/mxcli/mdl/visitor" +) + +func parseQueuedCallProgram(t *testing.T, src string) *ast.Program { + t.Helper() + prog, errs := visitor.Build(src) + if len(errs) > 0 { + t.Fatalf("parse: %v", errs) + } + return prog +} + +func runQueuedCallCheck(t *testing.T, src string) []string { + t.Helper() + var out []string + for _, v := range ValidateQueuedCallReturnType(parseQueuedCallProgram(t, src)) { + if v.RuleID == "MDL088" { + out = append(out, v.Message) + } + } + return out +} + +// mendixlabs/mxcli#1064, verbatim: `CALL MICROFLOW M.F(…) IN QUEUE M.Q` where F +// returns Boolean — `mxcli check --references` says "Check passed!" and `mx +// check` says CE7033 "A microflow used for background execution must have a +// Microflow return type of 'Nothing'." +func TestQueuedCall_MicroflowReturningBooleanIsReported(t *testing.T) { + got := runQueuedCallCheck(t, ` +create microflow M.F () returns boolean +begin + return true; +end; +/ +create microflow M.Caller () +begin + call microflow M.F() in queue M.Q; +end; +/ +`) + if len(got) != 1 { + t.Fatalf("MDL088 violations = %v, want exactly one", got) + } + for _, want := range []string{"M.F", "Boolean", "CE7033", "M.Q"} { + if !strings.Contains(got[0], want) { + t.Errorf("message %q does not mention %q", got[0], want) + } + } +} + +// CONTROL: the identical call on a microflow that returns nothing is what +// Mendix accepts, and must stay silent — otherwise the test above would pass +// against a rule that flags every queued call. +func TestQueuedCall_VoidMicroflowIsClean(t *testing.T) { + if got := runQueuedCallCheck(t, ` +create microflow M.F () +begin + log info node 'M' 'x'; +end; +/ +create microflow M.Caller () +begin + call microflow M.F() in queue M.Q; +end; +/ +`); len(got) != 0 { + t.Errorf("a void queued microflow was rejected: %v", got) + } +} + +// CONTROL: the same Boolean microflow called WITHOUT a queue is an ordinary call. +func TestQueuedCall_UnqueuedCallIsNotChecked(t *testing.T) { + if got := runQueuedCallCheck(t, ` +create microflow M.F () returns boolean +begin + return true; +end; +/ +create microflow M.Caller () +begin + $ok = call microflow M.F(); +end; +/ +`); len(got) != 0 { + t.Errorf("an unqueued call was type-checked: %v", got) + } +} + +// The queued call can sit anywhere in the body — a hand-written switch over +// statement kinds misses whichever nesting was added last. +func TestQueuedCall_NestedCallIsFound(t *testing.T) { + got := runQueuedCallCheck(t, ` +create microflow M.F () returns string +begin + return 'x'; +end; +/ +create microflow M.Caller ($b: boolean) +begin + if $b then + while $b + begin + call microflow M.F() in queue M.Q; + end while; + end if; +end; +/ +`) + if len(got) != 1 || !strings.Contains(got[0], "String") { + t.Errorf("MDL088 violations = %v, want one naming String", got) + } +} + +// A target the script does not define needs the project; the project-less pass +// must not guess. +func TestQueuedCall_UndefinedTargetIsLeftToReferences(t *testing.T) { + if got := runQueuedCallCheck(t, ` +create microflow M.Caller () +begin + call microflow Other.F() in queue M.Q; +end; +/ +`); len(got) != 0 { + t.Errorf("an unknown target was reported without a project: %v", got) + } +} + +// The --references half: a target already stored in the project, with the +// return type the backend lists for it. +func TestQueuedCall_StoredTarget(t *testing.T) { + body := parseQueuedCallProgram(t, ` +create microflow M.Caller () +begin + call microflow M.Stored() in queue M.Q; +end; +/ +`).Statements[0].(*ast.CreateMicroflowStmt).Body + + for _, tc := range []struct { + stored string + wantErr bool + }{ + {"Boolean", true}, + {"Object", true}, + {"Void", false}, // how a stored "returns nothing" reads back + {"", false}, // ReturnType absent altogether + } { + errs := validateQueuedMicroflowTargets(body, map[string]string{"M.Stored": tc.stored}, newScriptContext()) + if (len(errs) > 0) != tc.wantErr { + t.Errorf("stored return %q: errors = %v, wantErr %v", tc.stored, errs, tc.wantErr) + } + } + + // CONTROL: return type unavailable → nothing is known to be wrong. + if errs := validateQueuedMicroflowTargets(body, nil, newScriptContext()); len(errs) != 0 { + t.Errorf("unknown return type was reported: %v", errs) + } + + // A target the script itself (re)creates is the project-less pass's to + // report; reporting it here too would print it twice. + sc := newScriptContext() + sc.microflows["M.Stored"] = true + if errs := validateQueuedMicroflowTargets(body, map[string]string{"M.Stored": "Boolean"}, sc); len(errs) != 0 { + t.Errorf("script-defined target reported on the project path too: %v", errs) + } +} diff --git a/mdl/types/entity_move.go b/mdl/types/entity_move.go new file mode 100644 index 0000000000..e02fd24d5d --- /dev/null +++ b/mdl/types/entity_move.go @@ -0,0 +1,46 @@ +// SPDX-License-Identifier: Apache-2.0 + +package types + +// MovedAssociation reports one association that a cross-module entity move +// converted into a cross-module association, and the qualified name it holds +// afterwards. +// +// The old and new names are both carried because the conversion is asymmetric and +// the caller must not guess which happened. Mendix stores an association in the +// module of its FROM entity, so: +// +// - the FROM (parent) entity moved → the cross-association travels with it, and +// its qualified name changes to the target module; +// - the TO (child) entity moved → the cross-association stays in the source +// module and its qualified name does not change. +// +// Measured on a blank 11.13 app: moving the parent left 9 of 33 CE1613s naming the +// association, moving the child left 0. A reference sweep driven by the module +// names alone would therefore corrupt the second case, and one driven by these +// fields is a no-op there instead of a special case (ako/mxcli#605). +type MovedAssociation struct { + // Name is the association's own (unqualified) name, for reporting to the user. + Name string + // OldQualifiedName is what it was called before the move. + OldQualifiedName string + // NewQualifiedName is what it is called after it. Equal to OldQualifiedName + // when the association did not change module. + NewQualifiedName string +} + +// Moved reports whether the association's qualified name changed, i.e. whether +// references to it elsewhere in the project have to be rewritten. +func (m MovedAssociation) Moved() bool { + return m.OldQualifiedName != m.NewQualifiedName +} + +// MovedAssociationNames returns the association names, for the user-facing +// "Converted N association(s)" report. +func MovedAssociationNames(ms []MovedAssociation) []string { + out := make([]string, 0, len(ms)) + for _, m := range ms { + out = append(out, m.Name) + } + return out +} diff --git a/modelsdk/canon/identity.go b/modelsdk/canon/identity.go index 02ba3df7de..8c9143d9ba 100644 --- a/modelsdk/canon/identity.go +++ b/modelsdk/canon/identity.go @@ -33,7 +33,10 @@ import ( // the common rebuild case — is unaffected. type Option func(*reconcileOpts) -type reconcileOpts struct{ contentsOwnTranslations bool } +type reconcileOpts struct { + contentsOwnTranslations bool + contentsOwnStorageGUIDs bool +} // ContentsOwnTranslations tells Reconcile that the write already accounts for // every translation in the document, so it must not carry the stored ones. @@ -50,6 +53,37 @@ func ContentsOwnTranslations() Option { return func(o *reconcileOpts) { o.contentsOwnTranslations = true } } +// ContentsOwnStorageGUIDs tells the write path that this write is AUTHORITATIVE +// about the storage GUIDs in the document, so the #1119 guard must not refuse it. +// +// Exactly one caller is: the marketplace module update transplants the stored +// GUIDs onto a module's replacement documents (marketplace.ApplyIdentities), +// which is the same thing Studio Pro's own update does and the reason an update +// does not destroy that module's data. That write's whole purpose is to move +// GUIDs onto elements that kept their $ID — the pattern the guard is built to +// catch — so it has to be able to say so. +// +// It is a narrow exemption on purpose. Every other write either leaves a GUID +// alone or has no business changing one, and "the guard was in the way" is not a +// reason to pass this: a write that trips the guard without being a deliberate +// identity transplant is the #1119 defect, and silencing it there would lose +// production data. +func ContentsOwnStorageGUIDs() Option { + return func(o *reconcileOpts) { o.contentsOwnStorageGUIDs = true } +} + +// OwnsStorageGUIDs reports whether these options carry the storage-GUID +// exemption. The guard itself runs at the writer's choke point, where both +// documents are in hand, rather than inside Reconcile — which returns no error — +// so the writer needs to read the option back out. +func OwnsStorageGUIDs(opts ...Option) bool { + var o reconcileOpts + for _, fn := range opts { + fn(&o) + } + return o.contentsOwnStorageGUIDs +} + func Reconcile(contents, stored []byte, opts ...Option) (out []byte, unchanged bool) { if len(stored) == 0 { return contents, false diff --git a/modelsdk/canon/storageguid.go b/modelsdk/canon/storageguid.go new file mode 100644 index 0000000000..259107ca65 --- /dev/null +++ b/modelsdk/canon/storageguid.go @@ -0,0 +1,255 @@ +// SPDX-License-Identifier: Apache-2.0 + +package canon + +import ( + "fmt" + "sort" + + "go.mongodb.org/mongo-driver/v2/bson" +) + +// An element's GUID is the DATABASE's identity for it, and it is not +// interchangeable with its $ID. The runtime keys mendixsystem$entity.id and +// mendixsystem$attribute.id on the GUID verbatim, so changing only an +// attribute's GUID — same name, same type, same entity — makes the synchroniser +// treat it as an attribute deleted and a new one added, and DROP its column on +// the next deploy (CLAUDE.md, "A GUID Is the Database's Identity"). +// +// That is a data-loss bug with no diagnostic behind it. The model stays +// perfectly valid, so mx check is clean and the build passes; mxcli's own reader +// never surfaces the GUID, so DESCRIBE is byte-identical before and after. It +// only becomes visible when the package meets a database that already holds +// data — which is to say, in production. Issue #1119 lost 28 attributes of 607 +// rows that way, from one ALTER of one entity. +// +// Worse, the damage does not repeat: the codec writes GUID = $ID, and the $ID is +// held stable by TransplantIDs, so the SECOND identical write produces +// byte-identical bytes and is elided. The corruption happens exactly once, on +// the first write, and every check afterwards — including a re-run of the same +// script — reports "Unchanged". +// +// So the three guards that already stand at this choke point cannot see it: +// +// - No-op elision compares a document with itself and finds it stable. +// - TestFreshGUIDFieldsHaveAnIdentityDecision sees only codec FreshGUIDFields. +// A GUID derived from the $ID is registered as EmitGUID, a different +// mechanism, so it was never in that guard's view — the same blind spot that +// let Workflows$*.PersistentId through in #949. +// - identityFields/CarryIdentity reach only top-level properties of the +// document root, and these GUIDs sit on elements nested inside it. +// +// Hence a guard here, in the same spirit as DuplicateElementIDError: cheap, at +// the moment the bytes would land, and phrased as the message the user would +// otherwise never get. It converts silent data loss into one refusal naming the +// element. +// +// It is deliberately NOT a repair. Carrying the stored GUID here would mean +// trusting the structural pairing TransplantIDs is built on, whose correctness +// bar is explicitly low — a wrong $ID match only makes a diff bigger, but a +// wrong GUID match makes the runtime adopt another column's data under a new +// name and type. The fix belongs where the write knows which element is which: +// see carryChildIdentity in mdl/backend/modelsdk, which pairs on the $ID the +// executor tracked through the statement. +// +// The check runs AFTER TransplantIDs, because before it a rebuilt element carries +// a freshly minted $ID that appears in no stored document and there would be +// nothing to compare against. +// +// But a shared $ID after the transplant does NOT by itself mean the same member, +// and an earlier version of this comment claimed it did ("unambiguously"). The +// transplant pairs STRUCTURALLY — by $Type and shape, LCS-anchored — and its own +// correctness bar is low on purpose, because a wrong $ID match only makes a diff +// bigger. Feed that pairing to a guard and a wrong match becomes a refusal. +// +// Measured: `CREATE OR MODIFY PERSISTENT ENTITY BusinessEvents.PublishedBusinessEvent +// (EventId: long)` against the marketplace module, whose entity has six +// differently-named attributes. The statement drops all six and adds one; the +// transplant paired the NEW attribute with one of the REMOVED ones and handed it that +// stored $ID; the codec had written GUID = $ID, and the transplant substitutes over +// every 16-byte binary, so the GUID followed. The guard then saw a stored $ID whose +// GUID had "changed" and refused a write that corrupts nothing — blocking a documented +// statement on a doctype script that had been passing for months. +// +// So the pairing here is $ID **plus the member's identity**: the same Name where the +// element has one, and the same $Type where it has none (see sameMember). +// +// The $Type is deliberately not compared for a named element. An earlier version +// did, and it went blind to the one conversion mxcli performs: MOVE ENTITY re-types +// an Association as a CrossAssociation in place, under the same $ID and Name — the +// very arm that exposed ako/mxcli#503. The transplant never pairs across a $Type, so +// comparing it guarded against nothing the transplant can produce. +// +// What the guard does NOT cover, each checked instead where it is decidable: +// +// - A RENAME that re-minted a GUID, because the name is what changed. The carry +// tests in mdl/backend/modelsdk cover it (TestIssue1119_AlterPreservesAttributeGUIDs, +// RenameAttribute case). +// - Any element that changes UNIT — MOVE ENTITY's entity and attributes, or the +// cross-association when the FROM side moves. The guard compares one unit's +// written bytes with that same unit's stored bytes, and in the target unit the +// moved $ID pairs with nothing. TestIssue503_MovePreservesStorageGUIDs covers it. +// - A pair where only one side has a Name. Nothing converts between those shapes, +// so calling them one member would be a guess. +// +// A backstop that refuses correct writes is worse than a backstop with a hole: the +// first makes the tool unusable for work the user is entitled to do, and this one had +// already done so. The holes are listed so nobody mistakes a quiet guard for coverage. + +// GUIDChange is one element that kept its $ID across a write while its GUID +// changed — the shape of the #1119 defect. +type GUIDChange struct { + ElementID string + Type string + Stored string + Written string +} + +// StorageGUIDChanges reports every element that appears in both documents under +// the same $ID but with a different GUID, in a stable order. +// +// Only an element is considered: a sub-document carrying both $ID and $Type. +// A pointer to another element is a primitive property holding the same 16-byte +// shape under a different key, and a containment walk meets plenty of those. +// +// An element present on only one side is not a change: a genuinely new element +// has an $ID no stored element holds, and a deleted one is simply absent. Nor is +// a GUID that only one side carries — an optional property Mendix fills in on +// load must not be invented, and one it stopped writing must not be preserved. +// +// Nor is a pair that is not the same member. Sharing an $ID after the transplant +// does not make two elements the same MEMBER — see the note above the type — so the +// member's own identity has to agree before a GUID difference means anything: its +// Name where it has one (whatever its $Type), its $Type where it has none. +// +// A document that cannot be unmarshalled yields no changes rather than an error. +// This runs on the write path, where failing a write because the guard could not +// read the bytes would be worse than the defect it prevents. +func StorageGUIDChanges(contents, stored []byte) []GUIDChange { + now := elementGUIDs(contents) + if len(now) == 0 { + return nil + } + was := elementGUIDs(stored) + if len(was) == 0 { + return nil + } + + ids := make([]string, 0, len(now)) + for id := range now { + ids = append(ids, id) + } + sort.Strings(ids) + + var out []GUIDChange + for _, id := range ids { + stored, ok := was[id] + if !ok { + continue + } + written := now[id] + if written.guid == stored.guid { + continue + } + if !sameMember(stored, written) { + continue + } + out = append(out, GUIDChange{ + ElementID: id, + Type: written.typ, + Stored: stored.guid, + Written: written.guid, + }) + } + return out +} + +// sameMember reports whether two elements sharing an $ID are the same member, and +// so whether a GUID difference between them is a rewrite rather than the +// transplant having paired two unrelated elements. +// +// A named element is its name. The $Type is deliberately NOT compared there: the +// transplant never pairs across a $Type (pairDoc stops at the mismatch), so two +// types sharing an $ID can only be an element a writer converted in place and +// kept the $ID of — MOVE ENTITY re-typing an Association as a CrossAssociation +// (ako/mxcli#503). That is one member, and its GUID is still the database's +// identity for it. +// +// A nameless element (an index) has nothing but its $Type to go on. And a pair +// where only one side is named is not the same member: nothing converts between +// those shapes, so agreeing on it would be a guess. +func sameMember(a, b elementGUID) bool { + switch { + case a.name != "" && b.name != "": + return a.name == b.name + case a.name == "" && b.name == "": + return a.typ == b.typ + default: + return false + } +} + +// elementGUID is one GUID-bearing element: its GUID, and the two properties that +// say which member it is. The name is "" for an element that has none. +type elementGUID struct { + guid string + typ string + name string +} + +// elementGUIDs maps element $ID -> elementGUID for every element in raw that +// carries both an $ID and a GUID. +func elementGUIDs(raw []byte) map[string]elementGUID { + var d bson.D + if err := bson.Unmarshal(raw, &d); err != nil { + return nil + } + out := map[string]elementGUID{} + var walk func(any) + walk = func(v any) { + if doc, ok := asDoc(v); ok { + if id, ok := elementID(doc); ok && hasType(doc) { + if g, ok := binary16(doc, "GUID"); ok { + name, _ := doc["Name"].(string) + out[id] = elementGUID{guid: blobToUUID(g), typ: typeOf(doc), name: name} + } + } + for _, k := range sortedKeys(doc) { + walk(doc[k]) + } + return + } + if s, ok := asSlice(v); ok { + for _, e := range s { + walk(e) + } + } + } + walk(d) + return out +} + +// StorageGUIDError returns the error a write should fail with, or nil. +// unitLabel is whatever the caller can name the unit by — an id is enough, a +// qualified name is better. +func StorageGUIDError(unitLabel string, contents, stored []byte) error { + changes := StorageGUIDChanges(contents, stored) + if len(changes) == 0 { + return nil + } + const maxReported = 3 + msg := fmt.Sprintf("refusing to write unit %s: %d element(s) kept their $ID but would be "+ + "written with a different GUID. The runtime keys the database on that GUID "+ + "(mendixsystem$entity.id / mendixsystem$attribute.id), so deploying this would make "+ + "the synchroniser drop and recreate the affected columns and lose their data. "+ + "The model would still be valid and `mx check` would still pass", + unitLabel, len(changes)) + for i, c := range changes { + if i == maxReported { + msg += fmt.Sprintf("\n ... and %d more", len(changes)-maxReported) + break + } + msg += fmt.Sprintf("\n %s (%s): stored %s, would write %s", c.ElementID, c.Type, c.Stored, c.Written) + } + return fmt.Errorf("%s", msg) +} diff --git a/modelsdk/canon/storageguid_test.go b/modelsdk/canon/storageguid_test.go new file mode 100644 index 0000000000..5b999a8939 --- /dev/null +++ b/modelsdk/canon/storageguid_test.go @@ -0,0 +1,462 @@ +// SPDX-License-Identifier: Apache-2.0 + +package canon + +import ( + "strings" + "testing" + + "go.mongodb.org/mongo-driver/v2/bson" +) + +// dmDoc builds the shape the #1119 guard exists for: a domain model holding one +// entity holding two attributes, each element carrying both an $ID and the +// separate GUID the runtime keys the database on. Each id/guid is a byte, so the +// same structure can be built twice with chosen identities. +// +// attrPointer is a plain 16-byte binary property that happens to hold an +// element's $ID — the shape a reference takes in a stored document. It is here +// so the guard is measured against a document that contains one. +func dmDoc(t *testing.T, entGUID, a1ID, a1GUID, a2ID, a2GUID byte) []byte { + t.Helper() + return marshal(t, bson.D{ + {Key: "$Type", Value: "DomainModels$DomainModel"}, + {Key: "$ID", Value: bin(1)}, + {Key: "Entities", Value: bson.A{ + bson.D{ + {Key: "$Type", Value: "DomainModels$EntityImpl"}, + {Key: "$ID", Value: bin(2)}, + {Key: "GUID", Value: bin(entGUID)}, + {Key: "Name", Value: "Organization"}, + {Key: "Attributes", Value: bson.A{ + bson.D{ + {Key: "$Type", Value: "DomainModels$Attribute"}, + {Key: "$ID", Value: bin(a1ID)}, + {Key: "GUID", Value: bin(a1GUID)}, + {Key: "Name", Value: "Name"}, + }, + bson.D{ + {Key: "$Type", Value: "DomainModels$Attribute"}, + {Key: "$ID", Value: bin(a2ID)}, + {Key: "GUID", Value: bin(a2GUID)}, + {Key: "Name", Value: "Status"}, + }, + }}, + }, + }}, + {Key: "Associations", Value: bson.A{ + bson.D{ + {Key: "$Type", Value: "DomainModels$Association"}, + {Key: "$ID", Value: bin(9)}, + {Key: "GUID", Value: bin(90)}, + {Key: "Name", Value: "Org_Person"}, + // A reference, not a containment edge: the same 16-byte shape + // under a different key. + {Key: "ParentPointer", Value: bin(2)}, + }, + }}, + }) +} + +// TestStorageGUIDChanges_FiresOnTheReportedShape is the guard's positive +// control. The written document is what #1119 produces: every $ID preserved +// (TransplantIDs put them back) and every attribute's GUID replaced by its own +// $ID. If this does not fire, the guard is inert and the rest of the cases prove +// nothing. +func TestStorageGUIDChanges_FiresOnTheReportedShape(t *testing.T) { + stored := dmDoc(t, 20, 3, 30, 4, 40) + // GUID = $ID on both attributes, exactly as the codec's EmitGUID default writes it. + written := dmDoc(t, 20, 3, 3, 4, 4) + + changes := StorageGUIDChanges(written, stored) + if len(changes) != 2 { + t.Fatalf("got %d changes, want 2: %+v", len(changes), changes) + } + for _, c := range changes { + if c.Type != "DomainModels$Attribute" { + t.Errorf("change on %s, want DomainModels$Attribute", c.Type) + } + if c.Stored == c.Written { + t.Errorf("change reported with equal GUIDs: %+v", c) + } + } + + err := StorageGUIDError("Sales.DomainModel", written, stored) + if err == nil { + t.Fatal("StorageGUIDError returned nil for a document that changes two GUIDs") + } + // The message has to name the unit and say what the consequence is — the + // whole point is that the user would otherwise meet this as missing data + // after a deploy, with nothing connecting it to an MDL edit. + for _, want := range []string{"Sales.DomainModel", "mendixsystem$attribute.id", "DomainModels$Attribute"} { + if !strings.Contains(err.Error(), want) { + t.Errorf("error message does not mention %q: %v", want, err) + } + } +} + +// TestStorageGUIDChanges_QuietWhenNothingMoved: the guard must be silent on an +// unchanged document, and on one whose CONTENT changed while identities held. +func TestStorageGUIDChanges_QuietWhenNothingMoved(t *testing.T) { + stored := dmDoc(t, 20, 3, 30, 4, 40) + + if got := StorageGUIDChanges(stored, stored); len(got) != 0 { + t.Errorf("identical documents reported %d changes: %+v", len(got), got) + } + + // An entity rename: same identities, different content. + renamed := marshal(t, bson.D{ + {Key: "$Type", Value: "DomainModels$DomainModel"}, + {Key: "$ID", Value: bin(1)}, + {Key: "Entities", Value: bson.A{ + bson.D{ + {Key: "$Type", Value: "DomainModels$EntityImpl"}, + {Key: "$ID", Value: bin(2)}, + {Key: "GUID", Value: bin(20)}, + {Key: "Name", Value: "Renamed"}, + }, + }}, + }) + if got := StorageGUIDChanges(renamed, stored); len(got) != 0 { + t.Errorf("a rename reported %d GUID changes: %+v", len(got), got) + } +} + +// TestStorageGUIDChanges_NewAndDroppedElementsAreNotChanges: an element the +// write adds has an $ID no stored element holds, and one it removes is simply +// absent. Neither is a GUID that moved, and reporting either would make the +// guard fire on every ADD ATTRIBUTE — which is how a guard gets switched off. +func TestStorageGUIDChanges_NewAndDroppedElementsAreNotChanges(t *testing.T) { + stored := dmDoc(t, 20, 3, 30, 4, 40) + + // Same two attributes plus a third, with identities of its own. + added := marshal(t, bson.D{ + {Key: "$Type", Value: "DomainModels$DomainModel"}, + {Key: "$ID", Value: bin(1)}, + {Key: "Entities", Value: bson.A{ + bson.D{ + {Key: "$Type", Value: "DomainModels$EntityImpl"}, + {Key: "$ID", Value: bin(2)}, + {Key: "GUID", Value: bin(20)}, + {Key: "Attributes", Value: bson.A{ + bson.D{ + {Key: "$Type", Value: "DomainModels$Attribute"}, + {Key: "$ID", Value: bin(3)}, + {Key: "GUID", Value: bin(30)}, + }, + bson.D{ + {Key: "$Type", Value: "DomainModels$Attribute"}, + {Key: "$ID", Value: bin(4)}, + {Key: "GUID", Value: bin(40)}, + }, + bson.D{ + {Key: "$Type", Value: "DomainModels$Attribute"}, + {Key: "$ID", Value: bin(5)}, + {Key: "GUID", Value: bin(50)}, + }, + }}, + }, + }}, + }) + if got := StorageGUIDChanges(added, stored); len(got) != 0 { + t.Errorf("adding an attribute reported %d GUID changes: %+v", len(got), got) + } + + // And dropping one: the survivor keeps its GUID, the removed one is gone. + dropped := marshal(t, bson.D{ + {Key: "$Type", Value: "DomainModels$DomainModel"}, + {Key: "$ID", Value: bin(1)}, + {Key: "Entities", Value: bson.A{ + bson.D{ + {Key: "$Type", Value: "DomainModels$EntityImpl"}, + {Key: "$ID", Value: bin(2)}, + {Key: "GUID", Value: bin(20)}, + {Key: "Attributes", Value: bson.A{ + bson.D{ + {Key: "$Type", Value: "DomainModels$Attribute"}, + {Key: "$ID", Value: bin(3)}, + {Key: "GUID", Value: bin(30)}, + }, + }}, + }, + }}, + }) + if got := StorageGUIDChanges(dropped, stored); len(got) != 0 { + t.Errorf("dropping an attribute reported %d GUID changes: %+v", len(got), got) + } +} + +// TestStorageGUIDChanges_OneSidedGUIDIsNotAChange: a GUID only one side carries +// is left alone in both directions. Inventing an optional property Studio Pro +// fills in on load is how a document becomes unopenable (CLAUDE.md on overlay +// writes), and the guard must not demand one be written. +func TestStorageGUIDChanges_OneSidedGUIDIsNotAChange(t *testing.T) { + withGUID := marshal(t, bson.D{ + {Key: "$Type", Value: "DomainModels$Attribute"}, + {Key: "$ID", Value: bin(3)}, + {Key: "GUID", Value: bin(30)}, + }) + withoutGUID := marshal(t, bson.D{ + {Key: "$Type", Value: "DomainModels$Attribute"}, + {Key: "$ID", Value: bin(3)}, + }) + + if got := StorageGUIDChanges(withoutGUID, withGUID); len(got) != 0 { + t.Errorf("dropping the GUID key reported %d changes: %+v", len(got), got) + } + if got := StorageGUIDChanges(withGUID, withoutGUID); len(got) != 0 { + t.Errorf("adding the GUID key reported %d changes: %+v", len(got), got) + } +} + +// TestStorageGUIDChanges_PointerIsNotAnElement: a 16-byte binary under some +// other key is a reference, and the document in these tests holds one that +// points at the entity. Treating it as an element would make the guard report +// on references rather than identities. +func TestStorageGUIDChanges_PointerIsNotAnElement(t *testing.T) { + stored := dmDoc(t, 20, 3, 30, 4, 40) + // Repoint the association at a different element. Only ParentPointer moves; + // no GUID does. + written := marshal(t, bson.D{ + {Key: "$Type", Value: "DomainModels$DomainModel"}, + {Key: "$ID", Value: bin(1)}, + {Key: "Associations", Value: bson.A{ + bson.D{ + {Key: "$Type", Value: "DomainModels$Association"}, + {Key: "$ID", Value: bin(9)}, + {Key: "GUID", Value: bin(90)}, + {Key: "Name", Value: "Org_Person"}, + {Key: "ParentPointer", Value: bin(7)}, + }, + }}, + }) + if got := StorageGUIDChanges(written, stored); len(got) != 0 { + t.Errorf("repointing a reference reported %d GUID changes: %+v", len(got), got) + } +} + +// TestStorageGUIDChanges_UnreadableBytesDoNotBlockAWrite: the guard sits on the +// write path, so failing a write because it could not parse the bytes would be a +// worse failure than the one it prevents. Mirrors DuplicateElementIDs. +func TestStorageGUIDChanges_UnreadableBytesDoNotBlockAWrite(t *testing.T) { + good := dmDoc(t, 20, 3, 30, 4, 40) + garbage := []byte{0x01, 0x02, 0x03} + + if got := StorageGUIDChanges(garbage, good); got != nil { + t.Errorf("unreadable contents reported changes: %+v", got) + } + if got := StorageGUIDChanges(good, garbage); got != nil { + t.Errorf("unreadable stored bytes reported changes: %+v", got) + } + if err := StorageGUIDError("u", garbage, good); err != nil { + t.Errorf("unreadable contents refused the write: %v", err) + } +} + +// TestStorageGUIDChanges_ReportsAreStable: the order must not depend on map +// iteration, or the error message reorders between runs. +func TestStorageGUIDChanges_ReportsAreStable(t *testing.T) { + stored := dmDoc(t, 20, 3, 30, 4, 40) + written := dmDoc(t, 21, 3, 3, 4, 4) + + first := StorageGUIDError("u", written, stored).Error() + for i := 0; i < 20; i++ { + if got := StorageGUIDError("u", written, stored).Error(); got != first { + t.Fatalf("message not stable across runs:\n%s\n%s", first, got) + } + } + // All three GUIDs moved, so all three are reported. + if n := len(StorageGUIDChanges(written, stored)); n != 3 { + t.Errorf("got %d changes, want 3 (entity + two attributes)", n) + } +} + +// TestStorageGUIDChanges_TransplantPairingIsNotMemberIdentity is the case that made +// the guard refuse correct writes, and its positive half is the reason the guard is +// still worth having. +// +// TransplantIDs pairs STRUCTURALLY — by $Type and shape, LCS-anchored — so a +// genuinely new element can be handed the $ID of a removed one. Measured on +// `CREATE OR MODIFY PERSISTENT ENTITY BusinessEvents.PublishedBusinessEvent +// (EventId: long)` against the marketplace module, whose entity carries six +// differently-named attributes: the statement drops all six and adds one, the +// transplant paired the new attribute with a removed one, and because the codec had +// written GUID = $ID and the transplant substitutes over every 16-byte binary, the +// GUID followed the $ID. The guard saw a "changed" GUID on a "kept" $ID and refused a +// write that corrupts nothing — on a doctype script that had been passing for months. +// +// The discriminator is the member's own identity. Both halves are asserted here +// together, because either one alone is satisfiable by a guard that is simply wrong: +// dropping the name check makes the first case fire, and never firing at all makes the +// second pass. +func TestStorageGUIDChanges_TransplantPairingIsNotMemberIdentity(t *testing.T) { + attr := func(id, guid byte, name, typ string) bson.D { + return bson.D{ + {Key: "$Type", Value: typ}, + {Key: "$ID", Value: bin(id)}, + {Key: "GUID", Value: bin(guid)}, + {Key: "Name", Value: name}, + } + } + wrap := func(t *testing.T, a bson.D) []byte { + t.Helper() + return marshal(t, bson.D{ + {Key: "$Type", Value: "DomainModels$DomainModel"}, + {Key: "$ID", Value: bin(1)}, + {Key: "Entities", Value: bson.A{ + bson.D{ + {Key: "$Type", Value: "DomainModels$EntityImpl"}, + {Key: "$ID", Value: bin(2)}, + {Key: "GUID", Value: bin(20)}, + {Key: "Name", Value: "PublishedBusinessEvent"}, + {Key: "Attributes", Value: bson.A{a}}, + }, + }}, + }) + } + + for _, tc := range []struct { + name string + stored, next bson.D + wantChange bool + why string + }{ + { + // The reported false positive: same $ID, different member. + name: "DifferentName_NotAChange", + stored: attr(3, 30, "ServiceName", "DomainModels$Attribute"), + next: attr(3, 3, "EventId", "DomainModels$Attribute"), + wantChange: false, + why: "a new member the transplant paired with a removed one is not a rewrite", + }, + { + // The guard's reason to exist, unchanged: same member, GUID replaced by + // its own $ID. This is #1119 exactly. + name: "SameName_IsAChange", + stored: attr(3, 30, "ServiceName", "DomainModels$Attribute"), + next: attr(3, 3, "ServiceName", "DomainModels$Attribute"), + wantChange: true, + why: "the member survived and its database identity was replaced", + }, + { + // For a NAMED element the $Type is not part of the test. The transplant + // never pairs across a $Type, so a kept $ID with a new $Type is a writer's + // in-place conversion — MOVE ENTITY re-typing an Association as a + // CrossAssociation (#503). An earlier version pinned this as "not a + // change" on the grounds that nothing authored it; MoveEntity did, and the + // guard went blind to exactly the arm that had exposed #503. The realistic + // shape is in TestStorageGUIDChanges_TypeConversionIsStillTheSameMember. + name: "DifferentTypeSameName_IsAChange", + stored: attr(3, 30, "Same", "DomainModels$Association"), + next: attr(3, 3, "Same", "DomainModels$CrossAssociation"), + wantChange: true, + why: "a re-typed member that kept its $ID and name is still that member", + }, + { + // A nameless element has only its $Type, so there a different $Type is a + // different member. + name: "NamelessDifferentType_NotAChange", + stored: bson.D{{Key: "$Type", Value: "DomainModels$EntityIndex"}, {Key: "$ID", Value: bin(3)}, {Key: "GUID", Value: bin(30)}}, + next: bson.D{{Key: "$Type", Value: "DomainModels$Attribute"}, {Key: "$ID", Value: bin(3)}, {Key: "GUID", Value: bin(3)}}, + wantChange: false, + why: "a nameless element is identified by its $Type alone", + }, + { + // A name on only one side: nothing converts between those shapes, so + // calling them one member would be a guess. + name: "OneSideNamed_NotAChange", + stored: bson.D{{Key: "$Type", Value: "DomainModels$EntityIndex"}, {Key: "$ID", Value: bin(3)}, {Key: "GUID", Value: bin(30)}}, + next: attr(3, 3, "Code", "DomainModels$Attribute"), + wantChange: false, + why: "one named and one nameless element are not known to be one member", + }, + { + // A nameless element (an index) has only its $Type, so it must still be + // compared — otherwise the index arm of the carry loses its backstop. + name: "NamelessElement_StillCompared", + stored: bson.D{{Key: "$Type", Value: "DomainModels$EntityIndex"}, {Key: "$ID", Value: bin(3)}, {Key: "GUID", Value: bin(30)}}, + next: bson.D{{Key: "$Type", Value: "DomainModels$EntityIndex"}, {Key: "$ID", Value: bin(3)}, {Key: "GUID", Value: bin(3)}}, + wantChange: true, + why: "an index has no name to distinguish it, so $Type is the whole test", + }, + } { + t.Run(tc.name, func(t *testing.T) { + got := StorageGUIDChanges(wrap(t, tc.next), wrap(t, tc.stored)) + if tc.wantChange && len(got) == 0 { + t.Errorf("no change reported, want one: %s", tc.why) + } + if !tc.wantChange && len(got) != 0 { + t.Errorf("reported %d change(s), want none: %s\n %+v", len(got), tc.why, got) + } + }) + } +} + +// TestStorageGUIDChanges_TypeConversionIsStillTheSameMember pins the arm that +// ako/mxcli#503 was found through. MOVE ENTITY of an association's TO side +// converts that association IN PLACE, in the same unit and under the same $ID, +// from DomainModels$Association to DomainModels$CrossAssociation. It is the same +// member with the same database identity, only re-typed, so a re-minted GUID on it +// loses the association's data exactly as it would on an ALTER. +// +// Requiring an equal $Type made the guard skip this pair. That requirement bought +// nothing: TransplantIDs never pairs across a $Type (pairDoc stops at the +// mismatch — TestTransplantIgnoresMismatchedTypes), so an $ID shared by two +// DIFFERENT types can only be one a writer kept on purpose, which is a conversion. +// The mis-pairing the member check exists for is same-type, different-name, and +// that case stays quiet (TestStorageGUIDChanges_TransplantPairingIsNotMemberIdentity). +func TestStorageGUIDChanges_TypeConversionIsStillTheSameMember(t *testing.T) { + // assocDM is the source domain model: the association either still regular, + // or already converted, carrying the given GUID. + assocDM := func(t *testing.T, converted bool, name string, guid byte) []byte { + t.Helper() + list, typ, target := "Associations", "DomainModels$Association", bson.E{Key: "ChildPointer", Value: bin(5)} + if converted { + list, typ, target = "CrossAssociations", "DomainModels$CrossAssociation", bson.E{Key: "Child", Value: "Other.Account"} + } + return marshal(t, bson.D{ + {Key: "$Type", Value: "DomainModels$DomainModel"}, + {Key: "$ID", Value: bin(1)}, + {Key: list, Value: bson.A{ + bson.D{ + {Key: "$Type", Value: typ}, + {Key: "$ID", Value: bin(9)}, + {Key: "GUID", Value: bin(guid)}, + {Key: "Name", Value: name}, + {Key: "ParentPointer", Value: bin(2)}, + target, + }, + }}, + }) + } + stored := assocDM(t, false, "AccountPasswordData_Account", 90) + + t.Run("ConvertedWithReMintedGUID_IsAChange", func(t *testing.T) { + // The pre-#503 crossAssocFromGenAssoc: $ID kept, GUID = $ID. + got := StorageGUIDChanges(assocDM(t, true, "AccountPasswordData_Account", 9), stored) + if len(got) != 1 { + t.Fatalf("got %d change(s), want 1 — a re-typed association that lost its GUID "+ + "is the #503 data loss: %+v", len(got), got) + } + if got[0].Type != "DomainModels$CrossAssociation" { + t.Errorf("change reported on %s, want the written type DomainModels$CrossAssociation", got[0].Type) + } + if err := StorageGUIDError("Administration.DomainModel", assocDM(t, true, "AccountPasswordData_Account", 9), stored); err == nil { + t.Error("StorageGUIDError returned nil for the #503 shape") + } + }) + + t.Run("ConvertedWithGUIDKept_IsNotAChange", func(t *testing.T) { + // The fixed conversion (a raw transform) keeps the GUID and must be let through. + if got := StorageGUIDChanges(assocDM(t, true, "AccountPasswordData_Account", 90), stored); len(got) != 0 { + t.Errorf("reported %d change(s) for a conversion that kept its GUID: %+v", len(got), got) + } + }) + + t.Run("ConvertedUnderADifferentName_IsNotAChange", func(t *testing.T) { + // Type AND name both differ: nothing says this is the same member. + if got := StorageGUIDChanges(assocDM(t, true, "SomethingElse", 9), stored); len(got) != 0 { + t.Errorf("reported %d change(s) for a different member: %+v", len(got), got) + } + }) +} diff --git a/modelsdk/mpr/writer_core.go b/modelsdk/mpr/writer_core.go index 43316fd12b..01e090bfcd 100644 --- a/modelsdk/mpr/writer_core.go +++ b/modelsdk/mpr/writer_core.go @@ -214,7 +214,10 @@ func (wt *WriteTransaction) WriteUnit(unitID string, contents []byte) error { // same storage (codec.Store.SaveUnit / FlushUnits reach it), so leaving it // out would mean the codec engine's own document writes kept churning while // UpdateRawUnit's stopped — a difference no caller could reason about. - contents, unchanged := wt.writer.reconcileWithStored(unitID, contents) + contents, unchanged, err := wt.writer.reconcileWithStored(unitID, contents) + if err != nil { + return err + } if unchanged { return nil } @@ -254,7 +257,7 @@ func (wt *WriteTransaction) WriteUnit(unitID string, contents []byte) error { } // V1: Update in database directly - _, err := wt.tx.Exec(` + _, err = wt.tx.Exec(` UPDATE Unit SET Contents = ? WHERE UnitID = ? `, contents, unitIDBlob) return err @@ -614,7 +617,10 @@ func (w *Writer) updateUnit(unitID string, contents []byte, opts ...canon.Option return w.sessionBuf(unitID, contents) } - contents, unchanged := w.reconcileWithStored(unitID, contents, opts...) + contents, unchanged, err := w.reconcileWithStored(unitID, contents, opts...) + if err != nil { + return err + } if unchanged { return nil } @@ -661,7 +667,7 @@ func (w *Writer) updateUnit(unitID string, contents []byte, opts ...canon.Option } // MPR v1: Update in database - _, err := w.reader.db.Exec(` + _, err = w.reader.db.Exec(` UPDATE Unit SET Contents = ? WHERE UnitID = ? `, contents, unitIDBlob) return err @@ -669,18 +675,40 @@ func (w *Writer) updateUnit(unitID string, contents []byte, opts ...canon.Option // reconcileWithStored applies the shared no-op-elision policy (canon.Reconcile, // ADR-0008 decision 1) to a write against this project. -func (w *Writer) reconcileWithStored(unitID string, contents []byte, opts ...canon.Option) (out []byte, unchanged bool) { +func (w *Writer) reconcileWithStored(unitID string, contents []byte, opts ...canon.Option) (out []byte, unchanged bool, err error) { w.writesOffered++ - stored, err := w.reader.GetRawUnitBytes(unitID) - if err != nil { + stored, readErr := w.reader.GetRawUnitBytes(unitID) + if readErr != nil { w.writesLanded++ - return contents, false // new unit, or unreadable — write it + return contents, false, nil // new unit, or unreadable — write it } out, unchanged = canon.Reconcile(contents, stored, opts...) + + // The storage-GUID guard runs here, on the reconciled bytes, because it is + // the transplant inside Reconcile that establishes which written element is + // which stored one: before it, every rebuilt element carries a freshly minted + // $ID that matches nothing. An element that came out of Reconcile sharing an + // $ID with a stored element but carrying a different GUID is a rewrite that + // dropped the database's identity for it (#1119) — refuse rather than write. + // + // Placed in reconcileWithStored rather than beside the duplicate-$ID guard in + // updateUnit so that BOTH choke points get it: WriteTransaction.WriteUnit is + // how codec.Store reaches storage, and a guard on only one of them is the + // inconsistency CLAUDE.md's "adding a write path means wiring it to + // canon.Reconcile" exists to prevent. + // + // A deliberate identity transplant (the marketplace module update) is the one + // write whose purpose is to move GUIDs, and it says so. + if !canon.OwnsStorageGUIDs(opts...) { + if guardErr := canon.StorageGUIDError(unitID, out, stored); guardErr != nil { + return nil, false, guardErr + } + } + if !unchanged { w.writesLanded++ } - return out, unchanged + return out, unchanged, nil } // rememberRemovedUnit captures a unit's bytes on the way out, so an insert of @@ -791,6 +819,19 @@ func (w *Writer) UpdateRawUnitOwningTranslations(unitID string, contents []byte) return w.updateUnit(unitID, contents, canon.ContentsOwnTranslations()) } +// UpdateRawUnitOwningStorageGUIDs is UpdateRawUnit for a write that deliberately +// transplants storage GUIDs onto elements that keep their $ID — the marketplace +// module update, which carries a module's existing GUIDs onto the documents +// replacing it so the next deploy does not destroy that module's data. +// +// That is the exact pattern the #1119 guard refuses, and rightly: for every other +// write it means the database's identity for an element was dropped. Only a +// caller that captured those GUIDs from the stored model and is putting them back +// may use this. See canon.ContentsOwnStorageGUIDs. +func (w *Writer) UpdateRawUnitOwningStorageGUIDs(unitID string, contents []byte) error { + return w.updateUnit(unitID, contents, canon.ContentsOwnStorageGUIDs()) +} + // InsertUnit creates a new unit in the project database. // This is the exported version of insertUnit for use by TreeWriter and other packages. func (w *Writer) InsertUnit(unitID, containerID, containmentName, unitType string, contents []byte) error { diff --git a/modelsdk/mpr/writer_storage_guid_test.go b/modelsdk/mpr/writer_storage_guid_test.go new file mode 100644 index 0000000000..03fdbd693d --- /dev/null +++ b/modelsdk/mpr/writer_storage_guid_test.go @@ -0,0 +1,225 @@ +// SPDX-License-Identifier: Apache-2.0 + +package mpr + +import ( + "os" + "strings" + "testing" + + "go.mongodb.org/mongo-driver/v2/bson" +) + +// Re-minting an element's GUID while keeping its $ID makes the Mendix runtime's +// database synchroniser treat the element as deleted and re-added, so it drops +// and recreates the column and loses its data (issue #1119). Nothing else +// notices: the model stays valid, `mx check` passes, and because the new GUID is +// derived from a stable $ID the second identical write is elided. +// +// So the guard has to sit at the write choke point, and these tests go through +// the Writer rather than calling the detector — the wiring is the half that can +// silently come undone, which is what these cover. (Mirrors the duplicate-$ID +// guard's tests next door.) + +// guidUnit builds a one-entity domain model whose entity carries an $ID and a +// separate GUID, so the two can be moved independently. +func guidUnit(t *testing.T, entityID, entityGUID, attrID, attrGUID string) []byte { + t.Helper() + bin := func(id string) bson.Binary { + return bson.Binary{Subtype: 0x00, Data: uuidToBlob(id)} + } + b, err := bson.Marshal(bson.D{ + {Key: "$Type", Value: "DomainModels$DomainModel"}, + {Key: "$ID", Value: bin("11111111-1111-1111-1111-111111111111")}, + {Key: "Entities", Value: bson.A{ + bson.D{ + {Key: "$Type", Value: "DomainModels$EntityImpl"}, + {Key: "$ID", Value: bin(entityID)}, + {Key: "GUID", Value: bin(entityGUID)}, + {Key: "Name", Value: "Organization"}, + {Key: "Attributes", Value: bson.A{ + bson.D{ + {Key: "$Type", Value: "DomainModels$Attribute"}, + {Key: "$ID", Value: bin(attrID)}, + {Key: "GUID", Value: bin(attrGUID)}, + {Key: "Name", Value: "Name"}, + }, + }}, + }, + }}, + }) + if err != nil { + t.Fatalf("marshal: %v", err) + } + return b +} + +const ( + guidEntityID = "22222222-2222-2222-2222-222222222222" + guidEntityGUID = "33333333-3333-3333-3333-333333333333" + guidAttrID = "44444444-4444-4444-4444-444444444444" + guidAttrGUID = "55555555-5555-5555-5555-555555555555" +) + +func TestUpdateUnitRefusesAChangedStorageGUID(t *testing.T) { + const unitID = "66666666-6666-6666-6666-666666666666" + stored := guidUnit(t, guidEntityID, guidEntityGUID, guidAttrID, guidAttrGUID) + w, unitPath := newV2WriterForCommitTest(t, unitID, stored) + + before, err := os.ReadFile(unitPath) + if err != nil { + t.Fatalf("read seeded unit: %v", err) + } + + // Exactly what #1119 produced: the $IDs held (TransplantIDs put them back), + // the attribute's GUID replaced by its own $ID. + err = w.UpdateRawUnit(unitID, guidUnit(t, guidEntityID, guidEntityGUID, guidAttrID, guidAttrID)) + if err == nil { + t.Fatal("write accepted a unit that changes an attribute's storage GUID") + } + for _, want := range []string{unitID, "DomainModels$Attribute", "mendixsystem$attribute.id"} { + if !strings.Contains(err.Error(), want) { + t.Errorf("message missing %q: %v", want, err) + } + } + + // A guard that reports the problem after the bytes have landed is the + // failure it exists to prevent. + after, err := os.ReadFile(unitPath) + if err != nil { + t.Fatalf("read unit after refusal: %v", err) + } + if string(after) != string(before) { + t.Error("the refused write still changed the stored unit") + } +} + +// The control, and the one that matters most: the guard must not refuse the +// ordinary writes. A guard that fires on a legitimate edit gets switched off, +// and then it protects nothing. +func TestUpdateUnitAcceptsOrdinaryEditsThatKeepGUIDs(t *testing.T) { + bin := func(id string) bson.Binary { + return bson.Binary{Subtype: 0x00, Data: uuidToBlob(id)} + } + + cases := []struct { + name string + // contents is the document offered for writing against the same stored unit. + contents func(t *testing.T) []byte + }{ + { + // Content changed, identities held — the shape of every correct ALTER. + name: "RenameKeepingIdentities", + contents: func(t *testing.T) []byte { + b, err := bson.Marshal(bson.D{ + {Key: "$Type", Value: "DomainModels$DomainModel"}, + {Key: "$ID", Value: bin("11111111-1111-1111-1111-111111111111")}, + {Key: "Entities", Value: bson.A{ + bson.D{ + {Key: "$Type", Value: "DomainModels$EntityImpl"}, + {Key: "$ID", Value: bin(guidEntityID)}, + {Key: "GUID", Value: bin(guidEntityGUID)}, + {Key: "Name", Value: "Renamed"}, + {Key: "Attributes", Value: bson.A{ + bson.D{ + {Key: "$Type", Value: "DomainModels$Attribute"}, + {Key: "$ID", Value: bin(guidAttrID)}, + {Key: "GUID", Value: bin(guidAttrGUID)}, + {Key: "Name", Value: "RenamedAttr"}, + }, + }}, + }, + }}, + }) + if err != nil { + t.Fatalf("marshal: %v", err) + } + return b + }, + }, + { + // A new attribute, with identities of its own. This is ADD ATTRIBUTE, + // and it must not read as a GUID that moved. + name: "AddAttribute", + contents: func(t *testing.T) []byte { + b, err := bson.Marshal(bson.D{ + {Key: "$Type", Value: "DomainModels$DomainModel"}, + {Key: "$ID", Value: bin("11111111-1111-1111-1111-111111111111")}, + {Key: "Entities", Value: bson.A{ + bson.D{ + {Key: "$Type", Value: "DomainModels$EntityImpl"}, + {Key: "$ID", Value: bin(guidEntityID)}, + {Key: "GUID", Value: bin(guidEntityGUID)}, + {Key: "Name", Value: "Organization"}, + {Key: "Attributes", Value: bson.A{ + bson.D{ + {Key: "$Type", Value: "DomainModels$Attribute"}, + {Key: "$ID", Value: bin(guidAttrID)}, + {Key: "GUID", Value: bin(guidAttrGUID)}, + {Key: "Name", Value: "Name"}, + }, + bson.D{ + {Key: "$Type", Value: "DomainModels$Attribute"}, + {Key: "$ID", Value: bin("77777777-7777-7777-7777-777777777777")}, + {Key: "GUID", Value: bin("88888888-8888-8888-8888-888888888888")}, + {Key: "Name", Value: "Status"}, + }, + }}, + }, + }}, + }) + if err != nil { + t.Fatalf("marshal: %v", err) + } + return b + }, + }, + { + // DROP ATTRIBUTE: the removed attribute's GUID is absent, not moved. + name: "DropAttribute", + contents: func(t *testing.T) []byte { + b, err := bson.Marshal(bson.D{ + {Key: "$Type", Value: "DomainModels$DomainModel"}, + {Key: "$ID", Value: bin("11111111-1111-1111-1111-111111111111")}, + {Key: "Entities", Value: bson.A{ + bson.D{ + {Key: "$Type", Value: "DomainModels$EntityImpl"}, + {Key: "$ID", Value: bin(guidEntityID)}, + {Key: "GUID", Value: bin(guidEntityGUID)}, + {Key: "Name", Value: "Organization"}, + }, + }}, + }) + if err != nil { + t.Fatalf("marshal: %v", err) + } + return b + }, + }, + } + + for i, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + // A distinct unit id per case so each starts from the same stored bytes. + unitID := [...]string{ + "a0000000-0000-0000-0000-000000000001", + "a0000000-0000-0000-0000-000000000002", + "a0000000-0000-0000-0000-000000000003", + }[i] + stored := guidUnit(t, guidEntityID, guidEntityGUID, guidAttrID, guidAttrGUID) + w, unitPath := newV2WriterForCommitTest(t, unitID, stored) + + if err := w.UpdateRawUnit(unitID, tc.contents(t)); err != nil { + t.Fatalf("guard refused an ordinary edit: %v", err) + } + // And it really wrote: a "pass" that elided the write proves nothing. + after, err := os.ReadFile(unitPath) + if err != nil { + t.Fatalf("read unit: %v", err) + } + if string(after) == string(stored) { + t.Error("the write did not land, so the guard was never exercised") + } + }) + } +}