From 5b8506abadbc4bb6c84ec2b99f9182a899348ffd Mon Sep 17 00:00:00 2001 From: Ako Date: Sun, 4 Oct 2026 14:40:50 +0000 Subject: [PATCH 01/17] fix(check): E001/E002 only for literals in value position (#969) A literal compared with a String, passed to a function or used in an if-condition inside an enumeration-slot expression was reported as the value assigned to the enumeration, so exec refused valid microflows. Co-Authored-By: Claude Opus 5.5 --- mdl/exprcheck/parser.go | 83 ++++++++++++++++++++-------- mdl/exprcheck/value_position_test.go | 75 +++++++++++++++++++++++++ 2 files changed, 136 insertions(+), 22 deletions(-) create mode 100644 mdl/exprcheck/value_position_test.go diff --git a/mdl/exprcheck/parser.go b/mdl/exprcheck/parser.go index 2c69f0df9e..1aef924b18 100644 --- a/mdl/exprcheck/parser.go +++ b/mdl/exprcheck/parser.go @@ -59,6 +59,7 @@ func (p *parserImpl) Parse(src string, ctx Context) (RobustExpr, []Hint) { "Also check for glued keywords such as 'emptyor' (should be 'empty or').", }) } + hs = append(hs, checkValueLiterals(expr, ctx)...) hs = append(hs, checkSlotKind(expr, ctx)...) hs = append(hs, checkBareIdentifierValue(expr, ctx)...) return expr, hs @@ -365,26 +366,10 @@ func parsePrimary(s *Stream, ctx Context) (RobustExpr, []Hint) { if len(v) >= 2 && v[0] == '\'' && v[len(v)-1] == '\'' { v = v[1 : len(v)-1] } - node := &StringLit{baseNode: baseNode{P: t.Pos}, Value: v} - var hs []Hint - if v == "true" || v == "false" || v == "True" || v == "False" { - if sc, ok := slotKind(ctx); ok && sc.Kind == KindBoolean { - hs = append(hs, Hint{ - Code: "E002", Slug: "bool-string-mismatch", Severity: hints.SeverityError, - Where: hints.Location{ - Microflow: ctx.Microflow, - Context: SlotToContext(ctx.SlotPath), - Line: t.Pos.Line, - Column: t.Pos.Column, - }, - YouWrote: "'" + v + "'", - Problem: "Mendix Boolean expressions use the unquoted literals true and false; a quoted string is never equal to a Boolean.", - Fix: strings.ToLower(v), - }) - } - } - hs = append(hs, checkStringLitVsSlot(node, ctx, t)...) - return node, hs + // Slot checks (E001/E002) are not run here: a literal is only the + // slot's value in value position, which the parse cannot know yet. + // checkValueLiterals runs them over the finished tree. + return &StringLit{baseNode: baseNode{P: t.Pos}, Value: v}, nil case TokNumber: s.Consume() kind := KindInteger @@ -610,7 +595,61 @@ func matchKeyword(s *Stream, kw string) bool { return false } -func checkStringLitVsSlot(node *StringLit, ctx Context, tok Token) []Hint { +// checkValueLiterals runs the slot-literal checks (E002 quoted Boolean, E001 +// quoted enumeration value) on the string literals that ARE the slot's value: +// the whole expression, a parenthesised one, or a then/else result — +// recursively, since a result can itself be an if. +// +// They used to run on every string literal the parser met, so in +// `change $Log (Windrichting = if $Dir = 'NW' then E.NW else E.N)` the 'NW' +// compared with a String variable was reported as assigning a string to the +// enumeration, and every literal argument of find(...) as well — an error, so +// exec refused a valid microflow (ako/mxcli#969). A literal in a comparison, a +// function argument or a condition is an operand, not the value; the +// comparison form of the enum mistake has its own check +// (checkEnumComparedToString), which resolves the other operand instead. +func checkValueLiterals(expr RobustExpr, ctx Context) []Hint { + switch n := expr.(type) { + case *StringLit: + return checkValueStringLit(n, ctx) + case *ParenExpr: + return checkValueLiterals(n.Inner, ctx) + case *IfThenElseExpr: + var hs []Hint + if n.Then != nil { + hs = append(hs, checkValueLiterals(n.Then, ctx)...) + } + if n.Else != nil { + hs = append(hs, checkValueLiterals(n.Else, ctx)...) + } + return hs + } + return nil +} + +func checkValueStringLit(node *StringLit, ctx Context) []Hint { + var hs []Hint + v := node.Value + if v == "true" || v == "false" || v == "True" || v == "False" { + if sc, ok := slotKind(ctx); ok && sc.Kind == KindBoolean { + hs = append(hs, Hint{ + Code: "E002", Slug: "bool-string-mismatch", Severity: hints.SeverityError, + Where: hints.Location{ + Microflow: ctx.Microflow, + Context: SlotToContext(ctx.SlotPath), + Line: node.P.Line, + Column: node.P.Column, + }, + YouWrote: "'" + v + "'", + Problem: "Mendix Boolean expressions use the unquoted literals true and false; a quoted string is never equal to a Boolean.", + Fix: strings.ToLower(v), + }) + } + } + return append(hs, checkStringLitVsSlot(node, ctx)...) +} + +func checkStringLitVsSlot(node *StringLit, ctx Context) []Hint { if ctx.Catalog == nil || ctx.SlotPath == "" { return nil } @@ -629,7 +668,7 @@ func checkStringLitVsSlot(node *StringLit, ctx Context, tok Token) []Hint { Code: "E001", Slug: "enum-string-mismatch", Severity: hints.SeverityError, - Where: hintsLocation(ctx, tok.Pos), + Where: hintsLocation(ctx, node.P), YouWrote: "'" + node.Value + "'", Problem: "Comparing or assigning an Enumeration attribute against " + "a string literal. In Mendix expressions, enumeration values " + diff --git a/mdl/exprcheck/value_position_test.go b/mdl/exprcheck/value_position_test.go new file mode 100644 index 0000000000..bbe7f99e8b --- /dev/null +++ b/mdl/exprcheck/value_position_test.go @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: Apache-2.0 + +package exprcheck + +import "testing" + +// E001 (string literal in an enumeration slot) is about the VALUE assigned to +// the attribute. A literal compared with a String variable, passed to a +// function, or used as an if-condition operand is not the value, so it must not +// be reported (ako/mxcli#969 item 1: `if $Dir = 'NW' then E.NW else E.N` was +// refused, blocking exec). +func enumSlotCtx() Context { + return Context{ + SlotPath: "ChangeItem.Value:JTS.Log.Windrichting", + Slots: DefaultSlotResolver(), + Catalog: lookupCatalog{ + kinds: map[string]TypeKind{"JTS.Log|Windrichting": KindEnumeration}, + enums: map[string]string{"JTS.Log|Windrichting": "JTS.WindrichtingEnum"}, + cases: map[string][]string{"JTS.WindrichtingEnum": {"N", "NW"}}, + }, + } +} + +func countCode(hs []Hint, code string) int { + n := 0 + for _, h := range hs { + if h.Code == code { + n++ + } + } + return n +} + +func TestE001_NotInNonValuePositions(t *testing.T) { + for _, src := range []string{ + `if $Dir = 'NW' then JTS.WindrichtingEnum.NW else JTS.WindrichtingEnum.N`, + `if find('|NW|NORTHWEST|', '|' + $Dir + '|') >= 0 then JTS.WindrichtingEnum.NW else JTS.WindrichtingEnum.N`, + `if contains($Dir, 'N') and $Dir != 'NW' then JTS.WindrichtingEnum.N else empty`, + } { + _, hs := NewParser().Parse(src, enumSlotCtx()) + if n := countCode(hs, "E001"); n != 0 { + t.Errorf("%s: %d E001 for literals outside value position: %+v", src, n, hs) + } + } +} + +func TestE001_StillFiresInValuePositions(t *testing.T) { + for src, want := range map[string]int{ + `'NW'`: 1, + `('NW')`: 1, + `if $Dir = 'x' then 'NW' else JTS.WindrichtingEnum.N`: 1, + `if $Dir = 'x' then 'NW' else 'N'`: 2, + `if $A then JTS.WindrichtingEnum.N else if $B then 'NW' else empty`: 1, + } { + _, hs := NewParser().Parse(src, enumSlotCtx()) + if n := countCode(hs, "E001"); n != want { + t.Errorf("%s: %d E001, want %d: %+v", src, n, want, hs) + } + } +} + +func TestE002_OnlyInValuePosition(t *testing.T) { + ctx := Context{SlotPath: "IfStmt.Condition", Slots: DefaultSlotResolver()} + if sc, ok := slotKind(ctx); !ok || sc.Kind != KindBoolean { + t.Skip("IfStmt.Condition is not a Boolean slot in this table") + } + _, hs := NewParser().Parse(`$S = 'true'`, ctx) + if hasCode(hs, "E002") { + t.Errorf("'true' compared with a String is not a Boolean value: %+v", hs) + } + _, hs = NewParser().Parse(`'true'`, ctx) + if !hasCode(hs, "E002") { + t.Errorf("control: a whole-expression 'true' in a Boolean slot must be E002: %+v", hs) + } +} From 6b02dff6067c79d7f6b0be5e61f3023f4791070d Mon Sep 17 00:00:00 2001 From: Ako Date: Sun, 4 Oct 2026 14:40:52 +0000 Subject: [PATCH 02/17] fix(check): type-check against enums and attributes the script creates (#969) Co-Authored-By: Claude Opus 5.5 --- mdl/executor/typecheck.go | 64 ++++++++++++++++++++++++++++ mdl/executor/typecheck_test.go | 51 ++++++++++++++++++++++ mdl/exprcatalog/exprcatalog.go | 32 ++++++++++++++ mdl/exprcheck/value_position_test.go | 4 +- 4 files changed, 149 insertions(+), 2 deletions(-) diff --git a/mdl/executor/typecheck.go b/mdl/executor/typecheck.go index 0e00537301..0f9e688169 100644 --- a/mdl/executor/typecheck.go +++ b/mdl/executor/typecheck.go @@ -7,6 +7,7 @@ import ( "github.com/mendixlabs/mxcli/mdl/ast" "github.com/mendixlabs/mxcli/mdl/exprcatalog" + "github.com/mendixlabs/mxcli/mdl/exprcheck" "github.com/mendixlabs/mxcli/mdl/exprcheck/adapters" "github.com/mendixlabs/mxcli/mdl/linter" ) @@ -53,6 +54,8 @@ func (e *Executor) TypeCheckProgram(prog *ast.Program) []linter.Violation { return nil } + declareScriptTypes(reader, prog) + // microflowExprSource falls back to rendering the AST when the visitor did // not attach source text, which it does for some slots and not others. The // adapter's own default reads SourceExpr only, and would silently skip @@ -69,3 +72,64 @@ func (e *Executor) TypeCheckProgram(prog *ast.Program) []linter.Violation { } return out } + +// declareScriptTypes overlays the enumerations and attributes the script itself +// creates onto the catalog-backed reader. The catalog only knows the stored +// project, so a microflow assigning a quoted string to an enumeration attribute +// created a few statements earlier checked clean and failed at build time +// (ako/mxcli#969). Statements are applied in order, so a later redefinition +// wins, as it does when the script runs. +func declareScriptTypes(reader *exprcatalog.Reader, prog *ast.Program) { + for _, stmt := range prog.Statements { + switch s := stmt.(type) { + case *ast.CreateEnumerationStmt: + cases := make([]string, 0, len(s.Values)) + for _, v := range s.Values { + cases = append(cases, v.Name) + } + reader.DeclareEnumeration(s.Name.String(), cases) + case *ast.CreateEntityStmt: + for i := range s.Attributes { + declareScriptAttribute(reader, s.Name.String(), s.Attributes[i].Name, s.Attributes[i].Type) + } + case *ast.AlterEntityStmt: + switch { + case s.Operation == ast.AlterEntityAddAttribute && s.Attribute != nil: + declareScriptAttribute(reader, s.Name.String(), s.Attribute.Name, s.Attribute.Type) + case s.Operation == ast.AlterEntityModifyAttribute: + declareScriptAttribute(reader, s.Name.String(), s.AttributeName, s.DataType) + } + } + } +} + +func declareScriptAttribute(reader *exprcatalog.Reader, entityQN, attr string, dt ast.DataType) { + enumQN := "" + if dt.EnumRef != nil { + enumQN = dt.EnumRef.String() + } + reader.DeclareAttribute(entityQN, attr, scriptAttributeKind(dt.Kind), enumQN) +} + +// scriptAttributeKind mirrors exprcatalog's mapping of stored type names. +func scriptAttributeKind(k ast.DataTypeKind) exprcheck.TypeKind { + switch k { + case ast.TypeString, ast.TypeHashedString: + return exprcheck.KindString + case ast.TypeInteger: + return exprcheck.KindInteger + case ast.TypeLong, ast.TypeAutoNumber: + return exprcheck.KindLong + case ast.TypeDecimal: + return exprcheck.KindDecimal + case ast.TypeBoolean: + return exprcheck.KindBoolean + case ast.TypeDateTime, ast.TypeDate, ast.TypeAutoCreatedDate, ast.TypeAutoChangedDate: + return exprcheck.KindDateTime + case ast.TypeBinary: + return exprcheck.KindBinary + case ast.TypeEnumeration: + return exprcheck.KindEnumeration + } + return exprcheck.KindUnknown +} diff --git a/mdl/executor/typecheck_test.go b/mdl/executor/typecheck_test.go index 9c1967d496..7314a59222 100644 --- a/mdl/executor/typecheck_test.go +++ b/mdl/executor/typecheck_test.go @@ -524,3 +524,54 @@ END; t.Errorf("Mendix's string find() produced %+v, want none", stringFind) } } + +// TestTypeCheckProgramSeesScriptCreatedEnum covers the side finding of +// ako/mxcli#969 item 1: the enumeration and the attribute are created in the +// same script as the microflow, so the catalog (built from the stored project) +// knows neither, and the genuine E001 used to vanish. +func TestTypeCheckProgramSeesScriptCreatedEnum(t *testing.T) { + exec := typeCheckFixture(t) + + got := typeCheck(t, exec, ` +CREATE ENUMERATION MyFirstModule.WindEnum (N 'North', NW 'North west'); +CREATE PERSISTENT ENTITY MyFirstModule.Log (Dir: String(10), Wind: Enumeration(MyFirstModule.WindEnum)); +CREATE OR REPLACE MICROFLOW MyFirstModule.ACT_Wind ($Log: MyFirstModule.Log) +BEGIN + CHANGE $Log (Wind = 'NW'); +END; +`) + if len(got) != 1 || got[0].RuleID != "E001" { + t.Fatalf("got %+v, want one E001 for the quoted value", got) + } + if !strings.Contains(got[0].Suggestion, "MyFirstModule.WindEnum.NW") { + t.Errorf("suggestion is %q, want the script-declared enum value", got[0].Suggestion) + } +} + +// TestTypeCheckProgramEnumSlotOperandsAreNotValues is item 1 itself: literals +// compared with a String, passed to find(), or used in an if-condition are not +// the value assigned to the enumeration attribute. +func TestTypeCheckProgramEnumSlotOperandsAreNotValues(t *testing.T) { + exec := typeCheckFixture(t) + + got := typeCheck(t, exec, ` +CREATE OR REPLACE MICROFLOW MyFirstModule.ACT_Status ($T: MyFirstModule.Ticket, $Dir: String) +BEGIN + CHANGE $T (Status = if $Dir = 'NW' then MyFirstModule.OrderStatus.Open else MyFirstModule.OrderStatus.Closed); + CHANGE $T (Status = if find('|NW|NORTHWEST|', '|' + $Dir + '|') >= 0 then MyFirstModule.OrderStatus.Open else MyFirstModule.OrderStatus.Closed); +END; +`) + if len(got) != 0 { + t.Errorf("operands outside value position were flagged: %+v", got) + } + // Control: a quoted value in a then-branch is still the value. + got = typeCheck(t, exec, ` +CREATE OR REPLACE MICROFLOW MyFirstModule.ACT_Status2 ($T: MyFirstModule.Ticket, $Dir: String) +BEGIN + CHANGE $T (Status = if $Dir = 'NW' then 'Open' else MyFirstModule.OrderStatus.Closed); +END; +`) + if len(got) != 1 || got[0].RuleID != "E001" { + t.Errorf("then-branch 'Open' must be E001, got %+v", got) + } +} diff --git a/mdl/exprcatalog/exprcatalog.go b/mdl/exprcatalog/exprcatalog.go index 0c7cbd793a..b0214acbac 100644 --- a/mdl/exprcatalog/exprcatalog.go +++ b/mdl/exprcatalog/exprcatalog.go @@ -202,6 +202,38 @@ func (r *Reader) loadAssociations(db Querier) error { return rows.Err() } +// DeclareEnumeration records an enumeration a script creates, replacing what +// the catalog held under that name (a CREATE OR MODIFY redefines the cases). +// +// The catalog is built from the stored project, so without this an attribute +// or enumeration created earlier in the same script resolved to nothing, the +// kind came out Unknown, and every rule keyed on it stayed silent — a genuine +// `change $X (EnumAttr = 'NW')` passed check whenever the enum was new +// (ako/mxcli#969). +func (r *Reader) DeclareEnumeration(enumQN string, cases []string) { + if enumQN == "" { + return + } + r.enumCase[enumQN] = append([]string(nil), cases...) +} + +// DeclareAttribute records an attribute a script creates or retypes. An +// unknown kind removes any stored entry rather than keeping a stale type. +func (r *Reader) DeclareAttribute(entityQN, attrName string, kind exprcheck.TypeKind, enumQN string) { + if entityQN == "" || attrName == "" { + return + } + key := entityQN + "." + attrName + delete(r.attrKind, key) + delete(r.attrEnum, key) + if kind != exprcheck.KindUnknown { + r.attrKind[key] = kind + } + if kind == exprcheck.KindEnumeration && enumQN != "" { + r.attrEnum[key] = enumQN + } +} + // AssociationTarget returns the entity at the other end of an association. // // Both directions resolve: a Mendix expression follows an association from its diff --git a/mdl/exprcheck/value_position_test.go b/mdl/exprcheck/value_position_test.go index bbe7f99e8b..2ef65aeab9 100644 --- a/mdl/exprcheck/value_position_test.go +++ b/mdl/exprcheck/value_position_test.go @@ -48,8 +48,8 @@ func TestE001_StillFiresInValuePositions(t *testing.T) { for src, want := range map[string]int{ `'NW'`: 1, `('NW')`: 1, - `if $Dir = 'x' then 'NW' else JTS.WindrichtingEnum.N`: 1, - `if $Dir = 'x' then 'NW' else 'N'`: 2, + `if $Dir = 'x' then 'NW' else JTS.WindrichtingEnum.N`: 1, + `if $Dir = 'x' then 'NW' else 'N'`: 2, `if $A then JTS.WindrichtingEnum.N else if $B then 'NW' else empty`: 1, } { _, hs := NewParser().Parse(src, enumSlotCtx()) From 7bb835de6e7145a673b4b31b460d34414c897893 Mon Sep 17 00:00:00 2001 From: Ako Date: Sun, 4 Oct 2026 14:44:07 +0000 Subject: [PATCH 03/17] fix(check): alter page SET may name a flow or page the script creates (#969) Co-Authored-By: Claude Opus 5.5 --- mdl/executor/validate_alter_set.go | 47 +++++++- .../validate_alter_set_script_refs_test.go | 101 ++++++++++++++++++ 2 files changed, 146 insertions(+), 2 deletions(-) create mode 100644 mdl/executor/validate_alter_set_script_refs_test.go diff --git a/mdl/executor/validate_alter_set.go b/mdl/executor/validate_alter_set.go index b25da37808..3224d99c44 100644 --- a/mdl/executor/validate_alter_set.go +++ b/mdl/executor/validate_alter_set.go @@ -3,6 +3,7 @@ package executor import ( + "errors" "fmt" "sort" "strings" @@ -108,7 +109,7 @@ func validateAlterSetProperties(ctx *ExecContext, prog *ast.Program, sc *scriptC if grows[s.PageName.String()] && !probe.ResolvesTarget(set.Target.Widget, columnRefOf(set.Target)) { continue } - errs = append(errs, checkSetOp(ctx, probe, label, set, modName, containerID)...) + errs = append(errs, checkSetOp(ctx, probe, label, set, modName, containerID, sc)...) } } return errs @@ -135,7 +136,7 @@ func openPageProbe(ctx *ExecContext, unitID model.ID) pageProbe { // Per property rather than per op, on its own copy each time, so a statement // setting four properties reports all four mistakes instead of stopping at the // first — the difference between one round of correction and four. -func checkSetOp(ctx *ExecContext, p pageProbe, label string, op *ast.SetPropertyOp, modName string, modID model.ID) []error { +func checkSetOp(ctx *ExecContext, p pageProbe, label string, op *ast.SetPropertyOp, modName string, modID model.ID, sc *scriptContext) []error { names := make([]string, 0, len(op.Properties)) for name := range op.Properties { names = append(names, name) @@ -153,6 +154,9 @@ func checkSetOp(ctx *ExecContext, p pageProbe, label string, op *ast.SetProperty Properties: map[string]any{name: op.Properties[name]}, } if err := applySetPropertyMutator(ctx, probe, one, modName, modID); err != nil { + if scriptDeclaresMissing(sc, err, modName) { + continue + } errs = append(errs, mdlerrors.NewValidation(fmt.Sprintf( "%s: %v%s", label, err, declaredPropertyHint(p, op.Target, name)))) } @@ -160,6 +164,45 @@ func checkSetOp(ctx *ExecContext, p pageProbe, label string, op *ast.SetProperty return errs } +// scriptDeclaresMissing reports whether a dry-run failed only because the SET +// names a microflow, nanoflow or page the script itself creates. +// +// The dry run resolves against the stored project, and the session cache that +// lets exec find a document created a few statements earlier is filled only by +// executing — which check never does. So `set Action = microflow M.X on btn` +// after `create microflow M.X` was "microflow not found" in check while exec +// ran it fine, and since exec runs check first, the script could not run at all +// (ako/mxcli#969). What the setter would do with the resolved document is +// beyond a dry run without it; reference validation already covers a name the +// script does NOT declare, which still fails here. +func scriptDeclaresMissing(sc *scriptContext, err error, modName string) bool { + if sc == nil { + return false + } + var nf *mdlerrors.NotFoundError + if !errors.As(err, &nf) { + return false + } + var declared map[string]bool + switch nf.Kind { + case "microflow": + declared = sc.microflows + case "nanoflow": + declared = sc.nanoflows + case "page": + declared = sc.pages + case "snippet": + declared = sc.snippets + default: + return false + } + name := unquoteQualifiedName(nf.Name) + if !strings.Contains(name, ".") { + name = modName + "." + name + } + return declared[name] +} + // declaredPropertyHint names what the widget does have, when the widget resolves // and declares its own property keys. A near-miss is called out first: the // spelling of a pluggable key is the thing authors get wrong (it is lowerCamel diff --git a/mdl/executor/validate_alter_set_script_refs_test.go b/mdl/executor/validate_alter_set_script_refs_test.go new file mode 100644 index 0000000000..1bd6b86233 --- /dev/null +++ b/mdl/executor/validate_alter_set_script_refs_test.go @@ -0,0 +1,101 @@ +// SPDX-License-Identifier: Apache-2.0 + +package executor + +import ( + "strings" + "testing" + + "go.mongodb.org/mongo-driver/bson" + + "github.com/mendixlabs/mxcli/mdl/backend" + "github.com/mendixlabs/mxcli/mdl/backend/mock" + "github.com/mendixlabs/mxcli/mdl/backend/pagemutator" + "github.com/mendixlabs/mxcli/mdl/types" + "github.com/mendixlabs/mxcli/model" + "github.com/mendixlabs/mxcli/sdk/microflows" + "github.com/mendixlabs/mxcli/sdk/pages" +) + +// actionDeps serializes any action to a placeholder so the setter succeeds +// once the target resolves; the test is about resolution, not the BSON. +type actionDeps struct{ countingDeps } + +func (d *actionDeps) SerializeClientAction(pages.ClientAction) bson.D { + return bson.D{{Key: "$Type", Value: "Forms$NoAction"}} +} + +func storedButtonPage() bson.D { + btn := bson.D{ + {Key: "$Type", Value: "Forms$ActionButton"}, + {Key: "Name", Value: "btnGo"}, + {Key: "Action", Value: bson.D{{Key: "$Type", Value: "Forms$NoAction"}}}, + } + return bson.D{ + {Key: "$Type", Value: "Forms$Page"}, + {Key: "FormCall", Value: bson.D{ + {Key: "Arguments", Value: bson.A{ + int32(2), + bson.D{{Key: "Widgets", Value: bson.A{int32(2), btn}}}, + }}, + }}, + } +} + +func buttonPageCtx(t *testing.T) *ExecContext { + t.Helper() + mod := mkModule("MyModule") + pg := mkPage(mod.ID, "P_Btn") + deps := &actionDeps{} + mb := &mock.MockBackend{ + IsConnectedFunc: func() bool { return true }, + ListModulesFunc: func() ([]*model.Module, error) { return []*model.Module{mod}, nil }, + ListFoldersFunc: func() ([]*types.FolderInfo, error) { return nil, nil }, + ListPagesFunc: func() ([]*pages.Page, error) { return []*pages.Page{pg}, nil }, + ListMicroflowsFunc: func() ([]*microflows.Microflow, error) { return nil, nil }, + ListNanoflowsFunc: func() ([]*microflows.Nanoflow, error) { return nil, nil }, + OpenPageForMutationFunc: func(unitID model.ID) (backend.PageMutator, error) { + return pagemutator.New(storedButtonPage(), unitID, deps), nil + }, + } + ctx, _ := newMockCtx(t, withBackend(mb), withHierarchy(mkHierarchy(mod))) + return ctx +} + +// TestAlterSet_ActionTargetsCreatedInScript is ako/mxcli#969 item 2: a SET +// naming a microflow, nanoflow or page the same script creates was "not found" +// in check, because the dry run resolves against the stored project and the +// session cache exec would use is never filled by check. +func TestAlterSet_ActionTargetsCreatedInScript(t *testing.T) { + for name, src := range map[string]string{ + "microflow": `create microflow MyModule.ACT_New () begin end; +alter page MyModule.P_Btn { set Action = microflow MyModule.ACT_New on btnGo };`, + "nanoflow": `create nanoflow MyModule.NAV_New () begin end; +alter page MyModule.P_Btn { set Action = nanoflow MyModule.NAV_New on btnGo };`, + "page": `create page MyModule.P_New (Title: 'P', Layout: Atlas_Core.Atlas_Default) { dynamictext t1 (Content: 'x') }; +alter page MyModule.P_Btn { set Action = show page MyModule.P_New on btnGo };`, + } { + t.Run(name, func(t *testing.T) { + if errs := checkAlterSet(t, buttonPageCtx(t), src); len(errs) != 0 { + t.Errorf("script-created %s reported missing: %v", name, errs) + } + }) + } +} + +// TestAlterSet_ActionTargetsMissing is the control: a target neither stored +// nor created by the script is still refused. +func TestAlterSet_ActionTargetsMissing(t *testing.T) { + for name, src := range map[string]string{ + "microflow": `alter page MyModule.P_Btn { set Action = microflow MyModule.ACT_Missing on btnGo };`, + "nanoflow": `alter page MyModule.P_Btn { set Action = nanoflow MyModule.NAV_Missing on btnGo };`, + "page": `alter page MyModule.P_Btn { set Action = show page MyModule.P_Missing on btnGo };`, + } { + t.Run(name, func(t *testing.T) { + errs := checkAlterSet(t, buttonPageCtx(t), src) + if len(errs) != 1 || !strings.Contains(errs[0].Error(), "not found") { + t.Errorf("missing %s: got %v, want one not-found error", name, errs) + } + }) + } +} From 2f4634de9570118c6662fcd2a53de9c6fdd41994 Mon Sep 17 00:00:00 2001 From: Ako Date: Sun, 4 Oct 2026 14:51:10 +0000 Subject: [PATCH 04/17] feat(check): MDL-ASSOCDS01 predicts CE8812 for a list over a single-object association (#969) Measured on mxbuild 11.13.0 and 11.14.0: a list view, data grid or gallery over a Reference followed from its FROM entity, or over a Reference with owner Both from either end, is CE8812. ReferenceSets and the reverse of a default Reference build clean. Co-Authored-By: Claude Opus 5.5 --- mdl/executor/validate.go | 9 + mdl/executor/validate_assoc_list_source.go | 173 ++++++++++++++++++ .../validate_assoc_list_source_test.go | 122 ++++++++++++ .../validate_widget_attribute_scope.go | 5 + 4 files changed, 309 insertions(+) create mode 100644 mdl/executor/validate_assoc_list_source.go create mode 100644 mdl/executor/validate_assoc_list_source_test.go diff --git a/mdl/executor/validate.go b/mdl/executor/validate.go index b35d95407f..728989e983 100644 --- a/mdl/executor/validate.go +++ b/mdl/executor/validate.go @@ -104,6 +104,9 @@ type scriptContext struct { // (MDL-WIDGET39), and a retrieve constraint hopping over one resolves // against an entity the project already has. associationEnds map[string][2]string + // associationShapes holds the same associations' multiplicity, so a list + // widget over one can be judged before it is built (MDL-ASSOCDS01). + associationShapes map[string]assocShape // warnings are findings that do not block: dangling references in an // EXCLUDED document, which Mendix itself does not validate. Reported so @@ -141,6 +144,7 @@ func newScriptContext() *scriptContext { entityAttrs: map[string]map[string]bool{}, ambiguousAssc: map[string]bool{}, associationEnds: map[string][2]string{}, + associationShapes: map[string]assocShape{}, flowParams: make(map[string]*flowSignature), pageParams: make(map[string][]string), @@ -174,6 +178,11 @@ func (sc *scriptContext) recordAssociation(s *ast.CreateAssociationStmt) { } if from, to := s.Parent.String(), s.Child.String(); s.Parent.Module != "" && s.Child.Module != "" { sc.associationEnds[strings.ToLower(s.Name.String())] = [2]string{from, to} + sc.associationShapes[strings.ToLower(s.Name.String())] = assocShape{ + qn: s.Name.String(), from: from, to: to, + refSet: s.Type == ast.AssocReferenceSet, + owner: astOwnerName(s.Owner), + } } if prev, ok := sc.associations[s.Name.Name]; ok && prev != s.Name.String() { sc.ambiguousAssc[s.Name.Name] = true diff --git a/mdl/executor/validate_assoc_list_source.go b/mdl/executor/validate_assoc_list_source.go new file mode 100644 index 0000000000..a9b7c35888 --- /dev/null +++ b/mdl/executor/validate_assoc_list_source.go @@ -0,0 +1,173 @@ +// SPDX-License-Identifier: Apache-2.0 + +package executor + +import ( + "fmt" + "strings" + + "github.com/mendixlabs/mxcli/mdl/ast" + "github.com/mendixlabs/mxcli/model" + "github.com/mendixlabs/mxcli/sdk/domainmodel" +) + +// MDL-ASSOCDS01 — a list widget over an association that yields one object. +// +// A list view, data grid or gallery whose data source is `$currentObject/M.A` +// needs the association path to produce a LIST. mxbuild refuses it otherwise: +// CE8812 "A grid association path must result in a list." `check` passed it, +// `exec` wrote it, and the build failed (ako/mxcli#969 item 3). +// +// Measured on mxbuild 11.14.0, one page per shape, a list view, a data grid and +// a gallery in each (all three behave identically): +// +// Reference, owner Default, from the FROM entity → CE8812 +// Reference, owner Both, from the FROM entity → CE8812 +// Reference, owner Both, from the TO entity → CE8812 +// Reference, owner Default, from the TO entity → ok (the reverse is a list) +// ReferenceSet, owner Default or Both, either end → ok +// +// Only those shapes are judged. A context that is a specialization of an end, +// a self-association, a multi-hop path or an owner mxcli does not write is left +// alone rather than guessed at. + +type assocShape struct { + qn string + from, to string + refSet bool + owner string // "Default", "Both", or "" when not one of the measured two +} + +func astOwnerName(o ast.OwnerType) string { + switch o { + case ast.OwnerDefault: + return "Default" + case ast.OwnerBoth: + return "Both" + } + return "" +} + +func sdkOwnerName(o domainmodel.AssociationOwner) string { + switch o { + case domainmodel.AssociationOwnerDefault, "": + return "Default" + case domainmodel.AssociationOwnerBoth: + return "Both" + } + return "" +} + +// listWidgetTypes are the MDL widget keywords whose data source must be a list. +var listWidgetTypes = map[string]bool{ + "listview": true, "datagrid": true, "datagrid2": true, "gallery": true, "templategrid": true, +} + +// checkAssociationShapes indexes the project's associations, overlaid with the +// script's, by lower-cased qualified name. +func checkAssociationShapes(ctx *ExecContext, sc *scriptContext) map[string]assocShape { + out := map[string]assocShape{} + if ctx != nil && ctx.Backend != nil { + if modules, err := getModulesFromCache(ctx); err == nil { + moduleNames := make(map[model.ID]string, len(modules)) + for _, m := range modules { + moduleNames[m.ID] = m.Name + } + if dms, err := ctx.Backend.ListDomainModels(); err == nil { + byID := map[model.ID]string{} + for _, dm := range dms { + if mod := moduleNames[dm.ContainerID]; mod != "" { + for _, ent := range dm.Entities { + byID[ent.ID] = mod + "." + ent.Name + } + } + } + for _, dm := range dms { + mod := moduleNames[dm.ContainerID] + if mod == "" { + continue + } + for _, a := range dm.Associations { + if from, to := byID[a.ParentID], byID[a.ChildID]; from != "" && to != "" { + qn := mod + "." + a.Name + out[strings.ToLower(qn)] = assocShape{qn: qn, from: from, to: to, + refSet: a.Type == domainmodel.AssociationTypeReferenceSet, owner: sdkOwnerName(a.Owner)} + } + } + for _, a := range dm.CrossAssociations { + if from := byID[a.ParentID]; from != "" && a.ChildRef != "" { + qn := mod + "." + a.Name + out[strings.ToLower(qn)] = assocShape{qn: qn, from: from, to: a.ChildRef, + refSet: a.Type == domainmodel.AssociationTypeReferenceSet, owner: sdkOwnerName(a.Owner)} + } + } + } + } + } + } + if sc != nil { + for k, v := range sc.associationShapes { + out[k] = v + } + } + return out +} + +// checkAssocListSource reports MDL-ASSOCDS01 for one widget, given the data +// context it sits in. +func (v *attributeScopeValidator) checkAssocListSource(w *ast.WidgetV3, enclosing dataContext) { + if v.assocs == nil || !listWidgetTypes[strings.ToLower(w.Type)] { + return + } + ds := w.GetDataSource() + if ds == nil || ds.Type != "association" || ds.Reference == "" { + return + } + segs := strings.Split(ds.Reference, "/") + if len(segs) > 2 { + return + } + var ctxEntity string + switch cv := strings.ToLower(ds.ContextVariable); cv { + case "", "currentobject": + if enclosing.unresolved || len(enclosing.entities) == 0 { + return + } + ctxEntity = enclosing.entities[len(enclosing.entities)-1] + default: + qn, ok := v.pageParams[cv] + if !ok { + return + } + ctxEntity = qn + } + assoc := segs[0] + if !strings.Contains(assoc, ".") { + mod, _, _ := strings.Cut(ctxEntity, ".") + assoc = mod + "." + assoc + } + shape, ok := v.assocs[strings.ToLower(assoc)] + if !ok || shape.refSet || shape.owner == "" || strings.EqualFold(shape.from, shape.to) { + return + } + var why string + switch { + case strings.EqualFold(ctxEntity, shape.from): + why = fmt.Sprintf("%s is a Reference followed from its FROM entity %s, which yields one %s, not a list", shape.qn, shape.from, shape.to) + case strings.EqualFold(ctxEntity, shape.to) && shape.owner == "Both": + why = fmt.Sprintf("%s is a Reference with owner Both — one-to-one — so from %s it yields one %s, not a list", shape.qn, shape.to, shape.from) + default: + return + } + v.errs = append(v.errs, fmt.Sprintf( + "%s data source %s: %s — mxbuild rejects this with CE8812 \"A grid association path must result in a list\". "+ + "Show the single object in a data view, or make the association a ReferenceSet [MDL-ASSOCDS01]", + widgetLabel(w.Name, w.Type), dataSourceText(ds), why)) +} + +func dataSourceText(ds *ast.DataSourceV3) string { + if ds.ContextVariable != "" { + return "$" + ds.ContextVariable + "/" + ds.Reference + } + return "association " + ds.Reference +} diff --git a/mdl/executor/validate_assoc_list_source_test.go b/mdl/executor/validate_assoc_list_source_test.go new file mode 100644 index 0000000000..3e71e91058 --- /dev/null +++ b/mdl/executor/validate_assoc_list_source_test.go @@ -0,0 +1,122 @@ +// SPDX-License-Identifier: Apache-2.0 + +package executor + +import ( + "fmt" + "strings" + "testing" + + "github.com/mendixlabs/mxcli/mdl/ast" + "github.com/mendixlabs/mxcli/mdl/backend/mock" + "github.com/mendixlabs/mxcli/mdl/visitor" + "github.com/mendixlabs/mxcli/model" + "github.com/mendixlabs/mxcli/sdk/domainmodel" +) + +// assocShapeCtx is a project with C88.A and C88.B and one stored association +// per measured shape (validate_assoc_list_source.go). +func assocShapeCtx(t *testing.T) *ExecContext { + t.Helper() + mod := &model.Module{BaseElement: model.BaseElement{ID: "mod-c88"}, Name: "C88"} + assoc := func(name string, typ domainmodel.AssociationType, owner domainmodel.AssociationOwner) *domainmodel.Association { + return &domainmodel.Association{BaseElement: model.BaseElement{ID: model.ID("a-" + name)}, Name: name, + ParentID: "e-a", ChildID: "e-b", Type: typ, Owner: owner} + } + b := &mock.MockBackend{ + IsConnectedFunc: func() bool { return true }, + ListModulesFunc: func() ([]*model.Module, error) { return []*model.Module{mod}, nil }, + ListDomainModelsFunc: func() ([]*domainmodel.DomainModel, error) { + return []*domainmodel.DomainModel{{ + BaseElement: model.BaseElement{ID: "dm-c88"}, ContainerID: mod.ID, + Entities: []*domainmodel.Entity{ + {BaseElement: model.BaseElement{ID: "e-a"}, Name: "A"}, + {BaseElement: model.BaseElement{ID: "e-b"}, Name: "B"}, + }, + Associations: []*domainmodel.Association{ + assoc("R_def", domainmodel.AssociationTypeReference, domainmodel.AssociationOwnerDefault), + assoc("R_both", domainmodel.AssociationTypeReference, domainmodel.AssociationOwnerBoth), + assoc("RS_def", domainmodel.AssociationTypeReferenceSet, domainmodel.AssociationOwnerDefault), + assoc("RS_both", domainmodel.AssociationTypeReferenceSet, domainmodel.AssociationOwnerBoth), + }, + }}, nil + }, + } + ctx, _ := newMockCtx(t, withBackend(b)) + ctx.widgetRegistry = &WidgetRegistry{byMDLName: map[string]*WidgetDefinition{}, byWidgetID: map[string]*WidgetDefinition{}} + ctx.widgetRegistryLoaded = true + return ctx +} + +func assocListErrs(t *testing.T, ctx *ExecContext, src string) []string { + t.Helper() + prog, errs := visitor.Build(src) + if len(errs) > 0 { + t.Fatalf("parse: %v", errs) + } + sc := newScriptContext() + sc.collectDefinitions(prog) + var out []string + for _, st := range prog.Statements { + if s, ok := st.(*ast.CreatePageStmtV3); ok { + out = append(out, validatePluggableAttributeScopes(ctx, s.Layout, s.Parameters, allPageWidgets(s), sc)...) + } + } + return out +} + +func assocListPage(ent, widget, assoc string) string { + return fmt.Sprintf(`create page C88.P (title: 'P', layout: Atlas_Core.Atlas_Default, params: ($P: C88.%s)) { + dataview dv (datasource: $P) { + %s w (datasource: $currentObject/C88.%s) { } + } +}`, ent, widget, assoc) +} + +// TestAssocListSource_MeasuredShapes pins the 11.13.0/11.14.0 measurement: +// exactly the shapes mxbuild rejects with CE8812 are reported, for each of the +// three list widgets, and the shapes it accepts are not (the controls). +func TestAssocListSource_MeasuredShapes(t *testing.T) { + ctx := assocShapeCtx(t) + for _, tc := range []struct { + from, assoc string + ce8812 bool + }{ + {"A", "R_def", true}, + {"A", "R_both", true}, + {"B", "R_both", true}, + {"B", "R_def", false}, + {"A", "RS_def", false}, + {"B", "RS_def", false}, + {"A", "RS_both", false}, + {"B", "RS_both", false}, + } { + for _, widget := range []string{"listview", "datagrid", "gallery"} { + errs := assocListErrs(t, ctx, assocListPage(tc.from, widget, tc.assoc)) + got := len(errs) == 1 && strings.Contains(errs[0], "MDL-ASSOCDS01") && strings.Contains(errs[0], "CE8812") + if got != tc.ce8812 || (!tc.ce8812 && len(errs) != 0) { + t.Errorf("%s from %s over %s: got %q, want CE8812=%v", widget, tc.from, tc.assoc, errs, tc.ce8812) + } + } + } +} + +// A data view over the same single-object path is fine: only list widgets need +// a list. +func TestAssocListSource_DataViewIsNotJudged(t *testing.T) { + if errs := assocListErrs(t, assocShapeCtx(t), assocListPage("A", "dataview", "R_def")); len(errs) != 0 { + t.Errorf("a data view over a Reference was reported: %q", errs) + } +} + +// The association created by the same script is judged too — the issue's own +// shape, where everything is new. +func TestAssocListSource_ScriptDeclaredAssociation(t *testing.T) { + src := `create association C88.New_both from C88.A to C88.B type Reference owner Both; +create association C88.New_set from C88.A to C88.B type ReferenceSet owner Both; +` + assocListPage("B", "listview", "New_both") + ";\n" + strings.Replace(assocListPage("B", "listview", "New_set"), "C88.P ", "C88.P2 ", 1) + errs := assocListErrs(t, assocShapeCtx(t), src) + if len(errs) != 1 || !strings.Contains(errs[0], "C88.New_both") { + t.Errorf("want one MDL-ASSOCDS01 for New_both and none for the ReferenceSet control, got %q", errs) + } +} diff --git a/mdl/executor/validate_widget_attribute_scope.go b/mdl/executor/validate_widget_attribute_scope.go index 353de89d03..b733f28bca 100644 --- a/mdl/executor/validate_widget_attribute_scope.go +++ b/mdl/executor/validate_widget_attribute_scope.go @@ -73,6 +73,7 @@ func validatePluggableAttributeScopes(ctx *ExecContext, layout string, params [] index: checkAttributeIndex(ctx, sc), types: checkMemberTypeIndex(ctx, sc), reactClient: usesReactClient(ctx) && !layoutIsNative(ctx, layout), + assocs: checkAssociationShapes(ctx, sc), } for _, w := range widgets { v.walk(w, dataContext{}) @@ -91,6 +92,9 @@ type attributeScopeValidator struct { // The built-in input widget checks (validate_widget_attribute_type.go). types memberTypeIndex reactClient bool + + // assocs feeds MDL-ASSOCDS01 (validate_assoc_list_source.go). + assocs map[string]assocShape } func (v *attributeScopeValidator) walk(w *ast.WidgetV3, enclosing dataContext) { @@ -99,6 +103,7 @@ func (v *attributeScopeValidator) walk(w *ast.WidgetV3, enclosing dataContext) { } inner := childContext(enclosing, w.GetDataSource(), v.sigs, v.pageParams) v.checkInputBinding(w, enclosing) + v.checkAssocListSource(w, enclosing) v.checkReactUnsupported(w) if v.registry == nil { // No registry: the pluggable half cannot run, and a pluggable widget's From 5dc70e93f4dddb8406b6558a392487722b53803a Mon Sep 17 00:00:00 2001 From: Ako Date: Sun, 4 Oct 2026 14:53:06 +0000 Subject: [PATCH 05/17] feat(fix): mxcli fix hashes verifies and repairs the MPR v2 ContentsHash index Compares every Unit.ContentsHash with base64(SHA-256) of its .mxunit and reports mismatches, missing files and orphan files; --repair rewrites the mismatched hashes in one transaction under the writer's Studio Pro guard. exec and docker check warn when the index disagrees (~0.2 s on 900 units). Part of #972 (item 1). Co-Authored-By: Claude Opus 5.5 --- cmd/mxcli/cmd_exec.go | 3 + cmd/mxcli/cmd_fix_hashes.go | 209 +++++++++++++++++++++++++++++ cmd/mxcli/cmd_fix_hashes_test.go | 118 ++++++++++++++++ cmd/mxcli/docker.go | 3 + modelsdk/mpr/contents_hash.go | 184 +++++++++++++++++++++++++ modelsdk/mpr/contents_hash_test.go | 127 ++++++++++++++++++ 6 files changed, 644 insertions(+) create mode 100644 cmd/mxcli/cmd_fix_hashes.go create mode 100644 cmd/mxcli/cmd_fix_hashes_test.go create mode 100644 modelsdk/mpr/contents_hash.go create mode 100644 modelsdk/mpr/contents_hash_test.go diff --git a/cmd/mxcli/cmd_exec.go b/cmd/mxcli/cmd_exec.go index 34c5774b8b..5baddbe58c 100644 --- a/cmd/mxcli/cmd_exec.go +++ b/cmd/mxcli/cmd_exec.go @@ -100,6 +100,9 @@ Example: os.Exit(1) } } + // A stale MPR v2 ContentsHash index (units restored with git) + // is invisible to mx check; say so before writing (#972). + fmt.Fprint(os.Stderr, contentsHashDriftWarning(projectPath)) } // Parse and execute the file diff --git a/cmd/mxcli/cmd_fix_hashes.go b/cmd/mxcli/cmd_fix_hashes.go new file mode 100644 index 0000000000..28647742dd --- /dev/null +++ b/cmd/mxcli/cmd_fix_hashes.go @@ -0,0 +1,209 @@ +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "errors" + "fmt" + "io" + "os" + "path/filepath" + + mpr "github.com/mendixlabs/mxcli/modelsdk/mpr" + "github.com/spf13/cobra" + "go.mongodb.org/mongo-driver/v2/bson" +) + +// cmd_fix_hashes.go verifies and repairs the MPR v2 ContentsHash index +// (ako/mxcli#972). Unlike its siblings in cmd_fix.go it needs no mx: the index +// is mxcli's own to read and write. + +var fixHashesCmd = &cobra.Command{ + Use: "hashes", + Short: "Verify (and with --repair, rewrite) the MPR v2 ContentsHash index", + Long: `Verify that every unit's ContentsHash in the .mpr matches its .mxunit file. + +In an MPR v2 project the .mpr indexes each mprcontents/**/*.mxunit file by +base64(SHA-256(file)). mxcli and Studio Pro keep the two in step, but anything +that changes a .mxunit behind their back does not: restoring a unit with +'git checkout', 'git restore' or a merge leaves the index describing bytes that +are no longer on disk. 'mx check' does not notice; Studio Pro uses the index to +decide what changed. + +Reported: + MISMATCH the file's hash differs from the stored one (repairable) + MISSING the .mpr indexes a unit whose file does not exist + ORPHAN a .mxunit file that no unit in the .mpr indexes + +--repair rewrites every mismatched hash from the file on disk, in one +transaction, and leaves the files untouched — the files are the source of +truth. MISSING and ORPHAN have no correct hash to write and are only reported +('mxcli diag --check-units --fix' removes orphan files). Like every mxcli +write, the repair is refused while Studio Pro has the project open. + +Exits 1 when an issue remains after the command (so the verify can gate CI). +An MPR v1 project keeps unit contents inside the .mpr and has no index to +drift; the command says so and exits 0.`, + Example: ` mxcli fix hashes -p app.mpr + mxcli fix hashes -p app.mpr --repair`, + Args: cobra.NoArgs, + RunE: runFixHashes, + SilenceUsage: true, + SilenceErrors: true, +} + +func init() { + fixHashesCmd.Flags().StringP("project", "p", "", "path to the Mendix project (.mpr)") + _ = fixHashesCmd.MarkFlagRequired("project") + fixHashesCmd.Flags().Bool("repair", false, "rewrite mismatched ContentsHash values from the files on disk") + fixCmd.AddCommand(fixHashesCmd) +} + +// errHashIssuesRemain makes the command exit 1 after it printed the report. +var errHashIssuesRemain = errors.New("ContentsHash issues remain") + +func runFixHashes(cmd *cobra.Command, _ []string) error { + mprPath, _ := cmd.Flags().GetString("project") + repair, _ := cmd.Flags().GetBool("repair") + out := cmd.OutOrStdout() + if _, err := os.Stat(mprPath); err != nil { + return fmt.Errorf("project not found: %s", mprPath) + } + err := fixHashes(mprPath, repair, out) + if errors.Is(err, errHashIssuesRemain) { + cmd.SilenceErrors = true + os.Exit(1) + } + return err +} + +// fixHashes is the command without its exit: it prints the report and returns +// errHashIssuesRemain when the index and the files still disagree. +func fixHashes(mprPath string, repair bool, out io.Writer) error { + var ( + rep *mpr.ContentsHashReport + repaired int + err error + ) + if repair { + w, werr := mpr.NewWriter(mprPath) + if werr != nil { + return fmt.Errorf("open project: %w", werr) + } + rep, repaired, err = w.RepairContentsHashes() + _ = w.Close() + } else { + r, rerr := mpr.Open(mprPath) + if rerr != nil { + return fmt.Errorf("open project: %w", rerr) + } + rep, err = r.VerifyContentsHashes() + _ = r.Close() + } + if errors.Is(err, mpr.ErrNotMPRv2) { + fmt.Fprintf(out, "%s is an MPR v1 project: unit contents live inside the .mpr, so there is no ContentsHash index to verify.\n", filepath.Base(mprPath)) + return nil + } + if rep == nil { + return err + } + + fmt.Fprintf(out, "Checked %d unit(s) in %s against %d .mxunit file(s).\n", rep.Units, filepath.Base(mprPath), rep.Files) + contentsDir := filepath.Join(filepath.Dir(mprPath), "mprcontents") + rel := func(p string) string { + if r, e := filepath.Rel(contentsDir, p); e == nil { + return filepath.Join("mprcontents", r) + } + return p + } + for _, m := range rep.Mismatches { + fmt.Fprintf(out, " MISMATCH %s %s%s\n stored %s, file %s\n", m.UnitID, rel(m.Path), unitLabel(m.Path), orNone(m.Stored), m.Actual) + } + for _, m := range rep.MissingFiles { + fmt.Fprintf(out, " MISSING %s %s (indexed, no file)\n", m.UnitID, rel(m.Path)) + } + for _, f := range rep.OrphanFiles { + fmt.Fprintf(out, " ORPHAN %s%s (file, not indexed)\n", rel(f), unitLabel(f)) + } + fmt.Fprintf(out, "\n%d mismatch(es), %d missing file(s), %d orphan file(s).\n", + len(rep.Mismatches), len(rep.MissingFiles), len(rep.OrphanFiles)) + + if err != nil { // the repair itself failed (Studio Pro guard, SQLite) + return err + } + remaining := len(rep.MissingFiles) + len(rep.OrphanFiles) + switch { + case repair && repaired > 0: + fmt.Fprintf(out, "Repaired %d ContentsHash value(s) from the files on disk.\n", repaired) + case !repair && len(rep.Mismatches) > 0: + remaining += len(rep.Mismatches) + fmt.Fprintf(out, "Run 'mxcli fix hashes -p %s --repair' to rewrite the mismatched hashes from the files.\n", mprPath) + case rep.Clean(): + fmt.Fprintln(out, "The ContentsHash index matches every unit file.") + } + if len(rep.OrphanFiles) > 0 { + fmt.Fprintf(out, "Orphan files are not indexed by the .mpr; 'mxcli diag --check-units -p %s --fix' removes them.\n", mprPath) + } + if len(rep.MissingFiles) > 0 { + fmt.Fprintln(out, "Missing files cannot be repaired from here: restore them from version control.") + } + if remaining > 0 { + return errHashIssuesRemain + } + return nil +} + +func orNone(s string) string { + if s == "" { + return "(none)" + } + return s +} + +// unitLabel names the unit a .mxunit holds (" [Microflows$Microflow Name]"), +// or "" when the file does not read as BSON. +func unitLabel(path string) string { + b, err := os.ReadFile(path) + if err != nil || bson.Raw(b).Validate() != nil { + return "" + } + raw := bson.Raw(b) + typ, _ := raw.Lookup("$Type").StringValueOK() + name, _ := raw.Lookup("Name").StringValueOK() + switch { + case typ == "": + return "" + case name == "": + return " [" + typ + "]" + } + return " [" + typ + " " + name + "]" +} + +// contentsHashDriftWarning is the one-line warning exec and docker check print +// when the ContentsHash index disagrees with the unit files, or "" when it +// agrees. Measured on testapp (900 units): the whole `mxcli fix hashes` run, +// process start included, takes ~0.2 s, so it is cheap enough to run on every +// exec and check. It never fails the caller: an MPR v1 project or any error +// opening or reading the project yields "". +func contentsHashDriftWarning(mprPath string) string { + if mprPath == "" { + return "" + } + r, err := mpr.Open(mprPath) + if err != nil { + return "" + } + defer r.Close() + rep, err := r.VerifyContentsHashes() + if err != nil { + return "" + } + n := len(rep.Mismatches) + len(rep.MissingFiles) + if n == 0 { + return "" + } + return fmt.Sprintf("Warning: %d unit(s) in %s do not match the .mpr's ContentsHash index "+ + "(typically .mxunit files restored with git outside Studio Pro; mx check does not notice). "+ + "Run 'mxcli fix hashes -p %s' for the list, and --repair to fix it.\n", + n, filepath.Base(mprPath), mprPath) +} diff --git a/cmd/mxcli/cmd_fix_hashes_test.go b/cmd/mxcli/cmd_fix_hashes_test.go new file mode 100644 index 0000000000..7f6fd4bf15 --- /dev/null +++ b/cmd/mxcli/cmd_fix_hashes_test.go @@ -0,0 +1,118 @@ +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "bytes" + "errors" + "os" + "path/filepath" + "strings" + "testing" +) + +// ako/mxcli#972 item 1: `mxcli fix hashes` on a real MPR v2 project. + +func copyPedApp(t *testing.T) string { + t.Helper() + src := filepath.Join("..", "..", "testdata", "pedapp") + if _, err := os.Stat(filepath.Join(src, "PedApp.mpr")); err != nil { + t.Skip("testdata/pedapp not available") + } + dst := t.TempDir() + if err := copyTree(src, dst); err != nil { + t.Fatal(err) + } + return filepath.Join(dst, "PedApp.mpr") +} + +// revertOneUnit overwrites one .mxunit with different bytes, as `git checkout` +// of an older revision would, and returns its path. +func revertOneUnit(t *testing.T, mprPath string) string { + t.Helper() + files, _ := filepath.Glob(filepath.Join(filepath.Dir(mprPath), "mprcontents", "*", "*", "*.mxunit")) + if len(files) == 0 { + t.Fatal("no .mxunit files in the copy") + } + f := files[len(files)/2] + b, err := os.ReadFile(f) + if err != nil { + t.Fatal(err) + } + // Same content plus a trailing byte: different hash, same unit. + if err := os.WriteFile(f, append(b, 0), 0o644); err != nil { + t.Fatal(err) + } + return f +} + +func TestFixHashes_UntouchedProjectIsClean(t *testing.T) { + mprPath := copyPedApp(t) + var out bytes.Buffer + if err := fixHashes(mprPath, false, &out); err != nil { + t.Fatalf("control: %v\n%s", err, out.String()) + } + if !strings.Contains(out.String(), "matches every unit file") { + t.Fatalf("control not reported clean:\n%s", out.String()) + } + if w := contentsHashDriftWarning(mprPath); w != "" { + t.Fatalf("control warned: %s", w) + } +} + +func TestFixHashes_DetectsAndRepairsRevertedUnit(t *testing.T) { + mprPath := copyPedApp(t) + f := revertOneUnit(t, mprPath) + id := strings.TrimSuffix(filepath.Base(f), ".mxunit") + + var out bytes.Buffer + err := fixHashes(mprPath, false, &out) + if !errors.Is(err, errHashIssuesRemain) { + t.Fatalf("verify: want errHashIssuesRemain, got %v\n%s", err, out.String()) + } + if !strings.Contains(out.String(), "MISMATCH "+id) || !strings.Contains(out.String(), "1 mismatch(es)") { + t.Fatalf("mismatch not reported:\n%s", out.String()) + } + if w := contentsHashDriftWarning(mprPath); !strings.Contains(w, "1 unit(s)") || !strings.Contains(w, "mxcli fix hashes") { + t.Fatalf("drift warning = %q", w) + } + + out.Reset() + if err := fixHashes(mprPath, true, &out); err != nil { + t.Fatalf("repair: %v\n%s", err, out.String()) + } + if !strings.Contains(out.String(), "Repaired 1 ContentsHash") { + t.Fatalf("repair not reported:\n%s", out.String()) + } + + out.Reset() + if err := fixHashes(mprPath, false, &out); err != nil { + t.Fatalf("after repair: %v\n%s", err, out.String()) + } + if w := contentsHashDriftWarning(mprPath); w != "" { + t.Fatalf("warned after repair: %s", w) + } +} + +func TestFixHashes_V1NotApplicable(t *testing.T) { + src := filepath.Join("..", "..", "modelsdk", "mpr", "testdata", "v1-project") + files, _ := filepath.Glob(filepath.Join(src, "*.mpr")) + if len(files) == 0 { + t.Skip("no v1 fixture") + } + dst := t.TempDir() + if err := copyTree(src, dst); err != nil { + t.Fatal(err) + } + mprPath := filepath.Join(dst, filepath.Base(files[0])) + var out bytes.Buffer + if err := fixHashes(mprPath, false, &out); err != nil { + t.Fatalf("v1: %v", err) + } + if !strings.Contains(out.String(), "MPR v1") { + t.Fatalf("v1 not explained:\n%s", out.String()) + } + if w := contentsHashDriftWarning(mprPath); w != "" { + t.Fatalf("v1 warned: %s", w) + } +} diff --git a/cmd/mxcli/docker.go b/cmd/mxcli/docker.go index bba7c5990c..1b2d5b5b0d 100644 --- a/cmd/mxcli/docker.go +++ b/cmd/mxcli/docker.go @@ -215,6 +215,9 @@ Examples: Stderr: os.Stderr, } + // A stale ContentsHash index is not something mx check reports (#972). + fmt.Fprint(os.Stderr, contentsHashDriftWarning(projectPath)) + if err := docker.Check(opts); err != nil { fmt.Fprintf(os.Stderr, "Error: %v\n", err) os.Exit(1) diff --git a/modelsdk/mpr/contents_hash.go b/modelsdk/mpr/contents_hash.go new file mode 100644 index 0000000000..6058b01c95 --- /dev/null +++ b/modelsdk/mpr/contents_hash.go @@ -0,0 +1,184 @@ +// SPDX-License-Identifier: Apache-2.0 + +package mpr + +import ( + "crypto/sha256" + "encoding/base64" + "errors" + "fmt" + "os" + "path/filepath" + "sort" + "strings" +) + +// ContentsHash verification (ako/mxcli#972). +// +// In MPR v2 every Unit row carries ContentsHash = base64(SHA-256(.mxunit)). +// The writer keeps the two in step (WriteTransaction.WriteUnit, updateUnit), but +// anything that changes a .mxunit behind its back does not: restoring one with +// `git checkout` leaves the row describing bytes that are no longer on disk, and +// `mx check` does not notice. VerifyContentsHashes finds the drift and +// RepairContentsHashes rewrites the index from the files. + +// ErrNotMPRv2 is returned for an MPR v1 project, which has no ContentsHash index +// to drift: the unit contents live in the Unit table itself. +var ErrNotMPRv2 = errors.New("not an MPR v2 project: ContentsHash only indexes mprcontents files") + +// ContentsHashIssue is one unit whose stored hash does not describe its file. +type ContentsHashIssue struct { + UnitID string // Mendix UUID form, as everywhere else in this package + Path string // the .mxunit the row points at + Stored string // ContentsHash in the .mpr ("" when the row has none) + Actual string // hash of the file on disk ("" when the file is missing) + + blob []byte // the row's UnitID, for the repair's UPDATE +} + +// ContentsHashReport is the result of VerifyContentsHashes. +type ContentsHashReport struct { + Units int // rows in the Unit table + Files int // .mxunit files under mprcontents + Mismatches []ContentsHashIssue // file present, hash differs + MissingFiles []ContentsHashIssue // row present, file absent + OrphanFiles []string // file present, no row +} + +// Clean reports whether the index and the files agree completely. +func (r *ContentsHashReport) Clean() bool { + return len(r.Mismatches) == 0 && len(r.MissingFiles) == 0 && len(r.OrphanFiles) == 0 +} + +// contentsHashOf is the hash the writer stores for contents. +func contentsHashOf(contents []byte) string { + sum := sha256.Sum256(contents) + return base64.StdEncoding.EncodeToString(sum[:]) +} + +// unitFilePath is where the writer puts a unit's file. +func unitFilePath(contentsDir string, unitIDBlob []byte) string { + swapped := blobToUUIDSwapped(unitIDBlob) + return filepath.Join(contentsDir, swapped[0:2], swapped[2:4], swapped+".mxunit") +} + +// VerifyContentsHashes compares every Unit.ContentsHash with the SHA-256 of the +// .mxunit file it indexes, and lists .mxunit files no row indexes. Read-only. +// Returns ErrNotMPRv2 for an MPR v1 project. +func (r *Reader) VerifyContentsHashes() (*ContentsHashReport, error) { + if r.version != MPRVersionV2 || r.contentsDir == "" { + return nil, ErrNotMPRv2 + } + rows, err := r.db.Query(`SELECT UnitID, COALESCE(ContentsHash, '') FROM Unit`) + if err != nil { + return nil, fmt.Errorf("query unit hashes: %w", err) + } + type row struct { + blob []byte + hash string + } + var units []row + for rows.Next() { + var rw row + if err := rows.Scan(&rw.blob, &rw.hash); err != nil { + rows.Close() + return nil, fmt.Errorf("scan unit row: %w", err) + } + units = append(units, rw) + } + rows.Close() + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("query unit hashes: %w", err) + } + + rep := &ContentsHashReport{Units: len(units)} + indexed := make(map[string]bool, len(units)) + for _, u := range units { + if len(u.blob) != 16 { + continue + } + path := unitFilePath(r.contentsDir, u.blob) + indexed[filepath.Clean(path)] = true + issue := ContentsHashIssue{UnitID: blobToUUID(u.blob), Path: path, Stored: u.hash, blob: u.blob} + contents, err := os.ReadFile(path) + if err != nil { + if os.IsNotExist(err) { + rep.MissingFiles = append(rep.MissingFiles, issue) + continue + } + return nil, fmt.Errorf("read %s: %w", path, err) + } + issue.Actual = contentsHashOf(contents) + if issue.Actual != u.hash { + rep.Mismatches = append(rep.Mismatches, issue) + } + } + + files, err := filepath.Glob(filepath.Join(r.contentsDir, "*", "*", "*.mxunit")) + if err != nil { + return nil, fmt.Errorf("scan mprcontents: %w", err) + } + rep.Files = len(files) + for _, f := range files { + if !indexed[filepath.Clean(f)] { + rep.OrphanFiles = append(rep.OrphanFiles, f) + } + } + + sortIssues(rep.Mismatches) + sortIssues(rep.MissingFiles) + sort.Strings(rep.OrphanFiles) + return rep, nil +} + +func sortIssues(issues []ContentsHashIssue) { + sort.Slice(issues, func(i, j int) bool { return issues[i].Path < issues[j].Path }) +} + +// RepairContentsHashes rewrites every mismatched ContentsHash from the file on +// disk in one SQLite transaction, and bumps _Transaction.LastTransactionID in +// that transaction as every other write does (Studio Pro reads it to detect an +// external change). It takes the writer's Studio Pro open-project guard: a +// running Studio Pro would overwrite the index on its next save. +// +// Missing files and orphan files are reported, not repaired: neither has a +// correct hash to write (`mxcli diag --check-units --fix` removes orphans). +// Returns the report the repair was based on and the number of rows rewritten. +func (w *Writer) RepairContentsHashes() (*ContentsHashReport, int, error) { + rep, err := w.reader.VerifyContentsHashes() + if err != nil { + return nil, 0, err + } + if len(rep.Mismatches) == 0 { + return rep, 0, nil + } + if err := w.guardWrite(); err != nil { + return rep, 0, err + } + + tx, err := w.reader.db.Begin() + if err != nil { + return rep, 0, err + } + n := 0 + for _, m := range rep.Mismatches { + res, err := tx.Exec(`UPDATE Unit SET ContentsHash = ? WHERE UnitID = ?`, m.Actual, m.blob) + if err != nil { + _ = tx.Rollback() + return rep, 0, fmt.Errorf("update ContentsHash of %s: %w", m.UnitID, err) + } + if c, _ := res.RowsAffected(); c > 0 { + n += int(c) + } + } + if _, err := tx.Exec(`UPDATE _Transaction SET LastTransactionID = ?`, generateUUID()); err != nil && + !strings.Contains(err.Error(), "no such table") { + _ = tx.Rollback() + return rep, 0, fmt.Errorf("update _Transaction: %w", err) + } + if err := tx.Commit(); err != nil { + return rep, 0, err + } + w.reader.InvalidateCache() + return rep, n, nil +} diff --git a/modelsdk/mpr/contents_hash_test.go b/modelsdk/mpr/contents_hash_test.go new file mode 100644 index 0000000000..2af0dadf34 --- /dev/null +++ b/modelsdk/mpr/contents_hash_test.go @@ -0,0 +1,127 @@ +// SPDX-License-Identifier: Apache-2.0 + +package mpr + +import ( + "errors" + "os" + "path/filepath" + "testing" +) + +// ako/mxcli#972: restoring a .mxunit with `git checkout` leaves the .mpr's +// ContentsHash describing bytes that are no longer on disk. + +const hashTestUnit = "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee" + +func TestVerifyContentsHashesCleanControl(t *testing.T) { + w, _ := newV2WriterForCommitTest(t, hashTestUnit, unitDoc(t, "Stored")) + rep, err := w.ConcreteReader().VerifyContentsHashes() + if err != nil { + t.Fatal(err) + } + if !rep.Clean() || rep.Units != 1 || rep.Files != 1 { + t.Fatalf("untouched project not clean: %+v", rep) + } +} + +func TestVerifyContentsHashesDetectsRevertedFile(t *testing.T) { + w, unitPath := newV2WriterForCommitTest(t, hashTestUnit, unitDoc(t, "Stored")) + reverted := unitDoc(t, "RevertedByGit") + if err := os.WriteFile(unitPath, reverted, 0644); err != nil { + t.Fatal(err) + } + rep, err := w.ConcreteReader().VerifyContentsHashes() + if err != nil { + t.Fatal(err) + } + if len(rep.Mismatches) != 1 { + t.Fatalf("want 1 mismatch, got %+v", rep) + } + m := rep.Mismatches[0] + if m.UnitID != hashTestUnit || m.Path != unitPath || m.Actual != hashOf(reverted) || m.Stored == m.Actual { + t.Fatalf("wrong mismatch: %+v", m) + } +} + +func TestVerifyContentsHashesMissingAndOrphanFiles(t *testing.T) { + w, unitPath := newV2WriterForCommitTest(t, hashTestUnit, unitDoc(t, "Stored")) + if err := os.Remove(unitPath); err != nil { + t.Fatal(err) + } + orphan := filepath.Join(w.ConcreteReader().ContentsDir(), "12", "34", "12345678-0000-0000-0000-000000000000.mxunit") + if err := os.MkdirAll(filepath.Dir(orphan), 0755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(orphan, []byte("x"), 0644); err != nil { + t.Fatal(err) + } + rep, err := w.ConcreteReader().VerifyContentsHashes() + if err != nil { + t.Fatal(err) + } + if len(rep.MissingFiles) != 1 || rep.MissingFiles[0].UnitID != hashTestUnit { + t.Fatalf("missing file not reported: %+v", rep) + } + if len(rep.OrphanFiles) != 1 || rep.OrphanFiles[0] != orphan { + t.Fatalf("orphan file not reported: %+v", rep) + } + if len(rep.Mismatches) != 0 { + t.Fatalf("unexpected mismatches: %+v", rep.Mismatches) + } +} + +func TestRepairContentsHashesRewritesIndex(t *testing.T) { + w, unitPath := newV2WriterForCommitTest(t, hashTestUnit, unitDoc(t, "Stored")) + reverted := unitDoc(t, "RevertedByGit") + if err := os.WriteFile(unitPath, reverted, 0644); err != nil { + t.Fatal(err) + } + rep, n, err := w.RepairContentsHashes() + if err != nil { + t.Fatal(err) + } + if n != 1 || len(rep.Mismatches) != 1 { + t.Fatalf("want 1 row repaired, got %d (%+v)", n, rep) + } + if got := storedHash(t, w, hashTestUnit); got != hashOf(reverted) { + t.Fatalf("ContentsHash = %q, want %q", got, hashOf(reverted)) + } + after, err := w.ConcreteReader().VerifyContentsHashes() + if err != nil { + t.Fatal(err) + } + if !after.Clean() { + t.Fatalf("not clean after repair: %+v", after) + } + // The file itself is the source of truth and must be left alone. + if b, _ := os.ReadFile(unitPath); string(b) != string(reverted) { + t.Fatal("repair modified the unit file") + } +} + +func TestRepairContentsHashesRefusedWhileStudioProOpen(t *testing.T) { + w, unitPath := newV2WriterForCommitTest(t, hashTestUnit, unitDoc(t, "Stored")) + stored := storedHash(t, w, hashTestUnit) + if err := os.WriteFile(unitPath, unitDoc(t, "RevertedByGit"), 0644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(w.ConcreteReader().Path()+".lock", nil, 0644); err != nil { + t.Fatal(err) + } + _, _, err := w.RepairContentsHashes() + var open *StudioProOpenError + if !errors.As(err, &open) { + t.Fatalf("want StudioProOpenError, got %v", err) + } + if got := storedHash(t, w, hashTestUnit); got != stored { + t.Fatal("hash rewritten despite the guard") + } +} + +func TestVerifyContentsHashesV1NotApplicable(t *testing.T) { + r := &Reader{version: MPRVersionV1} + if _, err := r.VerifyContentsHashes(); !errors.Is(err, ErrNotMPRv2) { + t.Fatalf("want ErrNotMPRv2, got %v", err) + } +} From 7090e7226ba1ceebe790b8fdb41c2ed2c2448bcc Mon Sep 17 00:00:00 2001 From: Ako Date: Sun, 4 Oct 2026 14:53:07 +0000 Subject: [PATCH 06/17] feat: warn about git states that crash Studio Pro on open Studio Pro 11.13 fails with "Unable to find 'system' property in 'system'" when the branch has no upstream or git reports dubious ownership. docker check, run --local and diag -p now warn with the remedy. Never fatal; silent outside a repo, on a detached HEAD, under CI or with MXCLI_NO_GIT_WARNINGS=1. Part of #972 (item 3). Co-Authored-By: Claude Opus 5.5 --- cmd/mxcli/cmd_run.go | 2 + cmd/mxcli/diag.go | 23 +++++++ cmd/mxcli/docker.go | 3 +- cmd/mxcli/gitstate.go | 136 ++++++++++++++++++++++++++++++++++++ cmd/mxcli/gitstate_test.go | 137 +++++++++++++++++++++++++++++++++++++ 5 files changed, 300 insertions(+), 1 deletion(-) create mode 100644 cmd/mxcli/gitstate.go create mode 100644 cmd/mxcli/gitstate_test.go diff --git a/cmd/mxcli/cmd_run.go b/cmd/mxcli/cmd_run.go index aec8758042..9404c22a82 100644 --- a/cmd/mxcli/cmd_run.go +++ b/cmd/mxcli/cmd_run.go @@ -141,6 +141,8 @@ Examples: if abs, err := filepath.Abs(projectPath); err == nil { projectPath = abs } + // Git states that crash Studio Pro on open (#972): advice only. + printGitStateWarnings(projectPath, os.Stderr) watch, _ := cmd.Flags().GetBool("watch") testEndpoint, _ := cmd.Flags().GetBool("test-endpoint") diff --git a/cmd/mxcli/diag.go b/cmd/mxcli/diag.go index 909c8a5fc8..dfb20cd799 100644 --- a/cmd/mxcli/diag.go +++ b/cmd/mxcli/diag.go @@ -30,6 +30,7 @@ Examples: mxcli diag --log-path # Print log directory path mxcli diag --tail 20 # Show last 20 log entries mxcli diag --bundle # Create tar.gz with logs for bug reports + mxcli diag -p app.mpr # Also check the project: ContentsHash index, git state `, Run: func(cmd *cobra.Command, args []string) { logPath, _ := cmd.Flags().GetBool("log-path") @@ -66,6 +67,9 @@ Examples: } runDiagInfo(logDir) + if projectPath, _ := cmd.Flags().GetString("project"); projectPath != "" { + runDiagProject(os.Stdout, projectPath) + } }, } @@ -104,6 +108,25 @@ func runDiagInfo(logDir string) { } } +// runDiagProject reports the project problems no mx check sees (#972): a +// stale MPR v2 ContentsHash index, and git states that crash Studio Pro. +func runDiagProject(w io.Writer, projectPath string) { + fmt.Fprintln(w) + fmt.Fprintf(w, "Project: %s\n", projectPath) + n := 0 + if msg := contentsHashDriftWarning(projectPath); msg != "" { + fmt.Fprint(w, " "+msg) + n++ + } + for _, msg := range gitStateWarnings(projectPath) { + fmt.Fprintln(w, " "+msg) + n++ + } + if n == 0 { + fmt.Fprintln(w, " No ContentsHash drift; no git state known to crash Studio Pro.") + } +} + // runDiagTail shows the last N log entries. func runDiagTail(logDir string, n int) { files, _ := listLogFiles(logDir) diff --git a/cmd/mxcli/docker.go b/cmd/mxcli/docker.go index 1b2d5b5b0d..62c859052a 100644 --- a/cmd/mxcli/docker.go +++ b/cmd/mxcli/docker.go @@ -215,8 +215,9 @@ Examples: Stderr: os.Stderr, } - // A stale ContentsHash index is not something mx check reports (#972). + // Neither of these is something mx check reports (#972). fmt.Fprint(os.Stderr, contentsHashDriftWarning(projectPath)) + printGitStateWarnings(projectPath, os.Stderr) if err := docker.Check(opts); err != nil { fmt.Fprintf(os.Stderr, "Error: %v\n", err) diff --git a/cmd/mxcli/gitstate.go b/cmd/mxcli/gitstate.go new file mode 100644 index 0000000000..ea428690af --- /dev/null +++ b/cmd/mxcli/gitstate.go @@ -0,0 +1,136 @@ +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "bytes" + "context" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "slices" + "strings" + "time" +) + +// gitstate.go warns about git states that crash Studio Pro (ako/mxcli#972). +// +// Studio Pro 11.13 (reported on macOS) fails to open a project with "Unable to +// find 'system' property in 'system'" when the project folder is a git +// repository whose checked-out branch has no upstream, or which git refuses +// with "detected dubious ownership" (a folder shared between the host and a +// devcontainer, owned by a different uid on each side). Both are Studio Pro +// bugs; mxcli cannot fix them, but it can say so before the user opens the +// project and has to work out what the crash means. +// +// Policy — the warnings are advice, so they are quiet whenever they could be +// wrong or unwanted: +// - never fail the command; any error running git yields no warning; +// - nothing outside a git repository, and nothing when git is not installed; +// - nothing on a detached HEAD: that is how CI and `git checkout ` leave +// a repository, there is no branch to push, and nobody opens such a +// checkout in Studio Pro; +// - nothing when $CI is set, or when MXCLI_NO_GIT_WARNINGS=1. +// +// Dubious ownership is judged by the git this process runs. In a devcontainer +// that is the container's git, and the host's Studio Pro may see ownership +// differently — so the check catches the case when the container sees it, and +// the docs give the host-side remedy either way. + +// noGitWarningsEnv silences the warnings. +const noGitWarningsEnv = "MXCLI_NO_GIT_WARNINGS" + +// gitRunner runs git in dir and returns stdout, stderr and the error. +type gitRunner func(dir string, args ...string) (string, string, error) + +// runGitState is the default gitRunner, bounded so a hung git cannot stall a check. +func runGitState(dir string, args ...string) (string, string, error) { + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + cmd := exec.CommandContext(ctx, "git", append([]string{"-C", dir}, args...)...) + var out, errb bytes.Buffer + cmd.Stdout, cmd.Stderr = &out, &errb + // Never prompt (credential helpers, pagers) from a warning. + cmd.Env = append(os.Environ(), "GIT_TERMINAL_PROMPT=0", "GIT_PAGER=cat") + err := cmd.Run() + return strings.TrimSpace(out.String()), errb.String(), err +} + +// projectDirOf returns the folder of a project path (.mpr file or folder). +func projectDirOf(projectPath string) string { + if fi, err := os.Stat(projectPath); err == nil && fi.IsDir() { + return projectPath + } + return filepath.Dir(projectPath) +} + +// gitStateWarnings returns the warnings for projectPath's git state, if any. +func gitStateWarnings(projectPath string) []string { + if projectPath == "" || os.Getenv("CI") != "" || os.Getenv(noGitWarningsEnv) == "1" { + return nil + } + if _, err := exec.LookPath("git"); err != nil { + return nil + } + return gitStateWarningsWith(projectDirOf(projectPath), runGitState) +} + +func gitStateWarningsWith(dir string, git gitRunner) []string { + out, stderr, err := git(dir, "rev-parse", "--is-inside-work-tree") + if err != nil { + if strings.Contains(stderr, "dubious ownership") { + return []string{dubiousOwnershipWarning(dir, stderr)} + } + return nil // not a repository (or git failed): say nothing + } + if out != "true" { + return nil // inside .git, or a bare repository + } + branch, _, err := git(dir, "symbolic-ref", "--quiet", "--short", "HEAD") + if err != nil || branch == "" { + return nil // detached HEAD: see the policy above + } + if _, _, err := git(dir, "rev-parse", "--abbrev-ref", "--symbolic-full-name", "@{upstream}"); err == nil { + return nil + } + remotes, _, _ := git(dir, "remote") + remote := "origin" + if rs := strings.Fields(remotes); len(rs) > 0 && !slices.Contains(rs, "origin") { + remote = rs[0] + } + remedy := fmt.Sprintf("git push -u %s %s", remote, branch) + if strings.TrimSpace(remotes) == "" { + remedy = fmt.Sprintf("git remote add origin && git push -u origin %s", branch) + } + return []string{fmt.Sprintf( + "Warning: git branch %q has no upstream. Studio Pro 11.13 fails to open a project in this state "+ + "(\"Unable to find 'system' property in 'system'\"). Before opening it in Studio Pro, run: %s", + branch, remedy)} +} + +// dubiousOwnershipWarning reuses the safe.directory command git itself +// suggests (it names the repository root, which may be above dir). +func dubiousOwnershipWarning(dir, stderr string) string { + remedy := "" + for _, line := range strings.Split(stderr, "\n") { + if i := strings.Index(line, "git config --global --add safe.directory"); i >= 0 { + remedy = strings.TrimSpace(line[i:]) + break + } + } + if remedy == "" { + remedy = fmt.Sprintf("git config --global --add safe.directory %s", dir) + } + return "Warning: git reports \"detected dubious ownership\" for this project (typically a folder shared " + + "with a devcontainer). Studio Pro 11.13 fails to open a project in this state " + + "(\"Unable to find 'system' property in 'system'\"). On the machine that runs Studio Pro, run: " + remedy +} + +// printGitStateWarnings writes each warning on its own line to w. +func printGitStateWarnings(projectPath string, w io.Writer) { + for _, msg := range gitStateWarnings(projectPath) { + fmt.Fprintln(w, msg) + } +} diff --git a/cmd/mxcli/gitstate_test.go b/cmd/mxcli/gitstate_test.go new file mode 100644 index 0000000000..2fe9c88b62 --- /dev/null +++ b/cmd/mxcli/gitstate_test.go @@ -0,0 +1,137 @@ +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "errors" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +// ako/mxcli#972 item 3: warn about git states that crash Studio Pro 11.13. + +func requireGit(t *testing.T) { + t.Helper() + if _, err := exec.LookPath("git"); err != nil { + t.Skip("git not installed") + } + t.Setenv("CI", "") + t.Setenv(noGitWarningsEnv, "") +} + +func gitIn(t *testing.T, dir string, args ...string) { + t.Helper() + cmd := exec.Command("git", append([]string{"-C", dir}, args...)...) + cmd.Env = append(os.Environ(), + "GIT_AUTHOR_NAME=t", "GIT_AUTHOR_EMAIL=t@example.com", + "GIT_COMMITTER_NAME=t", "GIT_COMMITTER_EMAIL=t@example.com") + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("git %v: %v\n%s", args, err, out) + } +} + +// newRepoWithProject makes a repository holding App.mpr on branch "feature", +// with one commit, and returns the .mpr path. +func newRepoWithProject(t *testing.T) string { + t.Helper() + dir := t.TempDir() + // Keep git from discovering a repository above the temp dir. + t.Setenv("GIT_CEILING_DIRECTORIES", filepath.Dir(dir)) + gitIn(t, dir, "init", "-q", "-b", "feature") + mprPath := filepath.Join(dir, "App.mpr") + if err := os.WriteFile(mprPath, []byte("x"), 0o644); err != nil { + t.Fatal(err) + } + gitIn(t, dir, "add", "App.mpr") + gitIn(t, dir, "commit", "-q", "-m", "init") + return mprPath +} + +func TestGitState_NoUpstreamWarns(t *testing.T) { + requireGit(t) + mprPath := newRepoWithProject(t) + remote := t.TempDir() + gitIn(t, remote, "init", "-q", "--bare") + gitIn(t, filepath.Dir(mprPath), "remote", "add", "origin", remote) + + got := gitStateWarnings(mprPath) + if len(got) != 1 || !strings.Contains(got[0], `branch "feature" has no upstream`) || + !strings.Contains(got[0], "git push -u origin feature") { + t.Fatalf("want a no-upstream warning with the push remedy, got %q", got) + } +} + +func TestGitState_NoRemoteSaysAddOne(t *testing.T) { + requireGit(t) + mprPath := newRepoWithProject(t) + got := gitStateWarnings(mprPath) + if len(got) != 1 || !strings.Contains(got[0], "git remote add origin && git push -u origin feature") { + t.Fatalf("want a no-upstream warning with the add-remote remedy, got %q", got) + } +} + +func TestGitState_WithUpstreamIsSilent(t *testing.T) { + requireGit(t) + mprPath := newRepoWithProject(t) + remote := t.TempDir() + gitIn(t, remote, "init", "-q", "--bare") + dir := filepath.Dir(mprPath) + gitIn(t, dir, "remote", "add", "origin", remote) + gitIn(t, dir, "push", "-q", "-u", "origin", "feature") + + if got := gitStateWarnings(mprPath); len(got) != 0 { + t.Fatalf("control: branch with upstream warned: %q", got) + } +} + +func TestGitState_NotARepoIsSilent(t *testing.T) { + requireGit(t) + dir := t.TempDir() + t.Setenv("GIT_CEILING_DIRECTORIES", filepath.Dir(dir)) + if got := gitStateWarnings(filepath.Join(dir, "App.mpr")); len(got) != 0 { + t.Fatalf("outside a repository warned: %q", got) + } +} + +func TestGitState_DetachedHeadIsSilent(t *testing.T) { + requireGit(t) + mprPath := newRepoWithProject(t) + gitIn(t, filepath.Dir(mprPath), "checkout", "-q", "--detach") + if got := gitStateWarnings(mprPath); len(got) != 0 { + t.Fatalf("detached HEAD warned: %q", got) + } +} + +func TestGitState_CIAndOptOutAreSilent(t *testing.T) { + requireGit(t) + mprPath := newRepoWithProject(t) + t.Setenv("CI", "true") + if got := gitStateWarnings(mprPath); len(got) != 0 { + t.Fatalf("CI warned: %q", got) + } + t.Setenv("CI", "") + t.Setenv(noGitWarningsEnv, "1") + if got := gitStateWarnings(mprPath); len(got) != 0 { + t.Fatalf("opt-out warned: %q", got) + } +} + +// Dubious ownership needs a repository owned by another uid, which a test +// cannot create; git's refusal is replayed through the runner instead. The +// stderr is git 2.43's, verbatim. +func TestGitState_DubiousOwnershipWarnsWithGitsRemedy(t *testing.T) { + const stderr = "fatal: detected dubious ownership in repository at '/workspaces/app'\n" + + "To add an exception for this directory, call:\n\n" + + "\tgit config --global --add safe.directory /workspaces/app\n" + git := func(dir string, args ...string) (string, string, error) { + return "", stderr, errors.New("exit status 128") + } + got := gitStateWarningsWith("/workspaces/app/sub", git) + if len(got) != 1 || !strings.Contains(got[0], "dubious ownership") || + !strings.Contains(got[0], "git config --global --add safe.directory /workspaces/app") { + t.Fatalf("want a dubious-ownership warning with git's remedy, got %q", got) + } +} From 49cae40e156c05d731b899660474690430910dd9 Mon Sep 17 00:00:00 2001 From: Ako Date: Sun, 4 Oct 2026 14:53:08 +0000 Subject: [PATCH 07/17] =?UTF-8?q?docs:=20Working=20Outside=20Studio=20Pro?= =?UTF-8?q?=20=E2=80=94=20fix=20hashes=20and=20git-state=20warnings?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Part of #972. Co-Authored-By: Claude Opus 5.5 --- .claude/commands/mendix/diff-local.md | 3 + CHANGELOG.md | 2 + docs-site/src/SUMMARY.md | 1 + docs-site/src/tools/docker-check.md | 5 ++ docs-site/src/tools/outside-studio-pro.md | 71 +++++++++++++++++++++++ docs-site/src/tools/run-local.md | 4 ++ 6 files changed, 86 insertions(+) create mode 100644 docs-site/src/tools/outside-studio-pro.md diff --git a/.claude/commands/mendix/diff-local.md b/.claude/commands/mendix/diff-local.md index a78e327228..b99ce06e8a 100644 --- a/.claude/commands/mendix/diff-local.md +++ b/.claude/commands/mendix/diff-local.md @@ -122,6 +122,9 @@ Summary: 2 new, 3 modified, 1 deleted - Use `--format side` when comparing large objects with subtle differences - Use `--ref HEAD~5` to see what changed in the last 5 commits - Use `--ref feature-branch` to compare against a different branch +- After restoring a `.mxunit` with `git checkout`/`git restore`, run + `mxcli fix hashes -p app.mpr --repair`: git does not update the `.mpr`'s + ContentsHash index, and `mx check` does not notice the stale entry ## Requirements diff --git a/CHANGELOG.md b/CHANGELOG.md index 8a83d6a4e7..cccee63942 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -138,6 +138,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). ### Added +- **`mxcli fix hashes` verifies and repairs the MPR v2 ContentsHash index** (ako/mxcli#972) — every `Unit.ContentsHash` in the `.mpr` is compared with `base64(SHA-256)` of its `.mxunit`, and mismatches, missing files and orphan files are reported; the command exits 1 when any remain. `--repair` rewrites the mismatched hashes from the files in one transaction, under the same Studio Pro open-project guard as every write. Restoring a unit with `git checkout` leaves the index stale and `mx check` does not notice; measured on a TestApp copy, an entity added by `exec` and then reverted with a byte-level restore is reported as one `DomainModels$DomainModel` mismatch, and after `--repair` the verify is clean and `mx check` reports 0 errors. `exec` and `docker check` print a one-line warning when the index disagrees (about 0.2 s on 900 units). An MPR v1 project has no index, and the command says so. +- **Warnings for git states that crash Studio Pro** (ako/mxcli#972) — Studio Pro 11.13 fails to open a project ("Unable to find 'system' property in 'system'") when its git branch has no upstream or git reports "detected dubious ownership". `docker check`, `run --local` and the new `mxcli diag -p app.mpr` project section warn with the remedy (`git push -u origin `; `git config --global --add safe.directory ` on the Studio Pro machine). Never fatal, and silent outside a git repository, on a detached HEAD, under `CI`, or with `MXCLI_NO_GIT_WARNINGS=1`. See the new "Working Outside Studio Pro" page. - **A `commit` reference kind in the catalog** (ako/mxcli#963) — `refs` has a `commit` edge from a flow to the entity it commits: a commit action, or a create / change that commits (`Yes` or `YesWithoutEvents`), the latter beside its `create` / `change` edge. The entity of a committed variable is resolved like `change` / `delete`, loop iterators included, so `commit $Order` inside `loop $Order in $Orders` now has a row. `refs_to("M.Order")` in a Starlark rule answers which flows commit an order. `commit` is not in the analysis graph and not a caller kind. The catalog schema version is bumped, so a cached catalog rebuilds. - **`total_activity_count` on the Starlark microflow struct** (ako/mxcli#963) — the catalog's `TotalActivityCount` (loop bodies included, at any depth) for every flow `microflows()` yields: microflows, nanoflows and rules. `activity_count` keeps counting a loop as one activity. - **The catalog's `widgets` table records the widget tree, appearance and primary action** (mendixlabs/mxcli#1268) — `ParentWidgetId` (the nearest catalogued ancestor; skipped wrappers, layout grid rows and columns, tab pages and data grid 2 columns are transparent), `Depth` (0 at the page or snippet root), `Class`, `Style`, `DynamicClasses`, `ActionType` (the stored type of the button, on-click or click action, e.g. `Forms$DeleteClientAction`) and `HasConfirmation`. Starlark `widgets()` exposes them as `parent_widget_id`, `depth`, `class_name`, `style`, `dynamic_classes`, `action_type`, `has_confirmation`, plus `page_ref`, so a lint rule can flag deep nesting, inline styles, classes outside an allow-list and delete buttons. A delete action has no confirmation setting in Mendix; the enforceable rule is "no button uses the delete action directly". The catalog schema version is bumped, so a cached catalog rebuilds. diff --git a/docs-site/src/SUMMARY.md b/docs-site/src/SUMMARY.md index 53470ec9a8..3b63b00c3b 100644 --- a/docs-site/src/SUMMARY.md +++ b/docs-site/src/SUMMARY.md @@ -169,6 +169,7 @@ - [OQL Queries](tools/oql.md) - [Dev Container Setup](tools/devcontainer.md) - [Live Studio Pro Sync (MCP)](tools/mcp-connect.md) +- [Working Outside Studio Pro](tools/outside-studio-pro.md) --- diff --git a/docs-site/src/tools/docker-check.md b/docs-site/src/tools/docker-check.md index 8cd97c3891..b0d56bf459 100644 --- a/docs-site/src/tools/docker-check.md +++ b/docs-site/src/tools/docker-check.md @@ -16,6 +16,11 @@ mxcli docker check -p app.mpr If the project has errors, the command exits with a non-zero status code. +Before `mx check` it also warns, without failing, about two problems `mx check` +cannot see: an MPR v2 `ContentsHash` index that disagrees with the `.mxunit` +files (`mxcli fix hashes`), and git states that make Studio Pro 11.13 fail to +open the project. See [Working Outside Studio Pro](outside-studio-pro.md). + ## Auto-Download If mxbuild is not installed locally, you can download it first: diff --git a/docs-site/src/tools/outside-studio-pro.md b/docs-site/src/tools/outside-studio-pro.md new file mode 100644 index 0000000000..fa1b2a9894 --- /dev/null +++ b/docs-site/src/tools/outside-studio-pro.md @@ -0,0 +1,71 @@ +# Working Outside Studio Pro + +A project edited with mxcli, git and an editor is sometimes opened in Studio Pro +afterwards. Two kinds of state that Studio Pro depends on are invisible to +`mx check`, and mxcli checks both. + +## The ContentsHash index: `mxcli fix hashes` + +In an MPR v2 project (Mendix 10.18 and later) the `.mpr` file indexes every +`mprcontents/**/*.mxunit` file by its hash: `Unit.ContentsHash` is +`base64(SHA-256(file))`. mxcli's writer and Studio Pro keep the two in step. +Anything else that changes a `.mxunit` does not: restoring a unit with +`git checkout`, `git restore`, a revert or a merge leaves the index describing +bytes that are no longer on disk. `mx check` and MxBuild read the files and do +not notice; Studio Pro uses the index to decide what changed. + +```bash +mxcli fix hashes -p app.mpr # verify; exits 1 when something is off +mxcli fix hashes -p app.mpr --repair # rewrite mismatched hashes from the files +``` + +| Reported | Meaning | `--repair` | +|----------|---------|------------| +| `MISMATCH` | the file's hash differs from the stored one | rewrites the stored hash from the file | +| `MISSING` | the `.mpr` indexes a unit whose file does not exist | reported only — restore the file | +| `ORPHAN` | a `.mxunit` file no unit in the `.mpr` indexes | reported only — `mxcli diag --check-units --fix` removes it | + +The repair treats the files as the source of truth and never changes them. It +updates every hash in one SQLite transaction, and, like every mxcli write, is +refused while Studio Pro has the project open (the `.mpr.lock` beside it). An +MPR v1 project keeps unit contents inside the `.mpr`, has no index to drift, +and the command says so. + +`mxcli exec` and `mxcli docker check` run the verify too and print a one-line +warning when it finds a mismatch or a missing file. It never fails them; +measured on a 900-unit project, the whole verify takes about 0.2 s. + +## Git states that crash Studio Pro + +Studio Pro 11.13 fails to open a project with *"Unable to find 'system' +property in 'system'"* when the project folder is a git repository and + +- the checked-out branch has **no upstream** — remedy: + `git push -u origin ` before opening the project (or + `git remote add origin ` first, if the repository has no remote); or +- git reports **"detected dubious ownership"** — typical for a folder shared + between the host and a devcontainer, owned by a different user on each side. + Remedy, on the machine that runs Studio Pro: + `git config --global --add safe.directory `. + +`mxcli docker check`, `mxcli run --local` and `mxcli diag -p app.mpr` warn about +both. The warnings are advice and never fail a command. They are silent: + +- outside a git repository, or when git is not installed; +- on a **detached HEAD** — how CI systems and `git checkout ` leave a + repository; there is no branch to push, and nobody opens such a checkout in + Studio Pro; +- when the `CI` environment variable is set, or `MXCLI_NO_GIT_WARNINGS=1`. + +Dubious ownership is judged by the git mxcli runs. Inside a devcontainer that is +the container's git, which may see ownership differently from the host's Studio +Pro — so a clean report from inside the container does not rule it out on the +host. + +## One report: `mxcli diag -p` + +```bash +mxcli diag -p app.mpr +``` + +prints mxcli's diagnostics followed by a *Project* section with both checks. diff --git a/docs-site/src/tools/run-local.md b/docs-site/src/tools/run-local.md index 348a028e4c..a4375baa24 100644 --- a/docs-site/src/tools/run-local.md +++ b/docs-site/src/tools/run-local.md @@ -63,6 +63,10 @@ so structural changes need a restart; behavioural changes do not. createdb -h 127.0.0.1 -U mendix app1112 ``` +At start, `run --local` warns (without stopping) when the project's git state +would make Studio Pro 11.13 fail to open it — a branch with no upstream, or +"dubious ownership". See [Working Outside Studio Pro](outside-studio-pro.md). + ## Flags | Flag | Default | Purpose | From bbd65f15b98f035983ed844453fc88b9ece08c5f Mon Sep 17 00:00:00 2001 From: Ako Date: Sun, 4 Oct 2026 15:04:31 +0000 Subject: [PATCH 08/17] feat(check): info hint for a quoted template parameter that reads like an expression (#969) Co-Authored-By: Claude Opus 5.5 --- .../validate_quoted_template_params.go | 96 +++++++++++++++++++ .../validate_quoted_template_params_test.go | 49 ++++++++++ mdl/executor/validate_widgets.go | 3 + 3 files changed, 148 insertions(+) create mode 100644 mdl/executor/validate_quoted_template_params.go create mode 100644 mdl/executor/validate_quoted_template_params_test.go diff --git a/mdl/executor/validate_quoted_template_params.go b/mdl/executor/validate_quoted_template_params.go new file mode 100644 index 0000000000..4f885c0b7b --- /dev/null +++ b/mdl/executor/validate_quoted_template_params.go @@ -0,0 +1,96 @@ +// SPDX-License-Identifier: Apache-2.0 + +package executor + +import ( + "fmt" + "regexp" + "strings" + + "github.com/mendixlabs/mxcli/mdl/ast" + "github.com/mendixlabs/mxcli/mdl/exprcheck" + "github.com/mendixlabs/mxcli/mdl/linter" +) + +// validateQuotedTemplateParams emits an info hint (MDL-PARAMQUOTE01) for a +// template parameter whose value is a quoted string that reads like an +// expression: `{1} = 'formatDateTime($Log/Date, ''d MMM'')'`. +// +// The quotes make it a String literal, which is exactly what is stored — the +// page then shows the expression's TEXT. That is valid, builds clean and is +// occasionally intended, so it is a hint and not an error; but nothing said so, +// and the author saw the source text on the rendered page (ako/mxcli#969). +// +// The heuristic is deliberately narrow, so plain text with parentheses or a +// dollar sign is left alone: +// - the whole literal is a call of a known Mendix built-in function, with +// the parenthesis directly after the name (`toString(...)`, +// `formatDateTime(...)`; prose writes "length (cm)" with a space), or +// - it contains a variable path, `$Name/Attr`, or is exactly `$Name`. +// +// "Total (incl. VAT)", "Price in $" and "Cost: $5" match neither. +func validateQuotedTemplateParams(w *ast.WidgetV3, locationPrefix string) []linter.Violation { + if w == nil { + return nil + } + var out []linter.Violation + for _, set := range []struct { + prop string + params []ast.ParamAssignmentV3 + }{ + {"ContentParams", w.GetContentParams()}, + {"CaptionParams", w.GetCaptionParams()}, + } { + for _, p := range set.params { + s, ok := p.Value.(string) + if !ok { + continue + } + inner, quoted := singleStringLiteral(s) + if !quoted || !looksLikeExpression(inner) { + continue + } + out = append(out, linter.Violation{ + RuleID: "MDL-PARAMQUOTE01", + Severity: linter.SeverityInfo, + Message: fmt.Sprintf( + "%s: %s {%d} = %s is quoted, so it is stored as literal text and the page shows it verbatim — "+ + "this quoted parameter looks like an expression; drop the quotes to make it one", + locationPrefix+" "+widgetLabel(w.Name, w.Type), set.prop, p.Index, s), + Suggestion: fmt.Sprintf("{%d} = %s", p.Index, strings.ReplaceAll(inner, "''", "'")), + }) + } + } + return out +} + +// singleStringLiteral reports whether s is exactly one Mendix string literal +// ('…' with '' as the escaped quote) and returns its raw body. +func singleStringLiteral(s string) (string, bool) { + s = strings.TrimSpace(s) + if len(s) < 2 || s[0] != '\'' || s[len(s)-1] != '\'' { + return "", false + } + body := s[1 : len(s)-1] + // A lone quote inside would end the literal early: 'a' + 'b' is not one. + if strings.Count(strings.ReplaceAll(body, "''", ""), "'") != 0 { + return "", false + } + return body, true +} + +var ( + quotedCallRe = regexp.MustCompile(`^\s*([A-Za-z][A-Za-z0-9]*)\(.*\)\s*$`) + quotedPathRe = regexp.MustCompile(`\$[A-Za-z_][A-Za-z0-9_]*/[A-Za-z_]`) + quotedVarRe = regexp.MustCompile(`^\s*\$[A-Za-z_][A-Za-z0-9_]*\s*$`) +) + +func looksLikeExpression(body string) bool { + if m := quotedCallRe.FindStringSubmatch(body); m != nil { + // `not` is a keyword-shaped function that prose uses too ("not(yet)"). + if _, ok := exprcheck.FuncReturnKind(m[1]); ok && m[1] != "not" { + return true + } + } + return quotedPathRe.MatchString(body) || quotedVarRe.MatchString(body) +} diff --git a/mdl/executor/validate_quoted_template_params_test.go b/mdl/executor/validate_quoted_template_params_test.go new file mode 100644 index 0000000000..7108edb29a --- /dev/null +++ b/mdl/executor/validate_quoted_template_params_test.go @@ -0,0 +1,49 @@ +// SPDX-License-Identifier: Apache-2.0 + +package executor + +import ( + "strings" + "testing" + + "github.com/mendixlabs/mxcli/mdl/ast" +) + +func quotedParamWidget(t *testing.T, param string) *ast.WidgetV3 { + t.Helper() + prog := parseMDL(t, `create page M.P (Title: 'P', Layout: Atlas_Core.Atlas_Default) { + dynamictext d (Content: 'x {1}', ContentParams: ({1} = `+param+`)) +};`) + s := prog.Statements[0].(*ast.CreatePageStmtV3) + return allPageWidgets(s)[0] +} + +// ako/mxcli#969 item 5: a quoted parameter that reads like an expression is +// stored as its text; say so. The negatives are the false hints a looser +// heuristic would give on ordinary text. +func TestQuotedTemplateParam_Hint(t *testing.T) { + for param, want := range map[string]bool{ + `'formatDateTime($Log/Date, ''d MMM'')'`: true, + `'toString($Order/Total)'`: true, + `'$currentObject/Name'`: true, + `'$Order'`: true, + // controls — text, not expressions + `'Total (incl. VAT)'`: false, + `'length (cm)'`: false, + `'not(yet)'`: false, + `'Price in $'`: false, + `'Cost: $5'`: false, + `'Sum(of parts)'`: false, + `'a' + 'b'`: false, + `toString($Order/Total)`: false, + } { + got := validateQuotedTemplateParams(quotedParamWidget(t, param), "page M.P") + if (len(got) == 1) != want || len(got) > 1 { + t.Errorf("%s: got %v, want hint=%v", param, got, want) + continue + } + if want && (got[0].RuleID != "MDL-PARAMQUOTE01" || !strings.Contains(got[0].Message, "drop the quotes")) { + t.Errorf("%s: unexpected violation %+v", param, got[0]) + } + } +} diff --git a/mdl/executor/validate_widgets.go b/mdl/executor/validate_widgets.go index 7d70d68813..766dc11d42 100644 --- a/mdl/executor/validate_widgets.go +++ b/mdl/executor/validate_widgets.go @@ -251,6 +251,9 @@ func validateWidgetTreeIn(widgets []*ast.WidgetV3, registry *WidgetRegistry, loc out = append(out, validateImageSource(w, locationPrefix)...) out = append(out, validateStaticWidget(w, locationPrefix)...) out = append(out, validateDynamicTextFormatting(w, locationPrefix)...) + // A quoted template parameter that reads like an expression is stored + // as its text (ako/mxcli#969). + out = append(out, validateQuotedTemplateParams(w, locationPrefix)...) out = append(out, validateDatasourceXPathAssociationEmpty(w, locationPrefix)...) out = append(out, validateComboBoxAssociation(w, locationPrefix)...) // #631: inputs inside a list view that will be written read-only. From 0744ca3912e5c33d8930cb6948eb297ed8edcdea Mon Sep 17 00:00:00 2001 From: Ako Date: Sun, 4 Oct 2026 15:07:07 +0000 Subject: [PATCH 09/17] =?UTF-8?q?fix(lint):=20MPR006=20is=20an=20info=20no?= =?UTF-8?q?te=20=E2=80=94=20an=20empty=20container=20builds=20and=20render?= =?UTF-8?q?s=20(#969)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Measured on Mendix 11.13.0 (React client) with run --local and a headless browser: a bare and a styled empty container render, the widgets after them are present, no console errors. The crash claim had no evidence. Co-Authored-By: Claude Opus 5.5 --- .claude/skills/mendix/create-page/SKILL.md | 17 +++++++---------- mdl/linter/rules/empty_container.go | 18 ++++++++++++------ mdl/linter/rules/empty_container_test.go | 20 +++++++++++++++++--- 3 files changed, 36 insertions(+), 19 deletions(-) diff --git a/.claude/skills/mendix/create-page/SKILL.md b/.claude/skills/mendix/create-page/SKILL.md index 364e807022..2e11cd06d1 100644 --- a/.claude/skills/mendix/create-page/SKILL.md +++ b/.claude/skills/mendix/create-page/SKILL.md @@ -364,16 +364,13 @@ The following features are NOT implemented in mxcli and require manual configura ### Runtime Pitfalls -> **Empty CONTAINER crashes at runtime.** A CONTAINER with no child widgets compiles and builds successfully but crashes when the page loads with "Did not expect an argument to be undefined". Always include at least one child widget: -> ```sql -> -- Wrong: crashes at runtime -> CONTAINER spacer1 (Style: 'height: 6px;') -> -> -- Correct: include a child (even a space) -> CONTAINER spacer1 (Style: 'height: 6px;') { -> DYNAMICTEXT spacerText (Content: ' ', RenderMode: Paragraph) -> } -> ``` +> **An empty CONTAINER is valid.** It passes `mx check`, builds, and renders: +> measured on Mendix 11.13.0 (React client) under `run --local`, both a bare +> `container c1` and a styled spacer `container spacer1 (Style: 'height: 6px;')` +> rendered with the widgets after them present and no console errors +> (ako/mxcli#969). An older note here said it crashed with "Did not expect an +> argument to be undefined"; that was never measured. `mxcli lint` still reports +> one as MPR006 at info level, since an empty container is usually a leftover. > **`content: ''` (empty string) fails MxBuild.** An empty Content on DYNAMICTEXT causes a misleading error: "Place holder index 1 is greater than 0, the number of parameter(s)." Use a single space instead: > ```sql diff --git a/mdl/linter/rules/empty_container.go b/mdl/linter/rules/empty_container.go index 0cfe4431e6..20e23c2f5e 100644 --- a/mdl/linter/rules/empty_container.go +++ b/mdl/linter/rules/empty_container.go @@ -10,7 +10,13 @@ import ( ) // EmptyContainerRule checks for CONTAINER widgets with no children. -// Empty containers crash the runtime with "Did not expect an argument to be undefined". +// +// It used to say an empty container "crashes at runtime" with "Did not expect +// an argument to be undefined", a claim dating from the initial commit with no +// measurement behind it. Measured for ako/mxcli#969 on Mendix 11.13.0: a page +// with an empty container passes `mx check`, builds, and renders under +// `run --local` with the widgets after it present and no console errors. So it +// is an info note about a probably-unintended leftover, not a defect. type EmptyContainerRule struct{} // NewEmptyContainerRule creates a new empty container rule. @@ -20,15 +26,15 @@ func NewEmptyContainerRule() *EmptyContainerRule { func (r *EmptyContainerRule) ID() string { return "MPR006" } func (r *EmptyContainerRule) Name() string { return "EmptyContainer" } -func (r *EmptyContainerRule) Category() string { return "correctness" } -func (r *EmptyContainerRule) DefaultSeverity() linter.Severity { return linter.SeverityWarning } +func (r *EmptyContainerRule) Category() string { return "quality" } +func (r *EmptyContainerRule) DefaultSeverity() linter.Severity { return linter.SeverityInfo } // RequiredCatalogMode: ctx.Widgets() reads widgets_data, which only a full // catalog build fills; under the default fast build the rule found nothing. func (r *EmptyContainerRule) RequiredCatalogMode() linter.CatalogMode { return linter.CatalogFull } func (r *EmptyContainerRule) Description() string { - return "Checks for CONTAINER widgets with no children (crashes at runtime)" + return "Checks for CONTAINER widgets with no children (valid and renders, but usually a leftover)" } // emptyContainerInfo holds information about a found empty container widget. @@ -88,7 +94,7 @@ func (r *EmptyContainerRule) Check(ctx *linter.LintContext) []linter.Violation { violations = append(violations, linter.Violation{ RuleID: r.ID(), Severity: r.DefaultSeverity(), - Message: fmt.Sprintf("CONTAINER '%s' in %s has no children and will crash at runtime", + Message: fmt.Sprintf("CONTAINER '%s' in %s is empty — valid and it renders, but probably unintended", e.Name, c.QualifiedName), Location: linter.Location{ Module: c.ModuleName, @@ -96,7 +102,7 @@ func (r *EmptyContainerRule) Check(ctx *linter.LintContext) []linter.Violation { DocumentName: docNameFromQualified(c.QualifiedName), DocumentID: c.ID, }, - Suggestion: "Add a child widget (e.g., DYNAMICTEXT with Content: ' ') or remove the empty container", + Suggestion: "Remove the container, or give it the content it was meant to hold", }) } } diff --git a/mdl/linter/rules/empty_container_test.go b/mdl/linter/rules/empty_container_test.go index 66f4181446..b911432979 100644 --- a/mdl/linter/rules/empty_container_test.go +++ b/mdl/linter/rules/empty_container_test.go @@ -2,7 +2,12 @@ package rules -import "testing" +import ( + "strings" + "testing" + + "github.com/mendixlabs/mxcli/mdl/linter" +) func TestFindEmptyContainers_PageWithEmpty(t *testing.T) { rawData := map[string]any{ @@ -231,10 +236,19 @@ func TestEmptyContainerRule_Metadata(t *testing.T) { if r.ID() != "MPR006" { t.Errorf("ID = %q, want MPR006", r.ID()) } - if r.Category() != "correctness" { - t.Errorf("Category = %q, want correctness", r.Category()) + if r.Category() != "quality" { + t.Errorf("Category = %q, want quality", r.Category()) } if r.Name() != "EmptyContainer" { t.Errorf("Name = %q, want EmptyContainer", r.Name()) } + // ako/mxcli#969: an empty container was measured to build and render + // (11.13.0, run --local, no console errors); the old "crashes at runtime" + // warning had no evidence behind it. + if r.DefaultSeverity() != linter.SeverityInfo { + t.Errorf("DefaultSeverity = %v, want info", r.DefaultSeverity()) + } + if strings.Contains(strings.ToLower(r.Description()), "crash") { + t.Errorf("Description %q still claims a crash", r.Description()) + } } From 68b3197cb73cf7bfb6e13f6cfc019bae4209c82d Mon Sep 17 00:00:00 2001 From: Ako Date: Sun, 4 Oct 2026 15:11:42 +0000 Subject: [PATCH 10/17] fix(theme): a base-only design leaves the alternate variant's mixin alone (#970) theme create --from wrote a design's base palette into the base theme's alt-variant mixin, so a light ground and ink landed in a dark palette whose other surfaces stayed dark. Without a block for that variant the mixin is now left as the base ships it and create prints a note. Also: applyTokens matched the indent with \s*, which under (?m) swallowed the preceding newline, putting the first token on the @mixin line. Co-Authored-By: Claude Opus 5.5 --- .../skills/fix-issue/findings/cmd-mxcli.jsonl | 1 + .claude/skills/mendix/theme-styling/SKILL.md | 5 +- CHANGELOG.md | 1 + cmd/mxcli/cmd_theme.go | 15 ++- cmd/mxcli/cmd_theme_test.go | 29 +++++ cmd/mxcli/theme/create.go | 17 +++ cmd/mxcli/theme/create_variant_test.go | 101 ++++++++++++++++++ cmd/mxcli/theme/tokens.go | 11 +- docs-site/src/tools/theme.md | 7 ++ 9 files changed, 183 insertions(+), 4 deletions(-) create mode 100644 cmd/mxcli/theme/create_variant_test.go diff --git a/.claude/skills/fix-issue/findings/cmd-mxcli.jsonl b/.claude/skills/fix-issue/findings/cmd-mxcli.jsonl index a8ae7deedc..d825e231e9 100644 --- a/.claude/skills/fix-issue/findings/cmd-mxcli.jsonl +++ b/.claude/skills/fix-issue/findings/cmd-mxcli.jsonl @@ -158,3 +158,4 @@ {"area": "cmd/mxcli", "date": "2026-10-03", "symptom": "`mxcli report` could not score a project's own modules: `lint` has --modules, `report` had only --exclude", "cause": "Feature gap; and the LintContext module filter alone would not make the score exact, because project-level findings (CONV008 role mappings, project security) carry no module and are reported regardless", "file": "`cmd/mxcli/cmd_report.go`, `mdl/linter/report.go` (`ScopeToModules`, Report.Modules)", "insight": "Filter the scored violations to those located in a selected module, and print the selection in every format so a module score is not mistaken for the project's", "refs": ["ako/mxcli#953"]} {"area": "cmd/mxcli/docker", "date": "2026-10-03", "symptom": "`mxcli docker build` (and docker run / reload, which call it) rewrote an MPRv1 project's .mpr, rewrote every MPRv2 .mxunit (restored with new mtimes), and wrote theme-cache/, deployment/, javasource/ proxies, the .launch file, .classpath and .project into the project; the TUI checker and `mxcli eval`'s mx_check wrote theme-cache/web/ and deployment/sass/ on every run", "cause": "update-widgets ran on the project under a snapshot that restored only v2 storage, and mx check / MxBuild ran on the project itself; the TUI and eval runner ran a bare `mx check `", "file": "`cmd/mxcli/docker/build.go` (`buildOnCopy`), `cmd/mxcli/docker/check.go` (`MxCheckOnCopy`), `cmd/mxcli/tui/checker.go` (`runMxCheck`), `cmd/mxcli/evalrunner/checks.go` (`checkMxCheck`)", "insight": "Run update-widgets, mx check and MxBuild on one temporary copy (copyProjectToTemp) and write only the output directory. Measured on the 11.14 testapp: the PAD from the copy differs from the in-place build in the same 9 files as two in-place builds of identical copies differ (cache-bust stamps, operation ids, the native metro paths), so the output is equivalent; rebuild time unchanged (58s vs 59s), because a PAD build gains nothing from the project's deployment/. 11.14's blank app needs JDK 25, so the real-mx build test skips without it; run it with 11.13's mx.", "refs": ["ako/mxcli#961", "ako/mxcli#951", "mendixlabs/mxcli#808"]} {"area": "cmd/mxcli", "date": "2026-10-03", "symptom": "ako/mxcli#962 item 3: in VS Code, two calls to a stored void JavaScript action with the same output name (Studio Pro's `$ReturnValueName`/`$RefreshEntity` shape) are squiggled MDL063, while `mxcli check -p` passes them", "cause": "runSemanticValidation called executor.ValidateMicroflow/ValidateNanoflow, which pass a nil void-action resolver: without the project every call output counts as a declaration", "file": "`cmd/mxcli/lsp_diagnostics.go` (runSemanticValidation); `mdl/executor/validate_void_code_calls.go` (FlowRules, CodeActionCache)", "fix": "executor.NewFlowRules(prog, s.findMprPath(), s.codeActions): resolves actions through the script and the workspace project (opened lazily), treats an unresolvable action as possibly void (unknownIsVoid) for MDL063 but never for MDL093, and shares project answers across keystrokes for 30s since one action read costs ~300ms on PedApp", "insight": "An entry point that wraps a richer internal API with nil arguments (ValidateMicroflow = validateMicroflowWith(stmt, nil)) silently gives every caller the weakest behaviour; a fix landed in the richer API (#958) does not reach them. Grep the exported wrapper's callers when the internal one gains a parameter. Measure the cost before putting project I/O on a keystroke path", "test": "`cmd/mxcli/lsp_void_calls_test.go` (PedApp: void pair clean with and without project, Boolean pair control, MDL093 only with project); `mdl/executor/validate_void_code_calls_test.go` (TestCodeActionCache_SharesProjectAnswersAcrossRuns)"} +{"area": "cmd/mxcli", "date": "2026-10-04", "symptom": "ako/mxcli#970 item 2: `theme create --from design.css` with only a :root block (no dark block) wrote the design's light --mxt-ground/--mxt-ink/--mxt-brand into the base theme's dark mixin, whose other surfaces stayed dark; the first injected token also sat on the `@mixin … {` line, unindented", "cause": "Tokens.forVariant returned the base declarations for EITHER variant, and seedTokens applied it to the alt-palette mixin unconditionally; applyTokens matched `(?m)^(\\s*)name`, and \\s* at ^ swallows the preceding newline (and blank line), so the replacement lost its line break and indent", "file": "`cmd/mxcli/theme/create.go` (seedTokens, Create/CreateResult.UnseededVariant); `cmd/mxcli/theme/tokens.go` (applyTokens, Tokens.declares); `cmd/mxcli/cmd_theme.go` (note)", "fix": "seed the alt mixin only when the design declared a block for that variant; otherwise leave it byte-identical to the base and report UnseededVariant, which the CLI prints as a note; match the indent with [ \\t]* instead of \\s*", "insight": "A base palette is the default variant's palette, not 'both': a token set that does not say which variant it describes must not seed the other one. And under (?m), ^\\s* is not 'leading indentation' — it crosses lines; use [ \\t]*. The control for 'mixin untouched' is the same scaffold with no design at all, compared byte for byte", "test": "`cmd/mxcli/theme/create_variant_test.go` (all three bases, base-only vs variant block, indentation); `cmd/mxcli/cmd_theme_test.go` (TestThemeCreate_NotesTheVariantABaseOnlyDesignDidNotSeed)"} diff --git a/.claude/skills/mendix/theme-styling/SKILL.md b/.claude/skills/mendix/theme-styling/SKILL.md index 26aff020c3..f2cff62ac5 100644 --- a/.claude/skills/mendix/theme-styling/SKILL.md +++ b/.claude/skills/mendix/theme-styling/SKILL.md @@ -151,7 +151,10 @@ an HTML export: A dark block (`prefers-color-scheme: dark`, `.theme-dark`, `[data-theme="dark"]`) seeds the dark palette; everything else seeds the light one. Tokens the design -does not name keep the base theme's value. +does not name keep the base theme's value. **A design with only a base palette +does not touch the other variant** — it keeps the base theme's palette, and +`create` says so. To retheme dark mode too, emit a dark block (a `light` one for +a dark-first base such as console). If you are driving a design step (`/design` or similar) that will feed this, ask it to **emit a token block** rather than inferring one from the mockup. Two greys diff --git a/CHANGELOG.md b/CHANGELOG.md index 8a83d6a4e7..fc8b3a11b3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -56,6 +56,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). ### Fixed +- **`theme create --from` no longer paints a dark palette with a design's light colours** (ako/mxcli#970) — a design that declares only a base palette describes one variant, but its values were also written into the base theme's alternate-variant mixin: `--mxt-ground: #f7f9fb` and `--mxt-ink: #1b2733` landed in the dark mixin while its surfaces stayed dark, an unreadable mix. With no block for that variant the mixin is now left exactly as the base theme ships it, and `create` prints a note naming it and how to seed it (a `prefers-color-scheme: dark` block, or `light` for a dark-first base). Also fixed: the first rewritten token of a block landed on the `@mixin … {` line, unindented, and a rewritten token after a blank line swallowed the blank line. Verified with mxbuild 11.14.0's bundled sass on signal, console and ledger, with and without a variant block. - **`list workflows` and `show structure` count every activity of a workflow** (ako/mxcli#963) — including those on a boundary-event path and in an event sub-process, as the catalog's `workflows_data` has since #937. TestApp `Workflow1` listed 5 activities where the catalog counted 8; all three now share one walk (`wfnames.WalkActivities`). - **`docker build`, the TUI checker and `mxcli eval` no longer modify the project** (ako/mxcli#961) — `docker build` (and `docker run` / `docker reload`, which use it) now runs `mx update-widgets`, `mx check` and MxBuild on one temporary copy and writes only its output directory (`.docker/build/` or `-o`). Before, it rewrote an MPR v1 project's `.mpr`, rewrote every MPR v2 `.mxunit` (restored with new mtimes), and wrote `theme-cache/`, `deployment/`, `javasource/` proxies, the `.launch` file, `.classpath` and `.project` into the project. The build still uses the widget-normalised model, from the copy, and the package it produces is the same; `mxcli fix widgets` applies the normalisation to the project. **The project's `deployment/` is no longer refreshed by `docker build`** — `run --local` builds its own. The TUI auto-check and eval's `mx_check` ran a plain `mx check` on the project, which wrote `theme-cache/web/` and `deployment/sass/` every time; they now check a copy too. - **MPR010 no longer tells a native page to use a layout grid** (ako/mxcli#962) — the "wrap the form in a layoutgrid" advice is about Bootstrap columns, which a native page does not have: a form DataView directly on an `Atlas_Core.NativePhone_Default` page builds clean in mxbuild 11.13.0, and following the advice there is CE6858 ("update Atlas UI … to use Layout Grid on Native pages"). `lint` skips pages on a native layout and snippets of type Native (MPR010 now needs the full catalog, which a default `lint` builds anyway); `check -p` skips pages whose layout the project says is native. diff --git a/cmd/mxcli/cmd_theme.go b/cmd/mxcli/cmd_theme.go index f8d2ca13e0..e72ee3f49d 100644 --- a/cmd/mxcli/cmd_theme.go +++ b/cmd/mxcli/cmd_theme.go @@ -189,7 +189,10 @@ Seeding reads plain CSS custom properties, wherever they appear in the file: @media (prefers-color-scheme: dark) { :root { --mxt-ground: #16161a; } } Declarations inside a dark block seed the dark palette; everything else seeds -the light one. A token the base theme does not declare is an error, not a +the light one. A design with no block for the base theme's other variant +leaves that variant as the base ships it (and says so): its base palette +describes one variant, and copying a light ground into a dark palette whose +other surfaces stay dark is unreadable. A token the base theme does not declare is an error, not a silent no-op — run 'mxcli theme show signal' for the vocabulary. Nothing is applied. Edit the scaffold, then: @@ -247,6 +250,16 @@ Examples: res.Tokens.Count(), res.Tokens.Source, len(res.Tokens.Base), len(res.Tokens.Dark), len(res.Tokens.Light)) } + // A base-only design describes one variant. The other keeps the base + // theme's palette rather than a light/dark mix (#970) — say so, and + // say how to seed it. + if v := res.UnseededVariant; v != "" { + fmt.Printf("\nNote: %s declares no %s block, so the %s variant keeps %s's palette "+ + "(mixin mxcli-%s-%s in %s/files/theme/web/_mxcli-%s.scss). To seed it, add "+ + "`@media (prefers-color-scheme: %s) { :root { --mxt-…: …; } }` to the design and re-run "+ + "with --force, or edit the mixin.\n", + res.Tokens.Source, v, v, res.Base, res.Name, v, filepath.ToSlash(root), res.Name, v) + } // A seeded family mxcli does not vendor gets no @font-face and no file, // so it renders only on machines that happen to have it (#944). for _, fam := range res.UnvendoredFonts { diff --git a/cmd/mxcli/cmd_theme_test.go b/cmd/mxcli/cmd_theme_test.go index 7c605d7832..4e100a9d16 100644 --- a/cmd/mxcli/cmd_theme_test.go +++ b/cmd/mxcli/cmd_theme_test.go @@ -286,3 +286,32 @@ func TestThemeCreate_SeedsFromADesignFileAndReportsWhatItRead(t *testing.T) { t.Errorf("the design's brand colour did not reach the palette:\n%s", body) } } + +// ako/mxcli#970: a base-only design leaves the alternate variant's mixin as +// the base ships it, and create must say so — silently keeping the base's +// dark palette would read as "the design did not take". +func TestThemeCreate_NotesTheVariantABaseOnlyDesignDidNotSeed(t *testing.T) { + for _, tc := range []struct { + css string + wantNote bool + }{ + {":root{--mxt-brand:#10069f;--mxt-ground:#f7f9fb;}", true}, + {":root{--mxt-brand:#10069f;}\n@media (prefers-color-scheme: dark){:root{--mxt-brand:#a78bfa;}}", false}, + } { + dir := themeProject(t) + design := filepath.Join(dir, "design.css") + if err := os.WriteFile(design, []byte(tc.css), 0o644); err != nil { + t.Fatal(err) + } + out, err := captureStdout(t, func() error { + _, e := runTheme(t, "create", "acme", "-p", dir, "--from", design, "--base", "signal") + return e + }) + if err != nil { + t.Fatalf("create --from design: %v\n%s", err, out) + } + if got := strings.Contains(out, "declares no dark block, so the dark variant keeps signal's palette"); got != tc.wantNote { + t.Errorf("note present = %v, want %v for %q:\n%s", got, tc.wantNote, tc.css, out) + } + } +} diff --git a/cmd/mxcli/theme/create.go b/cmd/mxcli/theme/create.go index 48a2b3914f..807d9f7651 100644 --- a/cmd/mxcli/theme/create.go +++ b/cmd/mxcli/theme/create.go @@ -65,6 +65,9 @@ func Create(projectDir, name string, opts CreateOptions) (*CreateResult, error) res := &CreateResult{Name: name, Base: base, Dir: dest} if tokens != nil { res.Tokens = tokens + if alt := baseTheme.AltVariant(); !tokens.declares(alt) { + res.UnseededVariant = alt + } } rewrite := newRewriter(baseTheme, name, opts.Title, tokens) @@ -238,6 +241,12 @@ type CreateResult struct { // UnvendoredFonts are the seeded font families the theme names but does // not ship: no woff2, no @font-face. They render only where installed. UnvendoredFonts []string + // UnseededVariant is the base theme's alternate variant when the design + // declared no block for it. Its mixin is left exactly as the base ships + // it: the design's base palette describes the OTHER variant, and copying + // a light ground into a dark mixin whose remaining surfaces stay dark gives + // an unreadable mix (ako/mxcli#970). Empty when there was nothing to skip. + UnseededVariant Variant } // rewriter carries the renames that turn a copy of one theme into another: @@ -396,6 +405,14 @@ func seedTokens(rel, text string, base *Theme, newName string, tokens *Tokens) ( return "", fmt.Errorf("%s: no alt-palette mixin to seed", rel) } alt := base.AltVariant() + // A design with no block for the alternate variant says nothing about + // it. Its base palette is the DEFAULT variant's, and writing that + // into this mixin repainted the dark ground and ink light while the + // surfaces the design did not name stayed dark (ako/mxcli#970). The + // mixin is left as the base theme ships it; Create reports that. + if !tokens.declares(alt) { + return text, nil + } set := tokens.forVariant(alt) body := text[m[4]:m[5]] seeded, unplaced := applyTokens(body, set) diff --git a/cmd/mxcli/theme/create_variant_test.go b/cmd/mxcli/theme/create_variant_test.go new file mode 100644 index 0000000000..6b300846a9 --- /dev/null +++ b/cmd/mxcli/theme/create_variant_test.go @@ -0,0 +1,101 @@ +// SPDX-License-Identifier: Apache-2.0 + +package theme + +import ( + "path/filepath" + "regexp" + "strings" + "testing" +) + +// altMixin returns the alt-palette mixin of a scaffolded theme's partial, +// header line included, so a test can compare it and check its layout. +func altMixin(t *testing.T, partial, name string) string { + t.Helper() + m := regexp.MustCompile(`(?s)@mixin\s+mxcli-` + regexp.QuoteMeta(name) + `-(?:dark|light)\s*\{.*?\n\}`).FindString(partial) + if m == "" { + t.Fatalf("no alt-palette mixin in the scaffolded partial:\n%s", partial) + } + return m +} + +// scaffoldPartial creates a theme from the design css on base and returns its +// partial and the CreateResult. +func scaffoldPartial(t *testing.T, base, name, css string) (string, *CreateResult) { + t.Helper() + dir := newProject(t) + design := filepath.Join(dir, "design.css") + write(t, design, css) + res, err := Create(dir, name, CreateOptions{From: design, Base: base}) + if err != nil { + t.Fatal(err) + } + return read(t, filepath.Join(res.Dir, "files", "theme", "web", "_mxcli-"+name+".scss")), res +} + +// ako/mxcli#970: a design that declares only a base palette describes ONE +// variant. Copying its light ground and ink into the dark mixin, whose other +// surfaces stay dark, gave an unreadable mixed palette. With no block for the +// alt variant, the mixin must come out exactly as the base theme ships it. +func TestCreate_BaseOnlyDesignLeavesTheAltMixinAlone(t *testing.T) { + baseOnly := `:root { --mxt-brand: #10069f; --mxt-ground: #f7f9fb; --mxt-ink: #1b2733; }` + for _, base := range []string{"signal", "console", "ledger"} { + t.Run(base, func(t *testing.T) { + // The control: the same scaffold with no design at all. + ctlDir := newProject(t) + ctl, err := Create(ctlDir, "probe", CreateOptions{From: base}) + if err != nil { + t.Fatal(err) + } + want := altMixin(t, read(t, filepath.Join(ctl.Dir, "files", "theme", "web", "_mxcli-probe.scss")), "probe") + + partial, res := scaffoldPartial(t, base, "probe", baseOnly) + if got := altMixin(t, partial, "probe"); got != want { + t.Errorf("a base-only design rewrote the alt-palette mixin.\n got:\n%s\nwant:\n%s", got, want) + } + if res.UnseededVariant == "" { + t.Errorf("the result must name the variant the design did not seed, so the CLI can say so") + } + }) + } +} + +// The other half of the decision: a design that DOES declare the alt variant +// still seeds it — the fix keys on the block, not on giving up. +func TestCreate_DesignWithAVariantBlockSeedsTheAltMixin(t *testing.T) { + for base, block := range map[string]string{ + "signal": `@media (prefers-color-scheme: dark) { :root { --mxt-brand: #a78bfa; } }`, + "ledger": `@media (prefers-color-scheme: dark) { :root { --mxt-brand: #a78bfa; } }`, + "console": `@media (prefers-color-scheme: light) { :root { --mxt-brand: #a78bfa; } }`, + } { + t.Run(base, func(t *testing.T) { + partial, res := scaffoldPartial(t, base, "probe", `:root { --mxt-brand: #10069f; }`+"\n"+block) + mixin := altMixin(t, partial, "probe") + if !strings.Contains(mixin, "--mxt-brand: #a78bfa;") { + t.Errorf("the design's alt-variant brand did not reach the mixin:\n%s", mixin) + } + if strings.Contains(mixin, "#10069f") { + t.Errorf("the base palette leaked into the alt mixin:\n%s", mixin) + } + if res.UnseededVariant != "" { + t.Errorf("UnseededVariant = %q, want empty: the design declared that variant", res.UnseededVariant) + } + }) + } +} + +// The cosmetic half of #970: the first rewritten token used to land on the +// `@mixin … {` line itself, unindented, because the match's leading \s* +// swallowed the newline after the brace. +func TestApplyTokens_KeepsTheFirstDeclarationOnItsOwnLine(t *testing.T) { + body := "\n --mxt-brand: #2aa39f;\n --mxt-ink: #000;\n" + got, unplaced := applyTokens(body, TokenSet{"--mxt-brand": "#a78bfa"}) + if len(unplaced) != 0 { + t.Fatalf("unplaced = %v", unplaced) + } + want := "\n --mxt-brand: #a78bfa;\n --mxt-ink: #000;\n" + if got != want { + t.Errorf("applyTokens moved the declaration.\n got: %q\nwant: %q", got, want) + } +} diff --git a/cmd/mxcli/theme/tokens.go b/cmd/mxcli/theme/tokens.go index 22f841ce8c..1e60c00a63 100644 --- a/cmd/mxcli/theme/tokens.go +++ b/cmd/mxcli/theme/tokens.go @@ -69,9 +69,13 @@ func (t *Tokens) scoped(v Variant) TokenSet { return t.Dark } +// declares reports whether the artifact had a block for this variant. +func (t *Tokens) declares(v Variant) bool { return len(t.scoped(v)) > 0 } + // forVariant returns the token set that seeds a theme's given variant: the // base declarations, overlaid with anything the artifact scoped to that -// variant. A design that declares only a base palette seeds both. +// variant. Callers seeding the ALTERNATE variant check declares() first: a +// design with only a base palette describes one variant, not both (#970). func (t *Tokens) forVariant(v Variant) TokenSet { out := TokenSet{} for k, val := range t.Base { @@ -236,7 +240,10 @@ func applyTokens(scss string, tokens TokenSet) (string, []string) { out := scss var unplaced []string for _, name := range tokens.Names() { - re := regexp.MustCompile(`(?m)^(\s*)` + regexp.QuoteMeta(name) + `\s*:[^;]*;`) + // [ \t]*, not \s*: under (?m) a \s* at ^ also swallows the newline + // (and any blank line) before the declaration, so the first token of + // a block landed on the `{` line, unindented (ako/mxcli#970). + re := regexp.MustCompile(`(?m)^([ \t]*)` + regexp.QuoteMeta(name) + `\s*:[^;]*;`) if !re.MatchString(out) { unplaced = append(unplaced, name) continue diff --git a/docs-site/src/tools/theme.md b/docs-site/src/tools/theme.md index eee26b5ed0..5864660f82 100644 --- a/docs-site/src/tools/theme.md +++ b/docs-site/src/tools/theme.md @@ -222,6 +222,13 @@ Declarations inside a dark block (`prefers-color-scheme: dark`, `.theme-dark`, one. Tokens the design does not name keep the base theme's value, so a three-colour design still yields a complete, working palette. +A design with **no block for the base theme's other variant** — only a base +palette — leaves that variant exactly as the base theme ships it, and `create` +prints a note naming the mixin. The base palette describes one variant: copying +its light ground and ink into the dark palette, whose other surfaces stay dark, +gave an unreadable mix. Add a `prefers-color-scheme: dark` (or `light`, for a +dark-first base such as console) block to seed it. + `mxcli theme show signal` prints the vocabulary. A `--mxt-*` name the base theme does not declare is **an error**, not an extra: nothing would read it, so the theme would apply cleanly and render unchanged — the one failure mode this path From f689626e11327d2ecddcb99091452cdb4ff89675 Mon Sep 17 00:00:00 2001 From: Ako Date: Sun, 4 Oct 2026 15:12:02 +0000 Subject: [PATCH 11/17] feat(translations): warn on Marketplace writes, add 'without marketplace' (#970) An unscoped create translations reaches the whole project, Marketplace modules (and their Atlas templates/building blocks) included; a module update replaces those. The default is kept (ADR-0011: what a committed script writes does not change in place). The run now warns with a per-module count, and the additive clause 'without marketplace' (create and describe) keeps the run out of Marketplace modules and reports the entries it left alone. Co-Authored-By: Claude Opus 5.5 --- .../fix-issue/findings/mdl-executor.jsonl | 1 + .claude/skills/mendix/translations/SKILL.md | 17 +- CHANGELOG.md | 1 + cmd/mxcli/lsp_completions_gen.go | 1 + cmd/mxcli/syntax/features_misc.go | 11 +- docs-site/src/language/translations.md | 37 ++++- docs/01-project/MDL_QUICK_REFERENCE.md | 4 +- mdl/ast/ast_translations.go | 19 ++- mdl/executor/cmd_translations.go | 130 +++++++++++++-- .../cmd_translations_marketplace_test.go | 153 ++++++++++++++++++ mdl/executor/validate_exec_refusals.go | 2 +- mdl/grammar/MDLLexer.g4 | 1 + mdl/grammar/domains/MDLCatalog.g4 | 2 +- mdl/grammar/domains/MDLSettings.g4 | 12 +- mdl/translations/project.go | 14 ++ mdl/visitor/visitor_query.go | 1 + mdl/visitor/visitor_translations.go | 1 + mdl/visitor/visitor_translations_test.go | 29 ++++ 18 files changed, 403 insertions(+), 33 deletions(-) create mode 100644 mdl/executor/cmd_translations_marketplace_test.go diff --git a/.claude/skills/fix-issue/findings/mdl-executor.jsonl b/.claude/skills/fix-issue/findings/mdl-executor.jsonl index 8d3c87c883..6d70a6e628 100644 --- a/.claude/skills/fix-issue/findings/mdl-executor.jsonl +++ b/.claude/skills/fix-issue/findings/mdl-executor.jsonl @@ -853,3 +853,4 @@ {"area": "mdl/executor", "date": "2026-10-03", "symptom": "`list workflows` and `show structure` report fewer workflow activities than the catalog's workflows_data (TestApp Workflow1: 5 vs 8)", "cause": "cmd_workflows.go countFlowActivities and cmd_structure.go countStructureFlowActivities each recursed over outcome flows only, skipping boundary-event flows and event sub-processes; the catalog had moved to a shared walk in #937 and the executor copies were left behind", "file": "`mdl/backend/wfnames/walk.go` (`WalkActivities`, `CountActivities`), `mdl/executor/cmd_workflows.go`, `mdl/executor/cmd_structure.go`, `mdl/catalog/workflow_walk.go`", "insight": "Duplicate-resolver drift: fixing one copy of a traversal (#937) left two private copies answering differently. The walk now lives in wfnames, which both catalog and executor already import, so there is one place to add a new sub-flow slot. Grep for every recursion over `workflows.Flow` when one is fixed.", "refs": ["ako/mxcli#963", "ako/mxcli#937"]} {"area": "mdl/executor", "date": "2026-10-03", "symptom": "ako/mxcli#962 item 1: `$V = call java action M.VoidAction(...)` then `log ... + $V` (or `$V` as a JS call argument) passes `check` and `exec`, then mxbuild 11.13.0 fails CE0109 \"Undefined variable 'V'\"", "cause": "#953 taught MDL063 that a void call's output name declares nothing, but no rule read the other half: the name cannot be READ either. The resolver only answered void/not-void, so 'unknown' and 'non-void' were the same answer", "file": "`mdl/executor/validate_void_call_output.go` (checkVoidCallOutputUse, MDL093); `mdl/executor/validate_void_code_calls.go` (resolve -> voidness{void, known})", "fix": "MDL093: collect output names of calls KNOWN to be void, drop any name another statement defines flow-wide (declare, parameter, non-void producer), report each remaining name the flow reads (loopRefVars over every nested body). The resolver now returns known-ness, so 'possibly void' (the editor's policy) never produces MDL093", "insight": "A finding that says 'X declares nothing' has two consequences — no collision AND no definition; fixing the first and logging the second as follow-up left a CE gap. When a resolver's default is a policy (unknown counts as non-void), make the unknown state explicit before a second rule reads it: the CE0109 rule must use only knowledge, never the policy", "test": "`mdl/executor/validate_void_call_output_test.go`; `cmd/mxcli/check_void_calls_test.go` (TestCheck_ReadOfAStoredVoidCallOutput, PedApp stored void JS action, Boolean and unresolvable controls)"} {"area": "mdl/executor", "date": "2026-10-03", "symptom": "ako/mxcli#962 item 2: describe of a microflow with the same output name in each if/else branch (or inside a loop and again after it) printed no duplicate-variable warning; mxbuild 11.13.0 reports CE0111 for both", "cause": "duplicateOutputVariableWarnings only warned when one assignment could REACH another (a reachability walk written for #710's performance), so exclusive branches and a loop body vs. the flow after it were treated as separate scopes; a test pinned the branch scoping", "file": "`mdl/executor/cmd_microflows_show.go` (duplicateOutputVariableWarnings)", "fix": "Count non-void output names over every object collection (loop bodies included); warn for any name created twice. Linear, so #710's cost concern disappears with the reachability walk", "insight": "The reachability model encoded a belief (exclusive paths may reuse a name) that no one had measured; MDL063 had already been aligned to flow-wide names in #958, so two renderings of the same rule disagreed. When one rule is corrected against mxbuild, grep for the other places that encode the same rule — here describe's header warning", "test": "`mdl/executor/cmd_microflows_duplicate_output_test.go` (TestFormatMicroflowActivitiesWarnsForExclusiveBranchOutputs, TestFormatMicroflowActivitiesNamesAreFlowWide)"} +{"area": "mdl/executor", "date": "2026-10-04", "symptom": "ako/mxcli#970 item 1: unscoped `create or modify translations for nl_NL ('Cancel' as 'Annuleren')` on TestApp set 50 translations in 38 documents, 35 of them in Marketplace modules (WorkflowCommons, Atlas_Web_Content, FeedbackModule; 14 page templates/building blocks), silently — the next module update overwrites them", "cause": "translationScope returned nil (whole project) for an unscoped run, by design and documented; nothing told the author where the writes landed and there was no way to exclude Marketplace modules short of one `in ` run per own module", "file": "`mdl/executor/cmd_translations.go` (translationScope, reportMarketplaceWrites, reportOutOfScopeEntries); `mdl/translations/project.go` (Stats.Written); grammar MDLSettings.g4/MDLCatalog.g4 (WITHOUT MARKETPLACE)", "fix": "kept the default (ADR-0011: changing what a committed headerless or mdl 1 script writes is a meaning change, and a translated Marketplace page is a runtime-visible effect a user may want); warn per Marketplace module with the template/building-block count; added the additive clause `without marketplace` on create and describe, whose scope excludes units in modules with FromAppStore or AppStoreGuid (isMarketplaceModule, the same test alter page/layout use) and whose skipped entries are reported, not counted as drift", "insight": "When the fix the issue asks for is a change of default, check ADR-0011 first: a frozen language version turns 'skip by default' into 'warn + additive opt-out'. The precedent for the opt-out's semantics was already in ALTER ENTITIES (unscoped sweep skips Marketplace, naming a module means it). Per-unit write records (Stats.Written) are what let a caller say WHERE a project-wide walk wrote", "test": "`mdl/executor/cmd_translations_marketplace_test.go` (TestApp: unscoped still reaches Marketplace AND warns; without marketplace changes 0 Marketplace units and >0 own units; scoped runs do not warn; describe emits the clause); `mdl/visitor/visitor_translations_test.go` (TestTranslations_WithoutMarketplaceParses)"} diff --git a/.claude/skills/mendix/translations/SKILL.md b/.claude/skills/mendix/translations/SKILL.md index d42d38ce92..8a49c809b0 100644 --- a/.claude/skills/mendix/translations/SKILL.md +++ b/.claude/skills/mendix/translations/SKILL.md @@ -28,11 +28,11 @@ prompt. ## Statements ```sql -describe translations [in ] for ; +describe translations [in | without marketplace] for ; -create translations [in ] for ( 'src' as 'target', ... ); -create or modify translations [in ] for ( 'src' as 'target', ... ); -create or replace translations [in ] for ( 'src' as 'target', ... ); +create translations [in | without marketplace] for ( 'src' as 'target', ... ); +create or modify translations [in | without marketplace] for ( 'src' as 'target', ... ); +create or replace translations [in | without marketplace] for ( 'src' as 'target', ... ); ``` Entries use `as`, not a colon: a translation maps a user-provided name to another @@ -48,6 +48,15 @@ name. deletion** — without it, a set of per-module files would wipe each other on every run. +**Translating the app, not its Marketplace modules: add `without marketplace`.** +Without `in`, a run reaches the whole project — Marketplace modules, and the Atlas +page templates and building blocks in them, included. A module update replaces a +Marketplace module's contents, so translations written there are lost at the next +update and show up as unexpected diffs until then. The unscoped run warns with a +per-module count; `without marketplace` keeps the run in the app's own modules and +names the file's entries it left alone. Use it on `describe` as well, so the file +it emits carries the clause. + ## The trap: a language that is not enabled **A translation for a language the project has not enabled is stored, passes diff --git a/CHANGELOG.md b/CHANGELOG.md index fc8b3a11b3..5d11942959 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -139,6 +139,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). ### Added +- **`create translations … without marketplace`, and an unscoped translations run warns when it writes into Marketplace modules** (ako/mxcli#970) — without `in `, `create [or modify|or replace] translations` reaches the whole project, Marketplace modules and their Atlas page templates and building blocks included; on TestApp `'Cancel' as 'Annuleren'` changed 35 Marketplace documents of 38. A module update replaces those modules, so the translations are lost at the next update and show up as unexpected diffs until then. The run now warns with the count per module and how many are templates or building blocks. `without marketplace` (also on `describe translations`, which emits it) keeps the run, and an `or replace` deletion, to the app's own modules and names the file's entries it left alone. The default is unchanged: what an existing script writes does not change underneath it (ADR-0011). - **A `commit` reference kind in the catalog** (ako/mxcli#963) — `refs` has a `commit` edge from a flow to the entity it commits: a commit action, or a create / change that commits (`Yes` or `YesWithoutEvents`), the latter beside its `create` / `change` edge. The entity of a committed variable is resolved like `change` / `delete`, loop iterators included, so `commit $Order` inside `loop $Order in $Orders` now has a row. `refs_to("M.Order")` in a Starlark rule answers which flows commit an order. `commit` is not in the analysis graph and not a caller kind. The catalog schema version is bumped, so a cached catalog rebuilds. - **`total_activity_count` on the Starlark microflow struct** (ako/mxcli#963) — the catalog's `TotalActivityCount` (loop bodies included, at any depth) for every flow `microflows()` yields: microflows, nanoflows and rules. `activity_count` keeps counting a loop as one activity. - **The catalog's `widgets` table records the widget tree, appearance and primary action** (mendixlabs/mxcli#1268) — `ParentWidgetId` (the nearest catalogued ancestor; skipped wrappers, layout grid rows and columns, tab pages and data grid 2 columns are transparent), `Depth` (0 at the page or snippet root), `Class`, `Style`, `DynamicClasses`, `ActionType` (the stored type of the button, on-click or click action, e.g. `Forms$DeleteClientAction`) and `HasConfirmation`. Starlark `widgets()` exposes them as `parent_widget_id`, `depth`, `class_name`, `style`, `dynamic_classes`, `action_type`, `has_confirmation`, plus `page_ref`, so a lint rule can flag deep nesting, inline styles, classes outside an allow-list and delete buttons. A delete action has no confirmation setting in Mendix; the enforceable rule is "no button uses the delete action directly". The catalog schema version is bumped, so a cached catalog rebuilds. diff --git a/cmd/mxcli/lsp_completions_gen.go b/cmd/mxcli/lsp_completions_gen.go index 1d4339a5c4..8b9a1b455b 100644 --- a/cmd/mxcli/lsp_completions_gen.go +++ b/cmd/mxcli/lsp_completions_gen.go @@ -506,6 +506,7 @@ var mdlGeneratedKeywords = []protocol.CompletionItem{ {Label: "SLOT", Kind: protocol.CompletionItemKindKeyword, Detail: "Utility keyword"}, {Label: "LANGUAGES", Kind: protocol.CompletionItemKindKeyword, Detail: "Utility keyword"}, {Label: "TRANSLATIONS", Kind: protocol.CompletionItemKindKeyword, Detail: "Utility keyword"}, + {Label: "MARKETPLACE", Kind: protocol.CompletionItemKindKeyword, Detail: "Utility keyword"}, {Label: "INSERT", Kind: protocol.CompletionItemKindKeyword, Detail: "Utility keyword"}, {Label: "BEFORE", Kind: protocol.CompletionItemKindKeyword, Detail: "Utility keyword"}, {Label: "AFTER", Kind: protocol.CompletionItemKindKeyword, Detail: "Utility keyword"}, diff --git a/cmd/mxcli/syntax/features_misc.go b/cmd/mxcli/syntax/features_misc.go index 0358269c4d..179d381958 100644 --- a/cmd/mxcli/syntax/features_misc.go +++ b/cmd/mxcli/syntax/features_misc.go @@ -485,8 +485,8 @@ CREATE OR MODIFY NAVIGATION TabletOffline "translations", "translate", "language", "languages", "i18n", "localisation", "localization", "multilingual", "nl_NL", "de_DE", }, - Syntax: `DESCRIBE TRANSLATIONS [IN ] FOR ; -CREATE [OR MODIFY|REPLACE] TRANSLATIONS [IN ] FOR ( + Syntax: `DESCRIBE TRANSLATIONS [IN | WITHOUT MARKETPLACE] FOR ; +CREATE [OR MODIFY|REPLACE] TRANSLATIONS [IN | WITHOUT MARKETPLACE] FOR ( '' AS '', ... ); @@ -510,6 +510,13 @@ which reads as a half-applied translation rather than a scoping decision. A scoped run now names the file's own entries it did not reach; re-run the same file without IN to land those too. +Without IN, the run reaches the WHOLE project — Marketplace modules +included, Atlas page templates and building blocks among them. A module +update replaces a Marketplace module's contents, so what lands there is +lost at the next update. The run warns with a count per module; add +WITHOUT MARKETPLACE to keep it to your own modules (it reports the file's +entries it left alone), or name one module with IN . + Keyed on the source string, so one entry translates every occurrence. DESCRIBE emits the CREATE form, and an untranslated string comes back with an empty target — which is what makes the output an LLM prompt: diff --git a/docs-site/src/language/translations.md b/docs-site/src/language/translations.md index 52a56fd784..4662a17162 100644 --- a/docs-site/src/language/translations.md +++ b/docs-site/src/language/translations.md @@ -23,11 +23,11 @@ a few hundred distinct source strings, so one file per language stays practical. ## Statements ```sql -DESCRIBE TRANSLATIONS [IN ] FOR ; +DESCRIBE TRANSLATIONS [IN | WITHOUT MARKETPLACE] FOR ; -CREATE TRANSLATIONS [IN ] FOR ( '' AS '', ... ); -CREATE OR MODIFY TRANSLATIONS [IN ] FOR ( ... ); -CREATE OR REPLACE TRANSLATIONS [IN ] FOR ( ... ); +CREATE TRANSLATIONS [IN | WITHOUT MARKETPLACE] FOR ( '' AS '', ... ); +CREATE OR MODIFY TRANSLATIONS [IN | WITHOUT MARKETPLACE] FOR ( ... ); +CREATE OR REPLACE TRANSLATIONS [IN | WITHOUT MARKETPLACE] FOR ( ... ); ``` Entries use `AS`, not a colon: a translation maps a user-provided name to another @@ -42,6 +42,35 @@ name. `IN ` scopes both directions, and under `OR REPLACE` it **bounds the deletion** — without it, per-module files would wipe each other on every run. +### Marketplace modules + +Without `IN`, a run reaches the **whole project**, Marketplace modules included — +and with them the Atlas page templates and building blocks. On a stock app, +`'Cancel' AS 'Annuleren'` set 50 translations in 38 documents, 35 of them in +Marketplace modules. A module update replaces a Marketplace module's contents, so +those translations are lost at the next update and show up as unexpected diffs +until then. The run warns, with a count per module: + +```text +Warning: 47 of these translation change(s) landed in 35 document(s) of Marketplace +modules (Atlas_Web_Content 13, FeedbackModule 2, WorkflowCommons 20; 14 of the documents are page templates or building blocks). +``` + +`WITHOUT MARKETPLACE` keeps the run in your own modules, and names the file's +entries that also occur in the modules it skipped. Under `OR REPLACE` it also +keeps the deletion out of them. `DESCRIBE` accepts it too, and emits it, so the +described file round-trips: + +```sql +mdl 1; +describe translations without marketplace for nl_NL; +create or modify translations without marketplace for nl_NL ( 'Cancel' as 'Annuleren' ); +``` + +Naming a Marketplace module with `IN ` is taken as meaning it, and does +not warn. The default stays the whole project: what an existing script writes +does not change underneath it. + ## A translation is only built if its language is enabled This is the one thing worth knowing before translating anything. diff --git a/docs/01-project/MDL_QUICK_REFERENCE.md b/docs/01-project/MDL_QUICK_REFERENCE.md index c56170a4c7..5a64da20e9 100644 --- a/docs/01-project/MDL_QUICK_REFERENCE.md +++ b/docs/01-project/MDL_QUICK_REFERENCE.md @@ -1839,8 +1839,8 @@ Bulk translation of every user-visible string, one file per language. Entries us | Statement | Syntax | Notes | |-----------|--------|-------| -| Describe | `describe translations [in Module] for ;` | Emits the CREATE form; an untranslated string comes back with an **empty** target, which is what makes the output an LLM prompt | -| Create | `create translations [in Module] for ( 'src' as 'target', ... );` | The language is the thing that exists — **errors** if it already has translations | +| Describe | `describe translations [in Module \| without marketplace] for ;` | Emits the CREATE form; an untranslated string comes back with an **empty** target, which is what makes the output an LLM prompt | +| Create | `create translations [in Module \| without marketplace] for ( 'src' as 'target', ... );` | The language is the thing that exists — **errors** if it already has translations. Without `in`, the run reaches Marketplace modules too and **warns** with a per-module count (an update replaces them); `without marketplace` keeps it to the app's own modules | | Merge | `create or modify translations ...` | A source the file does not name is left alone | | Replace | `create or replace translations ...` | The file is authoritative: a translation whose source it does not name is **REMOVED**, and the run says which. `in Module` **bounds** the deletion | | Remove a language's translations | `create or replace translations [in Module] for ( );` | An empty file is authoritative over nothing, so everything in scope goes — the only way to take a language's translations out of the model | diff --git a/mdl/ast/ast_translations.go b/mdl/ast/ast_translations.go index 633b067531..1a2bd58434 100644 --- a/mdl/ast/ast_translations.go +++ b/mdl/ast/ast_translations.go @@ -38,21 +38,26 @@ func (m TranslationMode) String() string { return "create" } -// CreateTranslationsStmt is CREATE [OR MODIFY|REPLACE] TRANSLATIONS [IN Module] -// FOR ( 'src' AS 'target', … ). +// CreateTranslationsStmt is CREATE [OR MODIFY|REPLACE] TRANSLATIONS +// [IN Module | WITHOUT MARKETPLACE] FOR ( 'src' AS 'target', … ). type CreateTranslationsStmt struct { Language string Module string // optional scope; empty means the whole project - Mode TranslationMode - Entries []TranslationEntry + // WithoutMarketplace (`without marketplace`) leaves Marketplace modules out + // of an unscoped run: the next module update replaces what is written + // there (ako/mxcli#970). Mutually exclusive with Module in the grammar. + WithoutMarketplace bool + Mode TranslationMode + Entries []TranslationEntry } func (s *CreateTranslationsStmt) isStatement() {} -// DescribeTranslationsStmt is DESCRIBE TRANSLATIONS [IN Module] FOR . +// DescribeTranslationsStmt is DESCRIBE TRANSLATIONS [IN Module | WITHOUT MARKETPLACE] FOR . type DescribeTranslationsStmt struct { - Language string - Module string + Language string + Module string + WithoutMarketplace bool } func (s *DescribeTranslationsStmt) isStatement() {} diff --git a/mdl/executor/cmd_translations.go b/mdl/executor/cmd_translations.go index b25dae21cd..2eef57887d 100644 --- a/mdl/executor/cmd_translations.go +++ b/mdl/executor/cmd_translations.go @@ -14,17 +14,33 @@ import ( "github.com/mendixlabs/mxcli/model" ) -// translationScope restricts the walk to one module's units, or nil for the -// whole project. Resolved through the container hierarchy, which is what knows -// that a document nested in folders still belongs to its module. -func translationScope(ctx *ExecContext, moduleName string) (translations.Scope, error) { - if moduleName == "" { +// translationScope restricts the walk to one module's units, to every unit +// outside a Marketplace module (`without marketplace`), or nil for the whole +// project. Resolved through the container hierarchy, which is what knows that a +// document nested in folders still belongs to its module. +func translationScope(ctx *ExecContext, moduleName string, withoutMarketplace bool) (translations.Scope, error) { + if moduleName == "" && !withoutMarketplace { return nil, nil } modules, err := ctx.Backend.ListModules() if err != nil { return nil, mdlerrors.NewBackend("list modules", err) } + if moduleName == "" { + marketplace := map[model.ID]bool{} + for _, m := range modules { + if isMarketplaceModule(ctx, m) { + marketplace[m.ID] = true + } + } + h, err := getHierarchy(ctx) + if err != nil { + return nil, mdlerrors.NewBackend("resolve module hierarchy", err) + } + return func(unitID model.ID) bool { + return !marketplace[unitID] && !marketplace[h.FindModuleID(unitID)] + }, nil + } var moduleID model.ID for _, m := range modules { if strings.EqualFold(m.Name, moduleName) { @@ -54,7 +70,7 @@ func execDescribeTranslations(ctx *ExecContext, s *ast.DescribeTranslationsStmt) if ctx.Backend == nil { return mdlerrors.NewValidation("no project connected") } - scope, err := translationScope(ctx, s.Module) + scope, err := translationScope(ctx, s.Module, s.WithoutMarketplace) if err != nil { return err } @@ -72,8 +88,11 @@ func execDescribeTranslations(ctx *ExecContext, s *ast.DescribeTranslationsStmt) } in := "" - if s.Module != "" { + switch { + case s.Module != "": in = " in " + s.Module + case s.WithoutMarketplace: + in = " without marketplace" } fmt.Fprintf(ctx.Output, "create or modify translations%s for %s (\n", in, s.Language) translated := 0 @@ -102,7 +121,7 @@ func execCreateTranslations(ctx *ExecContext, s *ast.CreateTranslationsStmt) err if ctx.Backend == nil { return mdlerrors.NewValidation("no project connected") } - scope, err := translationScope(ctx, s.Module) + scope, err := translationScope(ctx, s.Module, s.WithoutMarketplace) if err != nil { return err } @@ -138,6 +157,7 @@ func execCreateTranslations(ctx *ExecContext, s *ast.CreateTranslationsStmt) err reportTranslationStats(ctx, s, stats, src, scope, outOfScope) reportOutOfScopeEntries(ctx, s, outOfScope) + reportMarketplaceWrites(ctx, s, stats) // A translation for a language the project has not enabled is stored, passes // every check, and is then discarded by the build. Say so AFTER the stats, so @@ -235,7 +255,19 @@ func excludeStrings(ss, drop []string) []string { // true of every scoped run and would warn forever, which is exactly the // per-module workflow the scoping exists to support. func reportOutOfScopeEntries(ctx *ExecContext, s *ast.CreateTranslationsStmt, missed []string) { - if s.Module == "" || len(missed) == 0 { + if len(missed) == 0 { + return + } + if s.Module == "" && s.WithoutMarketplace { + fmt.Fprintf(ctx.Output, + "\nLeft alone: %d of this file's source string(s) also occur in Marketplace modules,\n"+ + "which `without marketplace` skipped:\n\n", len(missed)) + for _, srcStr := range quoteAll(ctx, missed) { + fmt.Fprintf(ctx.Output, " %s\n", srcStr) + } + return + } + if s.Module == "" { return } fmt.Fprintf(ctx.Output, @@ -250,6 +282,86 @@ func reportOutOfScopeEntries(ctx *ExecContext, s *ast.CreateTranslationsStmt, mi "\nRe-run the same file without `in %s` to land these as well.\n", s.Module) } +// reportMarketplaceWrites warns when an unscoped run wrote into Marketplace +// modules (ako/mxcli#970). +// +// An unscoped statement reaches the whole project — documented, and kept: under +// ADR-0011 what a committed script writes does not change without a new +// language version, and a translated Administration page is something a user +// translating their app may well want. But a Marketplace module's contents are +// replaced by its next update, Atlas page templates and building blocks +// included, so those translations are lost then and show up as unexpected +// diffs until they are. 'Cancel' alone landed in 38 documents of TestApp, most +// of them in modules the author never opened. So it is said, with the count +// per module and the additive clause that keeps the run out of them. +// +// Not said for `in `: naming a module is taken as meaning it, the same +// division ALTER ENTITIES and `mxcli layout` make. +func reportMarketplaceWrites(ctx *ExecContext, s *ast.CreateTranslationsStmt, stats translations.Stats) { + if s.Module != "" || s.WithoutMarketplace || len(stats.Written) == 0 { + return + } + modules, err := ctx.Backend.ListModules() + if err != nil { + return + } + marketplace := map[model.ID]string{} + for _, m := range modules { + if isMarketplaceModule(ctx, m) { + marketplace[m.ID] = m.Name + } + } + if len(marketplace) == 0 { + return + } + h, err := getHierarchy(ctx) + if err != nil { + return + } + perModule := map[string]int{} + docs, changes, templates := 0, 0, 0 + for _, w := range stats.Written { + name, ok := marketplace[w.ID] + if !ok { + name, ok = marketplace[h.FindModuleID(w.ID)] + } + if !ok { + continue + } + perModule[name]++ + docs++ + changes += w.Set + w.Removed + if w.Type == "Forms$PageTemplate" || w.Type == "Forms$BuildingBlock" { + templates++ + } + } + if docs == 0 { + return + } + names := make([]string, 0, len(perModule)) + for n := range perModule { + names = append(names, n) + } + sort.Strings(names) + parts := make([]string, 0, len(names)) + for _, n := range names { + parts = append(parts, fmt.Sprintf("%s %d", n, perModule[n])) + } + tmpl := "" + if templates > 0 { + tmpl = fmt.Sprintf("; %d of the documents are page templates or building blocks", templates) + } + fmt.Fprintf(ctx.Output, + "\nWarning: %d of these translation change(s) landed in %d document(s) of Marketplace\n"+ + "modules (%s%s).\n"+ + "A module update replaces a Marketplace module's contents, so these are lost at the\n"+ + "next update and show up as unexpected diffs until then. To keep the run in your own\n"+ + "modules, add `without marketplace`:\n\n"+ + " %s translations without marketplace for %s ( … );\n\n"+ + "or scope it to one module with `in `.\n", + changes, docs, strings.Join(parts, ", "), tmpl, s.Mode.String(), s.Language) +} + func quoteAll(ctx *ExecContext, ss []string) []string { out := make([]string, 0, len(ss)) for _, s := range ss { diff --git a/mdl/executor/cmd_translations_marketplace_test.go b/mdl/executor/cmd_translations_marketplace_test.go new file mode 100644 index 0000000000..1dcea361f8 --- /dev/null +++ b/mdl/executor/cmd_translations_marketplace_test.go @@ -0,0 +1,153 @@ +// SPDX-License-Identifier: Apache-2.0 + +package executor + +import ( + "context" + "strconv" + "strings" + "testing" + + "github.com/mendixlabs/mxcli/model" +) + +// translationUnitSnapshot snapshots every unit's stored bytes, keyed by unit and +// tagged with whether it lives in a Marketplace module. +func translationUnitSnapshot(t *testing.T, exec *Executor) (map[model.ID]string, map[model.ID]bool) { + t.Helper() + ctx := exec.newExecContext(context.Background()) + modules, err := ctx.Backend.ListModules() + if err != nil { + t.Fatal(err) + } + mp := map[model.ID]bool{} + for _, m := range modules { + if isMarketplaceModule(ctx, m) { + mp[m.ID] = true + } + } + h, err := getHierarchy(ctx) + if err != nil { + t.Fatal(err) + } + units, err := ctx.Backend.ListUnits() + if err != nil { + t.Fatal(err) + } + raw := map[model.ID]string{} + inMP := map[model.ID]bool{} + for _, u := range units { + b, err := ctx.Backend.GetRawUnitBytes(u.ID) + if err != nil { + continue + } + raw[u.ID] = string(b) + inMP[u.ID] = mp[u.ID] || mp[h.FindModuleID(u.ID)] + } + return raw, inMP +} + +func changedUnits(before, after map[model.ID]string, inMP map[model.ID]bool) (marketplace, own int) { + for id, b := range before { + if after[id] == b { + continue + } + if inMP[id] { + marketplace++ + } else { + own++ + } + } + return +} + +// ako/mxcli#970: on TestApp, `'Cancel' as 'Annuleren'` unscoped set 50 +// translations in 38 documents, 35 of them in Marketplace modules — Atlas page +// templates and building blocks among them — which the next module update +// overwrites. The unscoped statement keeps its documented whole-project reach +// (ADR-0011: what a committed script writes does not change in place), and must +// SAY where it wrote. +func TestCreateTranslations_UnscopedWarnsAboutMarketplaceWrites(t *testing.T) { + exec, out := openTestAppCopy(t) + before, inMP := translationUnitSnapshot(t, exec) + + if err := afRun(t, exec, "create or modify translations for nl_NL ('Cancel' as 'Annuleren');"); err != nil { + t.Fatalf("exec: %v\n%s", err, out.String()) + } + after, _ := translationUnitSnapshot(t, exec) + mpChanged, ownChanged := changedUnits(before, after, inMP) + // Control: the default still reaches Marketplace modules — this test pins a + // warning, not a change of meaning. + if mpChanged == 0 || ownChanged == 0 { + t.Fatalf("precondition: unscoped run changed %d marketplace and %d own unit(s); want both > 0", mpChanged, ownChanged) + } + got := out.String() + for _, want := range []string{ + "landed in " + strconv.Itoa(mpChanged) + " document(s) of Marketplace", + "WorkflowCommons", + "page templates or building blocks", + "create or modify translations without marketplace for nl_NL", + } { + if !strings.Contains(got, want) { + t.Errorf("output lacks %q:\n%s", want, got) + } + } +} + +// `without marketplace` writes nothing into a Marketplace module, still writes +// the app's own modules (the control: a fix that wrote nothing would pass the +// first half), says what it left alone, and does not warn. +func TestCreateTranslations_WithoutMarketplaceSkipsMarketplaceModules(t *testing.T) { + exec, out := openTestAppCopy(t) + before, inMP := translationUnitSnapshot(t, exec) + + if err := afRun(t, exec, "create or modify translations without marketplace for nl_NL ('Cancel' as 'Annuleren');"); err != nil { + t.Fatalf("exec: %v\n%s", err, out.String()) + } + after, _ := translationUnitSnapshot(t, exec) + mpChanged, ownChanged := changedUnits(before, after, inMP) + if mpChanged != 0 { + t.Errorf("without marketplace changed %d unit(s) in Marketplace modules, want 0", mpChanged) + } + if ownChanged == 0 { + t.Errorf("without marketplace changed none of the app's own units — it must still translate them") + } + got := out.String() + if strings.Contains(got, "of Marketplace\nmodules") { + t.Errorf("without marketplace must not warn about marketplace writes:\n%s", got) + } + if !strings.Contains(got, "which `without marketplace` skipped") { + t.Errorf("without marketplace must say what it left alone:\n%s", got) + } + // Not drift: 'Cancel' matched, just outside the scope. + if strings.Contains(got, "matched nothing in the project") { + t.Errorf("an entry skipped by without marketplace was reported as drift:\n%s", got) + } +} + +// Naming a module is taken as meaning it: no warning for `in `, even +// when that module is a Marketplace one. +func TestCreateTranslations_ScopedRunDoesNotWarn(t *testing.T) { + for _, mod := range []string{"MyFirstModule", "WorkflowCommons"} { + t.Run(mod, func(t *testing.T) { + exec, out := openTestAppCopy(t) + if err := afRun(t, exec, "create or modify translations in "+mod+" for nl_NL ('Cancel' as 'Annuleren');"); err != nil { + t.Fatalf("exec: %v\n%s", err, out.String()) + } + if strings.Contains(out.String(), "of Marketplace\nmodules") { + t.Errorf("a scoped run warned about marketplace writes:\n%s", out.String()) + } + }) + } +} + +// DESCRIBE emits the clause it was given, so the described file round-trips. +func TestDescribeTranslations_WithoutMarketplaceHeader(t *testing.T) { + exec, out := openTestAppCopy(t) + if err := afRun(t, exec, "describe translations without marketplace for nl_NL;"); err != nil { + t.Fatalf("describe: %v\n%s", err, out.String()) + } + if !strings.Contains(out.String(), "create or modify translations without marketplace for nl_NL (") { + t.Errorf("describe header lacks the clause:\n%s", out.String()) + } +} diff --git a/mdl/executor/validate_exec_refusals.go b/mdl/executor/validate_exec_refusals.go index a71db3de87..6592c66404 100644 --- a/mdl/executor/validate_exec_refusals.go +++ b/mdl/executor/validate_exec_refusals.go @@ -135,7 +135,7 @@ func checkTranslationTargets(ctx *ExecContext, prog *ast.Program) []error { continue } written[lang] = true - scope, err := translationScope(ctx, s.Module) + scope, err := translationScope(ctx, s.Module, s.WithoutMarketplace) if err != nil { continue // a missing module is reported by exec as itself } diff --git a/mdl/grammar/MDLLexer.g4 b/mdl/grammar/MDLLexer.g4 index f7656795f9..984193289b 100644 --- a/mdl/grammar/MDLLexer.g4 +++ b/mdl/grammar/MDLLexer.g4 @@ -749,6 +749,7 @@ FRAGMENTS: F R A G M E N T S; SLOT: S L O T; LANGUAGES: L A N G U A G E S; TRANSLATIONS: T R A N S L A T I O N S; // create/describe translations for +MARKETPLACE: M A R K E T P L A C E; // translations without marketplace (ako/mxcli#970) // ALTER PAGE keywords INSERT: I N S E R T; diff --git a/mdl/grammar/domains/MDLCatalog.g4 b/mdl/grammar/domains/MDLCatalog.g4 index 1993775b82..121bb40372 100644 --- a/mdl/grammar/domains/MDLCatalog.g4 +++ b/mdl/grammar/domains/MDLCatalog.g4 @@ -219,7 +219,7 @@ describeStatement | DESCRIBE DATA TRANSFORMER qualifiedName // DESCRIBE DATA TRANSFORMER Module.Name | DESCRIBE FRAGMENT identifierOrKeyword // DESCRIBE FRAGMENT Name | DESCRIBE JAR DEPENDENCY (qualifiedName | IDENTIFIER) STRING_LITERAL // DESCRIBE JAR DEPENDENCY ModuleName 'group:artifact' - | DESCRIBE TRANSLATIONS (IN identifierOrKeyword)? FOR identifierOrKeyword // DESCRIBE TRANSLATIONS [IN Module] FOR nl_NL + | DESCRIBE TRANSLATIONS (IN identifierOrKeyword | WITHOUT MARKETPLACE)? FOR identifierOrKeyword // DESCRIBE TRANSLATIONS [IN Module | WITHOUT MARKETPLACE] FOR nl_NL // R6: the single-thing reports that were `show` forms. Each builds the // same statement as its `show` spelling (MDL-DEPR090). | DESCRIBE APP SECURITY // DESCRIBE APP SECURITY diff --git a/mdl/grammar/domains/MDLSettings.g4 b/mdl/grammar/domains/MDLSettings.g4 index 57bbd77f0c..e228603e18 100644 --- a/mdl/grammar/domains/MDLSettings.g4 +++ b/mdl/grammar/domains/MDLSettings.g4 @@ -515,7 +515,7 @@ booleanLiteral ; /** - * CREATE [OR MODIFY|REPLACE] TRANSLATIONS [IN Module] FOR ( 'src' AS 'target', ... ); + * CREATE [OR MODIFY|REPLACE] TRANSLATIONS [IN Module | WITHOUT MARKETPLACE] FOR ( 'src' AS 'target', ... ); * * A translation maps a user-provided name to another name, so entries use AS * rather than COLON — the same rule CUSTOM NAME map follows @@ -529,10 +529,16 @@ booleanLiteral * scope is removed — and it is the only way to take a language's translations * out of the model, which `alter settings LANGUAGE remove` points at. It parsed * as an error before, so the documented way to do it did not exist. + * WITHOUT MARKETPLACE keeps an unscoped run out of Marketplace modules, whose + * contents — Atlas page templates and building blocks included — the next + * module update replaces (ako/mxcli#970). Additive: without it the statement + * still reaches the whole project, as it always has, and warns when it wrote + * into a Marketplace module. `IN ` already names one module, so the two + * are alternatives rather than combinable. * See docs/11-proposals/PROPOSAL_translations.md. */ createTranslationsStatement - : TRANSLATIONS (IN identifierOrKeyword)? FOR identifierOrKeyword + : TRANSLATIONS (IN identifierOrKeyword | WITHOUT MARKETPLACE)? FOR identifierOrKeyword LPAREN (translationEntry (COMMA translationEntry)* COMMA?)? RPAREN ; @@ -795,7 +801,7 @@ keyword // CLI commands | BUILD | CATALOG | CHECK | CLEAR | COMMENT | CUSTOM_NAME_MAP | DESIGN | DRY | EXEC | FEATURES | ADDED | SINCE | FORCE - | LANGUAGES | LINT | PROPERTIES | READ | RULES | RUN | SARIF | SCRIPT | TRANSLATIONS + | LANGUAGES | LINT | MARKETPLACE | PROPERTIES | READ | RULES | RUN | SARIF | SCRIPT | TRANSLATIONS | SHOW | USE | STATUS | WRITE | VIA | VIEWS | TABLES // Sequence flow anchors (for @anchor annotation) diff --git a/mdl/translations/project.go b/mdl/translations/project.go index cccd9ddf0f..4585130863 100644 --- a/mdl/translations/project.go +++ b/mdl/translations/project.go @@ -48,6 +48,19 @@ type Stats struct { // The signal that a source string was edited after the file was written — // see SuggestDrift. Unmatched []string + // Written lists every unit Apply wrote, with what it changed there, in + // walk order. A caller that has to say WHERE the writes landed — an + // unscoped run reaching into Marketplace modules (ako/mxcli#970) — needs + // the unit, not just the total. + Written []UnitWrite +} + +// UnitWrite is one unit Apply wrote. +type UnitWrite struct { + ID model.ID + Type string // the unit's stored $Type, e.g. "Forms$PageTemplate" + Set int + Removed int } // Collect returns every translatable text in scope, deduplicated by source @@ -158,6 +171,7 @@ func Apply(p Project, sourceLang, lang string, dict Dictionary, mode Mode, scope stats.Set += us.Set stats.Removed += us.Removed stats.Units++ + stats.Written = append(stats.Written, UnitWrite{ID: u.ID, Type: u.Type, Set: us.Set, Removed: us.Removed}) for _, s := range us.RemovedSources { removed[s] = true } diff --git a/mdl/visitor/visitor_query.go b/mdl/visitor/visitor_query.go index 512b47f566..a37ada6dc0 100644 --- a/mdl/visitor/visitor_query.go +++ b/mdl/visitor/visitor_query.go @@ -815,6 +815,7 @@ func (b *Builder) ExitDescribeStatement(ctx *parser.DescribeStatementContext) { if len(ids) > 1 { stmt.Module = identifierOrKeywordText(ids[0]) } + stmt.WithoutMarketplace = ctx.MARKETPLACE() != nil b.statements = append(b.statements, stmt) return } diff --git a/mdl/visitor/visitor_translations.go b/mdl/visitor/visitor_translations.go index 6271d4507d..f5fc7c7da0 100644 --- a/mdl/visitor/visitor_translations.go +++ b/mdl/visitor/visitor_translations.go @@ -27,6 +27,7 @@ func (b *Builder) ExitCreateTranslationsStatement(ctx *parser.CreateTranslations if len(ids) > 1 { stmt.Module = identifierOrKeywordText(ids[0]) } + stmt.WithoutMarketplace = ctx.MARKETPLACE() != nil if p, ok := ctx.GetParent().(*parser.CreateStatementContext); ok { switch { diff --git a/mdl/visitor/visitor_translations_test.go b/mdl/visitor/visitor_translations_test.go index 1bef8c1980..26ddf12c7b 100644 --- a/mdl/visitor/visitor_translations_test.go +++ b/mdl/visitor/visitor_translations_test.go @@ -66,3 +66,32 @@ func TestCreateTranslations_EntriesStillParse(t *testing.T) { t.Errorf("first entry = %+v", stmt.Entries[0]) } } + +// ako/mxcli#970: `without marketplace` is the additive clause that keeps an +// unscoped run out of Marketplace modules; describe takes it too, so a +// described file round-trips. `marketplace` stays usable as a name. +func TestTranslations_WithoutMarketplaceParses(t *testing.T) { + prog, errs := Build("create or modify translations without marketplace for nl_NL ('Cancel' as 'Annuleren');\n" + + "describe translations without marketplace for nl_NL;\n" + + "create or modify translations for nl_NL ('Cancel' as 'Annuleren');\n" + + "create or modify translations in Marketplace for nl_NL ('Cancel' as 'Annuleren');") + if len(errs) > 0 { + t.Fatalf("parse errors: %v", errs) + } + if len(prog.Statements) != 4 { + t.Fatalf("statements = %d, want 4", len(prog.Statements)) + } + if c := prog.Statements[0].(*ast.CreateTranslationsStmt); !c.WithoutMarketplace || c.Module != "" || len(c.Entries) != 1 { + t.Errorf("create without marketplace = %+v", c) + } + if d := prog.Statements[1].(*ast.DescribeTranslationsStmt); !d.WithoutMarketplace || d.Language != "nl_NL" { + t.Errorf("describe without marketplace = %+v", d) + } + // Control: the unscoped form is unchanged. + if c := prog.Statements[2].(*ast.CreateTranslationsStmt); c.WithoutMarketplace { + t.Errorf("unscoped create read as without marketplace") + } + if c := prog.Statements[3].(*ast.CreateTranslationsStmt); c.WithoutMarketplace || c.Module != "Marketplace" { + t.Errorf("a module named Marketplace = %+v", c) + } +} From d5c2a2c2d2928b063a3d2c22b92472fe75f49cc9 Mon Sep 17 00:00:00 2001 From: Ako Date: Sun, 4 Oct 2026 15:20:15 +0000 Subject: [PATCH 12/17] fix(run-local): supervise the --watch web client bundler (#971) watchAndApply held a bare *WebClientWatcher. A bundler that exited was never restarted, so every later page change failed with "watcher exited"; a failed incremental build (ENOTDIR on web/pages mid-rewrite, measured on 11.13 after adding an entity) dropped the change and the next one too; and ensureClientServed's recovery ran a one-shot production bundle in the same web/ directory while the watcher was still running. bundlerSupervisor now owns the bundler: EnsureAlive restarts an exited one with backoff, AwaitRebuild retries a failed incremental rebuild once with a fresh bundler, and a recovery re-bundle replaces the bundler (stop and reap, then start) instead of running beside it. The bundle-build limit is configurable (--web-client-timeout, MXCLI_WEB_CLIENT_TIMEOUT) and a timeout prints the tail of web-client-build.log. A change that restarts the runtime says that browser sessions were dropped. Co-Authored-By: Claude Opus 5.5 --- .../skills/fix-issue/findings/cmd-mxcli.jsonl | 1 + .claude/skills/mendix/run-local/SKILL.md | 10 + CHANGELOG.md | 1 + cmd/mxcli/cmd_run.go | 3 + cmd/mxcli/docker/clientgate_test.go | 4 +- cmd/mxcli/docker/runlocal.go | 96 ++-- cmd/mxcli/docker/webclient.go | 60 ++- cmd/mxcli/docker/webclient_classic_test.go | 2 +- cmd/mxcli/docker/webclient_pages_test.go | 2 +- cmd/mxcli/docker/webclient_supervisor.go | 245 +++++++++++ cmd/mxcli/docker/webclient_supervisor_test.go | 412 ++++++++++++++++++ cmd/mxcli/docker/webclient_watch.go | 76 +++- docs-site/src/tools/run-local.md | 5 +- 13 files changed, 850 insertions(+), 67 deletions(-) create mode 100644 cmd/mxcli/docker/webclient_supervisor.go create mode 100644 cmd/mxcli/docker/webclient_supervisor_test.go diff --git a/.claude/skills/fix-issue/findings/cmd-mxcli.jsonl b/.claude/skills/fix-issue/findings/cmd-mxcli.jsonl index a8ae7deedc..ba50ac8637 100644 --- a/.claude/skills/fix-issue/findings/cmd-mxcli.jsonl +++ b/.claude/skills/fix-issue/findings/cmd-mxcli.jsonl @@ -158,3 +158,4 @@ {"area": "cmd/mxcli", "date": "2026-10-03", "symptom": "`mxcli report` could not score a project's own modules: `lint` has --modules, `report` had only --exclude", "cause": "Feature gap; and the LintContext module filter alone would not make the score exact, because project-level findings (CONV008 role mappings, project security) carry no module and are reported regardless", "file": "`cmd/mxcli/cmd_report.go`, `mdl/linter/report.go` (`ScopeToModules`, Report.Modules)", "insight": "Filter the scored violations to those located in a selected module, and print the selection in every format so a module score is not mistaken for the project's", "refs": ["ako/mxcli#953"]} {"area": "cmd/mxcli/docker", "date": "2026-10-03", "symptom": "`mxcli docker build` (and docker run / reload, which call it) rewrote an MPRv1 project's .mpr, rewrote every MPRv2 .mxunit (restored with new mtimes), and wrote theme-cache/, deployment/, javasource/ proxies, the .launch file, .classpath and .project into the project; the TUI checker and `mxcli eval`'s mx_check wrote theme-cache/web/ and deployment/sass/ on every run", "cause": "update-widgets ran on the project under a snapshot that restored only v2 storage, and mx check / MxBuild ran on the project itself; the TUI and eval runner ran a bare `mx check `", "file": "`cmd/mxcli/docker/build.go` (`buildOnCopy`), `cmd/mxcli/docker/check.go` (`MxCheckOnCopy`), `cmd/mxcli/tui/checker.go` (`runMxCheck`), `cmd/mxcli/evalrunner/checks.go` (`checkMxCheck`)", "insight": "Run update-widgets, mx check and MxBuild on one temporary copy (copyProjectToTemp) and write only the output directory. Measured on the 11.14 testapp: the PAD from the copy differs from the in-place build in the same 9 files as two in-place builds of identical copies differ (cache-bust stamps, operation ids, the native metro paths), so the output is equivalent; rebuild time unchanged (58s vs 59s), because a PAD build gains nothing from the project's deployment/. 11.14's blank app needs JDK 25, so the real-mx build test skips without it; run it with 11.13's mx.", "refs": ["ako/mxcli#961", "ako/mxcli#951", "mendixlabs/mxcli#808"]} {"area": "cmd/mxcli", "date": "2026-10-03", "symptom": "ako/mxcli#962 item 3: in VS Code, two calls to a stored void JavaScript action with the same output name (Studio Pro's `$ReturnValueName`/`$RefreshEntity` shape) are squiggled MDL063, while `mxcli check -p` passes them", "cause": "runSemanticValidation called executor.ValidateMicroflow/ValidateNanoflow, which pass a nil void-action resolver: without the project every call output counts as a declaration", "file": "`cmd/mxcli/lsp_diagnostics.go` (runSemanticValidation); `mdl/executor/validate_void_code_calls.go` (FlowRules, CodeActionCache)", "fix": "executor.NewFlowRules(prog, s.findMprPath(), s.codeActions): resolves actions through the script and the workspace project (opened lazily), treats an unresolvable action as possibly void (unknownIsVoid) for MDL063 but never for MDL093, and shares project answers across keystrokes for 30s since one action read costs ~300ms on PedApp", "insight": "An entry point that wraps a richer internal API with nil arguments (ValidateMicroflow = validateMicroflowWith(stmt, nil)) silently gives every caller the weakest behaviour; a fix landed in the richer API (#958) does not reach them. Grep the exported wrapper's callers when the internal one gains a parameter. Measure the cost before putting project I/O on a keystroke path", "test": "`cmd/mxcli/lsp_void_calls_test.go` (PedApp: void pair clean with and without project, Boolean pair control, MDL093 only with project); `mdl/executor/validate_void_code_calls_test.go` (TestCodeActionCache_SharesProjectAnswersAcrossRuns)"} +{"area": "cmd/mxcli", "date": "2026-10-04", "symptom": "`run --local --watch` (Mendix <= 11.13, rollup bundler): after a while every page change fails with `web client rebuild failed: web client watcher exited` (or `client bundle not served after apply: web client re-bundle: web client build timed out after 5m0s`) and nothing reaches the browser until `run --local` is restarted; after adding an entity the next changes fail with `ENOTDIR: not a directory, stat '.../web/pages/.js/package.json'`", "cause": "watchAndApply held a bare *WebClientWatcher: (1) once it exited nothing restarted it (only recoverMissingPages did), so WaitForRebuild failed every later change; (2) a failed incremental build (rollup's commonjs resolver hitting web/pages mid-rewrite by the serve build) left the watcher erroring and the change was dropped; (3) ensureClientServed's recovery ran a one-shot NODE_ENV=production BuildWebClient in the same web/ dir while the watcher was still running — two rollups on web/dist; (4) the 5m limit was hard-coded and a timeout printed nothing about why", "file": "cmd/mxcli/docker/webclient_supervisor.go, cmd/mxcli/docker/runlocal.go (watchAndApply, ensureClientServed), cmd/mxcli/docker/webclient.go (webClientTimeout, webClientBuildLogTail)", "fix": "bundlerSupervisor owns the bundler: EnsureAlive restarts an exited one (backoff 2s..60s after failed starts), AwaitRebuild retries a failed/aborted incremental rebuild once with a fresh bundler, Rebundle = stop+reap then start (never two). ensureClientServed takes the rebundle func; clientRebundler hands it the supervisor under --watch and the one-shot otherwise. --web-client-timeout / MXCLI_WEB_CLIENT_TIMEOUT; timeout errors append the last 30 lines of deployment/log/web-client-build.log. sessionNotice prints that a restart dropped sessions", "insight": "Killing the runner (`kill `) reproduces the dead-watcher state in seconds — no need to wait for it to die on its own. A fresh bundler is the universal recovery under --watch: its first build is a full bundle of the current source, so it covers missing pages, dangling chunks, a dist/ deleted by Gradle, and a transient incremental failure alike, without a second process on web/dist. Note: exec.Cmd.Wait called a second time concurrently with the reaper did block until exit on this Go version, so the old Stop was not the overlap — the one-shot in ensureClientServed was", "test": "cmd/mxcli/docker/webclient_supervisor_test.go (TestBundlerSupervisor_RestartsExitedBundler, _AwaitRebuildRecovers, _RebundleNeverOverlaps, TestEnsureClientServed_WatchModeRebundleIsExclusive, TestBuildWebClient_TimeoutShowsLogTail); live: 11.13 scratch app, 7 consecutive changes incl. a killed runner"} diff --git a/.claude/skills/mendix/run-local/SKILL.md b/.claude/skills/mendix/run-local/SKILL.md index 847c09fb89..64501fdec5 100644 --- a/.claude/skills/mendix/run-local/SKILL.md +++ b/.claude/skills/mendix/run-local/SKILL.md @@ -220,6 +220,7 @@ Launch `run --local` as the **sole** command in its invocation (don't chain a tr | `--hub-secret` | — | Shared auth (`user:pass`) matching an **open** hub's `--secret` | | *(hub API key)* | — | For an **authenticated** hub: get one from `https:///cli`, set `MXCLI_HUB_KEY` (see below) | | `--watch` | off | Rebuild + hot-apply on each change | +| `--web-client-timeout` | `$MXCLI_WEB_CLIENT_TIMEOUT`, else `5m` | Limit for one web client bundle build; on timeout the tail of `deployment/log/web-client-build.log` is printed | | `--ensure-db` | off | Provision local Postgres + app database if missing | | `--setup` | off | Cache MxBuild+runtime + ensure DB, then exit (SessionStart bring-up) | | `--screenshot` | off | Playwright PNG after boot + each change | @@ -301,6 +302,15 @@ Playwright + the devcontainer's Chromium). `mxbuild --serve`): a page/widget edit re-bundles in ~3–4 s; a microflow/entity edit skips the bundle and just hot-reloads. It uses `CHOKIDAR_USEPOLLING` because inotify is silent on container filesystems. + The bundler is supervised: one that **exits** is restarted on the next change + (`web client bundler exited unexpectedly; restarting it...`, with backoff when it + cannot start), and an incremental rebuild that **fails** is retried once with a + fresh bundler before the change is reported as failed. A recovery re-bundle + (missing page, dangling chunk, `/dist/index.js` gone after a restart) replaces the + bundler — two rollups never write `web/dist` at once. +- **A structural change restarts the runtime and drops every browser session.** The + loop prints `runtime restarted for this change — browser sessions were dropped; + log in again`; a browser test that sees it must log in before the next step. - Without `--watch`, a single one-shot bundle (~7 s) runs before boot. - **The bundle is re-checked after the boot**, because bundling before it is not enough: the runtime's boot runs Gradle `clean-custom-classes compile package`, diff --git a/CHANGELOG.md b/CHANGELOG.md index 8a83d6a4e7..ee8b6c2a6b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -56,6 +56,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). ### Fixed +- **`run --local --watch` keeps its web client bundler alive and never runs two** (ako/mxcli#971) — a bundler that exited stayed dead, so every later page change failed with `web client watcher exited` and only restarting `run --local` recovered (reproduced on Mendix 11.13 by killing the runner: two changes in a row failed). It is now restarted on the next change, with backoff when it cannot start. An incremental rebuild that fails — measured after adding an entity: `ENOTDIR … web/pages/.js/package.json`, and the next change failed the same way — is retried once with a fresh bundler. A recovery re-bundle (a missing page, a dangling chunk, or `/dist/index.js` gone after a runtime restart) replaces the bundler instead of running a one-shot production bundle beside it in the same `web/` directory. The bundle-build limit is configurable (`--web-client-timeout`, or `MXCLI_WEB_CLIENT_TIMEOUT`; default 5m), and a timeout prints the tail of `deployment/log/web-client-build.log`. A change that restarts the runtime now says that browser sessions were dropped. - **`list workflows` and `show structure` count every activity of a workflow** (ako/mxcli#963) — including those on a boundary-event path and in an event sub-process, as the catalog's `workflows_data` has since #937. TestApp `Workflow1` listed 5 activities where the catalog counted 8; all three now share one walk (`wfnames.WalkActivities`). - **`docker build`, the TUI checker and `mxcli eval` no longer modify the project** (ako/mxcli#961) — `docker build` (and `docker run` / `docker reload`, which use it) now runs `mx update-widgets`, `mx check` and MxBuild on one temporary copy and writes only its output directory (`.docker/build/` or `-o`). Before, it rewrote an MPR v1 project's `.mpr`, rewrote every MPR v2 `.mxunit` (restored with new mtimes), and wrote `theme-cache/`, `deployment/`, `javasource/` proxies, the `.launch` file, `.classpath` and `.project` into the project. The build still uses the widget-normalised model, from the copy, and the package it produces is the same; `mxcli fix widgets` applies the normalisation to the project. **The project's `deployment/` is no longer refreshed by `docker build`** — `run --local` builds its own. The TUI auto-check and eval's `mx_check` ran a plain `mx check` on the project, which wrote `theme-cache/web/` and `deployment/sass/` every time; they now check a copy too. - **MPR010 no longer tells a native page to use a layout grid** (ako/mxcli#962) — the "wrap the form in a layoutgrid" advice is about Bootstrap columns, which a native page does not have: a form DataView directly on an `Atlas_Core.NativePhone_Default` page builds clean in mxbuild 11.13.0, and following the advice there is CE6858 ("update Atlas UI … to use Layout Grid on Native pages"). `lint` skips pages on a native layout and snippets of type Native (MPR010 now needs the full catalog, which a default `lint` builds anyway); `check -p` skips pages whose layout the project says is native. diff --git a/cmd/mxcli/cmd_run.go b/cmd/mxcli/cmd_run.go index aec8758042..cd9e63be92 100644 --- a/cmd/mxcli/cmd_run.go +++ b/cmd/mxcli/cmd_run.go @@ -143,6 +143,7 @@ Examples: } watch, _ := cmd.Flags().GetBool("watch") + webClientTimeout, _ := cmd.Flags().GetDuration("web-client-timeout") testEndpoint, _ := cmd.Flags().GetBool("test-endpoint") ensureDB, _ := cmd.Flags().GetBool("ensure-db") setupOnly, _ := cmd.Flags().GetBool("setup") @@ -208,6 +209,7 @@ Examples: ServePort: servePort, MxBuildPath: mxbuildPath, Watch: watch, + WebClientTimeout: webClientTimeout, EnsureDB: ensureDB, SetupOnly: setupOnly, Screenshot: screenshot, @@ -313,6 +315,7 @@ func init() { runCmd.Flags().String("hub-worktree", "", "Worktree label to distinguish multiple worktrees of one branch") runCmd.Flags().String("hub-session", "", "Session id to group this preview under in the hub overview (default: CLAUDE_CODE_REMOTE_SESSION_ID / MXCLI_HUB_SESSION)") runCmd.Flags().Bool("watch", false, "Rebuild and hot-apply on every project change") + runCmd.Flags().Duration("web-client-timeout", 0, "Limit for one web client bundle build, e.g. 10m (default $MXCLI_WEB_CLIENT_TIMEOUT, else 5m); on timeout the tail of deployment/log/web-client-build.log is printed") runCmd.Flags().Bool("test-endpoint", false, "Host mxcli's token-guarded test endpoint so 'mxcli test --attach' can run tests against this app without booting its own runtime (removed on exit)") runCmd.Flags().Bool("ensure-db", false, "Provision the local Postgres + app database if missing (fresh-session bootstrap)") runCmd.Flags().Bool("setup", false, "Prepare prerequisites (cache MxBuild+runtime, ensure DB) and exit without booting — for a SessionStart hook") diff --git a/cmd/mxcli/docker/clientgate_test.go b/cmd/mxcli/docker/clientgate_test.go index 75a570f280..42b7398d78 100644 --- a/cmd/mxcli/docker/clientgate_test.go +++ b/cmd/mxcli/docker/clientgate_test.go @@ -73,7 +73,7 @@ func TestEnsureClientServed_NoRecoveryWhenServed(t *testing.T) { })) defer srv.Close() - if err := ensureClientServed(dir, srv.URL+"/", "/nonexistent/mxbuild", io.Discard); err != nil { + if err := ensureClientServed(dir, srv.URL+"/", bogusOneShot(dir), io.Discard); err != nil { t.Fatalf("a present+served bundle should need no recovery, got: %v", err) } } @@ -93,7 +93,7 @@ func TestEnsureClientServed_RecoversWhenNotServed(t *testing.T) { })) defer srv.Close() - err := ensureClientServed(dir, srv.URL+"/", "/nonexistent/mxbuild", io.Discard) + err := ensureClientServed(dir, srv.URL+"/", bogusOneShot(dir), io.Discard) if err == nil { t.Fatal("expected recovery to run and fail (no rollup.config.mjs)") } diff --git a/cmd/mxcli/docker/runlocal.go b/cmd/mxcli/docker/runlocal.go index 143bfff469..cdb4dac383 100644 --- a/cmd/mxcli/docker/runlocal.go +++ b/cmd/mxcli/docker/runlocal.go @@ -99,6 +99,9 @@ type LocalRunOptions struct { SetupOnly bool // PollInterval is how often Watch checks for changes (default 1s). PollInterval time.Duration + // WebClientTimeout bounds one web client bundle build. Zero falls back to + // $MXCLI_WEB_CLIENT_TIMEOUT, then 5m (see webClientTimeout). + WebClientTimeout time.Duration // Screenshot, when set, captures a PNG of the app after boot and after each // applied change (requires the Playwright CLI + a browser). Screenshot bool @@ -685,22 +688,26 @@ func RunLocal(opts LocalRunOptions) error { // /dist/index.js and renders blank. In --watch we keep an incremental bundler // hot (warm rollup graph + polling file detection -> ~3-4s re-bundles); // otherwise a single one-shot build (~7s cold) suffices. - var watcher *WebClientWatcher + var bundler *bundlerSupervisor if opts.Watch { fmt.Fprintln(w, "Starting incremental web client bundler...") // A nil watcher is not a failure: on Mendix 11.14+ mxbuild's serve build - // writes web/dist itself, so there is no bundler to keep hot. Every - // watcher method is nil-safe, so the watch loop needs no branch. - watcher, err = StartWebClientWatch(WebClientOptions{DeployDir: opts.DeployDir, MxBuildPath: mxbuildPath, Stdout: w}) + // writes web/dist itself, so there is no bundler to keep hot. The + // supervisor is a no-op then, so the watch loop needs no branch. + watcher, err := StartWebClientWatch(WebClientOptions{DeployDir: opts.DeployDir, MxBuildPath: mxbuildPath, Timeout: opts.WebClientTimeout, Stdout: w}) if err != nil { return fmt.Errorf("starting web client bundler: %w", err) } - // Stop whichever watcher is current at exit: watchAndApply replaces it - // when a newly added page needs a fresh bundler (see missingPageChunks). - defer func() { _ = watcher.Stop() }() + // From here on the supervisor owns the bundler: it restarts one that + // exits and replaces it for a recovery re-bundle (#971), so stop + // whichever one is current at exit. + bundler = newBundlerSupervisor(watcher, func() (*WebClientWatcher, error) { + return StartWebClientWatch(WebClientOptions{DeployDir: opts.DeployDir, MxBuildPath: mxbuildPath, Timeout: opts.WebClientTimeout, Stdout: io.Discard}) + }, w) + defer bundler.Stop() } else { fmt.Fprintln(w, "Bundling web client...") - if err := BuildWebClient(WebClientOptions{DeployDir: opts.DeployDir, MxBuildPath: mxbuildPath, Stdout: w}); err != nil { + if err := BuildWebClient(WebClientOptions{DeployDir: opts.DeployDir, MxBuildPath: mxbuildPath, Timeout: opts.WebClientTimeout, Stdout: w}); err != nil { return fmt.Errorf("bundling web client: %w", err) } } @@ -797,7 +804,7 @@ func RunLocal(opts LocalRunOptions) error { // wrote seconds ago. Verify after the boot, because before it proves nothing // (mxcli-formula1 §35). Costs a stat when the bundle survived. if _, err := EnsureWebClientBundle(WebClientOptions{ - DeployDir: opts.DeployDir, MxBuildPath: mxbuildPath, Stdout: w, + DeployDir: opts.DeployDir, MxBuildPath: mxbuildPath, Timeout: opts.WebClientTimeout, Stdout: w, }); err != nil { // The app is up and its services answer; only the browser is broken. Say so // and keep running rather than tearing down a working runtime. @@ -927,7 +934,7 @@ func RunLocal(opts LocalRunOptions) error { // 7. Stay up until interrupted. With --watch, rebuild + hot-apply on every // project change; otherwise just keep the runtime serving. if opts.Watch { - return watchAndApply(opts, serve, rt, &watcher, mxbuildPath) + return watchAndApply(opts, serve, rt, bundler, mxbuildPath) } fmt.Fprintln(w, "(run with --watch to rebuild and hot-apply on changes; Ctrl-C to stop)") if waitForInterruptOrExit(rt.Exited()) { @@ -1170,7 +1177,11 @@ func clientBundleServedWithin(appURL string, window time.Duration) bool { // present+served, fall back to the reliable synchronous one-shot bundle (the same // path the non-watch boot uses) and re-probe. A no-op when the bundle is already // served (a pure model reload never touches web/dist). -func ensureClientServed(deployDir, appURL, mxbuildPath string, out io.Writer) error { +// +// rebundle performs the recovery bundle. Under --watch it must be the bundler +// supervisor's (see clientRebundler): a one-shot here ran a second rollup on +// web/dist alongside the incremental watcher (#971). +func ensureClientServed(deployDir, appURL string, rebundle func() error, out io.Writer) error { // Nothing below applies to the classic (Dojo) client: it has no bundle, no // chunks, and never serves /dist/index.js. Fixing only the boot path would // have moved this failure to every applied change under --watch rather than @@ -1185,7 +1196,7 @@ func ensureClientServed(deployDir, appURL, mxbuildPath string, out io.Writer) er if dangling := danglingClientChunks(deployDir); len(dangling) > 0 { fmt.Fprintf(out, " client bundle inconsistent — index.js imports %d chunk(s) that are not on disk (%s); re-bundling web client...\n", len(dangling), strings.Join(dangling, ", ")) - if err := BuildWebClient(WebClientOptions{DeployDir: deployDir, MxBuildPath: mxbuildPath, Stdout: out}); err != nil { + if err := rebundle(); err != nil { return fmt.Errorf("web client re-bundle: %w", err) } if still := danglingClientChunks(deployDir); len(still) > 0 { @@ -1196,16 +1207,14 @@ func ensureClientServed(deployDir, appURL, mxbuildPath string, out io.Writer) er // A page module with no bundled chunk is likewise invisible to the index.js // probe: pages are loaded by dynamic import. A one-shot bundle re-globs // web/pages and emits it. See missingPageChunks. - if _, err := recoverMissingPages(deployDir, func() error { - return BuildWebClient(WebClientOptions{DeployDir: deployDir, MxBuildPath: mxbuildPath, Stdout: out}) - }, out); err != nil { + if _, err := recoverMissingPages(deployDir, rebundle, out); err != nil { return fmt.Errorf("web client re-bundle: %w", err) } if clientBundlePresent(deployDir) && clientBundleServedWithin(appURL, clientProbeWindow) { return nil } fmt.Fprintln(out, " /dist/index.js not served after apply; re-bundling web client...") - if err := BuildWebClient(WebClientOptions{DeployDir: deployDir, MxBuildPath: mxbuildPath, Stdout: out}); err != nil { + if err := rebundle(); err != nil { return fmt.Errorf("web client re-bundle: %w", err) } if !clientBundleServedWithin(appURL, clientProbeWindow) { @@ -1263,19 +1272,13 @@ func settleSourceWith(projectPath string, seen time.Time, sigCh <-chan os.Signal // watchAndApply polls the project for changes and applies each rebuild until the // user interrupts (Ctrl-C). StartLocalRuntime already resolved the JVM; here we // only rebuild via serve and let the RuntimeController decide reload vs restart. -func watchAndApply(opts LocalRunOptions, serve *ServeServer, rt *LocalRuntime, watcherRef **WebClientWatcher, mxbuildPath string) error { +func watchAndApply(opts LocalRunOptions, serve *ServeServer, rt *LocalRuntime, bundler *bundlerSupervisor, mxbuildPath string) error { w := opts.Stdout - watcher := *watcherRef - // restartWatcher replaces the incremental bundler with a fresh one, whose - // first build re-globs web/pages. The caller's reference is updated so it - // stops the live watcher, not the one replaced here. - restartWatcher := func() error { - _ = watcher.Stop() - next, err := StartWebClientWatch(WebClientOptions{DeployDir: opts.DeployDir, MxBuildPath: mxbuildPath, Stdout: io.Discard}) - watcher = next - *watcherRef = next - return err - } + // Every recovery re-bundle goes through the supervisor under --watch, so a + // one-shot never runs alongside the incremental bundler (#971). + rebundle := clientRebundler(bundler, WebClientOptions{ + DeployDir: opts.DeployDir, MxBuildPath: mxbuildPath, Timeout: opts.WebClientTimeout, Stdout: opts.Stdout, + }) sigCh := make(chan os.Signal, 1) signal.Notify(sigCh, syscall.SIGINT, syscall.SIGTERM) @@ -1328,10 +1331,20 @@ func watchAndApply(opts LocalRunOptions, serve *ServeServer, rt *LocalRuntime, w fmt.Fprintf(w, "Change detected, rebuilding (build #%d)...\n", gen) start := time.Now() + // A bundler that exited since the last change is restarted here + // rather than waited on: waiting on a dead one failed every later + // change with "watcher exited" until `run --local` was restarted + // (#971). Its first build bundles the current source, so whatever it + // missed while down is in it. + bundled, err := bundler.EnsureAlive() + if err != nil { + fmt.Fprintf(opts.Stderr, " %v\n", err) + } + // Capture the client-bundle generation and the web-source mtime before // the serve build, so we can tell whether the change plausibly touched // client source and, if so, wait for the incremental re-bundle. - genBefore := watcher.Generation() + genBefore := bundler.Generation() webBefore := webClientSourceMTime(opts.DeployDir) build, err := serve.Build(BuildRequest{Target: TargetDeploy, ProjectFilePath: opts.ProjectPath}) @@ -1360,24 +1373,24 @@ func watchAndApply(opts LocalRunOptions, serve *ServeServer, rt *LocalRuntime, w // no rebuild materializes (the touched file isn't a rollup input — e.g. a // microflow edit that rewrites a web metadata file but no page/widget), so // this never hangs. A pure model change skips the wait entirely. - bundled := false if webClientSourceMTime(opts.DeployDir).After(webBefore) { // Detection is a reliable ~1s with polling, so a 2.5s settle is ample // margin to catch a rebuild that's going to start, while keeping the // no-rebuild case (a model edit that only grazed web/) snappy. - bundled, err = watcher.WaitForRebuild(genBefore, 2500*time.Millisecond, 90*time.Second) + rebuilt, err := bundler.AwaitRebuild(genBefore, 2500*time.Millisecond, 90*time.Second, opts.Stderr) if err != nil { fmt.Fprintf(opts.Stderr, " web client rebuild failed: %v\n", err) continue } + bundled = bundled || rebuilt } // The incremental bundler never picks up a page added after it // started (mxbuild's pages plugin globs once), so a new page would // 404 in the browser while everything above reports success. A fresh // bundler re-globs. Only meaningful with a live watcher; without one, // ensureClientServed's one-shot covers the same check. - if watcher != nil { - restarted, err := recoverMissingPages(opts.DeployDir, restartWatcher, w) + if bundler.Wanted() { + restarted, err := recoverMissingPages(opts.DeployDir, bundler.Restart, w) if err != nil { fmt.Fprintf(opts.Stderr, " %v\n", err) continue @@ -1390,17 +1403,20 @@ func watchAndApply(opts LocalRunOptions, serve *ServeServer, rt *LocalRuntime, w fmt.Fprintf(opts.Stderr, " apply (%s) failed: %v\n", action, err) continue } + if note := sessionNotice(action); note != "" { + fmt.Fprintln(w, note) + } // Gate the "applied" report on the browser bundle actually being served: // a structural change can leave the restarted runtime 404ing // /dist/index.js (see ensureClientServed). This recovers before we tell // the user the build is live. - if err := ensureClientServed(opts.DeployDir, rt.AppURL(), mxbuildPath, opts.Stdout); err != nil { + if err := ensureClientServed(opts.DeployDir, rt.AppURL(), rebundle, opts.Stdout); err != nil { fmt.Fprintf(opts.Stderr, " client bundle not served after apply: %v\n", err) continue } client := "" if bundled { - client = fmt.Sprintf(", client re-bundled (gen %d)", watcher.Generation()) + client = fmt.Sprintf(", client re-bundled (gen %d)", bundler.Generation()) } fmt.Fprintf(w, " build #%d applied via %s in %s%s -> %s\n", gen, action, time.Since(start).Round(time.Millisecond), client, rt.AppURL()) maybeScreenshot(opts, rt) @@ -1411,6 +1427,16 @@ func watchAndApply(opts LocalRunOptions, serve *ServeServer, rt *LocalRuntime, w } } +// sessionNotice is the line printed after an apply that restarted the runtime: +// a restart drops every browser session, which a reload does not, and a browser +// test that is not told so fails on a login page it did not expect (#971). +func sessionNotice(action ApplyAction) string { + if action != ActionRestart { + return "" + } + return " runtime restarted for this change — browser sessions were dropped; log in again" +} + // declaredJarDependencies collects the managed Java dependency coordinates the // model declares, across every module. func declaredJarDependencies(reader backend.FullBackend) []JarDependencyRef { diff --git a/cmd/mxcli/docker/webclient.go b/cmd/mxcli/docker/webclient.go index 4cf7f078f7..5bb7b2c18a 100644 --- a/cmd/mxcli/docker/webclient.go +++ b/cmd/mxcli/docker/webclient.go @@ -9,6 +9,7 @@ import ( "os/exec" "path/filepath" "runtime" + "strings" "time" ) @@ -32,7 +33,8 @@ type WebClientOptions struct { // MxBuildPath is /modeler/mxbuild; the node tooling is resolved from // its sibling tools/node directory. MxBuildPath string - // Timeout bounds the bundle build (default 5m). + // Timeout bounds the bundle build. Zero means webClientTimeout's default: + // $MXCLI_WEB_CLIENT_TIMEOUT when set, else 5m. Timeout time.Duration // Stdout receives a short progress line (default discarded). Stdout io.Writer @@ -123,10 +125,7 @@ func BuildWebClient(opts WebClientOptions) error { if err != nil { return err } - timeout := opts.Timeout - if timeout == 0 { - timeout = 5 * time.Minute - } + timeout := webClientTimeout(opts.Timeout) start := time.Now() cmd := exec.Command(nodeBin, runner) @@ -152,7 +151,10 @@ func BuildWebClient(opts WebClientOptions) error { case <-time.After(timeout): _ = cmd.Process.Kill() <-done - return fmt.Errorf("web client build timed out after %s", timeout) + // A bare "timed out" left the user nothing to act on (#971): say how to + // raise the limit and show what the bundler was doing when it was cut off. + return fmt.Errorf("web client build timed out after %s (raise it with --web-client-timeout or %s)%s", + timeout, webClientTimeoutEnv, webClientBuildLogTail(opts.DeployDir)) } if !WebClientBundled(opts.DeployDir) { @@ -163,6 +165,52 @@ func BuildWebClient(opts WebClientOptions) error { return nil } +// defaultWebClientTimeout bounds one bundle build when nothing overrides it. +const defaultWebClientTimeout = 5 * time.Minute + +// webClientTimeoutEnv overrides defaultWebClientTimeout (a Go duration, e.g. +// "10m"). The --web-client-timeout flag of `run --local` wins over it. +const webClientTimeoutEnv = "MXCLI_WEB_CLIENT_TIMEOUT" + +// webClientTimeout resolves the bundle-build limit: an explicit value first, +// then $MXCLI_WEB_CLIENT_TIMEOUT, then 5m. An unparsable or non-positive +// environment value is ignored rather than fatal — it bounds a build, and the +// default is a working bound. +func webClientTimeout(explicit time.Duration) time.Duration { + if explicit > 0 { + return explicit + } + if v := strings.TrimSpace(os.Getenv(webClientTimeoutEnv)); v != "" { + if d, err := time.ParseDuration(v); err == nil && d > 0 { + return d + } + } + return defaultWebClientTimeout +} + +// webClientLogTailLines is how much of web-client-build.log a timeout shows. +const webClientLogTailLines = 30 + +// webClientBuildLogTail returns the last lines of the bundler's own log +// (deployment/log/web-client-build.log), formatted to append to an error, or "" +// when there is no log. The runner writes its progress there, not to stdout, so +// it is where a stalled bundle shows which phase it stalled in. +func webClientBuildLogTail(deployDir string) string { + path := filepath.Join(deployDir, "log", "web-client-build.log") + data, err := os.ReadFile(path) + if err != nil { + return "" + } + lines := strings.Split(strings.TrimRight(string(data), "\n"), "\n") + if len(lines) == 1 && lines[0] == "" { + return "" + } + if len(lines) > webClientLogTailLines { + lines = lines[len(lines)-webClientLogTailLines:] + } + return fmt.Sprintf("\n last %d line(s) of %s:\n %s", len(lines), path, strings.Join(lines, "\n ")) +} + // webClientBundlePath is the one file whose absence is the black screen: the // shell loads, paints the theme's background, and never starts the client. func webClientBundlePath(deployDir string) string { diff --git a/cmd/mxcli/docker/webclient_classic_test.go b/cmd/mxcli/docker/webclient_classic_test.go index 18c1e94eec..4d5fdd9d70 100644 --- a/cmd/mxcli/docker/webclient_classic_test.go +++ b/cmd/mxcli/docker/webclient_classic_test.go @@ -167,7 +167,7 @@ func TestEnsureWebClientBundleSkipsClassic(t *testing.T) { // rather than removed it. appURL is deliberately unreachable: reaching the probe // at all is the defect. func TestEnsureClientServedSkipsClassic(t *testing.T) { - if err := ensureClientServed(classicDeployment(t), "http://127.0.0.1:1", "", io.Discard); err != nil { + if err := ensureClientServed(classicDeployment(t), "http://127.0.0.1:1", bogusOneShot(""), io.Discard); err != nil { t.Fatalf("classic deployment must not be probed for a bundle it has no concept of: %v", err) } } diff --git a/cmd/mxcli/docker/webclient_pages_test.go b/cmd/mxcli/docker/webclient_pages_test.go index bd580ca9e5..1746315a3c 100644 --- a/cmd/mxcli/docker/webclient_pages_test.go +++ b/cmd/mxcli/docker/webclient_pages_test.go @@ -163,7 +163,7 @@ func TestEnsureClientServed_RecoversWhenAPageIsMissing(t *testing.T) { })) defer srv.Close() - err := ensureClientServed(deploy, srv.URL+"/", "/nonexistent/mxbuild", io.Discard) + err := ensureClientServed(deploy, srv.URL+"/", bogusOneShot(deploy), io.Discard) if err == nil || !strings.Contains(err.Error(), "re-bundle") { t.Fatalf("expected the re-bundle branch to run for a missing page chunk, got: %v", err) } diff --git a/cmd/mxcli/docker/webclient_supervisor.go b/cmd/mxcli/docker/webclient_supervisor.go new file mode 100644 index 0000000000..d45fb9257a --- /dev/null +++ b/cmd/mxcli/docker/webclient_supervisor.go @@ -0,0 +1,245 @@ +// SPDX-License-Identifier: Apache-2.0 + +package docker + +import ( + "fmt" + "io" + "strings" + "time" +) + +// webclient_supervisor.go owns the incremental web client bundler for the +// lifetime of a `run --local --watch` loop (#971). Two things went wrong while the +// loop held a bare *WebClientWatcher: +// +// - A bundler that exited stayed dead. Every later change waited on it, got +// "watcher exited", and was skipped — the edit never reached the browser and +// only restarting `run --local` recovered. +// - A recovery re-bundle (a dangling chunk, or /dist/index.js gone after a +// runtime restart) ran a one-shot production bundle in the same web/ +// directory while the watcher was still running, so two rollup processes +// wrote web/dist at once. +// +// The supervisor is the one place a bundler is started or stopped, so both are +// structural: Rebundle replaces the watcher (stop, then start — never both +// running), and EnsureAlive restarts one that has exited, with backoff. + +// clientBundler is the part of *WebClientWatcher the watch loop drives — a seam +// so the lifecycle is testable with fakes. +type clientBundler interface { + Generation() int + WaitForRebuild(sinceGen int, settle, buildTimeout time.Duration) (bool, error) + Exited() bool + Log() string + Stop() error +} + +// bundlerSupervisor keeps one incremental bundler alive. +type bundlerSupervisor struct { + // start launches a fresh bundler and blocks until its first bundle. A nil + // bundler with a nil error means the deployment needs none. + start func() (clientBundler, error) + out io.Writer + + cur clientBundler // the live bundler; nil when none is running + want bool // the deployment has an incremental bundler to keep alive + + failures int // consecutive failed starts + nextTry time.Time // no restart attempt before this (backoff) + now func() time.Time +} + +// newBundlerSupervisor wraps the bundler the boot started. initial may be nil +// (Mendix 11.14+ or the classic client): the supervisor then has nothing to keep +// alive and every method is a no-op. +func newBundlerSupervisor(initial *WebClientWatcher, start func() (*WebClientWatcher, error), out io.Writer) *bundlerSupervisor { + s := &bundlerSupervisor{ + start: func() (clientBundler, error) { + wc, err := start() + if wc == nil { + return nil, err // a typed nil must not become a non-nil interface + } + return wc, err + }, + out: out, + now: time.Now, + } + if initial != nil { + s.cur, s.want = initial, true + } + return s +} + +// Generation is the live bundler's success count, 0 when none is running. +func (s *bundlerSupervisor) Generation() int { + if s == nil || s.cur == nil { + return 0 + } + return s.cur.Generation() +} + +// WaitForRebuild waits on the live bundler; with none it has nothing to wait for. +func (s *bundlerSupervisor) WaitForRebuild(sinceGen int, settle, buildTimeout time.Duration) (bool, error) { + if s == nil || s.cur == nil { + return false, nil + } + return s.cur.WaitForRebuild(sinceGen, settle, buildTimeout) +} + +// Wanted reports whether this deployment has an incremental bundler at all. +func (s *bundlerSupervisor) Wanted() bool { return s != nil && s.want } + +// Restart replaces the bundler: the old one is stopped and reaped BEFORE the new +// one starts, so two bundlers never write web/dist at once. The new bundler's +// first build is a full bundle of the current source (re-globbing web/pages), so +// it doubles as the recovery re-bundle. Returns once that bundle has landed. +func (s *bundlerSupervisor) Restart() error { + if s == nil || !s.want { + return nil + } + if s.cur != nil { + _ = s.cur.Stop() + s.cur = nil + } + next, err := s.start() + if err != nil { + if next != nil { + _ = next.Stop() + } + s.failures++ + s.nextTry = s.now().Add(bundlerRestartBackoff(s.failures)) + return err + } + if next == nil { + // The deployment no longer has a bundler to keep (it changed shape, e.g. + // an upgrade to a Mendix that bundles itself). Nothing to supervise. + s.want = false + return nil + } + s.cur = next + s.failures = 0 + s.nextTry = time.Time{} + return nil +} + +// Rebundle is the recovery re-bundle under --watch: a fresh bundler, never a +// second one alongside the first. +func (s *bundlerSupervisor) Rebundle() error { return s.Restart() } + +// EnsureAlive restarts the bundler when it has exited (or a previous start +// failed). It reports whether it restarted one. Within the backoff window after +// a failed start it does not retry and returns an error saying when it will. +func (s *bundlerSupervisor) EnsureAlive() (bool, error) { + if s == nil || !s.want { + return false, nil + } + if s.cur != nil && !s.cur.Exited() { + return false, nil + } + if wait := s.nextTry.Sub(s.now()); wait > 0 { + return false, fmt.Errorf("web client bundler is down after %d failed restart(s); next attempt in %s", + s.failures, wait.Round(time.Second)) + } + if s.cur != nil { + fmt.Fprintf(s.out, " web client bundler exited unexpectedly; restarting it...%s\n", indentTail(s.cur.Log(), 10)) + } else { + fmt.Fprintf(s.out, " restarting web client bundler (attempt %d)...\n", s.failures+1) + } + if err := s.Restart(); err != nil { + return false, fmt.Errorf("restarting web client bundler (next attempt in %s): %w", + bundlerRestartBackoff(s.failures), err) + } + if s.want { + fmt.Fprintln(s.out, " web client bundler restarted") + } + return s.want, nil +} + +// AwaitRebuild waits for the incremental re-bundle a serve build triggered, and +// recovers when the bundler cannot deliver it. Two failures used to drop the +// change outright (#971): +// +// - the bundler exited (killed, crashed) — every later change then failed with +// "watcher exited" too; +// - the incremental build failed on a transient state of the source the serve +// build was rewriting, e.g. "ENOTDIR: stat …/web/pages/.js/package.json" +// after an entity was added — and the next change failed the same way. +// +// Either way a fresh bundler bundles the source as it now stands, so it is +// restarted once; only when that fails too is the error the caller's. errOut +// receives the incremental failure the recovery is answering. +func (s *bundlerSupervisor) AwaitRebuild(sinceGen int, settle, buildTimeout time.Duration, errOut io.Writer) (bool, error) { + rebuilt, err := s.WaitForRebuild(sinceGen, settle, buildTimeout) + if err == nil || !s.Wanted() { + return rebuilt, err + } + fmt.Fprintf(errOut, " incremental web client rebuild failed: %s\n", firstLine(err.Error())) + fmt.Fprintln(s.out, " re-bundling with a fresh web client bundler...") + if rerr := s.Restart(); rerr != nil { + return false, fmt.Errorf("fresh bundler failed too (next attempt in %s): %w", bundlerRestartBackoff(s.failures), rerr) + } + return s.want, nil +} + +// firstLine is s up to its first newline. +func firstLine(s string) string { + if i := strings.IndexByte(s, '\n'); i >= 0 { + return s[:i] + } + return s +} + +// Stop stops the live bundler, if any. +func (s *bundlerSupervisor) Stop() { + if s == nil || s.cur == nil { + return + } + _ = s.cur.Stop() + s.cur = nil +} + +// bundlerRestartBackoff is the wait after the n-th consecutive failed start: +// 2s, 4s, 8s ... capped at a minute, so a bundler that cannot start does not +// cost a full start attempt on every rebuild. +func bundlerRestartBackoff(failures int) time.Duration { + if failures <= 0 { + return 0 + } + d := 2 * time.Second + for i := 1; i < failures && d < time.Minute; i++ { + d *= 2 + } + if d > time.Minute { + d = time.Minute + } + return d +} + +// clientRebundler picks the recovery re-bundle for ensureClientServed. Under +// --watch with a live incremental bundler it is the supervisor's Rebundle — a +// one-shot BuildWebClient there would run a second rollup on web/dist alongside +// the watcher (#971). Otherwise it is the one-shot build. +func clientRebundler(sup *bundlerSupervisor, opts WebClientOptions) func() error { + if sup.Wanted() { + return sup.Rebundle + } + return func() error { return oneShotWebClientBuild(opts) } +} + +// oneShotWebClientBuild is BuildWebClient, as a var so a test can observe when a +// one-shot bundle runs. +var oneShotWebClientBuild = BuildWebClient + +// indentTail formats the last n lines of a log for appending to a message. +func indentTail(log string, n int) string { + log = strings.TrimRight(log, "\n") + if log == "" { + return "" + } + lines := strings.Split(log, "\n") + if len(lines) > n { + lines = lines[len(lines)-n:] + } + return "\n " + strings.Join(lines, "\n ") +} diff --git a/cmd/mxcli/docker/webclient_supervisor_test.go b/cmd/mxcli/docker/webclient_supervisor_test.go new file mode 100644 index 0000000000..128405680b --- /dev/null +++ b/cmd/mxcli/docker/webclient_supervisor_test.go @@ -0,0 +1,412 @@ +// SPDX-License-Identifier: Apache-2.0 + +package docker + +import ( + "bytes" + "errors" + "io" + "net/http" + "net/http/httptest" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "sync" + "testing" + "time" +) + +// bogusOneShot is a one-shot re-bundle that fails visibly (no mxbuild there), so +// a test can tell that the recovery branch ran. +func bogusOneShot(deployDir string) func() error { + return func() error { + return BuildWebClient(WebClientOptions{DeployDir: deployDir, MxBuildPath: "/nonexistent/mxbuild"}) + } +} + +// bundlerPool tracks fake bundlers, so a test can assert how many ran at once. +type bundlerPool struct { + mu sync.Mutex + live int + maxLive int + started int + // onStart runs when a fake bundler starts — its "first build". + onStart func() +} + +func (p *bundlerPool) start() (clientBundler, error) { + p.mu.Lock() + p.live++ + p.started++ + if p.live > p.maxLive { + p.maxLive = p.live + } + p.mu.Unlock() + if p.onStart != nil { + p.onStart() + } + return &fakeBundler{pool: p, gen: 1}, nil +} + +func (p *bundlerPool) liveCount() int { + p.mu.Lock() + defer p.mu.Unlock() + return p.live +} + +type fakeBundler struct { + pool *bundlerPool + gen int + exited bool + stopped bool + waitErr error // WaitForRebuild's error while the process is alive +} + +func (f *fakeBundler) Generation() int { return f.gen } +func (f *fakeBundler) WaitForRebuild(int, time.Duration, time.Duration) (bool, error) { + if f.exited { + return false, errors.New("web client watcher exited:\nboom") + } + if f.waitErr != nil { + return false, f.waitErr + } + return true, nil +} +func (f *fakeBundler) Exited() bool { return f.exited } +func (f *fakeBundler) Log() string { return "runner log line\n" } + +// die simulates the bundler process exiting on its own. +func (f *fakeBundler) die() { + if !f.exited { + f.exited = true + f.pool.mu.Lock() + f.pool.live-- + f.pool.mu.Unlock() + } +} + +func (f *fakeBundler) Stop() error { + f.stopped = true + f.die() + return nil +} + +func newFakeSupervisor(pool *bundlerPool, out io.Writer) (*bundlerSupervisor, *fakeBundler) { + first, _ := pool.start() + return &bundlerSupervisor{start: pool.start, out: out, cur: first, want: true, now: time.Now}, first.(*fakeBundler) +} + +// The reported symptom: once the bundler had exited, every later change failed +// with "watcher exited" and nothing restarted it. EnsureAlive must start a fresh +// one, and say so. +func TestBundlerSupervisor_RestartsExitedBundler(t *testing.T) { + pool := &bundlerPool{} + var out bytes.Buffer + sup, first := newFakeSupervisor(pool, &out) + + if restarted, err := sup.EnsureAlive(); err != nil || restarted { + t.Fatalf("live bundler: EnsureAlive = (%v, %v), want (false, nil)", restarted, err) + } + first.die() + restarted, err := sup.EnsureAlive() + if err != nil || !restarted { + t.Fatalf("exited bundler: EnsureAlive = (%v, %v), want (true, nil)", restarted, err) + } + if pool.started != 2 || pool.liveCount() != 1 { + t.Fatalf("started=%d live=%d, want a second bundler and exactly one live", pool.started, pool.liveCount()) + } + if sup.cur == clientBundler(first) || sup.cur.Exited() { + t.Fatal("the supervisor still holds the dead bundler") + } + if !strings.Contains(out.String(), "exited unexpectedly") || !strings.Contains(out.String(), "runner log line") { + t.Fatalf("restart not reported with the dead bundler's log:\n%s", out.String()) + } +} + +// A bundler that cannot start must not be retried on every change: after a +// failure the next attempt waits out a growing backoff. +func TestBundlerSupervisor_BackoffAfterFailedStart(t *testing.T) { + now := time.Unix(1000, 0) + attempts := 0 + fail := true + pool := &bundlerPool{} + sup := &bundlerSupervisor{ + start: func() (clientBundler, error) { + attempts++ + if fail { + return nil, errors.New("node crashed") + } + return pool.start() + }, + out: io.Discard, want: true, now: func() time.Time { return now }, + cur: &fakeBundler{pool: pool, exited: true}, + } + + if _, err := sup.EnsureAlive(); err == nil || !strings.Contains(err.Error(), "node crashed") { + t.Fatalf("first restart: want the start error, got %v", err) + } + if _, err := sup.EnsureAlive(); err == nil || !strings.Contains(err.Error(), "next attempt in 2s") { + t.Fatalf("within backoff: want a 'next attempt' error, got %v", err) + } + if attempts != 1 { + t.Fatalf("restart retried inside the backoff window: %d attempts", attempts) + } + now = now.Add(3 * time.Second) + fail = false + if restarted, err := sup.EnsureAlive(); err != nil || !restarted { + t.Fatalf("after backoff: EnsureAlive = (%v, %v), want (true, nil)", restarted, err) + } + if attempts != 2 || sup.failures != 0 { + t.Fatalf("attempts=%d failures=%d, want 2 and a reset counter", attempts, sup.failures) + } +} + +func TestBundlerRestartBackoff(t *testing.T) { + want := map[int]time.Duration{0: 0, 1: 2 * time.Second, 2: 4 * time.Second, 3: 8 * time.Second, 10: time.Minute} + for n, d := range want { + if got := bundlerRestartBackoff(n); got != d { + t.Errorf("bundlerRestartBackoff(%d) = %s, want %s", n, got, d) + } + } +} + +// A recovery re-bundle under --watch must replace the bundler, never run beside it. +func TestBundlerSupervisor_RebundleNeverOverlaps(t *testing.T) { + pool := &bundlerPool{} + sup, _ := newFakeSupervisor(pool, io.Discard) + for i := 0; i < 3; i++ { + if err := sup.Rebundle(); err != nil { + t.Fatal(err) + } + } + if pool.maxLive != 1 { + t.Fatalf("%d bundlers ran at once, want 1", pool.maxLive) + } +} + +// The concurrency defect end to end: a page missing from the bundle under +// --watch must be recovered through the supervisor. A one-shot BuildWebClient +// while the incremental bundler is live puts two rollups on web/dist (#971). +func TestEnsureClientServed_WatchModeRebundleIsExclusive(t *testing.T) { + deploy := pagesDeployment(t, []string{"A.js", "B.js"}, []string{"A.js"}) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + })) + defer srv.Close() + + pool := &bundlerPool{} + sup, _ := newFakeSupervisor(pool, io.Discard) + // A fresh bundler's first build emits every page. + pool.onStart = func() { + _ = os.WriteFile(filepath.Join(deploy, "web", "dist", "pages", "B.js"), []byte("//"), 0o600) + } + + oneShotWhileLive := 0 + oneShots := 0 + old := oneShotWebClientBuild + oneShotWebClientBuild = func(WebClientOptions) error { + oneShots++ + if pool.liveCount() > 0 { + oneShotWhileLive++ + } + pool.onStart() + return nil + } + defer func() { oneShotWebClientBuild = old }() + + rebundle := clientRebundler(sup, WebClientOptions{DeployDir: deploy}) + if err := ensureClientServed(deploy, srv.URL+"/", rebundle, io.Discard); err != nil { + t.Fatalf("ensureClientServed: %v", err) + } + if oneShotWhileLive > 0 { + t.Fatalf("a one-shot bundle ran %d time(s) while the incremental bundler was live", oneShotWhileLive) + } + if pool.maxLive != 1 || pool.started != 2 { + t.Fatalf("maxLive=%d started=%d, want the bundler replaced (2 starts, 1 at a time)", pool.maxLive, pool.started) + } + + // Control: without an incremental bundler the one-shot is the right tool. + deploy2 := pagesDeployment(t, []string{"A.js", "B.js"}, []string{"A.js"}) + pool.onStart = func() { + _ = os.WriteFile(filepath.Join(deploy2, "web", "dist", "pages", "B.js"), []byte("//"), 0o600) + } + none := newBundlerSupervisor(nil, nil, io.Discard) + if err := ensureClientServed(deploy2, srv.URL+"/", clientRebundler(none, WebClientOptions{DeployDir: deploy2}), io.Discard); err != nil { + t.Fatalf("ensureClientServed without a bundler: %v", err) + } + if oneShots != 1 { + t.Fatalf("without a bundler the one-shot should run once, ran %d", oneShots) + } +} + +// A nil supervisor bundler (Mendix 11.14+, classic client) is inert. +func TestBundlerSupervisor_NoBundlerIsNoop(t *testing.T) { + sup := newBundlerSupervisor(nil, func() (*WebClientWatcher, error) { + t.Fatal("start must not be called") + return nil, nil + }, io.Discard) + if sup.Wanted() || sup.Generation() != 0 { + t.Fatal("a supervisor without a bundler reports one") + } + if r, err := sup.EnsureAlive(); r || err != nil { + t.Fatalf("EnsureAlive = (%v, %v)", r, err) + } + if err := sup.Restart(); err != nil { + t.Fatal(err) + } + sup.Stop() +} + +// Stop must not return while the bundler process is still running: the +// supervisor's restart starts the next bundler as soon as Stop returns. A guard, +// not a regression test — the earlier double-Wait Stop also blocked until exit +// on this Go version, so this passes against it too. +func TestWebClientWatcher_StopWaitsForExit(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("process-group signalling is unix-only") + } + // Takes ~1s to exit on SIGTERM, the way node finishes a bundle in flight. + cmd := exec.Command("sh", "-c", `trap 'sleep 1; exit 0' TERM; echo ready; while :; do sleep 0.05; done`) + setProcessGroup(cmd) + stdout, err := cmd.StdoutPipe() + if err != nil { + t.Fatal(err) + } + wc, err := launchWatcher(cmd, &syncBuffer{}, stdout) + if err != nil { + t.Fatal(err) + } + time.Sleep(200 * time.Millisecond) // let the shell install its trap + if wc.Exited() { + t.Fatal("process exited before Stop") + } + _ = wc.Stop() + if !wc.Exited() { + t.Fatal("Stop returned while the bundler process was still running") + } +} + +func TestWebClientTimeout(t *testing.T) { + t.Setenv(webClientTimeoutEnv, "") + if got := webClientTimeout(0); got != defaultWebClientTimeout { + t.Errorf("default = %s", got) + } + t.Setenv(webClientTimeoutEnv, "12m") + if got := webClientTimeout(0); got != 12*time.Minute { + t.Errorf("env = %s, want 12m", got) + } + if got := webClientTimeout(3 * time.Second); got != 3*time.Second { + t.Errorf("explicit = %s, want it to win over the env", got) + } + t.Setenv(webClientTimeoutEnv, "soon") + if got := webClientTimeout(0); got != defaultWebClientTimeout { + t.Errorf("unparsable env = %s, want the default", got) + } +} + +// A timed-out bundle used to say only "timed out after 5m0s". It must name the +// knob and show the bundler's own log, which is where the stall is visible. +func TestBuildWebClient_TimeoutShowsLogTail(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("fake node is a shell script") + } + deploy := t.TempDir() + mustWrite := func(p, content string, mode os.FileMode) { + t.Helper() + if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(p, []byte(content), mode); err != nil { + t.Fatal(err) + } + } + mustWrite(filepath.Join(deploy, "web", "rollup.config.mjs"), "export default {};", 0o644) + var log strings.Builder + for i := 1; i <= 40; i++ { + log.WriteString("line ") + log.WriteString(strings.Repeat("x", i%3)) + log.WriteString("\n") + } + log.WriteString("INFO Bundling started\n") + mustWrite(filepath.Join(deploy, "log", "web-client-build.log"), log.String(), 0o644) + + mx := t.TempDir() + mustWrite(filepath.Join(mx, "tools", "node", "rollup-runner.mjs"), "", 0o644) + mustWrite(filepath.Join(mx, "tools", "node", "node"), "#!/bin/sh\nexec sleep 30\n", 0o755) + + err := BuildWebClient(WebClientOptions{ + DeployDir: deploy, MxBuildPath: filepath.Join(mx, "mxbuild"), Timeout: 300 * time.Millisecond, + }) + if err == nil { + t.Fatal("expected a timeout") + } + msg := err.Error() + for _, want := range []string{"timed out after 300ms", "--web-client-timeout", webClientTimeoutEnv, "INFO Bundling started", "last 30 line(s)"} { + if !strings.Contains(msg, want) { + t.Errorf("timeout error lacks %q:\n%s", want, msg) + } + } +} + +func TestSessionNotice(t *testing.T) { + if sessionNotice(ActionReload) != "" { + t.Error("a reload keeps sessions; no notice") + } + if n := sessionNotice(ActionRestart); !strings.Contains(n, "sessions were dropped") { + t.Errorf("restart notice = %q", n) + } +} + +// Both shapes the reported loop died of: an incremental build that failed on the +// source the serve build was rewriting (measured on 11.13 after adding an entity: +// ENOTDIR on web/pages/.js/package.json, then the same on the next change), +// and a bundler that had exited. A fresh bundler must deliver the change. +func TestBundlerSupervisor_AwaitRebuildRecovers(t *testing.T) { + cases := map[string]func(*fakeBundler){ + "failed incremental build": func(f *fakeBundler) { + f.waitErr = errors.New("web client build failed: (plugin commonjs--resolver) Error: ENOTDIR: not a directory\nstack") + }, + "exited bundler": func(f *fakeBundler) { f.die() }, + } + for name, breakIt := range cases { + t.Run(name, func(t *testing.T) { + pool := &bundlerPool{} + sup, first := newFakeSupervisor(pool, io.Discard) + breakIt(first) + var errOut bytes.Buffer + rebuilt, err := sup.AwaitRebuild(0, time.Millisecond, time.Millisecond, &errOut) + if err != nil || !rebuilt { + t.Fatalf("AwaitRebuild = (%v, %v), want the change delivered by a fresh bundler", rebuilt, err) + } + if pool.started != 2 || pool.maxLive != 1 { + t.Fatalf("started=%d maxLive=%d, want one replacement and never two at once", pool.started, pool.maxLive) + } + if !strings.Contains(errOut.String(), "incremental web client rebuild failed") || strings.Contains(errOut.String(), "stack") { + t.Fatalf("the incremental failure is not reported on one line:\n%s", errOut.String()) + } + }) + } +} + +func TestBundlerSupervisor_AwaitRebuildFreshFailsToo(t *testing.T) { + pool := &bundlerPool{} + sup, first := newFakeSupervisor(pool, io.Discard) + first.waitErr = errors.New("web client build failed: syntax error") + sup.start = func() (clientBundler, error) { return nil, errors.New("initial web client build failed: syntax error") } + if _, err := sup.AwaitRebuild(0, time.Millisecond, time.Millisecond, io.Discard); err == nil || !strings.Contains(err.Error(), "fresh bundler failed too") { + t.Fatalf("a genuine build error must surface, got %v", err) + } +} + +func TestBundlerSupervisor_AwaitRebuildHealthyNoRestart(t *testing.T) { + pool := &bundlerPool{} + sup, _ := newFakeSupervisor(pool, io.Discard) + if rebuilt, err := sup.AwaitRebuild(0, time.Millisecond, time.Millisecond, io.Discard); err != nil || !rebuilt { + t.Fatalf("AwaitRebuild = (%v, %v)", rebuilt, err) + } + if pool.started != 1 { + t.Fatalf("a healthy rebuild restarted the bundler (%d starts)", pool.started) + } +} diff --git a/cmd/mxcli/docker/webclient_watch.go b/cmd/mxcli/docker/webclient_watch.go index 163bd6450a..edf791fbe1 100644 --- a/cmd/mxcli/docker/webclient_watch.go +++ b/cmd/mxcli/docker/webclient_watch.go @@ -80,6 +80,12 @@ type WebClientWatcher struct { lastErr string // message of the most recent failed bundle exited bool // the runner process has exited updated chan struct{} // closed+replaced on every state change (broadcast) + + // done is closed once the process has been reaped. Stop waits on it rather + // than calling cmd.Wait a second time beside the reaper goroutine; a restart + // relies on Stop returning only after the old bundler is gone, so two never + // write web/dist at once (#971). + done chan struct{} } // applyStatus folds a status line into the watcher state and broadcasts. @@ -182,7 +188,29 @@ func StartWebClientWatch(opts WebClientOptions) (*WebClientWatcher, error) { } cmd.Stderr = log - wc := &WebClientWatcher{cmd: cmd, log: log, updated: make(chan struct{})} + wc, err := launchWatcher(cmd, log, stdout) + if err != nil { + return nil, err + } + + timeout := webClientTimeout(opts.Timeout) + if err := wc.waitForFirstBuild(timeout); err != nil { + _ = wc.Stop() + return nil, fmt.Errorf("%w%s", err, webClientBuildLogTail(opts.DeployDir)) + } + dist := filepath.Join(webDir, "dist", "index.js") + if _, err := os.Stat(dist); err != nil { + _ = wc.Stop() + return nil, fmt.Errorf("web client watcher reported success but %s is missing:\n%s", dist, wc.log.String()) + } + return wc, nil +} + +// launchWatcher starts cmd and wires the status reader and the reaper. Split from +// StartWebClientWatch so the process lifecycle (Stop, Exited) is testable with +// an ordinary shell process instead of mxbuild's node tooling. +func launchWatcher(cmd *exec.Cmd, log *syncBuffer, stdout io.Reader) (*WebClientWatcher, error) { + wc := &WebClientWatcher{cmd: cmd, log: log, updated: make(chan struct{}), done: make(chan struct{})} if err := cmd.Start(); err != nil { return nil, fmt.Errorf("launching web client watcher: %w", err) } @@ -193,22 +221,20 @@ func StartWebClientWatch(opts WebClientOptions) (*WebClientWatcher, error) { wc.exited = true wc.broadcastLocked() wc.mu.Unlock() + close(wc.done) }() + return wc, nil +} - timeout := opts.Timeout - if timeout == 0 { - timeout = 5 * time.Minute - } - if err := wc.waitForFirstBuild(timeout); err != nil { - _ = wc.Stop() - return nil, err - } - dist := filepath.Join(webDir, "dist", "index.js") - if _, err := os.Stat(dist); err != nil { - _ = wc.Stop() - return nil, fmt.Errorf("web client watcher reported success but %s is missing:\n%s", dist, wc.log.String()) +// Exited reports whether the bundler process has exited. A nil watcher (no +// incremental bundler for this deployment) has nothing to exit. +func (wc *WebClientWatcher) Exited() bool { + if wc == nil { + return false } - return wc, nil + wc.mu.Lock() + defer wc.mu.Unlock() + return wc.exited } // readLoop parses the runner's stdout and folds each status into state. It also @@ -306,19 +332,27 @@ func (wc *WebClientWatcher) Log() string { return wc.log.String() } -// Stop terminates the watcher process. +// Stop terminates the watcher process and returns only once it has been reaped, +// so a caller that starts another bundler next never runs two at once. func (wc *WebClientWatcher) Stop() error { - if wc == nil || wc.cmd == nil || wc.cmd.Process == nil { + if wc == nil || wc.cmd == nil || wc.cmd.Process == nil || wc.done == nil { return nil } + select { + case <-wc.done: + return nil // already gone + default: + } _ = signalProcessGroup(wc.cmd.Process, syscall.SIGTERM) - done := make(chan error, 1) - go func() { done <- wc.cmd.Wait() }() select { - case <-done: - case <-time.After(5 * time.Second): + case <-wc.done: + case <-time.After(watcherStopGrace): _ = killProcessGroup(wc.cmd.Process) - <-done + <-wc.done } return nil } + +// watcherStopGrace is how long Stop lets the bundler exit on SIGTERM before it +// kills the process group. A var so a test can shorten it. +var watcherStopGrace = 5 * time.Second diff --git a/docs-site/src/tools/run-local.md b/docs-site/src/tools/run-local.md index 348a028e4c..8f29baa5fc 100644 --- a/docs-site/src/tools/run-local.md +++ b/docs-site/src/tools/run-local.md @@ -25,7 +25,9 @@ the container on every change (~30–60 s). `run --local` instead: | entity / view entity / association | runtime restart + DDL | ~9 s | The metamodel catalog (entities/associations) is reconciled only at runtime startup, -so structural changes need a restart; behavioural changes do not. +so structural changes need a restart; behavioural changes do not. A restart drops +every browser session — `--watch` prints `browser sessions were dropped; log in +again` when it happens, so a browser test knows to sign in before its next step. ## What it does @@ -72,6 +74,7 @@ so structural changes need a restart; behavioural changes do not. | `--hub-secret` | — | Shared auth (`user:pass`) matching an **open** hub's `--secret` | | *(hub API key)* | — | For a **GitHub-authenticated** hub: get one from `https:///cli`, set `MXCLI_HUB_KEY` (see below) | | `--watch` | off | Rebuild + hot-apply on every project change | +| `--web-client-timeout` | `$MXCLI_WEB_CLIENT_TIMEOUT`, else `5m` | Limit for one web client bundle build (e.g. `10m`); on timeout the tail of `deployment/log/web-client-build.log` is printed | | `--ensure-db` | off | Provision local Postgres + the app database if missing (fresh-session bootstrap) | | `--setup` | off | Prepare prerequisites (cache MxBuild+runtime, ensure DB) and exit without booting — for a SessionStart hook | | `--app-port` | 8080 | App HTTP port | From ef91ffb5d8c0d3b9c89182d423911844ec563e5a Mon Sep 17 00:00:00 2001 From: Ako Date: Sun, 4 Oct 2026 15:20:45 +0000 Subject: [PATCH 13/17] fix(run-local): keep an edit made while --watch applies a change After each apply watchAndApply moved its change baseline to the source's current mtime, so an exec or save that landed during the build was folded into the baseline and never rebuilt. Keep the baseline at the mtime the build settled on; the build writes nothing under the watched source. Found while reproducing #971. Co-Authored-By: Claude Opus 5.5 --- .claude/skills/fix-issue/findings/cmd-mxcli.jsonl | 1 + CHANGELOG.md | 1 + cmd/mxcli/docker/runlocal.go | 10 +++++++--- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/.claude/skills/fix-issue/findings/cmd-mxcli.jsonl b/.claude/skills/fix-issue/findings/cmd-mxcli.jsonl index ba50ac8637..390c40d53a 100644 --- a/.claude/skills/fix-issue/findings/cmd-mxcli.jsonl +++ b/.claude/skills/fix-issue/findings/cmd-mxcli.jsonl @@ -159,3 +159,4 @@ {"area": "cmd/mxcli/docker", "date": "2026-10-03", "symptom": "`mxcli docker build` (and docker run / reload, which call it) rewrote an MPRv1 project's .mpr, rewrote every MPRv2 .mxunit (restored with new mtimes), and wrote theme-cache/, deployment/, javasource/ proxies, the .launch file, .classpath and .project into the project; the TUI checker and `mxcli eval`'s mx_check wrote theme-cache/web/ and deployment/sass/ on every run", "cause": "update-widgets ran on the project under a snapshot that restored only v2 storage, and mx check / MxBuild ran on the project itself; the TUI and eval runner ran a bare `mx check `", "file": "`cmd/mxcli/docker/build.go` (`buildOnCopy`), `cmd/mxcli/docker/check.go` (`MxCheckOnCopy`), `cmd/mxcli/tui/checker.go` (`runMxCheck`), `cmd/mxcli/evalrunner/checks.go` (`checkMxCheck`)", "insight": "Run update-widgets, mx check and MxBuild on one temporary copy (copyProjectToTemp) and write only the output directory. Measured on the 11.14 testapp: the PAD from the copy differs from the in-place build in the same 9 files as two in-place builds of identical copies differ (cache-bust stamps, operation ids, the native metro paths), so the output is equivalent; rebuild time unchanged (58s vs 59s), because a PAD build gains nothing from the project's deployment/. 11.14's blank app needs JDK 25, so the real-mx build test skips without it; run it with 11.13's mx.", "refs": ["ako/mxcli#961", "ako/mxcli#951", "mendixlabs/mxcli#808"]} {"area": "cmd/mxcli", "date": "2026-10-03", "symptom": "ako/mxcli#962 item 3: in VS Code, two calls to a stored void JavaScript action with the same output name (Studio Pro's `$ReturnValueName`/`$RefreshEntity` shape) are squiggled MDL063, while `mxcli check -p` passes them", "cause": "runSemanticValidation called executor.ValidateMicroflow/ValidateNanoflow, which pass a nil void-action resolver: without the project every call output counts as a declaration", "file": "`cmd/mxcli/lsp_diagnostics.go` (runSemanticValidation); `mdl/executor/validate_void_code_calls.go` (FlowRules, CodeActionCache)", "fix": "executor.NewFlowRules(prog, s.findMprPath(), s.codeActions): resolves actions through the script and the workspace project (opened lazily), treats an unresolvable action as possibly void (unknownIsVoid) for MDL063 but never for MDL093, and shares project answers across keystrokes for 30s since one action read costs ~300ms on PedApp", "insight": "An entry point that wraps a richer internal API with nil arguments (ValidateMicroflow = validateMicroflowWith(stmt, nil)) silently gives every caller the weakest behaviour; a fix landed in the richer API (#958) does not reach them. Grep the exported wrapper's callers when the internal one gains a parameter. Measure the cost before putting project I/O on a keystroke path", "test": "`cmd/mxcli/lsp_void_calls_test.go` (PedApp: void pair clean with and without project, Boolean pair control, MDL093 only with project); `mdl/executor/validate_void_code_calls_test.go` (TestCodeActionCache_SharesProjectAnswersAcrossRuns)"} {"area": "cmd/mxcli", "date": "2026-10-04", "symptom": "`run --local --watch` (Mendix <= 11.13, rollup bundler): after a while every page change fails with `web client rebuild failed: web client watcher exited` (or `client bundle not served after apply: web client re-bundle: web client build timed out after 5m0s`) and nothing reaches the browser until `run --local` is restarted; after adding an entity the next changes fail with `ENOTDIR: not a directory, stat '.../web/pages/.js/package.json'`", "cause": "watchAndApply held a bare *WebClientWatcher: (1) once it exited nothing restarted it (only recoverMissingPages did), so WaitForRebuild failed every later change; (2) a failed incremental build (rollup's commonjs resolver hitting web/pages mid-rewrite by the serve build) left the watcher erroring and the change was dropped; (3) ensureClientServed's recovery ran a one-shot NODE_ENV=production BuildWebClient in the same web/ dir while the watcher was still running — two rollups on web/dist; (4) the 5m limit was hard-coded and a timeout printed nothing about why", "file": "cmd/mxcli/docker/webclient_supervisor.go, cmd/mxcli/docker/runlocal.go (watchAndApply, ensureClientServed), cmd/mxcli/docker/webclient.go (webClientTimeout, webClientBuildLogTail)", "fix": "bundlerSupervisor owns the bundler: EnsureAlive restarts an exited one (backoff 2s..60s after failed starts), AwaitRebuild retries a failed/aborted incremental rebuild once with a fresh bundler, Rebundle = stop+reap then start (never two). ensureClientServed takes the rebundle func; clientRebundler hands it the supervisor under --watch and the one-shot otherwise. --web-client-timeout / MXCLI_WEB_CLIENT_TIMEOUT; timeout errors append the last 30 lines of deployment/log/web-client-build.log. sessionNotice prints that a restart dropped sessions", "insight": "Killing the runner (`kill `) reproduces the dead-watcher state in seconds — no need to wait for it to die on its own. A fresh bundler is the universal recovery under --watch: its first build is a full bundle of the current source, so it covers missing pages, dangling chunks, a dist/ deleted by Gradle, and a transient incremental failure alike, without a second process on web/dist. Note: exec.Cmd.Wait called a second time concurrently with the reaper did block until exit on this Go version, so the old Stop was not the overlap — the one-shot in ensureClientServed was", "test": "cmd/mxcli/docker/webclient_supervisor_test.go (TestBundlerSupervisor_RestartsExitedBundler, _AwaitRebuildRecovers, _RebundleNeverOverlaps, TestEnsureClientServed_WatchModeRebundleIsExclusive, TestBuildWebClient_TimeoutShowsLogTail); live: 11.13 scratch app, 7 consecutive changes incl. a killed runner"} +{"area": "cmd/mxcli", "date": "2026-10-04", "symptom": "`run --local --watch`: an `mxcli exec` (or save) made while a change is still building/applying is never built — no `Change detected` follows, the app keeps the previous model, and re-running the script writes nothing (byte-idempotent) so nothing re-triggers", "cause": "watchAndApply set `last = sourceMTime(...)` after every successful apply, under a comment claiming it kept mid-build edits; it did the opposite — the edit's mtime was folded into the baseline, so the next tick saw nothing newer", "file": "cmd/mxcli/docker/runlocal.go (watchAndApply)", "fix": "keep `last` at the settled mtime the build was taken from; the build writes nothing under the watched model/theme source (checked with find -newer during a live run), so this cannot self-trigger", "insight": "Found while reproducing #971 with a script that waited for the first output line of a change instead of its `applied` line — the next exec landed during a 2-minute restart-apply and vanished. Any test of a watch loop should include an edit made DURING a build, not only between builds", "test": "live only (11.13 scratch app, hsqldb): exec an entity add, exec a page change 15s into its build; fixed binary builds it as the next build, the binary with the refresh restored shows no further build after 45s"} diff --git a/CHANGELOG.md b/CHANGELOG.md index ee8b6c2a6b..faff9c5dcd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -56,6 +56,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). ### Fixed +- **`run --local --watch` no longer loses an edit made while a change is being applied** — after each apply the loop moved its change baseline to the source's current mtime, so an `exec` or save that landed during the build (easily, while a structural change restarts the runtime) was never rebuilt: the app kept serving the previous model with nothing reported. The baseline now stays at the mtime the build was taken from, and the edit is built on the next poll. - **`run --local --watch` keeps its web client bundler alive and never runs two** (ako/mxcli#971) — a bundler that exited stayed dead, so every later page change failed with `web client watcher exited` and only restarting `run --local` recovered (reproduced on Mendix 11.13 by killing the runner: two changes in a row failed). It is now restarted on the next change, with backoff when it cannot start. An incremental rebuild that fails — measured after adding an entity: `ENOTDIR … web/pages/.js/package.json`, and the next change failed the same way — is retried once with a fresh bundler. A recovery re-bundle (a missing page, a dangling chunk, or `/dist/index.js` gone after a runtime restart) replaces the bundler instead of running a one-shot production bundle beside it in the same `web/` directory. The bundle-build limit is configurable (`--web-client-timeout`, or `MXCLI_WEB_CLIENT_TIMEOUT`; default 5m), and a timeout prints the tail of `deployment/log/web-client-build.log`. A change that restarts the runtime now says that browser sessions were dropped. - **`list workflows` and `show structure` count every activity of a workflow** (ako/mxcli#963) — including those on a boundary-event path and in an event sub-process, as the catalog's `workflows_data` has since #937. TestApp `Workflow1` listed 5 activities where the catalog counted 8; all three now share one walk (`wfnames.WalkActivities`). - **`docker build`, the TUI checker and `mxcli eval` no longer modify the project** (ako/mxcli#961) — `docker build` (and `docker run` / `docker reload`, which use it) now runs `mx update-widgets`, `mx check` and MxBuild on one temporary copy and writes only its output directory (`.docker/build/` or `-o`). Before, it rewrote an MPR v1 project's `.mpr`, rewrote every MPR v2 `.mxunit` (restored with new mtimes), and wrote `theme-cache/`, `deployment/`, `javasource/` proxies, the `.launch` file, `.classpath` and `.project` into the project. The build still uses the widget-normalised model, from the copy, and the package it produces is the same; `mxcli fix widgets` applies the normalisation to the project. **The project's `deployment/` is no longer refreshed by `docker build`** — `run --local` builds its own. The TUI auto-check and eval's `mx_check` ran a plain `mx check` on the project, which wrote `theme-cache/web/` and `deployment/sass/` every time; they now check a copy too. diff --git a/cmd/mxcli/docker/runlocal.go b/cmd/mxcli/docker/runlocal.go index cdb4dac383..4490daec43 100644 --- a/cmd/mxcli/docker/runlocal.go +++ b/cmd/mxcli/docker/runlocal.go @@ -1420,9 +1420,13 @@ func watchAndApply(opts LocalRunOptions, serve *ServeServer, rt *LocalRuntime, b } fmt.Fprintf(w, " build #%d applied via %s in %s%s -> %s\n", gen, action, time.Since(start).Round(time.Millisecond), client, rt.AppURL()) maybeScreenshot(opts, rt) - // Refresh the baseline AFTER the apply so an edit made mid-build is - // still caught on the next tick. - last = sourceMTime(opts.ProjectPath) + // The baseline stays at the mtime this build settled on. Moving it to + // "now" here — as this loop used to, under a comment claiming the + // opposite — swallowed any edit made while the build was applying: the + // next tick saw nothing newer than the refreshed baseline, so the edit + // was never built (measured: an exec during a restart-apply on 11.13, + // no rebuild in 45s). The build writes nothing under the watched model + // and theme source, so keeping the settled baseline cannot loop. } } } From c4fd586b33854672840394a96c2a80c26a75cbfb Mon Sep 17 00:00:00 2001 From: Ako Date: Sun, 4 Oct 2026 15:22:23 +0000 Subject: [PATCH 14/17] style: gofmt the quoted-template-param comments (#969) Co-Authored-By: Claude Opus 5.5 --- mdl/executor/validate_quoted_template_params.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mdl/executor/validate_quoted_template_params.go b/mdl/executor/validate_quoted_template_params.go index 4f885c0b7b..17b0cd8c4d 100644 --- a/mdl/executor/validate_quoted_template_params.go +++ b/mdl/executor/validate_quoted_template_params.go @@ -14,7 +14,7 @@ import ( // validateQuotedTemplateParams emits an info hint (MDL-PARAMQUOTE01) for a // template parameter whose value is a quoted string that reads like an -// expression: `{1} = 'formatDateTime($Log/Date, ''d MMM'')'`. +// expression, such as a quoted formatDateTime($Log/Date, …) call. // // The quotes make it a String literal, which is exactly what is stored — the // page then shows the expression's TEXT. That is valid, builds clean and is @@ -65,7 +65,7 @@ func validateQuotedTemplateParams(w *ast.WidgetV3, locationPrefix string) []lint } // singleStringLiteral reports whether s is exactly one Mendix string literal -// ('…' with '' as the escaped quote) and returns its raw body. +// (quoted, with a doubled quote as the escape) and returns its raw body. func singleStringLiteral(s string) (string, bool) { s = strings.TrimSpace(s) if len(s) < 2 || s[0] != '\'' || s[len(s)-1] != '\'' { From 7e5bc2c4dcd3a34b86a0d9569d5b78f771a476d8 Mon Sep 17 00:00:00 2001 From: Ako Date: Sun, 4 Oct 2026 15:22:24 +0000 Subject: [PATCH 15/17] docs: changelog, findings and widget reference for #969 Co-Authored-By: Claude Opus 5.5 --- .claude/skills/fix-issue/findings/mdl-executor.jsonl | 2 ++ .claude/skills/fix-issue/findings/mdl-other.jsonl | 3 +++ .claude/skills/mendix/create-page/reference/widgets.md | 2 ++ CHANGELOG.md | 5 +++++ 4 files changed, 12 insertions(+) diff --git a/.claude/skills/fix-issue/findings/mdl-executor.jsonl b/.claude/skills/fix-issue/findings/mdl-executor.jsonl index 8d3c87c883..80f1b2f134 100644 --- a/.claude/skills/fix-issue/findings/mdl-executor.jsonl +++ b/.claude/skills/fix-issue/findings/mdl-executor.jsonl @@ -853,3 +853,5 @@ {"area": "mdl/executor", "date": "2026-10-03", "symptom": "`list workflows` and `show structure` report fewer workflow activities than the catalog's workflows_data (TestApp Workflow1: 5 vs 8)", "cause": "cmd_workflows.go countFlowActivities and cmd_structure.go countStructureFlowActivities each recursed over outcome flows only, skipping boundary-event flows and event sub-processes; the catalog had moved to a shared walk in #937 and the executor copies were left behind", "file": "`mdl/backend/wfnames/walk.go` (`WalkActivities`, `CountActivities`), `mdl/executor/cmd_workflows.go`, `mdl/executor/cmd_structure.go`, `mdl/catalog/workflow_walk.go`", "insight": "Duplicate-resolver drift: fixing one copy of a traversal (#937) left two private copies answering differently. The walk now lives in wfnames, which both catalog and executor already import, so there is one place to add a new sub-flow slot. Grep for every recursion over `workflows.Flow` when one is fixed.", "refs": ["ako/mxcli#963", "ako/mxcli#937"]} {"area": "mdl/executor", "date": "2026-10-03", "symptom": "ako/mxcli#962 item 1: `$V = call java action M.VoidAction(...)` then `log ... + $V` (or `$V` as a JS call argument) passes `check` and `exec`, then mxbuild 11.13.0 fails CE0109 \"Undefined variable 'V'\"", "cause": "#953 taught MDL063 that a void call's output name declares nothing, but no rule read the other half: the name cannot be READ either. The resolver only answered void/not-void, so 'unknown' and 'non-void' were the same answer", "file": "`mdl/executor/validate_void_call_output.go` (checkVoidCallOutputUse, MDL093); `mdl/executor/validate_void_code_calls.go` (resolve -> voidness{void, known})", "fix": "MDL093: collect output names of calls KNOWN to be void, drop any name another statement defines flow-wide (declare, parameter, non-void producer), report each remaining name the flow reads (loopRefVars over every nested body). The resolver now returns known-ness, so 'possibly void' (the editor's policy) never produces MDL093", "insight": "A finding that says 'X declares nothing' has two consequences — no collision AND no definition; fixing the first and logging the second as follow-up left a CE gap. When a resolver's default is a policy (unknown counts as non-void), make the unknown state explicit before a second rule reads it: the CE0109 rule must use only knowledge, never the policy", "test": "`mdl/executor/validate_void_call_output_test.go`; `cmd/mxcli/check_void_calls_test.go` (TestCheck_ReadOfAStoredVoidCallOutput, PedApp stored void JS action, Boolean and unresolvable controls)"} {"area": "mdl/executor", "date": "2026-10-03", "symptom": "ako/mxcli#962 item 2: describe of a microflow with the same output name in each if/else branch (or inside a loop and again after it) printed no duplicate-variable warning; mxbuild 11.13.0 reports CE0111 for both", "cause": "duplicateOutputVariableWarnings only warned when one assignment could REACH another (a reachability walk written for #710's performance), so exclusive branches and a loop body vs. the flow after it were treated as separate scopes; a test pinned the branch scoping", "file": "`mdl/executor/cmd_microflows_show.go` (duplicateOutputVariableWarnings)", "fix": "Count non-void output names over every object collection (loop bodies included); warn for any name created twice. Linear, so #710's cost concern disappears with the reachability walk", "insight": "The reachability model encoded a belief (exclusive paths may reuse a name) that no one had measured; MDL063 had already been aligned to flow-wide names in #958, so two renderings of the same rule disagreed. When one rule is corrected against mxbuild, grep for the other places that encode the same rule — here describe's header warning", "test": "`mdl/executor/cmd_microflows_duplicate_output_test.go` (TestFormatMicroflowActivitiesWarnsForExclusiveBranchOutputs, TestFormatMicroflowActivitiesNamesAreFlowWide)"} +{"area": "mdl/executor", "date": "2026-10-04", "symptom": "ako/mxcli#969 item 2: `alter page … { set Action = microflow M.X on btn }` with M.X created earlier in the same script failed check (\"microflow not found\") and exec refused the script; the same for nanoflow and show page targets", "cause": "validateAlterSetProperties dry-runs the SET against the stored document, and resolveMicroflow / resolveNanoflowByName / resolvePageRef only know the session cache (createdMicroflows, …) that executing fills — which check never does", "file": "`mdl/executor/validate_alter_set.go` (scriptDeclaresMissing)", "insight": "A dry run of a mutator in check must treat a NotFound for a name the script declares (scriptContext.microflows/nanoflows/pages/snippets) as satisfied, matching on the typed mdlerrors.NotFoundError Kind+Name through errors.As rather than the message. Do not register fake IDs in ctx.Cache instead: exec runs check on the same executor and would resolve to them. Control: an undeclared target still fails", "refs": ["#969"]} +{"area": "mdl/executor", "date": "2026-10-04", "symptom": "ako/mxcli#969 item 3: a list view / data grid / gallery with `datasource: $currentObject/M.Assoc` over a single-object association passed check and exec, then mxbuild failed CE8812 \"A grid association path must result in a list\"", "cause": "No rule modelled association multiplicity for list widgets", "file": "`mdl/executor/validate_assoc_list_source.go` (MDL-ASSOCDS01), hooked into attributeScopeValidator.walk", "insight": "Measured 8 shapes x 3 widgets on 11.13.0 and 11.14.0, identical: CE8812 for a Reference followed from its FROM entity (owner Default or Both) and for a Reference with owner Both from the TO entity (one-to-one); the reverse of a default Reference and every ReferenceSet build clean. Judge only those shapes with an exact context entity; skip specializations, self-associations and multi-hop paths. The attribute-scope walk already carries the data context, so hook there", "refs": ["#969"]} diff --git a/.claude/skills/fix-issue/findings/mdl-other.jsonl b/.claude/skills/fix-issue/findings/mdl-other.jsonl index dbf0803841..7a9eaff339 100644 --- a/.claude/skills/fix-issue/findings/mdl-other.jsonl +++ b/.claude/skills/fix-issue/findings/mdl-other.jsonl @@ -96,3 +96,6 @@ {"area": "mdl/linter", "date": "2026-10-03", "symptom": "MPR002 'Microflow X has no activities' on a microflow or nanoflow whose only content is `return ;` (e.g. a label formatter, `return $currentUser;`)", "cause": "ActivityCount excludes start and end events, so a flow that computes its result in the end event's return value counts 0 activities", "file": "`mdl/linter/rules/empty.go` (`returnsValue`)", "insight": "A non-Void ReturnType is the catalog's witness that the end event returns a value (mxbuild requires it on every end event), so no new column was needed; '' and 'Void' stay reported", "refs": ["ako/mxcli#953"]} {"area": "mdl/catalog", "date": "2026-10-03", "symptom": "`commit $Order` (on a loop iterator, a parameter or a retrieved list) wrote no refs row; refs_to(entity) could not answer which flows commit an entity", "cause": "refs had no commit ref kind: microflowActionRef / microflowVarActionRef emitted create/change/delete only, and a create/change with commit carried no commit edge", "file": "`mdl/catalog/builder_references.go` (`microflowCommitRef`, `RefKindCommit`)", "insight": "Commit is a use of the entity type like change/delete, resolved through the same intra-flow varEntity map (so the loop-iterator fix of #1266 applies for free), and emitted as a second edge beside create/change rather than replacing them. It stays out of graphRefKinds (would double existing flow->entity edges) and callerRefKinds (a type use, not an invocation). The ref_kind vocabulary test now reads every RefKind constant from the declarations, so a new kind cannot ship undocumented.", "refs": ["ako/mxcli#963", "mendixlabs/mxcli#1266", "mendixlabs/mxcli#1267"]} {"area": "mdl/linter", "date": "2026-10-03", "symptom": "ako/mxcli#962 item 4: MPR010 'DataView contains input fields but is not inside a layout grid' fires on a page with layout Atlas_Core.NativePhone_Default (lint and check); the bare form builds clean there and following the advice is CE6858 'Please update Atlas UI to version 2.4 or higher to use Layout Grid on Native pages' (mxbuild 11.13.0, PedApp)", "cause": "Same web-only assumption MPR012 had (#953): the rule's premise is Bootstrap label/input columns, which only the web client renders, but it walked every page and snippet regardless of platform", "file": "`mdl/linter/rules/dataview_layout_grid.go` (skip ctx.NativePages(), snippets with raw Type 'Native', RequiredCatalogMode CatalogFull); `mdl/executor/validate_page_layout.go` (validatePageLayoutGridWith + projectNativeLayouts)", "fix": "Lint: skip native pages via LintContext.NativePages() (declares CatalogFull since LayoutRef is full-only — NativePages added to the catalog-mode guard list) and native snippets via the snippet's own Type. Check: ask the project whether the page's layout is native, only when the page has something to report", "insight": "A platform-specific rule needs its platform in the predicate, and there are two places a page's platform lives: the layout (pages) and the document's own Type (snippets). Also measure the ADVICE, not only the warning: here following it produced a new CE, which settles 'does it apply' faster than reasoning about rendering. A rule that starts reading NativePages() silently returns nothing on a fast catalog unless it declares CatalogFull — the guard test only catches it if the method is in its list", "test": "`mdl/linter/rules/dataview_layout_grid_test.go` (TestDataViewLayoutGridRule_SkipsNativePagesAndSnippets); `mdl/executor/validate_page_layout_test.go` (TestValidatePageLayoutGrid_SkipsNativeLayouts); `cmd/mxcli/check_native_layout_grid_test.go`"} +{"area": "mdl/exprcheck", "date": "2026-10-04", "symptom": "ako/mxcli#969 item 1: `change $Log (Windrichting = if $Dir = 'NW' then E.NW else E.N)` on an enumeration attribute was refused with E001 (\"assigning an Enumeration attribute against a string literal\") for the 'NW' compared with a String; `find('|NW|NORTHWEST|', …)` inside the expression gave 3x E001 with fixes like `E.|`. exec runs check first, so a valid microflow could not be written", "cause": "parsePrimary ran checkStringLitVsSlot (E001) and the quoted-Boolean E002 on EVERY string literal while the slot path was set, so comparison operands, function arguments and if-conditions were judged as the slot's value", "file": "`mdl/exprcheck/parser.go` (checkValueLiterals)", "insight": "A slot constrains the VALUE, and the value is only the whole expression, a parenthesised one or a then/else result (recursively). Run slot-literal rules over the finished tree at those positions instead of during the parse, where the position is not yet known. Measured on 11.13.0: the comparison form builds clean, while a quoted 'NW' in a then-branch is CE0117 — so the branch rule is real and must keep firing. Note the WHOLE-literal form (`Wind = 'NW'`) builds clean because the writer rewrites it to the enum value; E001 there is stricter than mxbuild", "refs": ["#969"]} +{"area": "mdl/exprcheck", "date": "2026-10-04", "symptom": "ako/mxcli#969 item 1 side finding: `change $Log (Wind = 'NW')` reported nothing in check when the enumeration and the entity were created in the same script; with them already stored it was E001", "cause": "TypeCheckProgram's CatalogReader is loaded from the catalog of the stored project only, so a script-declared attribute resolved to nothing, its kind was Unknown and every rule keyed on it stayed silent", "file": "`mdl/executor/typecheck.go` (declareScriptTypes), `mdl/exprcatalog/exprcatalog.go` (DeclareEnumeration, DeclareAttribute)", "insight": "Any catalog-backed check must overlay the script's own declarations, applied in statement order; Unknown is designed to suppress rules, so a missing overlay fails silent rather than loud. Test with the definitions in the SAME script as the use — the stored-project test passed all along", "refs": ["#969"]} +{"area": "mdl/linter", "date": "2026-10-04", "symptom": "ako/mxcli#969 item 4: MPR006 warned that an empty container \"will crash at runtime\" (\"Did not expect an argument to be undefined\"), and the create-page skill told authors to pad every container with a dynamictext", "cause": "Claim from the initial commit with no measurement behind it", "file": "`mdl/linter/rules/empty_container.go`, `.claude/skills/mendix/create-page/SKILL.md`", "insight": "Measured on 11.13.0 (React client) with run --local --db-type hsqldb and a headless Chromium: a bare and a styled empty container render, the widgets after them are present, 0 console errors; mx check 0 errors. A runtime claim needs a runtime measurement — run --local + playwright is the layer (hsqldb avoids needing Postgres; the devcontainer needed the chromium shared libs apt-installed). Classic (Dojo) client not measured", "refs": ["#969"]} diff --git a/.claude/skills/mendix/create-page/reference/widgets.md b/.claude/skills/mendix/create-page/reference/widgets.md index c55355aed3..3831dc464f 100644 --- a/.claude/skills/mendix/create-page/reference/widgets.md +++ b/.claude/skills/mendix/create-page/reference/widgets.md @@ -385,6 +385,8 @@ column (caption: 'Actions') { | `datasource: $currentObject/Module.Assoc` | Sugar for `association` — same semantics, reads more naturally | | `datasource: database from $Ctx/Module.Assoc/Module.Entity [where …] [sort by …] [search by …]` | **List view only.** A *database* retrieve of what the association reaches — keeps XPath, sort and search, which the association source above does not have. Name the entity after each association (it may be a specialization of the association's end) | +> **A list widget needs an association that yields a list.** A list view, data grid or gallery over `$currentObject/M.Assoc` is CE8812 ("A grid association path must result in a list") when the association is a Reference followed from its FROM entity, or a Reference with owner Both from either end (one-to-one). Use a data view for the single object, or a ReferenceSet. `check` reports it as MDL-ASSOCDS01. The reverse of a default Reference (from the TO entity) is a list and fine. + **With WHERE and SORT BY (inline in DataSource):** ```sql datagrid dgActive ( diff --git a/CHANGELOG.md b/CHANGELOG.md index 8a83d6a4e7..3c767f5bb3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -56,6 +56,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). ### Fixed +- **`check` no longer reports E001 for a string compared or passed inside an enumeration assignment** (ako/mxcli#969) — `change $Log (Wind = if $Dir = 'NW' then E.NW else E.N)` was refused because the `'NW'` compared with a String was read as the value assigned to the enumeration, and each literal argument of `find('|NW|…', …)` drew its own E001; since `exec` runs `check` first, the microflow could not be written. E001 and E002 now judge only the literal that *is* the value: the whole expression, or a `then`/`else` result. A quoted value in a branch (`then 'NW'`) is still E001 — measured CE0117 on 11.13.0. And when the enumeration and the attribute are created in the same script, a genuine `change $X (Wind = 'NW')` is now reported; the type checker used to know only the stored project and said nothing. +- **`alter page … set Action = microflow M.X on btn` accepts a flow or page the script creates** (ako/mxcli#969) — the dry run of the SET resolved only against the stored project, so a microflow, nanoflow or page created earlier in the same script was "not found" in `check` and `exec` refused the whole script. A target the script does not create is still reported. +- **MPR006 no longer says an empty container crashes at runtime** (ako/mxcli#969) — measured on 11.13.0 with `run --local` and a headless browser, an empty container (bare or styled) builds and renders with no console errors. MPR006 is now an info note in the quality category ("empty — valid and it renders, but probably unintended"), and the create-page skill no longer tells authors to pad containers. - **`list workflows` and `show structure` count every activity of a workflow** (ako/mxcli#963) — including those on a boundary-event path and in an event sub-process, as the catalog's `workflows_data` has since #937. TestApp `Workflow1` listed 5 activities where the catalog counted 8; all three now share one walk (`wfnames.WalkActivities`). - **`docker build`, the TUI checker and `mxcli eval` no longer modify the project** (ako/mxcli#961) — `docker build` (and `docker run` / `docker reload`, which use it) now runs `mx update-widgets`, `mx check` and MxBuild on one temporary copy and writes only its output directory (`.docker/build/` or `-o`). Before, it rewrote an MPR v1 project's `.mpr`, rewrote every MPR v2 `.mxunit` (restored with new mtimes), and wrote `theme-cache/`, `deployment/`, `javasource/` proxies, the `.launch` file, `.classpath` and `.project` into the project. The build still uses the widget-normalised model, from the copy, and the package it produces is the same; `mxcli fix widgets` applies the normalisation to the project. **The project's `deployment/` is no longer refreshed by `docker build`** — `run --local` builds its own. The TUI auto-check and eval's `mx_check` ran a plain `mx check` on the project, which wrote `theme-cache/web/` and `deployment/sass/` every time; they now check a copy too. - **MPR010 no longer tells a native page to use a layout grid** (ako/mxcli#962) — the "wrap the form in a layoutgrid" advice is about Bootstrap columns, which a native page does not have: a form DataView directly on an `Atlas_Core.NativePhone_Default` page builds clean in mxbuild 11.13.0, and following the advice there is CE6858 ("update Atlas UI … to use Layout Grid on Native pages"). `lint` skips pages on a native layout and snippets of type Native (MPR010 now needs the full catalog, which a default `lint` builds anyway); `check -p` skips pages whose layout the project says is native. @@ -138,6 +141,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). ### Added +- **`check` foresees CE8812 for a list over a single-object association** (ako/mxcli#969) — **MDL-ASSOCDS01**: a list view, data grid or gallery whose data source is `$currentObject/M.Assoc` (or `association M.Assoc`) over a Reference followed from its FROM entity, or over a Reference with owner Both from either end, is refused; mxbuild 11.13.0 and 11.14.0 reject both with "A grid association path must result in a list". A ReferenceSet and the reverse of a default Reference are accepted, as mxbuild accepts them. +- **An info hint for a quoted template parameter that reads like an expression** (ako/mxcli#969) — **MDL-PARAMQUOTE01**: `{1} = 'formatDateTime($Log/Date, ''d MMM'')'` is stored as literal text and the page shows it verbatim (measured with `run --local`); `check` now says so and suggests dropping the quotes. It fires on a quoted call of a known Mendix function (parenthesis right after the name) or a `$Var/Attr` path, and not on text such as `'Total (incl. VAT)'`, `'length (cm)'` or `'Price in $'`. - **A `commit` reference kind in the catalog** (ako/mxcli#963) — `refs` has a `commit` edge from a flow to the entity it commits: a commit action, or a create / change that commits (`Yes` or `YesWithoutEvents`), the latter beside its `create` / `change` edge. The entity of a committed variable is resolved like `change` / `delete`, loop iterators included, so `commit $Order` inside `loop $Order in $Orders` now has a row. `refs_to("M.Order")` in a Starlark rule answers which flows commit an order. `commit` is not in the analysis graph and not a caller kind. The catalog schema version is bumped, so a cached catalog rebuilds. - **`total_activity_count` on the Starlark microflow struct** (ako/mxcli#963) — the catalog's `TotalActivityCount` (loop bodies included, at any depth) for every flow `microflows()` yields: microflows, nanoflows and rules. `activity_count` keeps counting a loop as one activity. - **The catalog's `widgets` table records the widget tree, appearance and primary action** (mendixlabs/mxcli#1268) — `ParentWidgetId` (the nearest catalogued ancestor; skipped wrappers, layout grid rows and columns, tab pages and data grid 2 columns are transparent), `Depth` (0 at the page or snippet root), `Class`, `Style`, `DynamicClasses`, `ActionType` (the stored type of the button, on-click or click action, e.g. `Forms$DeleteClientAction`) and `HasConfirmation`. Starlark `widgets()` exposes them as `parent_widget_id`, `depth`, `class_name`, `style`, `dynamic_classes`, `action_type`, `has_confirmation`, plus `page_ref`, so a lint rule can flag deep nesting, inline styles, classes outside an allow-list and delete buttons. A delete action has no confirmation setting in Mendix; the enforceable rule is "no button uses the delete action directly". The catalog schema version is bumped, so a cached catalog rebuilds. From f7ce0192da01c19db09c4be33bd94240edd36258 Mon Sep 17 00:00:00 2001 From: Ako Date: Sun, 4 Oct 2026 15:26:45 +0000 Subject: [PATCH 16/17] fix: write, describe and alter a date picker's DateFormat (#968) A date picker's DateFormat / CustomDateFormat passed check, was stored as the default Date, and was never described, so describe -> exec turned a Studio Pro date-time picker into a date-only one (mendixlabs/mxcli#1263). The builder now reads them into the picker's FormattingInfo, the writer stores it, describe prints non-default values, alter page set changes them, and check refuses an unknown value, a pattern with no DateFormat and Custom with no pattern (mxbuild CE0493) as MDL-WIDGET18. A text box's DecimalPrecision / GroupDigits had the same drop and is fixed with it. The unknown-property exemption for the dynamic-text format keys now applies to dynamictext only, and a dynamic-text parameter's customDateFormat beside an explicit non-Custom dateFormat (stored by Studio Pro) is accepted. Closes #968 Co-Authored-By: Claude Opus 5.5 --- .../fix-issue/findings/mdl-executor.jsonl | 1 + CHANGELOG.md | 2 + mdl/backend/mcp/page_widgets.go | 24 ++- .../modelsdk/widget_formatting_write_test.go | 64 ++++++ mdl/backend/modelsdk/widget_write.go | 13 +- mdl/backend/pagemutator/mutator.go | 90 ++++++++ mdl/executor/cmd_pages_builder_v3_widgets.go | 16 ++ mdl/executor/cmd_pages_describe.go | 10 +- mdl/executor/cmd_pages_describe_output.go | 4 + mdl/executor/cmd_pages_describe_parse.go | 2 + mdl/executor/dynamictext_format_test.go | 6 + mdl/executor/input_formatting.go | 201 ++++++++++++++++++ mdl/executor/input_formatting_pedapp_test.go | 187 ++++++++++++++++ mdl/executor/input_formatting_test.go | 64 ++++++ mdl/executor/validate_widgets.go | 36 +++- sdk/pages/formatting.go | 39 ++++ sdk/pages/pages_widgets_input.go | 10 +- 17 files changed, 750 insertions(+), 19 deletions(-) create mode 100644 mdl/backend/modelsdk/widget_formatting_write_test.go create mode 100644 mdl/executor/input_formatting.go create mode 100644 mdl/executor/input_formatting_pedapp_test.go create mode 100644 mdl/executor/input_formatting_test.go create mode 100644 sdk/pages/formatting.go diff --git a/.claude/skills/fix-issue/findings/mdl-executor.jsonl b/.claude/skills/fix-issue/findings/mdl-executor.jsonl index 8d3c87c883..0fc55a9fc5 100644 --- a/.claude/skills/fix-issue/findings/mdl-executor.jsonl +++ b/.claude/skills/fix-issue/findings/mdl-executor.jsonl @@ -853,3 +853,4 @@ {"area": "mdl/executor", "date": "2026-10-03", "symptom": "`list workflows` and `show structure` report fewer workflow activities than the catalog's workflows_data (TestApp Workflow1: 5 vs 8)", "cause": "cmd_workflows.go countFlowActivities and cmd_structure.go countStructureFlowActivities each recursed over outcome flows only, skipping boundary-event flows and event sub-processes; the catalog had moved to a shared walk in #937 and the executor copies were left behind", "file": "`mdl/backend/wfnames/walk.go` (`WalkActivities`, `CountActivities`), `mdl/executor/cmd_workflows.go`, `mdl/executor/cmd_structure.go`, `mdl/catalog/workflow_walk.go`", "insight": "Duplicate-resolver drift: fixing one copy of a traversal (#937) left two private copies answering differently. The walk now lives in wfnames, which both catalog and executor already import, so there is one place to add a new sub-flow slot. Grep for every recursion over `workflows.Flow` when one is fixed.", "refs": ["ako/mxcli#963", "ako/mxcli#937"]} {"area": "mdl/executor", "date": "2026-10-03", "symptom": "ako/mxcli#962 item 1: `$V = call java action M.VoidAction(...)` then `log ... + $V` (or `$V` as a JS call argument) passes `check` and `exec`, then mxbuild 11.13.0 fails CE0109 \"Undefined variable 'V'\"", "cause": "#953 taught MDL063 that a void call's output name declares nothing, but no rule read the other half: the name cannot be READ either. The resolver only answered void/not-void, so 'unknown' and 'non-void' were the same answer", "file": "`mdl/executor/validate_void_call_output.go` (checkVoidCallOutputUse, MDL093); `mdl/executor/validate_void_code_calls.go` (resolve -> voidness{void, known})", "fix": "MDL093: collect output names of calls KNOWN to be void, drop any name another statement defines flow-wide (declare, parameter, non-void producer), report each remaining name the flow reads (loopRefVars over every nested body). The resolver now returns known-ness, so 'possibly void' (the editor's policy) never produces MDL093", "insight": "A finding that says 'X declares nothing' has two consequences — no collision AND no definition; fixing the first and logging the second as follow-up left a CE gap. When a resolver's default is a policy (unknown counts as non-void), make the unknown state explicit before a second rule reads it: the CE0109 rule must use only knowledge, never the policy", "test": "`mdl/executor/validate_void_call_output_test.go`; `cmd/mxcli/check_void_calls_test.go` (TestCheck_ReadOfAStoredVoidCallOutput, PedApp stored void JS action, Boolean and unresolvable controls)"} {"area": "mdl/executor", "date": "2026-10-03", "symptom": "ako/mxcli#962 item 2: describe of a microflow with the same output name in each if/else branch (or inside a loop and again after it) printed no duplicate-variable warning; mxbuild 11.13.0 reports CE0111 for both", "cause": "duplicateOutputVariableWarnings only warned when one assignment could REACH another (a reachability walk written for #710's performance), so exclusive branches and a loop body vs. the flow after it were treated as separate scopes; a test pinned the branch scoping", "file": "`mdl/executor/cmd_microflows_show.go` (duplicateOutputVariableWarnings)", "fix": "Count non-void output names over every object collection (loop bodies included); warn for any name created twice. Linear, so #710's cost concern disappears with the reachability walk", "insight": "The reachability model encoded a belief (exclusive paths may reuse a name) that no one had measured; MDL063 had already been aligned to flow-wide names in #958, so two renderings of the same rule disagreed. When one rule is corrected against mxbuild, grep for the other places that encode the same rule — here describe's header warning", "test": "`mdl/executor/cmd_microflows_duplicate_output_test.go` (TestFormatMicroflowActivitiesWarnsForExclusiveBranchOutputs, TestFormatMicroflowActivitiesNamesAreFlowWide)"} +{"area": "mdl/executor", "date": "2026-10-04", "symptom": "ako/mxcli#968 / mendixlabs/mxcli#1263: `datepicker d (DateFormat: Time)` or `DateFormat: Custom, CustomDateFormat: '\u2026'` passes check and exec, mx check 0 errors, but every picker is stored FormattingInfo.DateFormat=Date; describe prints no format, so describe \u2192 exec silently turns a Studio Pro date-time picker into a date-only one. Same drop for a text box's DecimalPrecision/GroupDigits", "cause": "Three hops each dropped it: buildDatePickerV3/buildTextBoxV3 never read the properties, widget_write.go hard-coded newFormattingInfo() on DatePicker and TextBox, and describe never extracted FormattingInfo. Check stayed silent because validateStaticWidgetUnknownProps exempted the dynamic-text format keys (dateformat, customdateformat, \u2026) on EVERY widget type, not just dynamictext", "fix": "pages.DatePicker.FormattingInfo; executor input_formatting.go (inputFormattingInfo + inputFormattingProblems shared by builder and MDL-WIDGET18 check), writer formattingInfoToGen(x.FormattingInfo), describeInputFormatting, pagemutator setWidgetFormattingMut; per-widget allow-list pages.FormattingProperties. Measured: Custom with empty pattern = CE0493; Studio Pro stores CustomDateFormat beside DateFormat DateTime (TestApp WorkflowCommons), so only a pattern with NO DateFormat is refused \u2014 the param-format rule that refused it broke check on describe output", "file": "mdl/executor/input_formatting.go", "insight": "A key exempted from the unknown-property warning must be exempted per widget type: the dynamic-text format keys were skipped on every widget, which turned `DateFormat:` on a date picker (where nothing read it) into a silent drop. Before refusing a cross-field combination, scan Studio Pro-authored units for it \u2014 CustomDateFormat beside DateTime is stored by Studio Pro, and refusing it broke check on describe output.", "test": "mdl/executor/input_formatting_pedapp_test.go, input_formatting_test.go, mdl/backend/modelsdk/widget_formatting_write_test.go"} diff --git a/CHANGELOG.md b/CHANGELOG.md index 8a83d6a4e7..fd7d4ac5f4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -56,6 +56,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). ### Fixed +- **A date picker's `DateFormat` / `CustomDateFormat` is written, described and alterable** (ako/mxcli#968, mendixlabs/mxcli#1263) — `datepicker d (…, DateFormat: Time)` passed `check` and was stored as `Date`, so every picker was date-only, and `describe` printed neither property, so describe → exec turned a Studio Pro date-time picker into a date-only one. Both are now written to the picker's `FormattingInfo`, printed when not the default, settable with `alter page … set (DateFormat: …)`, and checked (MDL-WIDGET18): an unknown value, a `CustomDateFormat` with no `DateFormat`, and `DateFormat: Custom` with no pattern (mxbuild CE0493) are refused. The same drop on a text box's `DecimalPrecision` / `GroupDigits` is fixed with it. `DateFormat:` on a widget with no such format (a text area, a text box) now warns MDL-WIDGET07 instead of passing silently. A dynamic-text parameter's `customDateFormat` beside an explicit non-Custom `dateFormat` — what Studio Pro stores after switching away from Custom — is no longer refused, so describe's own output for TestApp's WorkflowCommons snippets checks clean. + - **`list workflows` and `show structure` count every activity of a workflow** (ako/mxcli#963) — including those on a boundary-event path and in an event sub-process, as the catalog's `workflows_data` has since #937. TestApp `Workflow1` listed 5 activities where the catalog counted 8; all three now share one walk (`wfnames.WalkActivities`). - **`docker build`, the TUI checker and `mxcli eval` no longer modify the project** (ako/mxcli#961) — `docker build` (and `docker run` / `docker reload`, which use it) now runs `mx update-widgets`, `mx check` and MxBuild on one temporary copy and writes only its output directory (`.docker/build/` or `-o`). Before, it rewrote an MPR v1 project's `.mpr`, rewrote every MPR v2 `.mxunit` (restored with new mtimes), and wrote `theme-cache/`, `deployment/`, `javasource/` proxies, the `.launch` file, `.classpath` and `.project` into the project. The build still uses the widget-normalised model, from the copy, and the package it produces is the same; `mxcli fix widgets` applies the normalisation to the project. **The project's `deployment/` is no longer refreshed by `docker build`** — `run --local` builds its own. The TUI auto-check and eval's `mx_check` ran a plain `mx check` on the project, which wrote `theme-cache/web/` and `deployment/sass/` every time; they now check a copy too. - **MPR010 no longer tells a native page to use a layout grid** (ako/mxcli#962) — the "wrap the form in a layoutgrid" advice is about Bootstrap columns, which a native page does not have: a form DataView directly on an `Atlas_Core.NativePhone_Default` page builds clean in mxbuild 11.13.0, and following the advice there is CE6858 ("update Atlas UI … to use Layout Grid on Native pages"). `lint` skips pages on a native layout and snippets of type Native (MPR010 now needs the full catalog, which a default `lint` builds anyway); `check -p` skips pages whose layout the project says is native. diff --git a/mdl/backend/mcp/page_widgets.go b/mdl/backend/mcp/page_widgets.go index 00c7105589..03d700b754 100644 --- a/mdl/backend/mcp/page_widgets.go +++ b/mdl/backend/mcp/page_widgets.go @@ -236,11 +236,11 @@ func (b *Backend) mapPageWidgetBody(w pages.Widget) (map[string]any, error) { case *pages.DataGrid: return nil, fmt.Errorf("legacy DataGrid is not supported by the MCP backend — pg_patch_page has no Pages$DataGrid type (use a ListView, or DataGrid 2 which is a pluggable widget)") case *pages.TextBox: - return inputWidget("Pages$TextBox", wd.Name, wd.Label, wd.AttributePath, wd.Class, wd.Style, wd.SourceVariable), nil + return withFormattingInfo(inputWidget("Pages$TextBox", wd.Name, wd.Label, wd.AttributePath, wd.Class, wd.Style, wd.SourceVariable), wd.FormattingInfo), nil case *pages.CheckBox: return inputWidget("Pages$CheckBox", wd.Name, wd.Label, wd.AttributePath, wd.Class, wd.Style, wd.SourceVariable), nil case *pages.DatePicker: - return inputWidget("Pages$DatePicker", wd.Name, wd.Label, wd.AttributePath, wd.Class, wd.Style, wd.SourceVariable), nil + return withFormattingInfo(inputWidget("Pages$DatePicker", wd.Name, wd.Label, wd.AttributePath, wd.Class, wd.Style, wd.SourceVariable), wd.FormattingInfo), nil case *pages.TextArea: return inputWidget("Pages$TextArea", wd.Name, wd.Label, wd.AttributePath, wd.Class, wd.Style, wd.SourceVariable), nil case *pages.RadioButtons: @@ -273,6 +273,26 @@ func inputWidget(typ, name, label, attribute, class, style string, sv *pages.Wid return w } +// withFormattingInfo adds an input widget's authored FormattingInfo — a date +// picker's DateFormat, a text box's precision — in the same Pages$FormattingInfo +// shape the template parameters send. nil leaves the slot to Studio Pro's +// defaults, as before; dropping an authored one silently turned a date-time +// picker into a date picker (ako/mxcli#968). +func withFormattingInfo(w map[string]any, fi *pages.FormattingInfo) map[string]any { + if fi == nil { + return w + } + w["formattingInfo"] = map[string]any{ + "$Type": "Pages$FormattingInfo", + "decimalPrecision": fi.DecimalPrecision, + "groupDigits": fi.GroupDigits, + "enumFormat": fi.EnumFormat, + "dateFormat": fi.DateFormat, + "customDateFormat": fi.CustomDateFormat, + } + return w +} + // pageVariable builds a Pages$PageVariable naming a data view and, beside it, // the data view's own variable. Studio Pro 11.14 fills that variable in itself // when given only the widget, so sending it is the same document either way. diff --git a/mdl/backend/modelsdk/widget_formatting_write_test.go b/mdl/backend/modelsdk/widget_formatting_write_test.go new file mode 100644 index 0000000000..20e995affc --- /dev/null +++ b/mdl/backend/modelsdk/widget_formatting_write_test.go @@ -0,0 +1,64 @@ +// SPDX-License-Identifier: Apache-2.0 + +package modelsdkbackend + +import ( + "testing" + + bsonv1 "go.mongodb.org/mongo-driver/bson" + + "github.com/mendixlabs/mxcli/sdk/pages" +) + +// ako/mxcli#968 (mendixlabs/mxcli#1263): the writer hard-coded the default +// Forms$FormattingInfo on every date picker and text box, so `DateFormat: Time` +// was stored as Date and a date-time picker came back date-only. + +func formattingOf(t *testing.T, doc bsonv1.D) map[string]any { + t.Helper() + fi, ok := docGet(doc, "FormattingInfo").(bsonv1.D) + if !ok { + t.Fatalf("FormattingInfo = %#v, want a Forms$FormattingInfo document", docGet(doc, "FormattingInfo")) + } + out := map[string]any{} + for _, e := range fi { + out[e.Key] = e.Value + } + return out +} + +func TestDatePickerFormattingInfoWritten(t *testing.T) { + for _, tc := range []struct { + name string + fi *pages.FormattingInfo + wantDF, wantCD string + }{ + {"default", nil, "Date", ""}, + {"time", &pages.FormattingInfo{DateFormat: "Time", DecimalPrecision: 2, EnumFormat: "Text"}, "Time", ""}, + {"datetime", &pages.FormattingInfo{DateFormat: "DateTime", DecimalPrecision: 2, EnumFormat: "Text"}, "DateTime", ""}, + {"custom", &pages.FormattingInfo{DateFormat: "Custom", CustomDateFormat: "dd-MM-yyyy H:mm", DecimalPrecision: 2, EnumFormat: "Text"}, "Custom", "dd-MM-yyyy H:mm"}, + } { + t.Run(tc.name, func(t *testing.T) { + dp := &pages.DatePicker{BaseWidget: pages.BaseWidget{Name: "dp"}, AttributePath: "M.Booking.Moment", FormattingInfo: tc.fi} + got := formattingOf(t, encodeWidget(t, dp)) + if got["DateFormat"] != tc.wantDF || got["CustomDateFormat"] != tc.wantCD { + t.Errorf("FormattingInfo DateFormat/CustomDateFormat = %v/%q, want %s/%q", + got["DateFormat"], got["CustomDateFormat"], tc.wantDF, tc.wantCD) + } + }) + } +} + +func TestTextBoxFormattingInfoWritten(t *testing.T) { + tb := &pages.TextBox{BaseWidget: pages.BaseWidget{Name: "tb"}, AttributePath: "M.Booking.Amount", + FormattingInfo: &pages.FormattingInfo{DateFormat: "Date", DecimalPrecision: 4, GroupDigits: true, EnumFormat: "Text"}} + got := formattingOf(t, encodeWidget(t, tb)) + if got["DecimalPrecision"] != int32(4) || got["GroupDigits"] != true { + t.Errorf("FormattingInfo DecimalPrecision/GroupDigits = %v/%v, want 4/true", got["DecimalPrecision"], got["GroupDigits"]) + } + // Control: no FormattingInfo keeps the defaults byte-for-byte. + def := formattingOf(t, encodeWidget(t, &pages.TextBox{BaseWidget: pages.BaseWidget{Name: "tb"}, AttributePath: "M.Booking.Amount"})) + if def["DecimalPrecision"] != int32(2) || def["GroupDigits"] != false || def["DateFormat"] != "Date" { + t.Errorf("default FormattingInfo = %v, want precision 2, no grouping, Date", def) + } +} diff --git a/mdl/backend/modelsdk/widget_write.go b/mdl/backend/modelsdk/widget_write.go index 33855d0c0c..e03b01f62e 100644 --- a/mdl/backend/modelsdk/widget_write.go +++ b/mdl/backend/modelsdk/widget_write.go @@ -491,7 +491,7 @@ func widgetToGen(w pages.Widget) (element.Element, error) { g.SetSourceVariable(sv) } g.SetEditable(pages.WidgetEditability(&x.BaseWidget)) - g.SetFormattingInfo(newFormattingInfo()) + g.SetFormattingInfo(formattingInfoToGen(x.FormattingInfo)) g.SetInputMask("") g.SetIsPasswordBox(x.IsPassword) g.SetKeyboardType("Default") @@ -627,7 +627,10 @@ func widgetToGen(w pages.Widget) (element.Element, error) { g.SetSourceVariable(sv) } g.SetEditable(pages.WidgetEditability(&x.BaseWidget)) - g.SetFormattingInfo(newFormattingInfo()) + // The picker's format IS its mode: DateFormat Time or DateTime is what + // makes it a time / date-time picker. The defaults were hard-coded here, + // so every picker was written date-only (ako/mxcli#968). + g.SetFormattingInfo(formattingInfoToGen(x.FormattingInfo)) if x.Label != "" { g.SetLabelTemplate(textAsClientTemplate(textFromString(x.Label))) } @@ -1336,12 +1339,6 @@ func inputSourceVariableToGen(sv *pages.WidgetVariable) element.Element { return pageVariableToGen(sv.Widget, sv.Variable, sv.Kind) } -// newFormattingInfo builds the default Forms$FormattingInfo (matches the legacy -// serializer; TimeFormat is intentionally omitted — it triggers CE0463). -func newFormattingInfo() element.Element { - return formattingInfoToGen(nil) -} - // formattingInfoToGen builds a Forms$FormattingInfo, using the parameter's // per-parameter formatting when present and the standard defaults otherwise. A // nil fi reproduces the previous hardcoded defaults, so every unformatted diff --git a/mdl/backend/pagemutator/mutator.go b/mdl/backend/pagemutator/mutator.go index 9e3494a588..4aef9b275d 100644 --- a/mdl/backend/pagemutator/mutator.go +++ b/mdl/backend/pagemutator/mutator.go @@ -2999,12 +2999,102 @@ func setRawWidgetPropertyMut(widget bson.D, propName string, value any) error { return nil case "attribute": return setWidgetAttributeRefMut(widget, value) + case "dateformat", "customdateformat", "decimalprecision", "groupdigits": + return setWidgetFormattingMut(widget, propName, value) default: // Try as pluggable widget property return setPluggableWidgetPropertyMut(widget, propName, value) } } +// setWidgetFormattingMut sets one field of an input widget's +// Forms$FormattingInfo — a date picker's DateFormat / CustomDateFormat, a text +// box's DecimalPrecision / GroupDigits (ako/mxcli#968). Which widget carries +// which is pages.FormattingProperties, the list CREATE PAGE reads too; any +// other widget refuses the key instead of falling through to the pluggable +// setter's "no Object" error, which named the wrong problem. +func setWidgetFormattingMut(widget bson.D, propName string, value any) error { + typ := bsonnav.DGetString(widget, "$Type") + field := "" + for _, p := range pages.FormattingProperties(typ) { + if strings.EqualFold(p, propName) { + field = p + } + } + if field == "" { + return fmt.Errorf("a %s has no %s property — DateFormat and CustomDateFormat are a date picker's, "+ + "DecimalPrecision and GroupDigits a text box's", widgetTypeLabel(typ), propName) + } + fi := bsonnav.DGetDoc(widget, "FormattingInfo") + if fi == nil { + return fmt.Errorf("this %s stores no FormattingInfo to set %s on", widgetTypeLabel(typ), field) + } + switch field { + case "DateFormat": + s, isString := value.(string) + canon, ok := pages.CanonicalDateFormat(s) + if !isString || !ok { + return fmt.Errorf("DateFormat is one of Date, Time, DateTime, Custom, not %v", value) + } + // mxbuild: CE0493 "Date format is custom but no format string is + // specified." Set the pattern in the same statement — properties are + // applied in name order, so CustomDateFormat lands first. + if canon == "Custom" && strings.TrimSpace(bsonnav.DGetString(fi, "CustomDateFormat")) == "" { + return fmt.Errorf("`DateFormat: Custom` needs a pattern: set CustomDateFormat in the same statement, " + + "e.g. `set (DateFormat: Custom, CustomDateFormat: 'dd-MM-yyyy HH:mm') on …` (mxbuild CE0493)") + } + bsonnav.DSet(fi, "DateFormat", canon) + case "CustomDateFormat": + s, isString := value.(string) + if !isString { + return fmt.Errorf("CustomDateFormat is a quoted pattern such as 'dd-MM-yyyy HH:mm', not %v", value) + } + bsonnav.DSet(fi, "CustomDateFormat", s) + case "DecimalPrecision": + n, ok := formattingInt(value) + if !ok || n < 0 { + return fmt.Errorf("DecimalPrecision is a non-negative integer, not %v", value) + } + // Keep the stored width: Studio Pro writes an int64, mxcli's codec an + // int32, and replacing one with the other is a change of its own. + if _, isInt32 := bsonnav.DGet(fi, "DecimalPrecision").(int32); isInt32 { + bsonnav.DSet(fi, "DecimalPrecision", int32(n)) + } else { + bsonnav.DSet(fi, "DecimalPrecision", int64(n)) + } + case "GroupDigits": + b, ok := editableBool(value) + if !ok { + return fmt.Errorf("GroupDigits is true or false, not %v", value) + } + bsonnav.DSet(fi, "GroupDigits", b) + } + return nil +} + +// formattingInt reads an integer property value in any of the forms the +// visitor produces. +func formattingInt(v any) (int, bool) { + switch n := v.(type) { + case int: + return n, true + case int32: + return int(n), true + case int64: + return int(n), true + case float64: + if n == math.Trunc(n) { + return int(n), true + } + case string: + var i int + if _, err := fmt.Sscanf(n, "%d", &i); err == nil && fmt.Sprint(i) == n { + return i, true + } + } + return 0, false +} + // --------------------------------------------------------------------------- // Design property (Atlas styling) mutation // --------------------------------------------------------------------------- diff --git a/mdl/executor/cmd_pages_builder_v3_widgets.go b/mdl/executor/cmd_pages_builder_v3_widgets.go index 21fe6404e5..71091bfd9c 100644 --- a/mdl/executor/cmd_pages_builder_v3_widgets.go +++ b/mdl/executor/cmd_pages_builder_v3_widgets.go @@ -488,6 +488,15 @@ func (pb *pageBuilder) buildTextBoxV3(w *ast.WidgetV3) (*pages.TextBox, error) { tb.ValidationExpression = w.GetStringProp("Validation") tb.ValidationMessage = w.GetStringProp("ValidationMessage") + // Forms$FormattingInfo: date format and numeric precision/grouping. The + // writer hard-coded the defaults, so a Studio Pro text box with a decimal + // precision lost it on describe → exec (ako/mxcli#968). + fi, err := inputFormattingInfo(w) + if err != nil { + return nil, err + } + tb.FormattingInfo = fi + // Handle Label if label := inputLabel(w); label != "" { tb.Label = label @@ -601,6 +610,13 @@ func (pb *pageBuilder) buildDatePickerV3(w *ast.WidgetV3) (*pages.DatePicker, er dp.Label = label } + // DateFormat / CustomDateFormat — the picker's mode (ako/mxcli#968). + fi, err := inputFormattingInfo(w) + if err != nil { + return nil, err + } + dp.FormattingInfo = fi + // Handle OnChange (the "On change" client action) if err := pb.applyOnChangeV3(w, &dp.OnChangeAction); err != nil { return nil, err diff --git a/mdl/executor/cmd_pages_describe.go b/mdl/executor/cmd_pages_describe.go index 2c3740b991..4a19cae80a 100644 --- a/mdl/executor/cmd_pages_describe.go +++ b/mdl/executor/cmd_pages_describe.go @@ -695,9 +695,13 @@ type rawWidget struct { // input widgets. ValidationExpression string ValidationMessage string - OnChange string // MDL rendering of the OnChangeAction client action - OnClick string // MDL rendering of a pluggable widget's onClick action (e.g. DataGrid2) - OnClickTrigger string // Gallery's onClickTrigger when not the default "single" (#842) + // Formatting is the widget's Forms$FormattingInfo as MDL properties + // (DateFormat, CustomDateFormat, DecimalPrecision, GroupDigits), only the + // non-default ones (ako/mxcli#968). + Formatting []string + OnChange string // MDL rendering of the OnChangeAction client action + OnClick string // MDL rendering of a pluggable widget's onClick action (e.g. DataGrid2) + OnClickTrigger string // Gallery's onClickTrigger when not the default "single" (#842) // Filter widget properties FilterAttributes []string // Attributes to filter on FilterExpression string // Default filter expression (contains, startsWith, etc.) diff --git a/mdl/executor/cmd_pages_describe_output.go b/mdl/executor/cmd_pages_describe_output.go index 12a3673635..ce94b0c370 100644 --- a/mdl/executor/cmd_pages_describe_output.go +++ b/mdl/executor/cmd_pages_describe_output.go @@ -585,6 +585,7 @@ func outputWidgetMDLV3(ctx *ExecContext, w rawWidget, indent int) { if w.OnChange != "" { props = append(props, actionProp("OnChange", w.OnChange)) } + props = append(props, w.Formatting...) props = appendInputValidationProps(ctx, props, w) props = appendAppearanceProps(ctx, props, w) formatWidgetProps(ctx.Output, prefix, header, props, "\n") @@ -616,6 +617,9 @@ func outputWidgetMDLV3(ctx *ExecContext, w rawWidget, indent int) { if w.Content != "" { props = append(props, fmt.Sprintf("Attribute: %s", w.Content)) } + // The picker's mode — without it a date-time picker described as a + // date-only one, and describe → exec wrote it that way (ako/mxcli#968). + props = append(props, w.Formatting...) if w.OnChange != "" { props = append(props, actionProp("OnChange", w.OnChange)) } diff --git a/mdl/executor/cmd_pages_describe_parse.go b/mdl/executor/cmd_pages_describe_parse.go index 4e8e408bb7..5a9f8284c1 100644 --- a/mdl/executor/cmd_pages_describe_parse.go +++ b/mdl/executor/cmd_pages_describe_parse.go @@ -369,6 +369,7 @@ func parseRawWidget(ctx *ExecContext, w map[string]any, parentEntityContext ...s widget.IsPassword, _ = w["IsPasswordBox"].(bool) widget.ValidationExpression, widget.ValidationMessage = extractWidgetValidation(ctx, w) widget.OnChange = extractOnChangeAction(ctx, w) + widget.Formatting = describeInputFormatting(ctx, "textbox", w) return []rawWidget{widget} case "Forms$TextArea", "Pages$TextArea": @@ -386,6 +387,7 @@ func parseRawWidget(ctx *ExecContext, w map[string]any, parentEntityContext ...s widget.Content = extractInputAttribute(ctx, w) widget.Editable = extractEditable(ctx, w) widget.OnChange = extractOnChangeAction(ctx, w) + widget.Formatting = describeInputFormatting(ctx, "datepicker", w) return []rawWidget{widget} case "Forms$RadioButtons", "Pages$RadioButtons", "Forms$RadioButtonGroup", "Pages$RadioButtonGroup": diff --git a/mdl/executor/dynamictext_format_test.go b/mdl/executor/dynamictext_format_test.go index 8f5dd6693f..c64fb09bfe 100644 --- a/mdl/executor/dynamictext_format_test.go +++ b/mdl/executor/dynamictext_format_test.go @@ -73,6 +73,12 @@ func TestValidateDynamicTextFormatting(t *testing.T) { ast.ParamFormatProp{Key: "enumformat", Value: "Nope"}), nil), "enumFormat must be"}, {"custom without Custom", dtWidget(fmtBlock( ast.ParamFormatProp{Key: "customdateformat", Value: "yyyy"}), nil), "requires `dateFormat: Custom`"}, + // Studio Pro keeps the pattern when the format is switched away from + // Custom, and describe prints both: refusing it failed check on + // describe's own output for TestApp's WorkflowCommons snippets (#968). + {"pattern beside explicit DateTime", dtWidget(fmtBlock( + ast.ParamFormatProp{Key: "dateformat", Value: "DateTime"}, + ast.ParamFormatProp{Key: "customdateformat", Value: "MM/dd/yyyy . hh:mma"}), nil), ""}, {"widget-level format key", dtWidget(nil, map[string]any{"decimalPrecision": 2}), "per-parameter format"}, } for _, tt := range tests { diff --git a/mdl/executor/input_formatting.go b/mdl/executor/input_formatting.go new file mode 100644 index 0000000000..86f212270e --- /dev/null +++ b/mdl/executor/input_formatting.go @@ -0,0 +1,201 @@ +// SPDX-License-Identifier: Apache-2.0 + +package executor + +import ( + "fmt" + "strings" + + "github.com/mendixlabs/mxcli/mdl/ast" + "github.com/mendixlabs/mxcli/sdk/pages" +) + +// Input-widget formatting (ako/mxcli#968, mendixlabs/mxcli#1263). +// +// A date picker and a text box each store a Forms$FormattingInfo. On a date +// picker it is the picker's mode: DateFormat Time or DateTime is what makes it +// a time or date-time picker, and Custom pairs with a CustomDateFormat pattern. +// On a text box it also carries the decimal precision and digit grouping of a +// numeric attribute. Neither was read: the builder ignored the properties, the +// writer hard-coded the defaults and describe printed nothing, so `DateFormat: +// Time` passed check and was written as Date, and a Studio Pro date-time picker +// came back from describe → exec as a date-only one. +// +// The MDL spelling is the widget-property form of the keys a dynamic-text +// parameter's `format (…)` block already uses — PascalCase, like every other +// widget property: `DateFormat: DateTime, CustomDateFormat: 'dd-MM-yyyy HH:mm'`. + +// inputFormatProps lists, per MDL widget type, the formatting properties the +// widget's FormattingInfo carries (pages.FormattingProperties, keyed by the +// storage type). A widget type that is absent has none MDL can set. +var inputFormatProps = map[string][]string{ + "datepicker": pages.FormattingProperties("Forms$DatePicker"), + "textbox": pages.FormattingProperties("Forms$TextBox"), +} + +// inputFormatProp reports whether key is a formatting property of widgetType, +// and returns its canonical spelling. +func inputFormatProp(widgetType, key string) (string, bool) { + for _, p := range inputFormatProps[strings.ToLower(widgetType)] { + if strings.EqualFold(p, key) { + return p, true + } + } + return "", false +} + +// inputFormattingInfo reads an input widget's formatting properties into the +// FormattingInfo the writer stores. It returns nil when the widget sets none, +// so the writer keeps the defaults byte-for-byte. Unset fields start from the +// Studio Pro defaults (Date, precision 2, Text, no grouping), as a dynamic-text +// parameter's format block does. Invalid values are an error; the check-time +// rule (MDL-WIDGET18) reports the same problems from the same function. +func inputFormattingInfo(w *ast.WidgetV3) (*pages.FormattingInfo, error) { + if problems := inputFormattingProblems(w); len(problems) > 0 { + return nil, fmt.Errorf("%s %s: %s", strings.ToLower(w.Type), w.Name, strings.Join(problems, "; ")) + } + var fi *pages.FormattingInfo + get := func() *pages.FormattingInfo { + if fi == nil { + fi = &pages.FormattingInfo{DateFormat: "Date", DecimalPrecision: 2, EnumFormat: "Text"} + } + return fi + } + for _, key := range inputFormatProps[strings.ToLower(w.Type)] { + v, ok := lookupPropCI(w, key) + if !ok { + continue + } + switch key { + case "DateFormat": + s, _ := v.(string) + get().DateFormat, _ = pages.CanonicalDateFormat(s) + case "CustomDateFormat": + s, _ := v.(string) + get().CustomDateFormat = s + case "DecimalPrecision": + n, _ := toFormatInt(v) + get().DecimalPrecision = n + case "GroupDigits": + b, _ := propBool(v) + get().GroupDigits = b + } + } + return fi, nil +} + +// inputFormattingProblems validates an input widget's formatting properties. +// Each problem is a sentence fragment naming the property; nil means valid. +func inputFormattingProblems(w *ast.WidgetV3) []string { + keys := inputFormatProps[strings.ToLower(w.Type)] + if len(keys) == 0 { + return nil + } + var out []string + dateFormat := "" + hasDateFormat := false + for _, key := range keys { + v, ok := lookupPropCI(w, key) + if !ok { + continue + } + switch key { + case "DateFormat": + hasDateFormat = true + s, isStr := v.(string) + canon, valid := pages.CanonicalDateFormat(s) + if !isStr || !valid { + out = append(out, fmt.Sprintf("DateFormat must be one of Date, Time, DateTime, Custom, got `%v`", v)) + continue + } + dateFormat = canon + case "CustomDateFormat": + if _, isStr := v.(string); !isStr { + out = append(out, fmt.Sprintf("CustomDateFormat must be a quoted pattern such as 'dd-MM-yyyy HH:mm', got `%v`", v)) + } + case "DecimalPrecision": + if n, isInt := toFormatInt(v); !isInt || n < 0 { + out = append(out, fmt.Sprintf("DecimalPrecision must be a non-negative integer, got `%v`", v)) + } + case "GroupDigits": + if _, err := propBool(v); err != nil { + out = append(out, fmt.Sprintf("GroupDigits must be true or false, got `%v`", v)) + } + } + } + custom, hasCustom := lookupPropCI(w, "CustomDateFormat") + // A pattern with no DateFormat never applies: the default is Date. An + // explicit other DateFormat beside a pattern is accepted — Studio Pro keeps + // the pattern when the format is switched away from Custom (TestApp's + // WorkflowCommons pickers store DateTime + 'dd/MM/yyyy HH:mm'), and + // describe prints both so the round trip preserves it. + if hasCustom && !hasDateFormat { + out = append(out, "CustomDateFormat applies only with `DateFormat: Custom` — add it, or the pattern is ignored and the widget shows a date") + } + if dateFormat == "Custom" { + if s, _ := custom.(string); strings.TrimSpace(s) == "" { + out = append(out, "`DateFormat: Custom` needs a non-empty CustomDateFormat pattern, e.g. `CustomDateFormat: 'dd-MM-yyyy HH:mm'`") + } + } + return out +} + +// toFormatInt accepts the integer forms the visitor produces. +func toFormatInt(v any) (int, bool) { + switch n := v.(type) { + case int: + return n, true + case int64: + return int(n), true + case float64: + if n == float64(int(n)) { + return int(n), true + } + } + return 0, false +} + +// describeInputFormatting renders a stored Forms$FormattingInfo as the MDL +// properties of an input widget of the given MDL type, emitting only what +// differs from the defaults the writer fills in (Date, precision 2, no +// grouping, no pattern), so an unformatted widget describes as before. +// +// A pattern beside a DateFormat other than Custom is printed with that +// DateFormat, even Date: Studio Pro keeps the pattern when the format is +// switched away from Custom, and check refuses a bare CustomDateFormat, so +// printing it alone would make describe's own output fail check. +func describeInputFormatting(ctx *ExecContext, mdlType string, w map[string]any) []string { + keys := inputFormatProps[mdlType] + fi, ok := w["FormattingInfo"].(map[string]any) + if len(keys) == 0 || !ok || fi == nil { + return nil + } + var props []string + for _, key := range keys { + switch key { + case "DateFormat": + df := extractString(fi["DateFormat"]) + if df == "" { + df = "Date" + } + if df != "Date" || extractString(fi["CustomDateFormat"]) != "" { + props = append(props, "DateFormat: "+df) + } + case "CustomDateFormat": + if cdf := extractString(fi["CustomDateFormat"]); cdf != "" { + props = append(props, "CustomDateFormat: "+mdlQuote(ctx, cdf)) + } + case "DecimalPrecision": + if v, present := fi["DecimalPrecision"]; present { + if dp := extractInt(v); dp != 2 { + props = append(props, fmt.Sprintf("DecimalPrecision: %d", dp)) + } + } + case "GroupDigits": + if gd, _ := fi["GroupDigits"].(bool); gd { + props = append(props, "GroupDigits: true") + } + } + } + return props +} diff --git a/mdl/executor/input_formatting_pedapp_test.go b/mdl/executor/input_formatting_pedapp_test.go new file mode 100644 index 0000000000..09ca9e5827 --- /dev/null +++ b/mdl/executor/input_formatting_pedapp_test.go @@ -0,0 +1,187 @@ +// SPDX-License-Identifier: Apache-2.0 + +package executor + +import ( + "context" + "strings" + "testing" + + "go.mongodb.org/mongo-driver/bson" +) + +// ako/mxcli#968 (mendixlabs/mxcli#1263): a date picker's DateFormat / +// CustomDateFormat — and a text box's DecimalPrecision / GroupDigits — passed +// check, were written as the defaults, and were never described. Run end to +// end on PedApp: exec, read the stored unit, describe, re-execute the +// description, alter. + +const bookingScript = `create non-persistent entity MyFirstModule.Booking968 ( + Moment: DateTime, + Amount: Decimal +); +create page MyFirstModule.Booking968_Edit ( + Title: 'Booking', + Layout: Atlas_Core.Atlas_Default, + Params: ( $Booking: MyFirstModule.Booking968 ) +) { + dataview dv (DataSource: $Booking) { + datepicker dpDefault (Label: 'Default', Attribute: Moment) + datepicker dpTime (Label: 'Time', Attribute: Moment, DateFormat: Time) + datepicker dpDateTime (Label: 'DateTime', Attribute: Moment, DateFormat: DateTime) + datepicker dpCustom (Label: 'Custom', Attribute: Moment, DateFormat: Custom, CustomDateFormat: 'dd-MM-yyyy H:mm') + datepicker dpStale (Label: 'Stale', Attribute: Moment, DateFormat: DateTime, CustomDateFormat: 'dd/MM/yyyy HH:mm') + textbox tbAmount (Label: 'Amount', Attribute: Amount, DecimalPrecision: 4, GroupDigits: true) + } +};` + +// storedFormatting returns each named widget's FormattingInfo fields from the +// stored page unit. +func storedFormatting(t *testing.T, exec *Executor, page string) map[string]map[string]any { + t.Helper() + ctx := exec.newExecContext(context.Background()) + unit, err := ctx.Backend.GetRawUnitByName("page", page) + if err != nil || unit == nil { + t.Fatalf("raw unit of %s: %v", page, err) + } + var doc bson.D + if err := bson.Unmarshal(unit.Contents, &doc); err != nil { + t.Fatalf("unmarshal %s: %v", page, err) + } + out := map[string]map[string]any{} + var walk func(v any) + walk = func(v any) { + switch x := v.(type) { + case bson.D: + var name string + var fi bson.D + for _, e := range x { + switch e.Key { + case "Name": + name, _ = e.Value.(string) + case "FormattingInfo": + fi, _ = e.Value.(bson.D) + } + } + if name != "" && fi != nil { + m := map[string]any{} + for _, e := range fi { + m[e.Key] = e.Value + } + out[name] = m + } + for _, e := range x { + walk(e.Value) + } + case bson.A: + for _, e := range x { + walk(e) + } + } + } + walk(doc) + return out +} + +func TestInputFormatting_WrittenDescribedAndRoundTripped(t *testing.T) { + exec, out, dir := openPedAppCopy(t) + if err := afRun(t, exec, bookingScript); err != nil { + t.Fatalf("exec: %v\n%s", err, out.String()) + } + const page = "MyFirstModule.Booking968_Edit" + + got := storedFormatting(t, exec, page) + for name, want := range map[string]map[string]any{ + "dpDefault": {"DateFormat": "Date", "CustomDateFormat": ""}, + "dpTime": {"DateFormat": "Time", "CustomDateFormat": ""}, + "dpDateTime": {"DateFormat": "DateTime", "CustomDateFormat": ""}, + "dpCustom": {"DateFormat": "Custom", "CustomDateFormat": "dd-MM-yyyy H:mm"}, + "dpStale": {"DateFormat": "DateTime", "CustomDateFormat": "dd/MM/yyyy HH:mm"}, + "tbAmount": {"DecimalPrecision": int32(4), "GroupDigits": true}, + } { + for k, v := range want { + if got[name][k] != v { + t.Errorf("%s: stored FormattingInfo.%s = %#v, want %#v", name, k, got[name][k], v) + } + } + } + + described := describeOn(t, exec, out, page) + for _, want := range []string{ + "DateFormat: Time", + "DateFormat: DateTime", + "DateFormat: Custom", + "CustomDateFormat: 'dd-MM-yyyy H:mm'", + "CustomDateFormat: 'dd/MM/yyyy HH:mm'", + "DecimalPrecision: 4", + "GroupDigits: true", + } { + if !strings.Contains(described, want) { + t.Errorf("describe should print %q:\n%s", want, described) + } + } + // The default is not printed: an unformatted picker describes as before. + if strings.Contains(described, "DateFormat: Date,") || strings.Contains(described, "DateFormat: Date\n") { + t.Errorf("describe printed the default DateFormat:\n%s", described) + } + + // GetPut: executing the description writes nothing. + before := projectFiles(t, dir) + out.Reset() + if err := afRun(t, exec, described); err != nil { + t.Fatalf("re-exec describe output: %v\n%s", err, out.String()) + } + if changed := diffProjectFiles(before, projectFiles(t, dir)); len(changed) != 0 { + t.Errorf("executing the describe output wrote %v", changed) + } + + // Control: an edited description IS a write, and lands. + edited := strings.Replace(described, "DateFormat: Time", "DateFormat: DateTime", 1) + if edited == described { + t.Fatal("control edit did not apply") + } + if err := afRun(t, exec, edited); err != nil { + t.Fatalf("exec edited describe output: %v", err) + } + if df := storedFormatting(t, exec, page)["dpTime"]["DateFormat"]; df != "DateTime" { + t.Errorf("edited dpTime DateFormat = %v, want DateTime", df) + } +} + +func TestInputFormatting_AlterPageSet(t *testing.T) { + exec, out := openPedAppFixture(t) + if err := afRun(t, exec, bookingScript); err != nil { + t.Fatalf("exec: %v\n%s", err, out.String()) + } + const page = "MyFirstModule.Booking968_Edit" + + if err := afRun(t, exec, `alter page `+page+` { set (DateFormat: Custom, CustomDateFormat: 'HH:mm') on dpDefault; set (DecimalPrecision: 0, GroupDigits: false) on tbAmount; set DateFormat = Time on dpDateTime; };`); err != nil { + t.Fatalf("alter page set: %v", err) + } + got := storedFormatting(t, exec, page) + if got["dpDefault"]["DateFormat"] != "Custom" || got["dpDefault"]["CustomDateFormat"] != "HH:mm" { + t.Errorf("dpDefault = %v, want Custom / HH:mm", got["dpDefault"]) + } + if got["dpDateTime"]["DateFormat"] != "Time" { + t.Errorf("dpDateTime = %v, want Time", got["dpDateTime"]) + } + if got["tbAmount"]["DecimalPrecision"] != int32(0) || got["tbAmount"]["GroupDigits"] != false { + t.Errorf("tbAmount = %v, want precision 0 (same int32 width), no grouping", got["tbAmount"]) + } + + for _, c := range []struct{ stmt, want string }{ + {`set DateFormat = Weekly on dpTime`, "DateFormat is one of Date, Time, DateTime, Custom"}, + // mxbuild CE0493 "Date format is custom but no format string is specified." + {`set DateFormat = Custom on dpTime`, "CE0493"}, + {`set DecimalPrecision = 3 on dpTime`, "no DecimalPrecision property"}, + {`set DateFormat = Time on tbAmount`, "no DateFormat property"}, + } { + err := afRun(t, exec, `alter page `+page+` { `+c.stmt+`; };`) + if err == nil || !strings.Contains(err.Error(), c.want) { + t.Errorf("%s: want an error containing %q, got %v", c.stmt, c.want, err) + } + } + if got := storedFormatting(t, exec, page)["dpTime"]["DateFormat"]; got != "Time" { + t.Errorf("a refused set changed dpTime to %v", got) + } +} diff --git a/mdl/executor/input_formatting_test.go b/mdl/executor/input_formatting_test.go new file mode 100644 index 0000000000..1c382907c9 --- /dev/null +++ b/mdl/executor/input_formatting_test.go @@ -0,0 +1,64 @@ +// SPDX-License-Identifier: Apache-2.0 + +package executor + +import ( + "strings" + "testing" + + "github.com/mendixlabs/mxcli/mdl/visitor" +) + +// ako/mxcli#968: `check` on a date picker's / text box's formatting. Before the +// fix every case below checked clean — the format keys were exempted from the +// unknown-property warning on every widget type, and nothing validated them. +func TestValidateInputFormatting(t *testing.T) { + page := func(widget string) string { + return `create page M.P (Title: 'x', Layout: Atlas_Core.Atlas_Default, Params: ($B: M.B)) { + dataview dv (DataSource: $B) { ` + widget + ` } +};` + } + for _, c := range []struct { + name, widget, rule, want string + }{ + {"valid time", `datepicker d (Attribute: Moment, DateFormat: Time)`, "", ""}, + {"valid custom", `datepicker d (Attribute: Moment, DateFormat: Custom, CustomDateFormat: 'dd-MM-yyyy')`, "", ""}, + {"stale pattern beside DateTime", `datepicker d (Attribute: Moment, DateFormat: DateTime, CustomDateFormat: 'dd-MM-yyyy')`, "", ""}, + {"valid precision", `textbox t (Attribute: Amount, DecimalPrecision: 0, GroupDigits: true)`, "", ""}, + {"unknown date format", `datepicker d (Attribute: Moment, DateFormat: Weekly)`, "MDL-WIDGET18", "DateFormat must be one of Date, Time, DateTime, Custom"}, + {"pattern without DateFormat", `datepicker d (Attribute: Moment, CustomDateFormat: 'HH:mm')`, "MDL-WIDGET18", "applies only with `DateFormat: Custom`"}, + // Measured: mxbuild CE0493 "Date format is custom but no format string is specified." + {"custom without pattern", `datepicker d (Attribute: Moment, DateFormat: Custom)`, "MDL-WIDGET18", "needs a non-empty CustomDateFormat"}, + {"negative precision", `textbox t (Attribute: Amount, DecimalPrecision: -1)`, "MDL-WIDGET18", "non-negative integer"}, + // A widget with no FormattingInfo still drops the key, and says so. + {"DateFormat on a text area", `textarea a (Attribute: Notes, DateFormat: Time)`, "MDL-WIDGET07", "DateFormat"}, + // A text box binds no dates (CE2421), so DateFormat means nothing there. + {"DateFormat on a text box", `textbox t (Attribute: Amount, DateFormat: Time)`, "MDL-WIDGET07", "DateFormat"}, + } { + t.Run(c.name, func(t *testing.T) { + prog, errs := visitor.Build(page(c.widget)) + if len(errs) > 0 { + t.Fatalf("parse: %v", errs) + } + vs := ValidateWidgetProperties(prog, "") + var hit []string + for _, v := range vs { + if strings.HasPrefix(v.RuleID, "MDL-WIDGET18") || strings.HasPrefix(v.RuleID, "MDL-WIDGET07") { + hit = append(hit, v.RuleID+": "+v.Message) + } + } + if c.rule == "" { + if len(hit) != 0 { + t.Errorf("want no formatting violation, got %v", hit) + } + return + } + for _, h := range hit { + if strings.HasPrefix(h, c.rule) && strings.Contains(h, c.want) { + return + } + } + t.Errorf("want %s containing %q, got %v", c.rule, c.want, hit) + }) + } +} diff --git a/mdl/executor/validate_widgets.go b/mdl/executor/validate_widgets.go index 7d70d68813..dcee1422b1 100644 --- a/mdl/executor/validate_widgets.go +++ b/mdl/executor/validate_widgets.go @@ -251,6 +251,7 @@ func validateWidgetTreeIn(widgets []*ast.WidgetV3, registry *WidgetRegistry, loc out = append(out, validateImageSource(w, locationPrefix)...) out = append(out, validateStaticWidget(w, locationPrefix)...) out = append(out, validateDynamicTextFormatting(w, locationPrefix)...) + out = append(out, validateInputFormatting(w, locationPrefix)...) out = append(out, validateDatasourceXPathAssociationEmpty(w, locationPrefix)...) out = append(out, validateComboBoxAssociation(w, locationPrefix)...) // #631: inputs inside a list view that will be written read-only. @@ -831,7 +832,15 @@ func validateStaticWidgetUnknownProps(w *ast.WidgetV3, locationPrefix string) [] // Dynamic-text format keys placed at the widget level are reported by // MDL-WIDGET18 (with actionable move-into-format-block guidance); don't // also warn about them here. - if paramFormatKeys[strings.ToLower(key)] { + if paramFormatKeys[strings.ToLower(key)] && strings.EqualFold(w.Type, "dynamictext") { + continue + } + // A date picker's or text box's own FormattingInfo (ako/mxcli#968). The + // exemption above used to cover every widget type, so `DateFormat:` on + // any widget passed silently — on a date picker, too, where nothing + // read it. It is per type: on a widget with no FormattingInfo the key + // is still dropped, and still warned about. + if _, ok := inputFormatProp(w.Type, key); ok { continue } hint := "" @@ -929,9 +938,15 @@ func validateDynamicTextFormatting(w *ast.WidgetV3, locationPrefix string) []lin } } } - // customDateFormat is only meaningful with dateFormat: Custom. + // customDateFormat is only meaningful with dateFormat: Custom. A pattern + // with NO dateFormat is a mistake — it never applies. One beside an + // explicit other dateFormat is what Studio Pro stores after the format + // is switched away from Custom (TestApp's + // WorkflowCommons.Snip_Workflow_CommentsAndAttachments: DateTime + + // 'MM/dd/yyyy . hh:mma'), and describe prints both, so refusing it + // failed check on describe's own output (ako/mxcli#968). if _, hasCustom := p.Format.Get("customdateformat"); hasCustom { - if df, _ := p.Format.Get("dateformat"); !strings.EqualFold(df, "Custom") { + if _, hasDF := p.Format.Get("dateformat"); !hasDF { out = append(out, violation18(locationPrefix, w, "customDateFormat requires `dateFormat: Custom`")) } @@ -940,6 +955,21 @@ func validateDynamicTextFormatting(w *ast.WidgetV3, locationPrefix string) []lin return out } +// validateInputFormatting (MDL-WIDGET18) checks the formatting properties of a +// date picker or text box — DateFormat, CustomDateFormat, DecimalPrecision, +// GroupDigits — with the same function the builder refuses them with, so check +// and exec cannot disagree (ako/mxcli#968). +func validateInputFormatting(w *ast.WidgetV3, locationPrefix string) []linter.Violation { + if w == nil { + return nil + } + var out []linter.Violation + for _, msg := range inputFormattingProblems(w) { + out = append(out, violation18(locationPrefix, w, msg)) + } + return out +} + func violation18(locationPrefix string, w *ast.WidgetV3, msg string) linter.Violation { return linter.Violation{ RuleID: "MDL-WIDGET18", diff --git a/sdk/pages/formatting.go b/sdk/pages/formatting.go new file mode 100644 index 0000000000..7336e2b2be --- /dev/null +++ b/sdk/pages/formatting.go @@ -0,0 +1,39 @@ +// SPDX-License-Identifier: Apache-2.0 + +package pages + +import "strings" + +// The formatting an input widget stores in its Forms$FormattingInfo, shared by +// the CREATE PAGE builder and the ALTER PAGE mutator so the two cannot accept +// different values (ako/mxcli#968). + +// dateFormats maps a lowercase DateFormat value to the stored enum value. +var dateFormats = map[string]string{ + "date": "Date", "time": "Time", "datetime": "DateTime", "custom": "Custom", +} + +// CanonicalDateFormat returns the stored spelling of a FormattingInfo +// DateFormat value (Date, Time, DateTime, Custom), matched case-insensitively. +func CanonicalDateFormat(s string) (string, bool) { + v, ok := dateFormats[strings.ToLower(s)] + return v, ok +} + +// formattingProperties lists, per widget storage type, the FormattingInfo +// fields MDL sets on it, in canonical spelling and describe order. +// +// A date picker's format is its mode: DateFormat Time or DateTime is what makes +// it a time or date-time picker. A text box binds only string and numeric +// attributes (CE2421 for a date), so of its FormattingInfo only the numeric +// fields mean anything. +var formattingProperties = map[string][]string{ + "Forms$DatePicker": {"DateFormat", "CustomDateFormat"}, + "Forms$TextBox": {"DecimalPrecision", "GroupDigits"}, +} + +// FormattingProperties returns the formatting properties an input widget of the +// given storage type ($Type) carries, or nil for one MDL gives none. +func FormattingProperties(storageType string) []string { + return formattingProperties[storageType] +} diff --git a/sdk/pages/pages_widgets_input.go b/sdk/pages/pages_widgets_input.go index f5c0d741f0..3fb392b410 100644 --- a/sdk/pages/pages_widgets_input.go +++ b/sdk/pages/pages_widgets_input.go @@ -76,9 +76,13 @@ type DatePicker struct { AttributeRefSteps []AttributeRefStep `json:"attributeRefSteps,omitempty"` // association hops when the attribute is reached over associations (AttributeRef.EntityRef) SourceVariable *WidgetVariable `json:"sourceVariable,omitempty"` // widget-scoped variable the attribute is read from (#826) Placeholder *model.Text `json:"placeholder,omitempty"` - DateFormat string `json:"dateFormat,omitempty"` - ReadOnly bool `json:"readOnly,omitempty"` - OnChangeAction ClientAction `json:"onChangeAction,omitempty"` + // FormattingInfo is the picker's Forms$FormattingInfo: DateFormat (Date, + // Time, DateTime, Custom) and the CustomDateFormat pattern. nil writes the + // defaults (Date). It replaced a DateFormat string nothing read, so every + // picker was written as date-only (ako/mxcli#968). + FormattingInfo *FormattingInfo `json:"formattingInfo,omitempty"` + ReadOnly bool `json:"readOnly,omitempty"` + OnChangeAction ClientAction `json:"onChangeAction,omitempty"` } // DropDown represents a drop-down selection widget. From 98e87cbeb168ed129982e38640af8f3a343e9658 Mon Sep 17 00:00:00 2001 From: Ako Date: Sun, 4 Oct 2026 15:26:46 +0000 Subject: [PATCH 17/17] docs: date picker DateFormat and text box precision in syntax, skill and quick reference (#968) Co-Authored-By: Claude Opus 5.5 --- .../mendix/create-page/reference/widgets.md | 16 ++++++++++++++++ cmd/mxcli/syntax/features_page.go | 7 ++++++- docs/01-project/MDL_QUICK_REFERENCE.md | 8 ++++++++ 3 files changed, 30 insertions(+), 1 deletion(-) diff --git a/.claude/skills/mendix/create-page/reference/widgets.md b/.claude/skills/mendix/create-page/reference/widgets.md index c55355aed3..674c0e05ed 100644 --- a/.claude/skills/mendix/create-page/reference/widgets.md +++ b/.claude/skills/mendix/create-page/reference/widgets.md @@ -481,7 +481,23 @@ radiobuttons rbStatus (label: 'Status', attribute: status) **DATEPICKER** - Date/time selection: ```sql datepicker dpCreated (label: 'Created Date', attribute: CreatedDate) +-- DateFormat is the picker's MODE: Date (default), Time, DateTime, or Custom. +-- A time-of-day or date-and-time field needs Time / DateTime, or it renders date-only. +datepicker dpStart (label: 'Start', attribute: StartTime, DateFormat: DateTime) +datepicker dpAt (label: 'At', attribute: StartTime, DateFormat: Custom, CustomDateFormat: 'dd-MM-yyyy HH:mm') ``` +`CustomDateFormat` needs `DateFormat: Custom` (written alone it is refused — it would +never apply), and `DateFormat: Custom` needs a non-empty pattern (mxbuild CE0493). +A pattern beside another explicit DateFormat is accepted: Studio Pro keeps it when the +format is switched away from Custom, and `describe` prints it. Change an existing +picker with `alter page … { set (DateFormat: DateTime) on dpStart; }`. + +**TEXTBOX numeric formatting** — on a Decimal/Integer/Long attribute: +```sql +textbox tbAmount (label: 'Amount', attribute: Amount, DecimalPrecision: 2, GroupDigits: true) +``` +A text box binds no dates (CE2421), so `DateFormat` on it is dropped and warned about +(MDL-WIDGET07), as on any widget other than a date picker. **COMBOBOX** - Combo box (pluggable widget): ```sql diff --git a/cmd/mxcli/syntax/features_page.go b/cmd/mxcli/syntax/features_page.go index ccb11450bb..bda76bf879 100644 --- a/cmd/mxcli/syntax/features_page.go +++ b/cmd/mxcli/syntax/features_page.go @@ -149,7 +149,12 @@ LIST IMPACT OF htmlelement; "-- A FILTER block written on a DATAGRID is not a column filter and not a\n" + "-- container the grid declares — it used to be dropped on write with no\n" + "-- diagnostic, and is now refused (MDL-WIDGET30).\n\n" + - "-- Inputs\nTEXTBOX name (Label: 'L', Attribute: Attr)\nTEXTAREA | DATEPICKER | COMBOBOX | CHECKBOX | RADIOBUTTONS\n\n" + + "-- Inputs\nTEXTBOX name (Label: 'L', Attribute: Attr)\nTEXTAREA | DATEPICKER | COMBOBOX | CHECKBOX | RADIOBUTTONS\n" + + "-- A date picker's DateFormat is its mode: Date (default) | Time | DateTime | Custom.\n" + + "DATEPICKER name (Attribute: Start, DateFormat: DateTime)\n" + + "DATEPICKER name (Attribute: Start, DateFormat: Custom, CustomDateFormat: 'dd-MM-yyyy HH:mm')\n" + + "-- A text box's numeric formatting:\n" + + "TEXTBOX name (Attribute: Amount, DecimalPrecision: 2, GroupDigits: true)\n\n" + "-- Conditional visibility / editability: a bare client expression, stored as\n" + "-- written (name attributes as $currentObject/Attr). A plain value is static.\n" + "TEXTBOX name (Attribute: Attr, Visible: $currentObject/IsActive, Editable: $currentObject/Status != 'Closed')\n" + diff --git a/docs/01-project/MDL_QUICK_REFERENCE.md b/docs/01-project/MDL_QUICK_REFERENCE.md index c56170a4c7..dffac80788 100644 --- a/docs/01-project/MDL_QUICK_REFERENCE.md +++ b/docs/01-project/MDL_QUICK_REFERENCE.md @@ -1675,6 +1675,14 @@ dynamictext due (content: '{1}', contentparams: ({1} = DueOn format (dateFormat ``` Keys: `decimalPrecision` (int), `groupDigits` (bool), `dateFormat` (`Date`|`DateTime`|`Time`|`Custom`), `customDateFormat` (pattern, with `dateFormat: Custom`), `enumFormat` (`Text`|`Image`). +**Input widget formatting** — the same fields as widget properties on the widgets that store a FormattingInfo: +```sql +datepicker dpStart (label: 'Start', attribute: StartTime, DateFormat: DateTime) -- Date | Time | DateTime | Custom +datepicker dpAt (label: 'At', attribute: StartTime, DateFormat: Custom, CustomDateFormat: 'dd-MM-yyyy HH:mm') +textbox tbAmount (label: 'Amount', attribute: Amount, DecimalPrecision: 2, GroupDigits: true) +``` +A date picker's `DateFormat` is its mode (time / date-time picker). `CustomDateFormat` requires an explicit `DateFormat` (and `Custom` requires a pattern — mxbuild CE0493); both are checked as MDL-WIDGET18. `alter page … { set (DateFormat: Time) on dp; }` changes them in place. + ## ALTER PAGE / ALTER SNIPPET Modify an existing page or snippet's widget tree in-place without full `create or replace`. Works directly on the raw BSON tree, preserving unsupported widget types.