From d5e43918df53e282bdb8bd7a36d51b11c674cd80 Mon Sep 17 00:00:00 2001 From: stagknee <241821830+stagknee@users.noreply.github.com> Date: Sat, 3 Oct 2026 19:37:43 -0500 Subject: [PATCH 1/5] fix: link the runtime into the mxbuild cache with a junction when Windows refuses a symlink (closes #1286) An ordinary Windows user cannot create a symlink without SeCreateSymbolicLinkPrivilege, so the first run --local / test --local for a new Mendix version failed linking the runtime into the mxbuild cache. linkDir tries os.Symlink and, on ERROR_PRIVILEGE_NOT_HELD, falls back to a directory junction (cmd /c mklink /J); both directory-link sites (ensureMxBuildRuntimeSibling, ensurePADFiles) use it. os.RemoveAll of the cache dir removes the junction, not the runtime behind it; a test pins that. Co-Authored-By: Claude Sonnet 5.5 --- ...ws-runtime-symlink-privilege-not-held.json | 1 + CHANGELOG.md | 1 + cmd/mxcli/docker/build.go | 4 +- cmd/mxcli/docker/linkdir_test.go | 136 ++++++++++++++++++ cmd/mxcli/docker/linkdir_unix.go | 14 ++ cmd/mxcli/docker/linkdir_windows.go | 57 ++++++++ cmd/mxcli/docker/runlocal.go | 6 +- 7 files changed, 215 insertions(+), 4 deletions(-) create mode 100644 .claude/skills/fix-issue/findings/cmd-mxcli/2026-10-03-windows-runtime-symlink-privilege-not-held.json create mode 100644 cmd/mxcli/docker/linkdir_test.go create mode 100644 cmd/mxcli/docker/linkdir_unix.go create mode 100644 cmd/mxcli/docker/linkdir_windows.go diff --git a/.claude/skills/fix-issue/findings/cmd-mxcli/2026-10-03-windows-runtime-symlink-privilege-not-held.json b/.claude/skills/fix-issue/findings/cmd-mxcli/2026-10-03-windows-runtime-symlink-privilege-not-held.json new file mode 100644 index 000000000..06f27ac6a --- /dev/null +++ b/.claude/skills/fix-issue/findings/cmd-mxcli/2026-10-03-windows-runtime-symlink-privilege-not-held.json @@ -0,0 +1 @@ +{"area": "cmd/mxcli", "date": "2026-10-03", "symptom": "mendixlabs/mxcli#1286: on Windows the first `run --local` / `test --local` for a new Mendix version fails with `linking runtime into mxbuild cache: symlink \\.mxcli\\runtime\\11.12.5\\runtime \\.mxcli\\mxbuild\\11.12.5\\runtime: A required privilege is not held by the client.`; hand-made `New-Item -ItemType Junction` links let the run proceed", "cause": "ensureMxBuildRuntimeSibling and ensurePADFiles linked a directory with os.Symlink; on Windows CreateSymbolicLink needs SeCreateSymbolicLinkPrivilege (Developer Mode or an elevated token), which an ordinary user lacks (ERROR_PRIVILEGE_NOT_HELD, 1314). A directory junction needs no privilege", "file": "`cmd/mxcli/docker/linkdir_windows.go` (linkDir); call sites `cmd/mxcli/docker/runlocal.go` (ensureMxBuildRuntimeSibling), `cmd/mxcli/docker/build.go` (ensurePADFiles)", "fix": "build-tagged linkDir(target, link): POSIX = os.Symlink; Windows = os.Symlink first, and only on errors.Is(err, syscall.Errno(1314)) fall back to `cmd /c mklink /J` (HideWindow, absolute target), returning mklink's output plus the original symlink error if that fails too. mklink rather than FSCTL_SET_REPARSE_POINT keeps golang.org/x/sys an indirect dependency", "insight": "Linux CI cannot see this: os.Symlink always works there, and a Windows CI runner or a Developer-Mode machine passes too, so the bug only shows on an ordinary Windows user account. The repo's own TestEnsureMxBuildRuntimeSibling already failed on such a machine (the #897 family: Windows-only failures nobody saw) - when a package has long-standing 'pre-existing' Windows failures, read their messages for a privilege/OS cause before writing them off. Safety property to test, not assume: download.go and build.go os.RemoveAll the mxbuild/runtime cache dirs that now contain a junction; Go (1.23+) reports a junction as ModeIrregular (a symlink is ModeSymlink) and RemoveAll deletes the junction itself, not the runtime behind it - measured with go1.26.6 on Windows, target file intact. The test's temp dir has a space in its name because mklink goes through cmd.exe quoting. Wrong turn to skip: do not touch the os.Symlink calls in check_copy.go / marketplace/refcache.go, they recreate existing symlinks while copying trees (a different concern)", "refs": ["mendixlabs/mxcli#1286", "#897"], "test": "`cmd/mxcli/docker/linkdir_test.go` (read through link, Stat follows, RemoveAll removes link not target, existing link path errors, Windows fallback proven when os.Symlink returns 1314 and the link is ModeIrregular); `TestEnsureMxBuildRuntimeSibling` now passes on Windows"} diff --git a/CHANGELOG.md b/CHANGELOG.md index c09ef26a8..f7a5f5f3a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,6 +28,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). ### Fixed +- **`run --local` and `test --local` work on Windows without Developer Mode or admin rights** (mendixlabs/mxcli#1286) — the first run for a new Mendix version failed with `linking runtime into mxbuild cache: symlink …: A required privilege is not held by the client.` mxcli links the downloaded runtime into the mxbuild cache, and an ordinary Windows user may create a symbolic link only with Developer Mode or an elevated token. When Windows refuses the symlink, mxcli now makes a directory junction instead, which needs no privilege (the same link `mklink /J` makes; it needs `cmd.exe`, which every Windows has). The other link site, the PAD files `docker build` links into the mxbuild cache, gets the same fallback. A cache that was already linked by hand with `New-Item -ItemType Junction` is left as it is. - **`run stop` and `run status` find a detached run's processes on every machine** — the guard against a reused session id rebuilt each process's start time from the kernel's boot time, which is whole seconds, and allowed only one second of slack. On a machine that booted late in a second, a run's own processes read as older than the run and were skipped, so `run stop` could report "was not running" for a live run. The slack is now two seconds. This was also the intermittent `TestRunStop_*` / `TestSessionMembers_*` failure on CI. - **The catalog names a call's and a delete's target** (mendixlabs/mxcli#1305) — `activities_for()` and `CATALOG.ACTIVITIES` returned every microflow, nanoflow, Java action and JavaScript action call with `action_ref=""`, and every delete with `entity_ref=""`, although `refs_from()` had both targets; a loop-scoped lint rule could not follow a call out of the loop. `action_ref` / `ActionRef` is now the called document, `entity_ref` / `EntityRef` a delete's entity (when the flow types the variable: a parameter, a create or retrieve output, a loop iterator), and the new `queue_ref` / `QueueRef` the task queue a microflow or Java action call runs in, so a rule can skip a call that runs asynchronously. A nanoflow's JavaScript action call now has a `refs_from()` `call` row (`target_type` `"JAVASCRIPT_ACTION"`), so `show callers` sees it. The catalog schema is bumped to 23; a cached catalog rebuilds. - **A DataGrid 2 column's `Visible:` expression is written** — `Visible: $showPrices`, `Visible: if … then … else …`, `visible: not(…)` and `Visible: [cond]` on a column passed `check` and `exec` and were stored as `true`, so the column was always visible; only the old quoted `Visible: ''` was kept. `alter page … set (Visible: ) on grid column(…)` was refused as "column property VisibleIf not found", and an inserted column dropped `Visible: false` too. `describe` now prints the bare expression. A column's visibility is evaluated once for the grid, with no row object, so `$currentObject` there is CE0117 at build; `check` refuses it as **MDL-WIDGET43** (measured on 11.14.0). Use a page variable or parameter. Projects regenerate their widget definitions (generator version 18). diff --git a/cmd/mxcli/docker/build.go b/cmd/mxcli/docker/build.go index 30d56d922..8a9848c5b 100644 --- a/cmd/mxcli/docker/build.go +++ b/cmd/mxcli/docker/build.go @@ -778,8 +778,8 @@ func ensurePADFiles(productVersion string, w io.Writer) error { return fmt.Errorf("creating runtime directory in mxbuild: %w", err) } - // Symlink runtime/pad into mxbuild - if err := os.Symlink(runtimePAD, mxbuildPAD); err != nil { + // Link runtime/pad into mxbuild (junction when Windows refuses a symlink) + if err := linkDir(runtimePAD, mxbuildPAD); err != nil { return fmt.Errorf("symlinking PAD files: %w", err) } fmt.Fprintf(w, " Linked PAD runtime files: %s -> %s\n", mxbuildPAD, runtimePAD) diff --git a/cmd/mxcli/docker/linkdir_test.go b/cmd/mxcli/docker/linkdir_test.go new file mode 100644 index 000000000..594c8321d --- /dev/null +++ b/cmd/mxcli/docker/linkdir_test.go @@ -0,0 +1,136 @@ +// SPDX-License-Identifier: Apache-2.0 + +package docker + +import ( + "errors" + "os" + "path/filepath" + "runtime" + "syscall" + "testing" +) + +// errPrivilegeNotHeld is Windows' ERROR_PRIVILEGE_NOT_HELD (1314), what +// os.Symlink returns for an ordinary user without Developer Mode. +const errPrivilegeNotHeld = syscall.Errno(1314) + +// newLinkFixture builds /target/f.txt and returns the target, the link +// path /cache/link (cache/ is created, link is not) and the cache dir. +// The root has a space in its name: mklink is driven through cmd.exe, whose +// quoting is the part most likely to break on such a path. +func newLinkFixture(t *testing.T) (target, link, cache string) { + t.Helper() + root := filepath.Join(t.TempDir(), "dir with space") + target = filepath.Join(root, "target") + cache = filepath.Join(root, "cache") + for _, d := range []string{target, cache} { + if err := os.MkdirAll(d, 0o755); err != nil { + t.Fatal(err) + } + } + if err := os.WriteFile(filepath.Join(target, "f.txt"), []byte("payload"), 0o644); err != nil { + t.Fatal(err) + } + return target, filepath.Join(cache, "link"), cache +} + +func TestLinkDir_TargetContentsReadableThroughLink(t *testing.T) { + target, link, _ := newLinkFixture(t) + if err := linkDir(target, link); err != nil { + t.Fatalf("linkDir: %v", err) + } + got, err := os.ReadFile(filepath.Join(link, "f.txt")) + if err != nil { + t.Fatalf("reading through the link: %v", err) + } + if string(got) != "payload" { + t.Errorf("read %q through the link, want %q", got, "payload") + } +} + +// ensureMxBuildRuntimeSibling's "already present" check is os.Stat(dst), which +// must follow the link to the directory it points at. +func TestLinkDir_StatFollowsLink(t *testing.T) { + target, link, _ := newLinkFixture(t) + if err := linkDir(target, link); err != nil { + t.Fatalf("linkDir: %v", err) + } + fi, err := os.Stat(link) + if err != nil { + t.Fatalf("os.Stat(link): %v", err) + } + if !fi.IsDir() { + t.Errorf("os.Stat(link) is not a directory: mode %v", fi.Mode()) + } +} + +// download.go and build.go os.RemoveAll the mxbuild/runtime cache directories, +// which contain this link. RemoveAll must delete the link itself, never the +// runtime it points at: following a junction here would wipe the downloaded +// runtime out of ~/.mxcli/runtime. +func TestLinkDir_RemoveAllRemovesLinkNotTarget(t *testing.T) { + target, link, cache := newLinkFixture(t) + if err := linkDir(target, link); err != nil { + t.Fatalf("linkDir: %v", err) + } + if err := os.RemoveAll(cache); err != nil { + t.Fatalf("RemoveAll(cache): %v", err) + } + if _, err := os.Lstat(link); !os.IsNotExist(err) { + t.Errorf("link still present after RemoveAll of its parent: %v", err) + } + if _, err := os.Stat(cache); !os.IsNotExist(err) { + t.Errorf("cache dir still present after RemoveAll: %v", err) + } + got, err := os.ReadFile(filepath.Join(target, "f.txt")) + if err != nil { + t.Fatalf("RemoveAll followed the link and damaged the target: %v", err) + } + if string(got) != "payload" { + t.Errorf("target file changed: %q", got) + } +} + +func TestLinkDir_FailsWhenLinkExists(t *testing.T) { + target, link, _ := newLinkFixture(t) + if err := os.Mkdir(link, 0o755); err != nil { + t.Fatal(err) + } + if err := linkDir(target, link); err == nil { + t.Error("expected an error when the link path already exists") + } +} + +// On Windows an ordinary user cannot create a symlink (#1286). When this +// machine is such a one, prove linkDir still works, i.e. the junction fallback +// ran; when it is not (Developer Mode or admin), there is no fallback to prove. +func TestLinkDir_WindowsFallsBackToJunctionWithoutSymlinkPrivilege(t *testing.T) { + if runtime.GOOS != "windows" { + t.Skip("Windows only") + } + target, link, cache := newLinkFixture(t) + + probe := filepath.Join(cache, "probe") + err := os.Symlink(target, probe) + if err == nil { + t.Skip("os.Symlink succeeds here (Developer Mode or elevated): the junction fallback cannot be exercised") + } + if !errors.Is(err, errPrivilegeNotHeld) { + t.Skipf("os.Symlink failed with %v, not ERROR_PRIVILEGE_NOT_HELD: the junction fallback cannot be exercised", err) + } + + if err := linkDir(target, link); err != nil { + t.Fatalf("linkDir without symlink privilege: %v", err) + } + fi, err := os.Lstat(link) + if err != nil { + t.Fatalf("os.Lstat(link): %v", err) + } + if fi.Mode()&os.ModeSymlink != 0 || fi.Mode()&os.ModeIrregular == 0 { + t.Errorf("link is not a junction (Go reports one as ModeIrregular): mode %v", fi.Mode()) + } + if _, err := os.ReadFile(filepath.Join(link, "f.txt")); err != nil { + t.Errorf("reading through the junction: %v", err) + } +} diff --git a/cmd/mxcli/docker/linkdir_unix.go b/cmd/mxcli/docker/linkdir_unix.go new file mode 100644 index 000000000..3c2f333d9 --- /dev/null +++ b/cmd/mxcli/docker/linkdir_unix.go @@ -0,0 +1,14 @@ +// SPDX-License-Identifier: Apache-2.0 + +//go:build !windows + +package docker + +import "os" + +// linkDir makes link a directory link to target. On POSIX systems that is a +// plain symlink; the Windows variant (linkdir_windows.go) has to cope with the +// symlink privilege, which is why this is a build-tagged pair. +func linkDir(target, link string) error { + return os.Symlink(target, link) +} diff --git a/cmd/mxcli/docker/linkdir_windows.go b/cmd/mxcli/docker/linkdir_windows.go new file mode 100644 index 000000000..26db92dd1 --- /dev/null +++ b/cmd/mxcli/docker/linkdir_windows.go @@ -0,0 +1,57 @@ +// SPDX-License-Identifier: Apache-2.0 + +//go:build windows + +package docker + +import ( + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "syscall" +) + +// errnoPrivilegeNotHeld is ERROR_PRIVILEGE_NOT_HELD: what CreateSymbolicLink +// reports when the caller lacks SeCreateSymbolicLinkPrivilege. +const errnoPrivilegeNotHeld = syscall.Errno(1314) + +// linkDir makes link a directory link to target. +// +// An ordinary Windows user may create a symlink only with Developer Mode on or +// an elevated token (SeCreateSymbolicLinkPrivilege); without either, +// os.Symlink fails with "A required privilege is not held by the client", which +// stopped the first `run --local` / `test --local` of every new Mendix version +// (mendixlabs/mxcli#1286). A directory junction needs no privilege and serves +// the same purpose here, so on that one error linkDir falls back to a junction. +// +// The junction is made with `cmd /c mklink /J`, the same shell-out style as +// killProcessTree, rather than FSCTL_SET_REPARSE_POINT through golang.org/x/sys, +// which is only an indirect dependency of this module and would become a direct +// one. Go reports a junction as os.ModeIrregular, not os.ModeSymlink, so +// os.RemoveAll deletes the junction itself and never follows it into target. +// +// A junction must name an absolute local path, so target is made absolute +// (the callers already pass absolute cache paths). Any other symlink failure, +// such as the link already existing, is returned unchanged. +func linkDir(target, link string) error { + symErr := os.Symlink(target, link) + if symErr == nil || !errors.Is(symErr, errnoPrivilegeNotHeld) { + return symErr + } + + absTarget, err := filepath.Abs(target) + if err != nil { + return fmt.Errorf("%w (junction fallback: resolving target: %v)", symErr, err) + } + cmd := exec.Command("cmd", "/c", "mklink", "/J", link, absTarget) + cmd.SysProcAttr = &syscall.SysProcAttr{HideWindow: true} + out, err := cmd.CombinedOutput() + if err != nil { + return fmt.Errorf("%w; junction fallback `mklink /J` failed: %v: %s", + symErr, err, strings.TrimSpace(string(out))) + } + return nil +} diff --git a/cmd/mxcli/docker/runlocal.go b/cmd/mxcli/docker/runlocal.go index ca0d3f4ee..21ee5896d 100644 --- a/cmd/mxcli/docker/runlocal.go +++ b/cmd/mxcli/docker/runlocal.go @@ -410,7 +410,9 @@ func deriveDBName(projectPath string) string { // runtime/ sibling of the mxbuild cache's modeler/ dir. mxbuild's Deploy/serve // javac step resolves the Mendix API from there; without it compilation fails // with "package com.mendix.* does not exist". It is a no-op if the sibling -// already exists (symlink or real dir). Mirrors ensurePADFiles' link pattern. +// already exists (link or real dir). Mirrors ensurePADFiles' link pattern. The +// link is a symlink, or a directory junction when Windows refuses a symlink +// (see linkDir, mendixlabs/mxcli#1286). func ensureMxBuildRuntimeSibling(version string, w io.Writer) error { mxbuildDir, err := MxBuildCacheDir(version) if err != nil { @@ -431,7 +433,7 @@ func ensureMxBuildRuntimeSibling(version string, w io.Writer) error { if err := os.MkdirAll(filepath.Dir(dst), 0o755); err != nil { return err } - if err := os.Symlink(src, dst); err != nil { + if err := linkDir(src, dst); err != nil { return fmt.Errorf("linking runtime into mxbuild cache: %w", err) } fmt.Fprintf(w, " Linked runtime into mxbuild cache: %s -> %s\n", dst, src) From 67e5e8dedbca7cb992f7dca062270eb7d3c31c75 Mon Sep 17 00:00:00 2001 From: Ako Date: Fri, 9 Oct 2026 13:06:17 +0200 Subject: [PATCH 2/5] docs: remove references to the closed-source reflection-data library (#1074) The reflection data comes from an internal Mendix library that is not publicly distributed, so docs pointing contributors at its directory under reference/ sent them to a path nobody can populate. Living docs and skills now point at the vendored metamodel types in modelsdk/gen (types.go for storage names, refs.go for reference kinds) and at Studio Pro-authored documents, which win on disagreement. cmd/codegen's -input no longer defaults to the library's path and is now required; CLAUDE.md's codegen example used a nonexistent -reflection-dir flag and now uses -input. At the maintainer's request, ADR-0008, the 2026-04-21 finding, the wiki sync log and gittree.txt are reworded too (wording only). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01LQcDcZKByMEd2tKJxmG11z --- .claude/commands/mxcli-dev/proposal.md | 4 +- .claude/skills/debug-bson.md | 5 +- ...ecacheunknowntypeexception-studio-pro.json | 2 +- .claude/skills/implement-mdl-bson | 92 ++++++++----------- .claude/skills/test-mdl.md | 16 ++-- .gitignore | 1 - CLAUDE.md | 5 +- CONTRIBUTING.md | 2 +- cmd/codegen/main.go | 12 ++- docs-site/src/appendixes/common-mistakes.md | 2 +- docs-site/src/appendixes/error-messages.md | 4 +- docs-site/src/internals/bson-structure.md | 23 +++-- docs-site/src/internals/packages.md | 1 - docs-site/src/internals/storage-names.md | 25 ++--- docs-wiki/SYNC_LOG.md | 4 +- docs-wiki/positioning/vs-typescript-sdk.md | 2 +- docs/01-project/SDK_EQUIVALENCE.md | 84 +++-------------- docs/03-development/BSON_TOOLING_GUIDE.md | 13 ++- .../03-development/PAGE_BSON_SERIALIZATION.md | 45 ++++----- docs/05-mdl-specification/10-bson-mapping.md | 6 +- .../11-proposals/PROPOSAL_backend_strategy.md | 2 +- .../PROPOSAL_codegen_ownership.md | 11 +-- ...OPOSAL_update_builtin_widget_properties.md | 2 +- .../archive/BSON_SCHEMA_REGISTRY_PROPOSAL.md | 2 +- .../archive/PROPOSAL_schema_extract.md | 6 +- .../archive/UNIFIED_SCHEMA_REGISTRY.md | 8 +- docs/11-proposals/odata-services-proposal.md | 2 +- docs/11-proposals/version-aware-mdl.md | 6 +- .../0008-identity-and-idempotence.md | 2 +- gittree.txt | 2 +- mdl/exprcheck/func_checker.go | 2 +- 31 files changed, 143 insertions(+), 250 deletions(-) diff --git a/.claude/commands/mxcli-dev/proposal.md b/.claude/commands/mxcli-dev/proposal.md index d1fba3c9f..849a79384 100644 --- a/.claude/commands/mxcli-dev/proposal.md +++ b/.claude/commands/mxcli-dev/proposal.md @@ -24,7 +24,7 @@ Ask the user: 5. **Does this touch BSON serialization?** (reading or writing Mendix documents) If the user isn't sure about version or BSON, help them find out: -- Version: check `reference/mendixmodellib/reflection-data/` or Mendix release notes +- Version: check the Mendix release notes - BSON: check if similar features exist in `mdl/backend/modelsdk/*_read.go` or `*_write.go` ### Phase 2: BSON Investigation (if applicable) @@ -51,7 +51,7 @@ in Studio Pro that are painful to debug. ``` 3. **Document the BSON structure** in the proposal: - - Storage name (`$Type` field) — verify against `reference/mendixmodellib/reflection-data/` + - Storage name (`$Type` field) — verify against `modelsdk/gen//types.go` and the dumped document - All fields with types and observed values - Any fields that use Mendix-internal IDs (pointers to other documents) - Note any counter-intuitive naming (like Parent/Child pointer inversion) diff --git a/.claude/skills/debug-bson.md b/.claude/skills/debug-bson.md index 85a1c19a1..f2971957b 100644 --- a/.claude/skills/debug-bson.md +++ b/.claude/skills/debug-bson.md @@ -237,9 +237,8 @@ The principle: for each `$type`, ALL instances must have the **exact same set of ### Version-Specific Properties Some properties only exist in certain Mendix versions. Before adding a property to a BSON writer: -1. Check `reference/mendixmodellib/reflection-data/` for the property definition -2. Check `min_version` if present -3. Test with `mx diff` self-diff on the target version +1. Check a Studio Pro-authored document **of the target version** for the property +2. Test with `mx diff` self-diff on the target version **Example**: `IsReusableComponent` exists in `Projects$ModuleImpl` in newer Mendix but NOT in 11.6.4. Writing it → crash. diff --git a/.claude/skills/fix-issue/findings/sdk/2026-04-21-typecacheunknowntypeexception-studio-pro.json b/.claude/skills/fix-issue/findings/sdk/2026-04-21-typecacheunknowntypeexception-studio-pro.json index e5e37a82d..9e02026e4 100644 --- a/.claude/skills/fix-issue/findings/sdk/2026-04-21-typecacheunknowntypeexception-studio-pro.json +++ b/.claude/skills/fix-issue/findings/sdk/2026-04-21-typecacheunknowntypeexception-studio-pro.json @@ -1 +1 @@ -{"area": "sdk/mpr", "date": "2026-04-21", "symptom": "`TypeCacheUnknownTypeException` in Studio Pro", "cause": "Wrong `$type` storage name in BSON write", "file": "`sdk/mpr/writer_microflow.go`", "insight": "Check the storage name table in CLAUDE.md; verify against `reference/mendixmodellib/reflection-data/`"} +{"area": "sdk/mpr", "date": "2026-04-21", "symptom": "`TypeCacheUnknownTypeException` in Studio Pro", "cause": "Wrong `$type` storage name in BSON write", "file": "`sdk/mpr/writer_microflow.go`", "insight": "Check the storage name table in CLAUDE.md; verify against `modelsdk/gen//types.go`"} diff --git a/.claude/skills/implement-mdl-bson b/.claude/skills/implement-mdl-bson index 4abbd3008..ed65ef88d 100644 --- a/.claude/skills/implement-mdl-bson +++ b/.claude/skills/implement-mdl-bson @@ -65,63 +65,51 @@ Use `dump-mpr` to answer: ### Find Type Definition +The vendored metamodel types in `modelsdk/gen//` carry the storage names. +`types.go` holds each type's `$Type` and its BSON field names; `refs.go` holds +the reference kind of each pointer property. + ```bash -# Search for a type in reflection data -grep -A 50 '"Microflows\$Microflow"' reference/mendixmodellib/reflection-data/11.6.0-structures.json | head -60 +# $Type storage name and BSON field names of a type +grep -n -A 12 'SetTypeName("Microflows\$Microflow")' modelsdk/gen/microflows/types.go ``` -### Key Metamodel Fields - -```json -{ - "Microflows$Microflow": { - "qualifiedName": "Microflows$Microflow", - "storageName": "Microflows$Microflow", // ← Use for $Type in BSON - "properties": { - "objectCollection": { - "name": "objectCollection", - "storageName": "ObjectCollection", // ← Use for BSON field name - "typeInfo": { - "type": "ELEMENT", - "elementType": "Microflows$MicroflowObjectCollection", - "kind": "PART" // ← Determines reference format - } - } - } - } -} +The `init()` constructor lists one property per BSON field, with the +storage name as the first argument: + +```go +o.SetTypeName("DataTypes$ListType") // ← $Type in BSON +o.entity = property.NewByNameRef[element.Element]("Entity", "DomainModels$Entity") // ← BSON field "Entity" ``` +`modelsdk/gen` is pinned to a Mendix release and does not know about anything +added after it. For anything newer, or whenever gen and a document disagree, +dump a Studio Pro-authored document; **the document wins**. + ### Reference Kinds -| Kind | BSON Format | Go Type | +| Kind (`codec.RefKind`) | BSON Format | Go Type | |------|-------------|---------| -| `PART` | Embedded document | Struct pointer | -| `BY_ID_REFERENCE` | Binary UUID | `model.ID` (serialize as BSON Binary) | -| `BY_NAME_REFERENCE` | String | `string` (qualified name like "Module.Entity") | -| `LOOKUP` | String | `string` | +| embedded part | Embedded document | Struct pointer | +| `RefById` | Binary UUID | `model.ID` (serialize as BSON Binary) | +| `RefByName` | String | `string` (qualified name like "Module.Entity") | +| `RefByNameList` | Array of strings | `[]string` | -### Looking Up Reference Kind in Metamodel +### Looking Up Reference Kind -**Critical:** Always check the `kind` field in the metamodel before implementing references. +**Critical:** Always check the reference kind before implementing references. Getting this wrong causes "Value cannot be null" or similar errors in `mx check`. ```bash -# Search for property definition -grep -A 20 '"entity"' reference/mendixmodellib/reflection-data/11.6.0-structures.json | grep -A 10 'DataTypes\$ListType' +# Reference properties of a type +grep -n -A 4 '"DataTypes\$ListType"' modelsdk/gen/datatypes/refs.go ``` -Example discovery - `DataTypes$ListType.entity`: -```json -"entity" : { - "name" : "entity", - "storageName" : "Entity", - "typeInfo" : { - "type" : "ELEMENT", - "elementType" : "DomainModels$Entity", - "kind" : "BY_NAME_REFERENCE" // ← This is the critical field! - } -} +Example discovery - `DataTypes$ListType.Entity`: +```go +codec.DefaultRefRegistry.RegisterRefs("DataTypes$ListType", []codec.RefMeta{ + {Prop: "Entity", Kind: codec.RefByName, Target: "DomainModels$Entity"}, // ← the critical field +}) ``` This means `Entity` must be serialized as `"MyModule.MyEntity"` (string), NOT as a binary UUID. @@ -131,24 +119,17 @@ This means `Entity` must be serialized as `"MyModule.MyEntity"` (string), NOT as - `mx check` error: `"Value cannot be null. (Parameter 'value') at Mendix.Modeler.DataTypes.EntityType.set_EntityId"` - Fix: Changed to string qualified name format -You can also check `reference/mendixmodellib/src/mpr/mprJsonConversion.js` for the TypeScript implementation: -```javascript -case "BY_NAME_REFERENCE": -case "LOCAL_BY_NAME_REFERENCE": - return value || ""; // Just passes string through, not binary -``` - ### Audit Similar Types -When you fix a BY_NAME_REFERENCE bug in one type, **immediately audit all similar types**. +When you fix a by-name reference bug in one type, **immediately audit all similar types**. Example: When we fixed `DataTypes$ListType.entity`, we should have checked: -- `DataTypes$ObjectType.entity` - Same bug! Also BY_NAME_REFERENCE -- `DataTypes$EnumerationType.enumeration` - Same pattern! Also BY_NAME_REFERENCE +- `DataTypes$ObjectType.entity` - Same bug! Also `RefByName` +- `DataTypes$EnumerationType.enumeration` - Same pattern! Also `RefByName` ```bash -# Find all entity references that might be BY_NAME_REFERENCE -grep -B 5 '"BY_NAME_REFERENCE"' reference/mendixmodellib/reflection-data/11.6.0-structures.json | grep -E '"(name|storageName)"' +# Find all by-name references to entities +grep -rn 'RefByName, Target: "DomainModels\$Entity"' modelsdk/gen/*/refs.go ``` ### Context-Specific Type Interpretation @@ -682,8 +663,7 @@ MxCheck tests only fail if errors are **related to newly created artifacts**: ## Reference Files -- Metamodel structures: `reference/mendixmodellib/reflection-data/11.6.0-structures.json` -- BSON conversion logic: `reference/mendixmodellib/src/mpr/mprJsonConversion.js` +- Metamodel types: `modelsdk/gen//types.go` (storage names) and `refs.go` (reference kinds) - BSON mapping docs: `docs/05-mdl-specification/10-bson-mapping.md` - Existing implementations: `sdk/domainmodel/`, `sdk/mpr/writer.go` - mx tools: `reference/mxbuild/modeler/mx` (check, create-project, build) diff --git a/.claude/skills/test-mdl.md b/.claude/skills/test-mdl.md index 0d3ac8b13..9d335a2da 100644 --- a/.claude/skills/test-mdl.md +++ b/.claude/skills/test-mdl.md @@ -118,7 +118,7 @@ The type cache does not contain a type with qualified name DomainModels$index **Cause**: Using `qualifiedName` instead of `storageName` for `$type` field. -**Fix**: Check `reference/mendixmodellib/reflection-data/-structures.json` for the correct `storageName`. +**Fix**: Use the `$Type` from the `SetTypeName(...)` call in `modelsdk/gen//types.go`, or from a Studio Pro-authored document. ### System.ArgumentNullException @@ -128,9 +128,9 @@ System.ArgumentNullException: value cannot be null. (parameter 'AttributeId') **Cause**: Wrong reference format. Using UUID for BY_NAME_REFERENCE or vice versa. -**Fix**: Check metamodel `typeInfo.kind` for the property: -- `BY_NAME_REFERENCE` → qualified name string (e.g., "Module.Entity.Attr") -- `BY_ID_REFERENCE` → binary UUID +**Fix**: Check the property's reference kind in `modelsdk/gen//refs.go`: +- `RefByName` → qualified name string (e.g., "Module.Entity.Attr") +- `RefById` → binary UUID ### Enumeration Not Displayed @@ -158,11 +158,11 @@ go run ./examples/debug_bson/main.go mx-test-projects/test1-go-app/test1-go.mpr ### Check Metamodel ```bash -# find type definition -grep -A 30 '"DomainModels\$Index"' reference/mendixmodellib/reflection-data/11.6.0-structures.json +# find type definition (storage name and BSON field names) +grep -n -A 12 'SetTypeName("DomainModels\$EntityIndex")' modelsdk/gen/domainmodels/types.go -# find property reference kind -grep -B 5 -A 10 '"storageName" : "Attribute"' reference/mendixmodellib/reflection-data/11.6.0-structures.json +# find property reference kinds +grep -n -A 6 '"DomainModels\$IndexedAttribute"' modelsdk/gen/domainmodels/refs.go ``` ## Test Project diff --git a/.gitignore b/.gitignore index 4f7c87398..27e173835 100644 --- a/.gitignore +++ b/.gitignore @@ -31,7 +31,6 @@ coverage.out coverage.html reference/mendix-repl/ reference/mxbuild/ -reference/mendixmodellib/ reference/mdl-grammar/ reference/mxbuild-11.6.0/ /reference/ diff --git a/CLAUDE.md b/CLAUDE.md index 9083bf8d9..cde52ea98 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -41,7 +41,7 @@ go run ./examples/read_project/main.go /path/to/project.mpr go run ./examples/modify_project/main.go /path/to/project.mpr # run the code generator -go run ./cmd/codegen/main.go -reflection-dir ./reference/mendixmodellib/reflection-data -version 10.0.0 -output ./generated/metamodel +go run ./cmd/codegen/main.go -input -version 10.0.0 -output ./generated/metamodel ``` **Note**: This project uses `modernc.org/sqlite` (pure Go) and does **not** require CGO. No C compiler is needed. @@ -118,8 +118,7 @@ that is not obvious from the layout: When adding new types, always verify the storage name by: 1. Examining existing MPR files with the `mx` tool or SQLite browser -2. Checking the reflection data in `reference/mendixmodellib/reflection-data/` -3. Looking at the decoder in `modelsdk/codec/` and the types in `modelsdk/gen/microflows/` +2. Looking at the decoder in `modelsdk/codec/` and the types in `modelsdk/gen/microflows/` **IMPORTANT**: When unsure about the correct BSON structure for a new feature, **ask the user to create a working example in Mendix Studio Pro** so you can compare the generated BSON against a known-good reference. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 229fd042a..7b1239cec 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -462,7 +462,7 @@ If you're contributing from a fork, this is the full cycle: ### BSON Storage Names -**Critical**: Mendix uses different storage names in BSON `$type` fields than the qualified names in SDK documentation. Always verify against `reference/mendixmodellib/reflection-data/` or existing MPR files. See the table in `CLAUDE.md` for common mismatches. +**Critical**: Mendix uses different storage names in BSON `$type` fields than the qualified names in SDK documentation. Always verify against the vendored types in `modelsdk/gen/` or existing MPR files. See the table in `CLAUDE.md` for common mismatches. ### BSON Tooling diff --git a/cmd/codegen/main.go b/cmd/codegen/main.go index 6b7b71d28..2abfce119 100644 --- a/cmd/codegen/main.go +++ b/cmd/codegen/main.go @@ -4,12 +4,12 @@ // // Usage: // -// go run cmd/codegen/main.go -version 10.0.0 -output ./generated +// go run cmd/codegen/main.go -input -version 10.0.0 -output ./generated // // Flags: // // -version Mendix version to generate for (default: "10.0.0") -// -input Path to reflection data directory (default: "libs/mendixmodellib/reflection-data") +// -input Path to reflection data directory (required) // -output Output directory for generated Go files (default: "generated") // -namespace Generate only specific namespace (comma-separated, or empty for all) // -single Generate all types in a single 'metamodel' package (avoids import cycles) @@ -29,12 +29,18 @@ import ( func main() { version := flag.String("version", "10.0.0", "Mendix version to generate for") - inputDir := flag.String("input", "libs/mendixmodellib/reflection-data", "Path to reflection data directory") + inputDir := flag.String("input", "", "Path to reflection data directory (required)") outputDir := flag.String("output", "generated", "Output directory for generated Go files") namespace := flag.String("namespace", "", "Generate only specific namespace (comma-separated, or empty for all)") singlePkg := flag.Bool("single", true, "Generate all types in a single 'metamodel' package") flag.Parse() + if *inputDir == "" { + fmt.Fprintln(os.Stderr, "Error: -input is required") + flag.Usage() + os.Exit(2) + } + if err := run(*version, *inputDir, *outputDir, *namespace, *singlePkg); err != nil { fmt.Fprintf(os.Stderr, "Error: %v\n", err) os.Exit(1) diff --git a/docs-site/src/appendixes/common-mistakes.md b/docs-site/src/appendixes/common-mistakes.md index 7d70bd926..7e4a55211 100644 --- a/docs-site/src/appendixes/common-mistakes.md +++ b/docs-site/src/appendixes/common-mistakes.md @@ -112,7 +112,7 @@ Mendix uses different "storage names" in BSON `$Type` fields than the "qualified When adding new types, always verify the storage name by: 1. Examining existing MPR files with the `mx` tool or SQLite browser -2. Checking the reflection data in the `reference/mendixmodellib/reflection-data/` directory +2. Checking the vendored types in `modelsdk/gen//types.go` 3. Looking at the read cases in `mdl/backend/modelsdk/microflow_read_actions.go` ## Mendix Expression String Escaping diff --git a/docs-site/src/appendixes/error-messages.md b/docs-site/src/appendixes/error-messages.md index 5e0883e9f..348f70d34 100644 --- a/docs-site/src/appendixes/error-messages.md +++ b/docs-site/src/appendixes/error-messages.md @@ -15,14 +15,14 @@ with qualified name DomainModels$Index **Cause:** The BSON `$Type` field uses the **qualifiedName** instead of the **storageName**. These are often identical, but not always. -**Solution:** Check the metamodel reflection data for the correct storage name: +**Solution:** Use the correct storage name: | qualifiedName (wrong) | storageName (correct) | |-----------------------|----------------------| | `DomainModels$Entity` | `DomainModels$EntityImpl` | | `DomainModels$Index` | `DomainModels$EntityIndex` | -Look up the type in `reference/mendixmodellib/reflection-data/-structures.json` and use the `storageName` field value. +Look up the type's `SetTypeName(...)` call in `modelsdk/gen//types.go`, or dump a Studio Pro-authored document. ### CE0463: Widget definition changed diff --git a/docs-site/src/internals/bson-structure.md b/docs-site/src/internals/bson-structure.md index 6389df5c3..9a1b63ed9 100644 --- a/docs-site/src/internals/bson-structure.md +++ b/docs-site/src/internals/bson-structure.md @@ -4,17 +4,16 @@ How Mendix project documents are serialized in BSON format, including the role o ## Source of Truth -The authoritative reference for BSON serialization is the **reflection-data** at: +The authoritative reference for BSON serialization is a document **authored by +Studio Pro** for the target Mendix version: dump it with `mxcli bson dump` and +compare. -``` -reference/mendixmodellib/reflection-data/{version}-structures.json -``` +The vendored metamodel types in `modelsdk/gen//` are the reference in +the tree: +- `types.go`: each type's BSON `$Type` (the `SetTypeName(...)` call, e.g. `Forms$DivContainer`) and its BSON field names +- `refs.go`: the reference kind of each pointer property (by name or by ID) -Each structure entry contains: -- `qualifiedName`: The API name (e.g., `Pages$DivContainer`) -- `storageName`: The BSON `$Type` value (e.g., `Forms$DivContainer`) -- `defaultSettings`: Required default property values -- `properties`: Property definitions with types and requirements +When `modelsdk/gen` and a Studio Pro document disagree, the document wins. ## Type Name Mapping @@ -273,10 +272,10 @@ There is no deduplication within a page. A page with 4 ComboBox widgets requires | Error | Cause | Solution | |-------|-------|----------| -| "The type cache does not contain a type with qualified name X" | Incorrect `$Type` value | Check reflection-data for correct storage name | +| "The type cache does not contain a type with qualified name X" | Incorrect `$Type` value | Check `modelsdk/gen` or a Studio Pro document for the correct storage name | | "No entity configured for the data source" | Missing or incorrect DataView DataSource | Configure `Forms$DataViewSource` with proper EntityRef | | CE0463 "widget definition has changed" | Object properties don't match Type PropertyTypes | Use template Object as base, only modify needed properties | -| "Project uses features that are no longer supported" | Missing widget default properties | Include all required defaults from reflection-data | +| "Project uses features that are no longer supported" | Missing widget default properties | Include all defaults Studio Pro writes for the widget | ## Files Reference @@ -290,4 +289,4 @@ There is no deduplication within a page. A page with 4 ComboBox widgets requires | `modelsdk/codec/decoder.go` | BSON → document (polymorphic types) | | `sdk/widgets/loader.go` | Embedded template loading | | `sdk/widgets/templates/mendix-11.6/*.json` | Embedded widget templates | -| `reference/mendixmodellib/reflection-data/*.json` | Type definitions | +| `modelsdk/gen/*/types.go` | Type definitions and storage names | diff --git a/docs-site/src/internals/packages.md b/docs-site/src/internals/packages.md index 78a33cdb7..b5fb3246a 100644 --- a/docs-site/src/internals/packages.md +++ b/docs-site/src/internals/packages.md @@ -153,7 +153,6 @@ github.com/mendixlabs/mxcli/ │ └── package.json # Extension manifest │ └── reference/ # Reference materials (not Go code) - ├── mendixmodellib/ # TypeScript library + reflection data ├── mendixmodelsdk/ # TypeScript SDK reference └── mdl-grammar/ # Comprehensive MDL grammar reference ``` diff --git a/docs-site/src/internals/storage-names.md b/docs-site/src/internals/storage-names.md index abc4722c9..19988acf1 100644 --- a/docs-site/src/internals/storage-names.md +++ b/docs-site/src/internals/storage-names.md @@ -54,28 +54,15 @@ Page client actions also have storage name differences: When adding new types, always verify the storage name by: 1. **Examining existing MPR files** with the `mx` tool or an SQLite browser -2. **Checking the reflection data** in `reference/mendixmodellib/reflection-data/` +2. **Checking the vendored types** in `modelsdk/gen//types.go` 3. **Looking at the read cases** in `mdl/backend/modelsdk/microflow_read_actions.go` -### Querying Reflection Data +### Looking Up a Storage Name -Use this Python snippet to check widget default settings: +Each generated type registers its BSON `$Type` with `SetTypeName`: -```python -import json - -with open('reference/mendixmodellib/reflection-data/11.0.0-structures.json') as f: - data = json.load(f) - -# Find widget by API name -widget = data.get('Pages$DivContainer', {}) -print('Storage name:', widget.get('storageName')) -print('Defaults:', json.dumps(widget.get('defaultSettings', {}), indent=2)) - -# Search by storage name -for key, val in data.items(): - if val.get('storageName') == 'Forms$NoAction': - print(f'{key}: {val.get("defaultSettings")}') +```bash +grep -rn 'SetTypeName("Forms\$DivContainer")' modelsdk/gen/ ``` ## Association Parent/Child Pointer Semantics @@ -95,4 +82,4 @@ This affects **entity access rules**: MemberAccess entries for associations must ## Key Takeaway -When unsure about the correct BSON structure for a new feature, create a working example in Mendix Studio Pro and compare the generated BSON against a known-good reference. The reflection data at `reference/mendixmodellib/reflection-data/` is the definitive source for storage names and default values. +When unsure about the correct BSON structure for a new feature, create a working example in Mendix Studio Pro and compare the generated BSON against a known-good reference. A Studio Pro-authored document is the definitive source for storage names and default values. diff --git a/docs-wiki/SYNC_LOG.md b/docs-wiki/SYNC_LOG.md index 36a59bb67..64824f1b8 100644 --- a/docs-wiki/SYNC_LOG.md +++ b/docs-wiki/SYNC_LOG.md @@ -17,12 +17,12 @@ not capture, because sources are upstream of the commit. | 2026-05-24 | architecture/mpr-read-write.md | sdk/mpr/reader.go, sdk/mpr/writer_core.go, sdk/mpr/parser.go, sdk/mpr/writer_widgets.go, modelsdk.go | Initial synthesis: v1/v2 format detection, reader/writer nesting, staged WriteTransaction, why the writer is BSON-aware. Dropped non-existent sdk/mpr/writer.go | | 2026-05-24 | architecture/widget-engine.md | sdk/widgets/definitions/loader.go, sdk/widgets/definitions/combobox.def.json, sdk/widgets/templates/README.md, sdk/mpr/writer_widgets.go, mdl/executor/cmd_pages_builder_v3_widgets.go, docs/03-development/{PAGE_BSON_SERIALIZATION,WIDGET_BSON_VERSION_COMPATIBILITY}.md | Initial synthesis: dual type/object template, .def.json mapping + 3-tier registry, two-layer version resilience | | 2026-05-24 | models/association-pointers.md | CLAUDE.md, sdk/mpr/writer_domainmodel.go, sdk/domainmodel/domainmodel.go | Initial synthesis: ParentPointer=FROM / ChildPointer=TO inversion and the CE0066 member-access consequence. Narrowed dir source to domainmodel.go | -| 2026-05-24 | models/storage-vs-qualified-names.md | CLAUDE.md, sdk/mpr/parser_microflow.go | Initial synthesis: two naming systems, Form=Page history, strict-write/lenient-read asymmetry. Dropped non-existent reference/mendixmodellib/reflection-data/ | +| 2026-05-24 | models/storage-vs-qualified-names.md | CLAUDE.md, sdk/mpr/parser_microflow.go | Initial synthesis: two naming systems, Form=Page history, strict-write/lenient-read asymmetry. Dropped non-existent reflection-data path | | 2026-05-24 | models/version-gating.md | sdk/versions/registry.go, sdk/versions/mendix-11.yaml, mdl/executor/cmd_features.go, .claude/skills/version-awareness.md | Initial synthesis: registry as source of truth, checkFeature() pre-check contract and its escape hatches. Narrowed dir sources to specific files | | 2026-05-24 | rationale/mdl-as-sql.md | docs/13-decisions/0003-mdl-is-sql-shaped.md, .claude/skills/design-mdl-syntax.md, docs/11-proposals/PROPOSAL_mdl_syntax_design_guidelines.md, docs/01-project/MDL_QUICK_REFERENCE.md | Initial synthesis: citizen-developer design tradition, PL/SQL pattern for imperative constructs, verbosity trade-off. Cites ADR-0003 | | 2026-05-24 | rationale/backend-abstraction.md | docs/13-decisions/0002-backend-abstraction.md, mdl/backend/{doc,backend,domainmodel}.go, CLAUDE.md | Initial synthesis: executor-is-wrong-layer-for-BSON seam, composed-domain interfaces, four-touch overhead. Cites ADR-0002 | | 2026-05-24 | positioning/vs-typescript-sdk.md | docs/01-project/SDK_EQUIVALENCE.md, README.md | Initial synthesis: local-vs-cloud, SQL-shaped DSL, pure-Go; "which tool for which job". Dropped non-existent reference/mendixmodelsdk/ | -| 2026-05-24 | glossary.md | CLAUDE.md, sdk/mpr/parser_microflow.go, README.md | Initial synthesis: three-vocabulary bridge (Mendix UI / mxcli-SDK / BSON storage names). Dropped non-existent reference/mendixmodellib/reflection-data/ | +| 2026-05-24 | glossary.md | CLAUDE.md, sdk/mpr/parser_microflow.go, README.md | Initial synthesis: three-vocabulary bridge (Mendix UI / mxcli-SDK / BSON storage names). Dropped non-existent reflection-data path | | 2026-05-24 | bug-patterns/bson-numeric-width.md | .claude/skills/fix-issue.md, sdk/mpr/parser.go | Initial synthesis: silent zero-defaulting from narrow int32 assertions; extractInt as canonical fix | | 2026-05-24 | bug-patterns/visitor-wiring-gaps.md | .claude/skills/fix-issue.md, mdl/visitor/visitor_enumeration.go | Initial synthesis: parsed-but-not-stored pattern; the one hand-written bridge with no compiler check. Narrowed dir source to visitor_enumeration.go | | 2026-05-24 | bug-patterns/widget-type-object-drift.md | .claude/skills/fix-issue.md, .claude/skills/debug-bson.md, sdk/widgets/templates/README.md, sdk/mpr/writer_widgets.go | Initial synthesis: Type↔Object coupling, CE0463 cascade, mx-check-tolerance trap | diff --git a/docs-wiki/positioning/vs-typescript-sdk.md b/docs-wiki/positioning/vs-typescript-sdk.md index 2eeab8bad..f60f8e92b 100644 --- a/docs-wiki/positioning/vs-typescript-sdk.md +++ b/docs-wiki/positioning/vs-typescript-sdk.md @@ -11,7 +11,7 @@ sources: ## What this is -mxcli is a local-first, pure-Go reimplementation of the capability behind Mendix's official TypeScript Model SDK (`mendixmodelsdk` + `mendixmodellib`): reading and modifying the model that lives inside a Mendix `.mpr` project. It targets the same metamodel and the same on-disk format, but reaches it from a different direction. +mxcli is a local-first, pure-Go reimplementation of the capability behind Mendix's official TypeScript Model SDK (`mendixmodelsdk`): reading and modifying the model that lives inside a Mendix `.mpr` project. It targets the same metamodel and the same on-disk format, but reaches it from a different direction. ## How it fits diff --git a/docs/01-project/SDK_EQUIVALENCE.md b/docs/01-project/SDK_EQUIVALENCE.md index 499ebc318..7d529fbfd 100644 --- a/docs/01-project/SDK_EQUIVALENCE.md +++ b/docs/01-project/SDK_EQUIVALENCE.md @@ -16,12 +16,11 @@ This document provides a comprehensive analysis of the original Mendix Model SDK ## Overview -The goal is to create a 100% functional equivalent of the Mendix Model SDK for local Mendix projects in Go. The original SDK consists of two main components: +The goal is to create a 100% functional equivalent of the Mendix Model SDK for local Mendix projects in Go. | Component | Purpose | Go Equivalent | |-----------|---------|---------------| | **mendixmodelsdk** | High-level OO API for model manipulation | `modelsdk-go` packages | -| **mendixmodellib** | Low-level format handling, metamodel definitions | `mpr/` package + generated types | ### Key Differences @@ -58,20 +57,15 @@ The high-level SDK providing: | `expressions` | Expression language | ~100 | | ... | 42 more domains | ... | -### mendixmodellib (v1.76.1) +### Metamodel Reflection Data -The low-level library providing: - -- **MPR file format handling** (SQLite + BSON) -- **Metamodel reflection data** for 88 Mendix versions (6.0.0 - 11.6.0) -- **Delta processing** for model transformations -- **GUID utilities** for binary encoding -- **Version compatibility** checking +`cmd/codegen` reads metamodel reflection data: per-version JSON type +definitions. This data is not publicly distributed, so it is not in the +repository and cannot be redistributed with it; see +[PROPOSAL_codegen_ownership.md](../11-proposals/PROPOSAL_codegen_ownership.md) §4. #### Reflection Data Structure -Located in `libs/mendixmodellib/reflection-data/`: - ``` reflection-data/ ├── 6.0.0-structures.json # Mendix 6.0.0 type definitions @@ -236,7 +230,7 @@ Even in implemented domains, many subtypes are missing: ### Recommended Approach: Generate from Reflection Data -The `mendixmodellib/reflection-data/` contains complete metamodel definitions in JSON format. We can generate Go code automatically from these definitions. +The reflection data contains complete metamodel definitions in JSON format. We can generate Go code automatically from these definitions. ### Benefits @@ -432,7 +426,7 @@ When a new Mendix version is released: ├─────────────────────────────────────────────────────────────┤ │ │ │ 1. Obtain new reflection data │ -│ └─▶ get {version}-structures.json from mendixmodellib │ +│ └─▶ {version}-structures.json for the new version │ │ │ │ 2. run generator │ │ └─▶ go run cmd/generate/main.go -version=11.7.0 │ @@ -456,16 +450,9 @@ When a new Mendix version is released: #### 1. Obtain Reflection Data -```bash -# Option A: from npm package -npm pack mendixmodellib@latest -tar -xzf mendixmodellib-*.tgz -cp package/reflection-data/*.json libs/mendixmodellib/reflection-data/ - -# Option B: from existing installation -cp ~/Projects/mcpmxsdk/node_modules/mendixmodellib/reflection-data/*.json \ - libs/mendixmodellib/reflection-data/ -``` +The reflection data is not publicly distributed; see +[PROPOSAL_codegen_ownership.md](../11-proposals/PROPOSAL_codegen_ownership.md) +§4 and §4A for the licence-clean replacement. #### 2. Run Generator @@ -506,54 +493,6 @@ var SupportedVersions = []string{ var LatestVersion = "11.7.0" ``` -### Automation with CI/CD - -```yaml -# .github/workflows/update-metamodel.yml -name: update Metamodel - -on: - schedule: - - cron: '0 0 * * 0' # Weekly - workflow_dispatch: - inputs: - version: - description: 'Mendix version to add' - required: true - -jobs: - update: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - - name: Setup Go - uses: actions/setup-go@v5 - with: - go-version: '1.24' - - - name: Setup node - uses: actions/setup-node@v4 - - - name: get latest mendixmodellib - run: | - npm pack mendixmodellib@latest - tar -xzf mendixmodellib-*.tgz - cp package/reflection-data/*.json libs/mendixmodellib/reflection-data/ - - - name: generate types - run: go run cmd/generate/main.go -latest -output=generated/ - - - name: run tests - run: go test ./... - - - name: create PR - uses: peter-evans/create-pull-request@v6 - with: - title: "update metamodel to latest version" - branch: update-metamodel -``` - --- ## Implementation Roadmap @@ -634,5 +573,4 @@ Total: 88 versions supported - [Mendix Model SDK Documentation](https://docs.mendix.com/apidocs-mxsdk/mxsdk/) - [mendixmodelsdk npm package](https://www.npmjs.com/package/mendixmodelsdk) -- [mendixmodellib npm package](https://www.npmjs.com/package/mendixmodellib) - [Mendix MPR File Format](https://docs.mendix.com/refguide/mpr-format/) diff --git a/docs/03-development/BSON_TOOLING_GUIDE.md b/docs/03-development/BSON_TOOLING_GUIDE.md index e1c7c5d29..d8c02d7e1 100644 --- a/docs/03-development/BSON_TOOLING_GUIDE.md +++ b/docs/03-development/BSON_TOOLING_GUIDE.md @@ -236,16 +236,15 @@ What are you trying to do? | --> mx check app.mpr ``` -## Reflection data reference +## Metamodel type reference -The Mendix type definitions live in `reference/mendixmodellib/reflection-data/`. Each JSON file defines one metamodel domain with: -- Type names and their storage names (`$type` values) -- Properties with types, defaults, and whether they're required -- Inheritance hierarchy +The vendored metamodel types live in `modelsdk/gen//`: +- `types.go`: each type's storage name (`$Type`, from `SetTypeName`) and its BSON field names +- `refs.go`: the reference kind of each pointer property -Check these when you're unsure whether a field belongs on a type. For example, `DomainModels.json` shows that `ParentConnection` exists on `DomainModels$association` but not on `DomainModels$CrossAssociation`. +Check these when you're unsure whether a field belongs on a type. For example, `modelsdk/gen/domainmodels/types.go` shows that `ParentConnection` exists on `DomainModels$Association` but not on `DomainModels$CrossAssociation`. -The generated Go metamodel in `generated/metamodel/types.go` mirrors these definitions and is used by `bson discover` for field coverage analysis. +The generated Go metamodel in `generated/metamodel/types.go` is used by `bson discover` for field coverage analysis. Both are pinned to a Mendix release; for anything newer, a Studio Pro-authored document is the reference. ## Related documentation diff --git a/docs/03-development/PAGE_BSON_SERIALIZATION.md b/docs/03-development/PAGE_BSON_SERIALIZATION.md index 8e8dafc6e..ed0577596 100644 --- a/docs/03-development/PAGE_BSON_SERIALIZATION.md +++ b/docs/03-development/PAGE_BSON_SERIALIZATION.md @@ -4,16 +4,16 @@ This document describes the BSON serialization format for Mendix pages, includin ## Source of Truth -The authoritative reference for BSON serialization is the **reflection-data** at: -``` -reference/mendixmodellib/reflection-data/{version}-structures.json -``` +The authoritative reference for BSON serialization is a page **authored by +Studio Pro** for the target Mendix version: dump it with `mxcli bson dump` and +compare. -Each structure entry contains: -- `qualifiedName`: The API name (e.g., `pages$DivContainer`) -- `storageName`: The BSON `$type` value (e.g., `Forms$DivContainer`) -- `defaultSettings`: Required default property values -- `properties`: Property definitions with types and requirements +The vendored metamodel types in `modelsdk/gen/pages/` are the reference in the +tree: +- `types.go`: each widget's BSON `$Type` (the `SetTypeName(...)` call, e.g. `Forms$DivContainer`) and its BSON field names +- `refs.go`: the reference kind of each pointer property + +When `modelsdk/gen` and a Studio Pro document disagree, the document wins. ## Type Name Mapping @@ -492,29 +492,18 @@ The DataView's `datasource` property is missing or incorrectly configured. A Dat ### "Project uses features that are no longer supported" -Widget properties are missing or have incorrect values. Check that all required default properties from the reflection-data are included. - -## Querying Reflection Data +Widget properties are missing or have incorrect values. Check that all default properties Studio Pro writes for the widget are included. -Use this Python snippet to check widget default settings: +## Looking Up a Widget Type -```python -import json +Each generated widget type registers its BSON `$Type` with `SetTypeName`, followed by its properties: -with open('reference/mendixmodellib/reflection-data/11.0.0-structures.json') as f: - data = json.load(f) - -# find widget by api name -widget = data.get('Pages$DivContainer', {}) -print('Storage name:', widget.get('storageName')) -print('Defaults:', json.dumps(widget.get('defaultSettings', {}), indent=2)) - -# search by storage name -for key, val in data.items(): - if val.get('storageName') == 'Forms$NoAction': - print(f'{key}: {val.get("defaultSettings")}') +```bash +grep -n -A 20 'SetTypeName("Forms\$DivContainer")' modelsdk/gen/pages/types.go ``` +For default values, dump the same widget from a page Studio Pro authored. + ## Files Reference | File | Purpose | @@ -529,7 +518,7 @@ for key, val in data.items(): | `sdk/widgets/templates/mendix-11.6/*.json` | Embedded widget templates | | `sdk/pages/pages_widgets_advanced.go` | CustomWidget Go types | | `mdl/executor/cmd_pages_builder_input.go` | Widget creation from MDL | -| `reference/mendixmodellib/reflection-data/*.json` | Type definitions | +| `modelsdk/gen/pages/types.go` | Widget type definitions and storage names | ## Pluggable Widgets (CustomWidgets) diff --git a/docs/05-mdl-specification/10-bson-mapping.md b/docs/05-mdl-specification/10-bson-mapping.md index 7bb8255aa..2f4ab1782 100644 --- a/docs/05-mdl-specification/10-bson-mapping.md +++ b/docs/05-mdl-specification/10-bson-mapping.md @@ -217,7 +217,7 @@ TypeCacheUnknownTypeException: The type cache does not contain a type with quali | `DomainModels$index` | `DomainModels$EntityIndex` | | `DomainModels$entity` | `DomainModels$EntityImpl` | -When adding support for new document types, always check the metamodel reflection data in `reference/mendixmodellib/reflection-data/-structures.json` to find the correct `storageName`. +When adding support for new document types, always check the `SetTypeName(...)` call in `modelsdk/gen//types.go`, or a Studio Pro-authored document, to find the correct storage name. ### Metamodel Reference Definition @@ -929,11 +929,11 @@ When adding support for new Mendix metamodel types (microflows, pages, workflows ### 1. Find the Metamodel Definition -Locate the type in `reference/mendixmodellib/reflection-data/-structures.json`: +Locate the type in the vendored metamodel, `modelsdk/gen//types.go`: ```bash # search for a type -grep -A 20 '"Microflows\$Microflow"' reference/mendixmodellib/reflection-data/11.6.0-structures.json +grep -n -A 20 'SetTypeName("Microflows\$Microflow")' modelsdk/gen/microflows/types.go ``` ### 2. Check storageName vs qualifiedName diff --git a/docs/11-proposals/PROPOSAL_backend_strategy.md b/docs/11-proposals/PROPOSAL_backend_strategy.md index 1b94648be..2e3511d8b 100644 --- a/docs/11-proposals/PROPOSAL_backend_strategy.md +++ b/docs/11-proposals/PROPOSAL_backend_strategy.md @@ -64,7 +64,7 @@ Studio Pro 11.11 (`test7-app`). | **MCP write semantics** | **On-disk files are stale-by-default while Studio Pro is open.** `ped_update_document` edits live in memory (shown unsaved) until the user saves; `ped_create_module` flushes immediately; `ped_check_errors` does not flush; no save tool. | | **Widgets** | His `modelsdk/widgets` is, by his own spec, an *early, less-complete fork* of our `sdk/widgets` (which he deleted); serialization *"does not change"*. Our **v0.12 serialization is absent** (~15 commits). His **real-time `.mpk` registry supersedes** our `generatorVersion` staleness fix. | | **Codegen source** | `dtsparser/jsparser.go` is **still regex** (23 patterns, no AST) over the public `mendixmodelsdk` npm. retran (PR #335) called it a *"fragile foundation"*; engalar agreed; only an `audit` gap-detector was added (test-only). | -| **Source reliability** | `mendixmodelsdk` IS current (4.112.0, 2026-05-21, ~monthly) **but structurally omits new doc types**: latest package has 55 domains, **zero agent/AI** → Agent Editor is **hand-coded**. `mendixmodellib` (internal Studio Pro source) is **404 on npm**. | +| **Source reliability** | `mendixmodelsdk` IS current (4.112.0, 2026-05-21, ~monthly) **but structurally omits new doc types**: latest package has 55 domains, **zero agent/AI** → Agent Editor is **hand-coded**. The reflection data (internal Studio Pro source) is **not on npm**. | | **Reverse delta (what switching loses)** | Integration features are **present/ahead** on his branch (he even shipped 3 microflow statements we only proposed). Genuine losses: **`docs-site` (305 files)**, **LSP + `vscode-mdl`**, **`api/`**, **v0.12 widget serialization**, **~7 CI workflows**. | | **doctype-tests** | ~90% shared corpus (45 vs 47 files, same numbering, mostly small diffs) → MDL syntax did **not** fork hard; usable as a parity harness. | diff --git a/docs/11-proposals/PROPOSAL_codegen_ownership.md b/docs/11-proposals/PROPOSAL_codegen_ownership.md index 893336bdc..c8fefe391 100644 --- a/docs/11-proposals/PROPOSAL_codegen_ownership.md +++ b/docs/11-proposals/PROPOSAL_codegen_ownership.md @@ -153,11 +153,10 @@ it is a reason to fix provenance (§4) rather than lean on the snapshot forever. Both generators are unrunnable here today. -- `/reference/` is gitignored (`.gitignore:37`, plus `reference/mendixmodellib/` - explicitly at line 34), so `reference/mendixmodellib/reflection-data/` — named - by `BSON_TOOLING_GUIDE.md` §"Reflection data reference" and - `PAGE_BSON_SERIALIZATION.md` as *the* authoritative source — is absent. -- `npm registry mendixmodellib` → **404**. A registry search for Mendix model +- `/reference/` is gitignored, so the reflection data — then named by + `BSON_TOOLING_GUIDE.md` and `PAGE_BSON_SERIALIZATION.md` as *the* + authoritative source — is absent. +- The npm registry does not publish it (**404**). A registry search for Mendix model packages returns the TypeScript SDK, the Platform SDK, widget tooling and third-party wrappers; **nothing publishes reflection/structures JSON**. - It is not in the mxbuild distribution we already download per version, at @@ -172,7 +171,7 @@ Pulling it on every Mendix release would keep gen current and keep it wrong. ### 4.1 Reflection data is disqualified on licence, not availability -`reference/mendixmodellib/` is an **internal, closed-source Mendix library**. Even +The reflection data comes from an **internal, closed-source Mendix library**. Even where a copy exists, building mxcli's correctness on it would mean the project could only be developed by people holding that artefact — which is disqualifying for an open-source community regardless of the technical fit. diff --git a/docs/11-proposals/PROPOSAL_update_builtin_widget_properties.md b/docs/11-proposals/PROPOSAL_update_builtin_widget_properties.md index 187f113e2..f7e44250e 100644 --- a/docs/11-proposals/PROPOSAL_update_builtin_widget_properties.md +++ b/docs/11-proposals/PROPOSAL_update_builtin_widget_properties.md @@ -38,7 +38,7 @@ read from MPR (BSON) → Parse to Go struct → modify struct → Serialize back ### How the reflection data solves this -The reflection data (`reference/mendixmodellib/reflection-data/{version}-structures.json`) already contains exact property metadata for every built-in widget type. For example, `pages$layoutgrid`: +The reflection data (`{version}-structures.json`) already contains exact property metadata for every built-in widget type. For example, `pages$layoutgrid`: ```json { diff --git a/docs/11-proposals/archive/BSON_SCHEMA_REGISTRY_PROPOSAL.md b/docs/11-proposals/archive/BSON_SCHEMA_REGISTRY_PROPOSAL.md index 35d2be7a1..dff44c3c2 100644 --- a/docs/11-proposals/archive/BSON_SCHEMA_REGISTRY_PROPOSAL.md +++ b/docs/11-proposals/archive/BSON_SCHEMA_REGISTRY_PROPOSAL.md @@ -509,7 +509,7 @@ The schema registry is a **complement**, not a replacement. It handles the mecha ## Reflection Data Version Count -The `reference/mendixmodellib/reflection-data/` directory contains two JSON files per Mendix minor release — `{version}-structures.json` and `{version}-storageNames.json`. The metamodel schema changes at the **minor release** level (e.g. 10.5 → 10.6), not at patch level (10.5.0 → 10.5.1), so there is one schema per minor. The approximate breakdown: +The reflection data contains two JSON files per Mendix minor release — `{version}-structures.json` and `{version}-storageNames.json`. The metamodel schema changes at the **minor release** level (e.g. 10.5 → 10.6), not at patch level (10.5.0 → 10.5.1), so there is one schema per minor. The approximate breakdown: | Major | Minor versions | Range | |-------|---------------|-------| diff --git a/docs/11-proposals/archive/PROPOSAL_schema_extract.md b/docs/11-proposals/archive/PROPOSAL_schema_extract.md index 2fd02a471..81fed459d 100644 --- a/docs/11-proposals/archive/PROPOSAL_schema_extract.md +++ b/docs/11-proposals/archive/PROPOSAL_schema_extract.md @@ -21,7 +21,7 @@ The BSON Schema Registry proposal (see `BSON_SCHEMA_REGISTRY_PROPOSAL.md`) requi per-version type metadata to drive serialization, validation, and default-value filling. That metadata currently comes from two sources, both with reliability problems: -1. **Reflection data** (`reference/mendixmodellib/reflection-data/`) — extracted from the +1. **Reflection data** — extracted from the TypeScript `mendixmodelsdk` npm package. Stale at Mendix 11.6. Requires manual update each release. Uses TypeScript SDK names that differ from BSON storage names (e.g. SDK says `CreateObjectAction`, BSON stores `CreateChangeAction`). @@ -66,7 +66,7 @@ JSON file. ```bash # Extract schema for whatever Studio Pro version is currently open -mxcli schema extract --output reference/mendixmodellib/reflection-data/ +mxcli schema extract --output reference/reflection-data/ # Extract only microflow and domain model domains mxcli schema extract --domains microflows,domainmodels --output ./schemas/ @@ -204,7 +204,7 @@ different Mendix version") acts as a version guard — the right `mx` binary mus The extractor writes one JSON file per Mendix version: ``` -reference/mendixmodellib/reflection-data/ +reference/reflection-data/ 11.9.0-extracted.json ← new format, one file replaces -structures + -storageNames pair 11.8.0-extracted.json ... diff --git a/docs/11-proposals/archive/UNIFIED_SCHEMA_REGISTRY.md b/docs/11-proposals/archive/UNIFIED_SCHEMA_REGISTRY.md index 28d19a891..396484f19 100644 --- a/docs/11-proposals/archive/UNIFIED_SCHEMA_REGISTRY.md +++ b/docs/11-proposals/archive/UNIFIED_SCHEMA_REGISTRY.md @@ -47,7 +47,7 @@ Three separate systems, none self-updating, overlapping responsibilities, diverg | System | Source | Update mechanism | Failure mode | |---|---|---|---| -| `reference/mendixmodellib/reflection-data/` | TS SDK extraction | Manual per release | Stale at 11.6, missing storage names / list encodings / ref kinds | +| Reflection data | TS SDK extraction | Manual per release | Stale at 11.6, missing storage names / list encodings / ref kinds | | `supplements.json` | Hand-curated overrides | Per-release human review | Gaps discovered at runtime when Studio Pro rejects output | | `sdk/widgets/templates/mendix-11.6/` + augmentation | Manual extraction | One-off per Mendix version | Frozen at 11.6; structural BSON shifts not handled — see [WIDGET_BSON_VERSION_COMPATIBILITY.md](../03-development/WIDGET_BSON_VERSION_COMPATIBILITY.md) for the five-fix patch sequence that was needed to support 11.9 | | `.mxcli/widgets/*.def.json` (per-project) | `mxcli widget init` parses MPK XML | User-triggered | Lossy: ignores object-list properties (Accordion `groups`, etc.); no integration with init/refresh | @@ -153,7 +153,7 @@ enough to ship separately, which is what the phasing reflects. What we drop: -- **TypeScript reflection data** (`reference/mendixmodellib/reflection-data/`) — lossy +- **TypeScript reflection data** — lossy intermediate, manually updated. `mx dump-mpr` provides the same data with no gaps. - **`supplements.json`** — exists only because TS reflection lacks storage names / list encodings / ref kinds. `mx dump-mpr` has all of these directly. @@ -510,7 +510,7 @@ validation, and skills. Migrated projects get a visible to-do list of legacy wid **Goal**: Replace TS reflection data with `mx dump-mpr` output, via build-time codegen. The existing `cmd/codegen/main.go` already generates Go types from a JSON-shaped reflection -input. This phase swaps its data source from `reference/mendixmodellib/reflection-data/` +input. This phase swaps its data source from the reflection data to `mx dump-mpr` output, and extends the generator to emit storage names, list encodings, and ref kinds (the fields TS reflection drops). @@ -529,7 +529,7 @@ or invalidate. `mxcli schema show entity` reads from generated Go types. **Goal**: Retire legacy data sources; ship cross-version inspection and migration commands. - Validate Phase 4 codegen has full coverage of properties currently in `supplements.json` -- Drop `supplements.json` and `reference/mendixmodellib/reflection-data/` +- Drop `supplements.json` and the reflection data - Implement `mxcli schema list/show/diff` reading from generated platform schemas + runtime widget schemas - Implement `mxcli check --post-migration`, `mxcli widget upgrade` with tier-based drift diff --git a/docs/11-proposals/odata-services-proposal.md b/docs/11-proposals/odata-services-proposal.md index 35e24b3e5..b8b4978f1 100644 --- a/docs/11-proposals/odata-services-proposal.md +++ b/docs/11-proposals/odata-services-proposal.md @@ -702,7 +702,7 @@ revoke access on odata service MyModule.CustomerAPI from MyModule.Guest; ## References -- `reference/mendixmodellib/reflection-data/11.6.0-structures.json` - Type definitions +- `modelsdk/gen/` - Type definitions - `mx-test-projects/QueryDemoApp-main/QueryDemoApp.mpr` - Working examples - Types: `rest$ConsumedODataService`, `ODataPublish$PublishedODataService2`, `ODataPublish$EntitySet`, `ODataPublish$EntityType` diff --git a/docs/11-proposals/version-aware-mdl.md b/docs/11-proposals/version-aware-mdl.md index 066394aa4..0f850cab9 100644 --- a/docs/11-proposals/version-aware-mdl.md +++ b/docs/11-proposals/version-aware-mdl.md @@ -7,7 +7,7 @@ status: proposed ## Problem Statement -The Mendix metamodel evolves across versions. The reflection data in `reference/mendixmodellib/reflection-data/` demonstrates that BSON document structures can change between Mendix versions: +The Mendix metamodel evolves across versions. The metamodel reflection data demonstrates that BSON document structures can change between Mendix versions: - **Field names** may change (storageName evolution) - **New properties** are added in newer versions @@ -315,7 +315,7 @@ sdk/mpr/ ├── checker.go └── warnings.go -reference/mendixmodellib/reflection-data/ +reflection-data/ ├── *.json # Existing reflection data └── README.md # documentation of schema format @@ -354,6 +354,6 @@ docs/05-mdl-specification/ ## References -- `reference/mendixmodellib/reflection-data/` - Metamodel definitions per version +- `modelsdk/gen/` - Vendored metamodel types - `docs/05-mdl-specification/10-bson-mapping.md` - Current BSON mapping docs - TypeScript Model SDK - Reference implementation for version handling diff --git a/docs/13-decisions/0008-identity-and-idempotence.md b/docs/13-decisions/0008-identity-and-idempotence.md index eb4f3d831..60e287232 100644 --- a/docs/13-decisions/0008-identity-and-idempotence.md +++ b/docs/13-decisions/0008-identity-and-idempotence.md @@ -224,7 +224,7 @@ in any future attempt's test plan. Freezing it was the last blocker, so "what breaks if we stop regenerating it?" had to be answered before decision 1 could be implemented. mxcli's own reference material is silent — the field appears nowhere in -`reference/mendixmodellib/reflection-data/`, `reference/mendixmodelsdk/`, or +the metamodel reflection data, `reference/mendixmodelsdk/`, or `generated/metamodel/`. That silence means only that mxcli never had a description of it, so the evidence below comes from Mendix's own binaries and from a build's output. diff --git a/gittree.txt b/gittree.txt index f1880950f..5f9c91f24 100644 --- a/gittree.txt +++ b/gittree.txt @@ -606,7 +606,7 @@ * f613a108 - (6 weeks ago) Fix VALIDATION FEEDBACK association path serialization (CE1613) - Andrej Koelewijn * 5b83d674 - (6 weeks ago) Fix duplicate ViewEntitySourceDocuments and batch view entity creation - Andrej Koelewijn * 77ad3fb6 - (6 weeks ago) Fix DROP VIEW ENTITY to also delete orphaned OQL source documents - Andrej Koelewijn -* ec168e24 - (6 weeks ago) Remove unused mendixmodellib tarball and fix script file modes - Andrej Koelewijn +* ec168e24 - (6 weeks ago) Remove unused reflection-data tarball and fix script file modes - Andrej Koelewijn * d51eef33 - (6 weeks ago) Update skills and MDL examples - Andrej Koelewijn * ce646d00 - (6 weeks ago) Simplify devcontainer: native Claude Code install, remove Node.js dependency - Andrej Koelewijn * 7bc95d61 - (6 weeks ago) Add CustomWithoutRollBack error handling type - Andrej Koelewijn diff --git a/mdl/exprcheck/func_checker.go b/mdl/exprcheck/func_checker.go index 646640415..82b9d2313 100644 --- a/mdl/exprcheck/func_checker.go +++ b/mdl/exprcheck/func_checker.go @@ -46,7 +46,7 @@ func (s funcSig) min() int { // funcTable lists every Mendix built-in function and its signature. // Extracted from Mendix 10/11 expression documentation. // When Mendix adds new built-ins, add them here and update the corresponding -// test in func_checker_test.go. Source: reference/mendixmodellib/reflection-data/. +// test in func_checker_test.go. var funcTable = map[string]funcSig{ // Boolean "not": {args: []TypeKind{KindBoolean}, ret: KindBoolean}, From 8feb10376fde51df7e9484bacd241ee217072ae2 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 9 Oct 2026 11:22:42 +0000 Subject: [PATCH 3/5] fix: refuse retrieve by association over System.owner/changedBy (MDL-RETRIEVE02) `retrieve $u from $task/System.owner;` passed check and exec, then mxbuild failed with [CE0136] "Retrieve object must specify the 'Entity' property." (mendixlabs/mxcli#1358). owner and changedBy are entity flags, not modelled associations: mxbuild resolves the name but derives no entity from it, with or without `owner: AutoOwner` (measured on 11.12.2; System.changedBy alike). check and exec now refuse it and name the form that builds clean: `retrieve $u from System.User where [id = $task/System.owner] first;`. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01P3PcLf2tEE4sTP5eamiVpt --- ...ve-by-association-system-owner-ce0136.json | 1 + CHANGELOG.md | 1 + cmd/mxcli/syntax/features_microflow.go | 3 + docs-site/src/appendixes/error-messages.md | 21 +++++ ...retrieve-system-owner-association.fail.mdl | 24 ++++++ ...1358-retrieve-system-owner-association.mdl | 28 +++++++ mdl/executor/validate.go | 3 + mdl/executor/validate_microflow.go | 1 + ...alidate_microflow_retrieve_system_assoc.go | 58 +++++++++++++ ...te_microflow_retrieve_system_assoc_test.go | 82 +++++++++++++++++++ 10 files changed, 222 insertions(+) create mode 100644 .claude/skills/fix-issue/findings/mdl-executor/2026-10-09-retrieve-by-association-system-owner-ce0136.json create mode 100644 mdl-examples/bug-tests/1358-retrieve-system-owner-association.fail.mdl create mode 100644 mdl-examples/bug-tests/1358-retrieve-system-owner-association.mdl create mode 100644 mdl/executor/validate_microflow_retrieve_system_assoc.go create mode 100644 mdl/executor/validate_microflow_retrieve_system_assoc_test.go diff --git a/.claude/skills/fix-issue/findings/mdl-executor/2026-10-09-retrieve-by-association-system-owner-ce0136.json b/.claude/skills/fix-issue/findings/mdl-executor/2026-10-09-retrieve-by-association-system-owner-ce0136.json new file mode 100644 index 000000000..8908b9860 --- /dev/null +++ b/.claude/skills/fix-issue/findings/mdl-executor/2026-10-09-retrieve-by-association-system-owner-ce0136.json @@ -0,0 +1 @@ +{"area":"mdl/executor","date":"2026-10-09","symptom":"`retrieve $u from $task/System.owner;` (or System.changedBy) passed `check` and `exec`; mxbuild then failed `[CE0136] \"Retrieve object must specify the 'Entity' property.\"` — with or without `owner: AutoOwner` on the entity. Issue mendixlabs/mxcli#1358 (11.12.2)","cause":"owner/changedBy are HasOwner/HasChangedBy flags, not modelled associations. mxbuild resolves the NAME in an AssociationRetrieveSource (System.Owner is CE1613 instead, so lookup works) but derives no target entity from it, so the RetrieveAction has none. Nothing mxcli can write in that source builds","file":"`mdl/executor/validate_microflow_retrieve_system_assoc.go` (MDL-RETRIEVE02, exec-enforced in `execEnforcedMicroflowRules`)","insight":"The issue proposed writing `Entity: System.User` — there is no such property: AssociationRetrieveSource has only StartVariableName + AssociationId, and CE0136's 'Entity' is the entity Studio Pro DERIVES from the association. Measure alternatives with mx check before designing a writer fix: `[System.owner/M.Task = $task]` on System.User is CE0161 (system associations do not traverse in reverse in XPath), but `retrieve $u from System.User where [id = $task/System.owner] first;` builds clean for owner and changedBy. So the fix is a project-free refusal naming that rewrite, not a builder change. Not measured for nanoflows (rule is microflow-only)","refs":["mendixlabs/mxcli#1358","mdl-examples/bug-tests/1358-retrieve-system-owner-association.mdl","mdl-examples/bug-tests/1358-retrieve-system-owner-association.fail.mdl"],"ce":["CE0136"],"rules":["MDL-RETRIEVE02"]} diff --git a/CHANGELOG.md b/CHANGELOG.md index c09ef26a8..983adcd86 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,6 +28,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). ### Fixed +- **`retrieve $u from $obj/System.owner` passed check and exec, then failed the build with CE0136** (mendixlabs/mxcli#1358) — `owner` and `changedBy` are entity flags, not modelled associations: mxbuild resolves the name in a retrieve by association but derives no entity from it, `[CE0136] "Retrieve object must specify the 'Entity' property."`, with or without `owner: AutoOwner` (measured on 11.12.2, `System.changedBy` alike). `check` and `exec` now refuse it as **MDL-RETRIEVE02** and name the form that builds: `retrieve $u from System.User where [id = $obj/System.owner] first;`. - **`run stop` and `run status` find a detached run's processes on every machine** — the guard against a reused session id rebuilt each process's start time from the kernel's boot time, which is whole seconds, and allowed only one second of slack. On a machine that booted late in a second, a run's own processes read as older than the run and were skipped, so `run stop` could report "was not running" for a live run. The slack is now two seconds. This was also the intermittent `TestRunStop_*` / `TestSessionMembers_*` failure on CI. - **The catalog names a call's and a delete's target** (mendixlabs/mxcli#1305) — `activities_for()` and `CATALOG.ACTIVITIES` returned every microflow, nanoflow, Java action and JavaScript action call with `action_ref=""`, and every delete with `entity_ref=""`, although `refs_from()` had both targets; a loop-scoped lint rule could not follow a call out of the loop. `action_ref` / `ActionRef` is now the called document, `entity_ref` / `EntityRef` a delete's entity (when the flow types the variable: a parameter, a create or retrieve output, a loop iterator), and the new `queue_ref` / `QueueRef` the task queue a microflow or Java action call runs in, so a rule can skip a call that runs asynchronously. A nanoflow's JavaScript action call now has a `refs_from()` `call` row (`target_type` `"JAVASCRIPT_ACTION"`), so `show callers` sees it. The catalog schema is bumped to 23; a cached catalog rebuilds. - **A DataGrid 2 column's `Visible:` expression is written** — `Visible: $showPrices`, `Visible: if … then … else …`, `visible: not(…)` and `Visible: [cond]` on a column passed `check` and `exec` and were stored as `true`, so the column was always visible; only the old quoted `Visible: ''` was kept. `alter page … set (Visible: ) on grid column(…)` was refused as "column property VisibleIf not found", and an inserted column dropped `Visible: false` too. `describe` now prints the bare expression. A column's visibility is evaluated once for the grid, with no row object, so `$currentObject` there is CE0117 at build; `check` refuses it as **MDL-WIDGET43** (measured on 11.14.0). Use a page variable or parameter. Projects regenerate their widget definitions (generator version 18). diff --git a/cmd/mxcli/syntax/features_microflow.go b/cmd/mxcli/syntax/features_microflow.go index 54bbc1f3c..bc30cd75d 100644 --- a/cmd/mxcli/syntax/features_microflow.go +++ b/cmd/mxcli/syntax/features_microflow.go @@ -167,6 +167,9 @@ func init() { "-- * FIRST is the one form that binds an object, in every language version.\n" + "-- HEAD() or COUNT() over it is CE0097 at build time and a LOOP CE0100;\n" + "-- mxcli reports both as MDL-RETRIEVE01 at check time.\n" + + "-- * $Obj/System.owner and $Obj/System.changedBy are not associations a\n" + + "-- retrieve can follow (CE0136, MDL-RETRIEVE02). Use\n" + + "-- RETRIEVE $U FROM System.User WHERE [id = $Obj/System.owner] FIRST;\n" + "-- * LIMIT 1 without OFFSET depends on the language version: under `mdl 1;`\n" + "-- it is a list of one, as in `import from mapping … limit 1`. Without the\n" + "-- header it keeps its old meaning, the object, and warns MDL-V1-LIMIT1 —\n" + diff --git a/docs-site/src/appendixes/error-messages.md b/docs-site/src/appendixes/error-messages.md index 5e0883e9f..177b69b14 100644 --- a/docs-site/src/appendixes/error-messages.md +++ b/docs-site/src/appendixes/error-messages.md @@ -311,6 +311,27 @@ Copy the parameter into a variable and change that (`declare $Value Integer = $N parameter is not refused — `set $L = $M` on one is a Change list Replace, which builds — and neither is a member change, `set $Param/Attr = …`. +### MDL-RETRIEVE02: Retrieve by association over `System.owner` / `System.changedBy` + +``` +retrieve $u from $task/System.owner: System.owner is not a modelled association, and a +retrieve by association over it has no entity — mxbuild rejects it with CE0136 "Retrieve +object must specify the 'Entity' property." [MDL-RETRIEVE02] +``` + +**Cause:** `owner` and `changedBy` are not associations in any domain model — an entity +carries them as the *Store owner* / *Store changedBy* flags (`AutoOwner`, `AutoChangedBy`). +XPath constraints and page paths resolve them, but a retrieve by association does not: +mxbuild finds the name and derives no entity from it, whether or not the entity stores the +member (measured on 11.12.2, mendixlabs/mxcli#1358). `check` and `exec` both refuse it. + +**Solution:** Retrieve the user from the database by id: + +```mdl +retrieve $u from System.User where [id = $task/System.owner] first; +retrieve $u from System.User where [id = $task/System.changedBy] first; +``` + ### MDL-EMAIL02 / 03: A `send email` setting Studio Pro would not allow ``` diff --git a/mdl-examples/bug-tests/1358-retrieve-system-owner-association.fail.mdl b/mdl-examples/bug-tests/1358-retrieve-system-owner-association.fail.mdl new file mode 100644 index 000000000..aa295ff2a --- /dev/null +++ b/mdl-examples/bug-tests/1358-retrieve-system-owner-association.fail.mdl @@ -0,0 +1,24 @@ +mdl 1; +-- ============================================================================ +-- Upstream #1358 (the refusal): retrieve by association over System.owner +-- ============================================================================ +-- +-- owner / changedBy are not modelled associations: an entity carries them as +-- the HasOwner / HasChangedBy flags. mxbuild resolves the name in a +-- retrieve-by-association but derives no entity from it, and reports +-- [CE0136] "Retrieve object must specify the 'Entity' property." — with or +-- without `owner: AutoOwner` (measured on 11.12.2). `mxcli check` must refuse +-- it as MDL-RETRIEVE02. See the companion +-- 1358-retrieve-system-owner-association.mdl for the form that builds. +-- +-- This file is expected to FAIL `mxcli check`. +-- ============================================================================ + +create module F1358F; +create persistent entity F1358F.Task ("Title": String(200), "owner": AutoOwner); + +create or modify microflow F1358F.T_OwnerPath ($task: F1358F.Task) +returns System.User as $u +begin + retrieve $u from $task/System.owner; +end; diff --git a/mdl-examples/bug-tests/1358-retrieve-system-owner-association.mdl b/mdl-examples/bug-tests/1358-retrieve-system-owner-association.mdl new file mode 100644 index 000000000..9a2e6c0e6 --- /dev/null +++ b/mdl-examples/bug-tests/1358-retrieve-system-owner-association.mdl @@ -0,0 +1,28 @@ +mdl 1; +-- ============================================================================ +-- Upstream #1358 (the form that builds): the owner / last editor of an object +-- ============================================================================ +-- +-- `retrieve $u from $task/System.owner;` is CE0136 (MDL-RETRIEVE02, see the +-- .fail.mdl companion). Retrieving System.User from the database by id builds +-- clean — measured with mx check on 11.12.2, for owner and changedBy alike. +-- ============================================================================ + +create module B1358; +create persistent entity B1358.Task ( + "Title": String(200), + "owner": AutoOwner, + "changedBy": AutoChangedBy +); + +create or modify microflow B1358.T_Owner ($task: B1358.Task) +returns System.User as $u +begin + retrieve $u from System.User where [id = $task/System.owner] first; +end; + +create or modify microflow B1358.T_LastEditor ($task: B1358.Task) +returns System.User as $u +begin + retrieve $u from System.User where [id = $task/System.changedBy] first; +end; diff --git a/mdl/executor/validate.go b/mdl/executor/validate.go index 753a3a03f..77249e9f2 100644 --- a/mdl/executor/validate.go +++ b/mdl/executor/validate.go @@ -1861,6 +1861,9 @@ var execEnforcedMicroflowRules = map[string]bool{ // measured on 11.14.0 (mendixlabs/mxcli#1323). The builder refuses an // object itself; a primitive parameter it would write. "MDL-SET01": true, + // MDL-RETRIEVE02: a retrieve by association over System.owner / + // System.changedBy is CE0136, measured on 11.12.2 (mendixlabs/mxcli#1358). + "MDL-RETRIEVE02": true, } // validateMicroflowRules runs the MDL0xx microflow rule set (ValidateMicroflow) diff --git a/mdl/executor/validate_microflow.go b/mdl/executor/validate_microflow.go index e9158cc61..dbe2cda16 100644 --- a/mdl/executor/validate_microflow.go +++ b/mdl/executor/validate_microflow.go @@ -120,6 +120,7 @@ func (v *microflowValidator) addViolation(ruleID string, severity linter.Severit func (v *microflowValidator) validate(body []ast.MicroflowStatement) { v.checkListOperationIterator(body) v.checkRetrieveLimitOneAsList(body) + v.checkRetrieveOverSystemAssociation(body) v.checkSetOnObjectVariable(v.params, body) v.checkListOperationSource(body) v.checkMergeJoinLabels(body) diff --git a/mdl/executor/validate_microflow_retrieve_system_assoc.go b/mdl/executor/validate_microflow_retrieve_system_assoc.go new file mode 100644 index 000000000..f80c1d5ca --- /dev/null +++ b/mdl/executor/validate_microflow_retrieve_system_assoc.go @@ -0,0 +1,58 @@ +// SPDX-License-Identifier: Apache-2.0 + +package executor + +import ( + "fmt" + "strings" + + "github.com/mendixlabs/mxcli/mdl/ast" + "github.com/mendixlabs/mxcli/mdl/linter" +) + +// retrieveSystemAssocRule is the rule ID for "an association retrieve over +// System.owner / System.changedBy". +const retrieveSystemAssocRule = "MDL-RETRIEVE02" + +// checkRetrieveOverSystemAssociation flags `retrieve $u from $obj/System.owner` +// (and System.changedBy) — mendixlabs/mxcli#1358. +// +// owner and changedBy are not associations any domain model lists: an entity +// carries them as the HasOwner / HasChangedBy flags on its generalization root. +// XPath and page paths resolve them (#641, #1338), but a retrieve-by-association +// does not. Measured on mxbuild 11.12.2, entity declaring `owner: AutoOwner, +// changedBy: AutoChangedBy`: +// +// retrieve $u from $task/System.owner; CE0136 "Retrieve object must specify the 'Entity' property." +// retrieve $u from $task/System.changedBy; CE0136 +// retrieve $u from $task/System.Owner; CE1613 (the name itself does not resolve) +// retrieve $u from System.User where [id = $task/System.owner] first; clean +// +// So the name resolves and the source still yields no entity — there is nothing +// to write that mxbuild accepts. The rule needs no project: no module but System +// can declare these names, and the entity flags do not change the outcome. +func (v *microflowValidator) checkRetrieveOverSystemAssociation(body []ast.MicroflowStatement) { + forEachMicroflowStatement(body, func(s ast.MicroflowStatement) { + r, ok := s.(*ast.RetrieveStmt) + if !ok || r.StartVariable == "" || !strings.EqualFold(r.Source.Module, "System") { + return + } + var member string + switch strings.ToLower(r.Source.Name) { + case "owner": + member = "System.owner" + case "changedby": + member = "System.changedBy" + default: + return + } + v.addViolation(retrieveSystemAssocRule, linter.SeverityError, + fmt.Sprintf("retrieve $%s from $%s/%s.%s: %s is not a modelled association, and a "+ + "retrieve by association over it has no entity — mxbuild rejects it with CE0136 "+ + "\"Retrieve object must specify the 'Entity' property.\"", + r.Variable, r.StartVariable, r.Source.Module, r.Source.Name, member), + fmt.Sprintf("Retrieve the user from the database instead: "+ + "`retrieve $%s from System.User where [id = $%s/%s] first;`", + r.Variable, r.StartVariable, member)) + }) +} diff --git a/mdl/executor/validate_microflow_retrieve_system_assoc_test.go b/mdl/executor/validate_microflow_retrieve_system_assoc_test.go new file mode 100644 index 000000000..97b3a2460 --- /dev/null +++ b/mdl/executor/validate_microflow_retrieve_system_assoc_test.go @@ -0,0 +1,82 @@ +// SPDX-License-Identifier: Apache-2.0 + +package executor + +import ( + "strings" + "testing" + + "github.com/mendixlabs/mxcli/mdl/ast" +) + +func systemAssocRetrieveViolations(t *testing.T, stmt *ast.CreateMicroflowStmt) []string { + t.Helper() + var out []string + for _, v := range ValidateMicroflow(stmt) { + if v.RuleID == retrieveSystemAssocRule { + out = append(out, v.Message+" | "+v.Suggestion) + } + } + return out +} + +func retrieveByAssoc(variable, start, module, name string) *ast.RetrieveStmt { + return &ast.RetrieveStmt{ + Variable: variable, + StartVariable: start, + Source: ast.QualifiedName{Module: module, Name: name}, + } +} + +// TestRetrieveOverSystemOwnerAssociation is mendixlabs/mxcli#1358: +// `retrieve $u from $task/System.owner;` passed check and exec, and mxbuild then +// reported `[CE0136] "Retrieve object must specify the 'Entity' property."`. +// Measured on 11.12.2: mxbuild resolves the name (System.Owner is CE1613 +// instead) but derives no entity from it, with or without `owner: AutoOwner`. +func TestRetrieveOverSystemOwnerAssociation(t *testing.T) { + t.Run("association retrieve over System.owner is refused", func(t *testing.T) { + got := systemAssocRetrieveViolations(t, mfWith(retrieveByAssoc("u", "task", "System", "owner"))) + if len(got) != 1 { + t.Fatalf("want one %s violation, got %v", retrieveSystemAssocRule, got) + } + for _, want := range []string{"CE0136", "System.owner", "[id = $task/System.owner]"} { + if !strings.Contains(got[0], want) { + t.Errorf("violation does not mention %q: %s", want, got[0]) + } + } + }) + + t.Run("System.changedBy, nested in a branch, is refused", func(t *testing.T) { + got := systemAssocRetrieveViolations(t, mfWith(&ast.IfStmt{ + ThenBody: []ast.MicroflowStatement{retrieveByAssoc("u", "task", "System", "changedBy")}, + })) + if len(got) != 1 || !strings.Contains(got[0], "[id = $task/System.changedBy]") { + t.Fatalf("want one violation naming the System.changedBy rewrite, got %v", got) + } + }) + + // Controls: the forms that build. + t.Run("a modelled association to System.User is not flagged", func(t *testing.T) { + if got := systemAssocRetrieveViolations(t, mfWith(retrieveByAssoc("u", "task", "MyFirstModule", "Task_User"))); len(got) != 0 { + t.Fatalf("unexpected violation: %v", got) + } + }) + t.Run("the database rewrite is not flagged", func(t *testing.T) { + db := &ast.RetrieveStmt{Variable: "u", Source: ast.QualifiedName{Module: "System", Name: "User"}, First: true} + if got := systemAssocRetrieveViolations(t, mfWith(db)); len(got) != 0 { + t.Fatalf("unexpected violation: %v", got) + } + }) + t.Run("System.UserRoles is a modelled System association and is not flagged", func(t *testing.T) { + if got := systemAssocRetrieveViolations(t, mfWith(retrieveByAssoc("roles", "user", "System", "UserRoles"))); len(got) != 0 { + t.Fatalf("unexpected violation: %v", got) + } + }) + + t.Run("exec refuses it too", func(t *testing.T) { + err := validateMicroflowRules(mfWith(retrieveByAssoc("u", "task", "System", "owner"))) + if err == nil || !strings.Contains(err.Error(), retrieveSystemAssocRule) { + t.Fatalf("exec must refuse the write, got %v", err) + } + }) +} From e0dd6f8b369a999c17b0aeef4d2f7a80b6313bc6 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 9 Oct 2026 12:32:51 +0000 Subject: [PATCH 4/5] fix: don't count an implicit catalog rebuild against the statement timeout (closes mendixlabs/mxcli#1329) After the project file changed, show references / search / show callers / show impact rebuilt the stale catalog at its cached source mode inside the statement's wall-clock guard. On a large app the default 5m fired mid-build ("statement timed out after 5m0s"); the unfinished build was never saved, so every later call repeated it. REFRESH CATALOG was exempt from the guard (#651) by statement type; the same build reached implicitly was not. ensureCatalog now marks its rebuild as untimed work, and Execute's timer subtracts that time from the statement's clock (chained through nested EXECUTE SCRIPT). An explicit MXCLI_EXEC_TIMEOUT still caps it, as it caps REFRESH CATALOG, and a timeout that cuts a rebuild off now says so and points at REFRESH CATALOG FULL SOURCE. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01WzwbPBB3HkMhu7c9bzTGqy --- ...-rebuild-cut-off-by-statement-timeout.json | 1 + .../1329-implicit-catalog-rebuild-timeout.mdl | 38 ++++++ mdl/executor/catalog_rebuild_timeout_test.go | 114 +++++++++++++++++ mdl/executor/cmd_catalog.go | 22 ++++ mdl/executor/executor.go | 80 ++++++++++-- mdl/executor/untimed_work.go | 118 ++++++++++++++++++ 6 files changed, 364 insertions(+), 9 deletions(-) create mode 100644 .claude/skills/fix-issue/findings/mdl-executor/2026-10-09-implicit-catalog-rebuild-cut-off-by-statement-timeout.json create mode 100644 mdl-examples/bug-tests/1329-implicit-catalog-rebuild-timeout.mdl create mode 100644 mdl/executor/catalog_rebuild_timeout_test.go create mode 100644 mdl/executor/untimed_work.go diff --git a/.claude/skills/fix-issue/findings/mdl-executor/2026-10-09-implicit-catalog-rebuild-cut-off-by-statement-timeout.json b/.claude/skills/fix-issue/findings/mdl-executor/2026-10-09-implicit-catalog-rebuild-cut-off-by-statement-timeout.json new file mode 100644 index 000000000..58a11ad12 --- /dev/null +++ b/.claude/skills/fix-issue/findings/mdl-executor/2026-10-09-implicit-catalog-rebuild-cut-off-by-statement-timeout.json @@ -0,0 +1 @@ +{"area": "mdl/executor", "date": "2026-10-09", "symptom": "After the .mpr changed, `show references` / `search` / `show callers` / `show impact` on a source-mode catalog failed with \"statement timed out after 5m0s\" on a 3,032-document app (build stopped at document 1,110), and every retry did the same: the cache stayed stale. Explicit `refresh catalog` worked", "cause": "ensureCatalog rebuilds a stale cache at its cached source mode (the #1081 never-narrow fix) inside the calling statement, and Execute's wall-clock guard counted that build. REFRESH CATALOG is exempt from the default guard (#651) by statement type; an implicit rebuild of the same catalog inside a ShowStmt was not. The build is only saved on completion, so a guard firing mid-build leaves the cache exactly as stale as before, guaranteeing the next call repeats it", "file": "`mdl/executor/untimed_work.go` (per-statement tracker of untimed sections, chained for EXECUTE SCRIPT), `mdl/executor/executor.go` (Execute: timer re-armed by the excluded time instead of context.WithTimeout; executeTimeoutError names an interrupted rebuild), `mdl/executor/cmd_catalog.go` (ensureCatalog marks its build untimed); tests `catalog_rebuild_timeout_test.go`; repro `mdl-examples/bug-tests/1329-implicit-catalog-rebuild-timeout.mdl`", "insight": "An exemption keyed on statement TYPE misses the same work reached implicitly: #651 exempted REFRESH CATALOG, and the identical build behind SHOW REFERENCES stayed capped. Exempt the work, not the statement — mark the bounded section where it runs and have the guard subtract it. Wrong turn to skip: building full instead of source for full consumers — the never-narrow guard then refuses to save it and the rebuild repeats on every call (the very thrash ensureCatalog's source upgrade was added to stop). A guard that discards half-finished work must say so, or each retry reads as a fresh failure. Reproduce with a test hook that lengthens the build and a lowered default limit; the fixture builds in under a second", "refs": ["mendixlabs/mxcli#1329", "mendixlabs/mxcli#651", "mendixlabs/mxcli#1081"], "ce": [], "rules": []} diff --git a/mdl-examples/bug-tests/1329-implicit-catalog-rebuild-timeout.mdl b/mdl-examples/bug-tests/1329-implicit-catalog-rebuild-timeout.mdl new file mode 100644 index 000000000..985bb35ef --- /dev/null +++ b/mdl-examples/bug-tests/1329-implicit-catalog-rebuild-timeout.mdl @@ -0,0 +1,38 @@ +mdl 1; +-- ============================================================================ +-- Bug #1329: an implicit catalog rebuild is cut off by the statement timeout +-- ============================================================================ +-- +-- Symptom (before fix): +-- After the project file changed, `show references` / `search` / +-- `show callers` / `show impact` rebuilt the stale catalog at its cached +-- source mode inside the statement's 5-minute wall-clock guard. On a +-- 3,032-document app the build reached document 1,110 and stopped: +-- "statement timed out after 5m0s" +-- The unfinished build was never saved, so every later call started the +-- same rebuild over. `refresh catalog` was exempt from the guard (#651); an +-- implicit rebuild of the same catalog was not. +-- +-- After fix: +-- The guard does not count time spent in an implicit rebuild, so the +-- statement finishes and the cache is current for the next one. Under an +-- explicit MXCLI_EXEC_TIMEOUT (which caps REFRESH CATALOG too) the rebuild +-- is still capped, and the error says a rebuild was cut off and to run +-- REFRESH CATALOG FULL SOURCE. +-- +-- The timing needs a large app; on a small one this script shows the shape: +-- the CREATE below changes the project file, so SHOW REFERENCES rebuilds. +-- +-- Usage: +-- mxcli -p app.mpr -c "refresh catalog full source" +-- mxcli exec mdl-examples/bug-tests/1329-implicit-catalog-rebuild-timeout.mdl -p app.mpr +-- mxcli -p app.mpr -c "show catalog status" -- Status: current, source mode +-- ============================================================================ + +create module BugTest1329; + +create persistent entity BugTest1329.Ticket ( + Title: String(100) +); + +list references to BugTest1329.Ticket; diff --git a/mdl/executor/catalog_rebuild_timeout_test.go b/mdl/executor/catalog_rebuild_timeout_test.go new file mode 100644 index 000000000..70060221d --- /dev/null +++ b/mdl/executor/catalog_rebuild_timeout_test.go @@ -0,0 +1,114 @@ +// SPDX-License-Identifier: Apache-2.0 + +package executor + +import ( + "strings" + "testing" + "time" + + "github.com/mendixlabs/mxcli/mdl/visitor" +) + +// mendixlabs/mxcli#1329: SHOW REFERENCES, SEARCH, SHOW CALLERS and SHOW IMPACT +// rebuild a stale catalog on the way to their answer, at the cached source +// mode. On a 3,000-document app that build outlasts the default 5m statement +// limit; the guard fired mid-build, the unfinished build was never saved, and +// every later call started the same rebuild over: +// +// statement timed out after 5m0s +// +// The fixture builds in well under a second, so the test lowers the default +// limit and makes the build outlast it — the same arithmetic as the report. + +// slowImplicitRebuilds makes every implicit catalog rebuild take at least d +// longer, and lowers the default statement limit to limit. The returned channel +// receives once per finished rebuild, so a test can wait for one the guard +// abandoned before its fixture is torn down. +func slowImplicitRebuilds(t *testing.T, d, limit time.Duration) <-chan struct{} { + t.Helper() + done := make(chan struct{}, 8) + prevHook, prevLimit := implicitCatalogBuildHook, executeTimeoutDefault + implicitCatalogBuildHook = func() func() { + time.Sleep(d) + return func() { done <- struct{}{} } + } + executeTimeoutDefault = limit + t.Cleanup(func() { + implicitCatalogBuildHook, executeTimeoutDefault = prevHook, prevLimit + }) + return done +} + +func execStatement(t *testing.T, exec *Executor, mdl string) error { + t.Helper() + prog, errs := visitor.Build(mdl) + if len(errs) > 0 || len(prog.Statements) != 1 { + t.Fatalf("parsing %q: %v", mdl, errs) + } + return exec.Execute(prog.Statements[0]) +} + +// TestImplicitRebuildIsNotCountedAgainstTheDefaultLimit is the report: a +// statement whose implicit rebuild outlasts the default limit must finish, and +// leave a current cache so the next call does not rebuild again. +func TestImplicitRebuildIsNotCountedAgainstTheDefaultLimit(t *testing.T) { + exec, cachePath := sourceCachedFixture(t) + t.Setenv("MXCLI_EXEC_TIMEOUT", "") + slowImplicitRebuilds(t, 1500*time.Millisecond, 500*time.Millisecond) + + touchProject(t, exec) + + if err := execStatement(t, exec, "SHOW REFERENCES TO MyFirstModule.Ticket"); err != nil { + t.Fatalf("SHOW REFERENCES after the project changed: %v", err) + } + + ctx := exec.newExecContext(t.Context()) + if valid, reason := isCacheValid(ctx, cachePath, "source"); !valid { + t.Errorf("cache still invalid after the implicit rebuild: %s — the next call rebuilds again", reason) + } +} + +// TestDefaultLimitStillCountsTheRestOfTheStatement is the control on the +// exclusion: only the rebuild is skipped. A statement that is itself slow — +// the runaway loop the guard is for — still times out. +func TestDefaultLimitStillCountsTheRestOfTheStatement(t *testing.T) { + exec := New(&strings.Builder{}) + w := exec.pushUntimed(true) + defer exec.popUntimed(w) + begun := time.Now() + time.Sleep(600 * time.Millisecond) + if counted := time.Since(begun) - w.excluded(time.Now()); counted < 500*time.Millisecond { + t.Errorf("counted %v of a statement with no rebuild, want the whole of it", counted) + } +} + +// TestExplicitLimitCapsTheRebuildAndSaysSo: an explicit MXCLI_EXEC_TIMEOUT caps +// everything, REFRESH CATALOG included, so it caps an implicit rebuild too. The +// error then has to say a rebuild was cut off, because the cache was not +// updated and the next call will rebuild again — without that, each retry +// reads as a new failure. +func TestExplicitLimitCapsTheRebuildAndSaysSo(t *testing.T) { + exec, _ := sourceCachedFixture(t) + t.Setenv("MXCLI_EXEC_TIMEOUT", "500ms") + rebuilt := slowImplicitRebuilds(t, 1500*time.Millisecond, 500*time.Millisecond) + + touchProject(t, exec) + + err := execStatement(t, exec, "SHOW REFERENCES TO MyFirstModule.Ticket") + if err == nil { + t.Fatal("SHOW REFERENCES finished under an explicit 500ms cap with a 1.5s rebuild") + } + msg := err.Error() + for _, want := range []string{"timed out", "rebuilding the catalog", "source mode", "REFRESH CATALOG FULL SOURCE"} { + if !strings.Contains(msg, want) { + t.Errorf("timeout error lacks %q:\n%s", want, msg) + } + } + // Let the abandoned build finish before the fixture's cleanup closes it. + select { + case <-rebuilt: + case <-time.After(time.Minute): + t.Fatal("the abandoned rebuild did not finish") + } +} diff --git a/mdl/executor/cmd_catalog.go b/mdl/executor/cmd_catalog.go index 914129748..40f7b8c12 100644 --- a/mdl/executor/cmd_catalog.go +++ b/mdl/executor/cmd_catalog.go @@ -343,10 +343,32 @@ func ensureCatalog(ctx *ExecContext, full bool) error { isSource = true } + // An implicit rebuild is a bounded build that happens to run inside some + // other statement, so the statement's wall-clock guard does not count it — + // as it does not count REFRESH CATALOG (#651). Counted, the default 5m + // fired mid-build on a 3,000-document app, the unfinished build was never + // saved, and every later SHOW REFERENCES / SEARCH started it over + // (mendixlabs/mxcli#1329). + mode := "fast" + if isSource { + mode = "source" + } else if full { + mode = "full" + } + defer beginUntimed(ctx.Context, mode+" mode")() + if implicitCatalogBuildHook != nil { + defer implicitCatalogBuildHook()() + } + // Build fresh catalog return buildCatalog(ctx, full, isSource, false, 0) } +// implicitCatalogBuildHook, when set, runs at the start of every implicit +// catalog rebuild, and the function it returns at the end. Tests use it to +// make a build outlast a short timeout and to wait for an abandoned one. +var implicitCatalogBuildHook func() (done func()) + // getCachePath returns the path to the catalog cache file for the current project. func getCachePath(ctx *ExecContext) string { if ctx.MprPath == "" { diff --git a/mdl/executor/executor.go b/mdl/executor/executor.go index c16e62c04..662076016 100644 --- a/mdl/executor/executor.go +++ b/mdl/executor/executor.go @@ -190,6 +190,11 @@ const ( defaultExecuteTimeout = 5 * time.Minute ) +// executeTimeoutDefault is the limit effectiveExecuteTimeout applies when no +// MXCLI_EXEC_TIMEOUT is set. A variable so a test can bring it below the +// duration of a catalog build. +var executeTimeoutDefault = defaultExecuteTimeout + // configuredExecuteTimeout returns the per-statement wall-clock timeout. The // value is read from the MXCLI_EXEC_TIMEOUT environment variable on every call // so long-running audits can opt into a higher ceiling without recompiling. @@ -225,7 +230,7 @@ func effectiveExecuteTimeout(stmt ast.Statement) time.Duration { if isExemptFromExecuteTimeout(stmt) { return 0 } - return defaultExecuteTimeout + return executeTimeoutDefault } // isExemptFromExecuteTimeout reports whether a statement is a known long-running @@ -267,6 +272,8 @@ type Executor struct { themeRegistry *ThemeRegistry // cached theme design property definitions (lazy init) registry *Registry // statement dispatch registry catalogMu sync.RWMutex // protects catalog field from background goroutine writes + untimedMu sync.Mutex // protects untimed + untimed *untimedWork // the running statement's untimed-work tracker (see untimed_work.go) catalogGen uint64 // monotonic generation counter for catalog swaps } @@ -359,31 +366,54 @@ func (e *Executor) Execute(stmt ast.Statement) error { e.guard.reset() } - // Enforce wall-clock timeout via context.WithTimeout. + // Enforce the wall-clock timeout with a timer that skips untimed work. // The goroutine pattern is retained because handlers are not yet // context-aware; threading context through handlers is a follow-up. executeTimeout := effectiveExecuteTimeout(stmt) + // Time spent rebuilding the catalog implicitly is not counted against the + // default limit, for the reason REFRESH CATALOG is exempt from it + // (mendixlabs/mxcli#1329). An explicit MXCLI_EXEC_TIMEOUT caps it, as it + // caps REFRESH CATALOG. + untimed := e.pushUntimed(os.Getenv("MXCLI_EXEC_TIMEOUT") == "") + defer e.popUntimed(untimed) + baseCtx := withUntimedWork(context.Background(), untimed) + var err error if executeTimeout <= 0 { // No wall-clock guard for exempt statements (e.g. REFRESH CATALOG on a // large project). The output-line guard still bounds runaway output. - err = e.executeInner(context.Background(), stmt) + err = e.executeInner(baseCtx, stmt) } else { - ctx, cancel := context.WithTimeout(context.Background(), executeTimeout) + ctx, cancel := context.WithCancel(baseCtx) defer cancel() type result struct{ err error } ch := make(chan result, 1) + begun := time.Now() go func() { ch <- result{e.executeInner(ctx, stmt)} }() - select { - case r := <-ch: - err = r.err - case <-ctx.Done(): - err = mdlerrors.NewValidationf("statement timed out after %v (raise the limit with MXCLI_EXEC_TIMEOUT, e.g. MXCLI_EXEC_TIMEOUT=30m)", executeTimeout) + timer := time.NewTimer(executeTimeout) + defer timer.Stop() + wait: + for { + select { + case r := <-ch: + err = r.err + break wait + case <-timer.C: + now := time.Now() + counted := now.Sub(begun) - untimed.excluded(now) + if remaining := executeTimeout - counted; remaining > 0 { + timer.Reset(remaining) + continue + } + cancel() + err = executeTimeoutError(executeTimeout, untimed.inProgress()) + break wait + } } } @@ -393,6 +423,38 @@ func (e *Executor) Execute(stmt ast.Statement) error { return err } +// executeTimeoutError is the error for a statement the wall-clock guard +// stopped. When it stopped an implicit catalog rebuild it says so: the build +// is only saved once it finishes, so the next statement that needs the catalog +// starts the same rebuild over — without the hint, every retry looks like a new +// failure (mendixlabs/mxcli#1329). +func executeTimeoutError(limit time.Duration, building string) error { + if building == "" { + return mdlerrors.NewValidationf("statement timed out after %v (raise the limit with MXCLI_EXEC_TIMEOUT, e.g. MXCLI_EXEC_TIMEOUT=30m)", limit) + } + return mdlerrors.NewValidationf("statement timed out after %v while rebuilding the catalog (%s); the catalog cache was not updated, so the next statement that needs it rebuilds again. Rebuild it once with REFRESH CATALOG FULL SOURCE (with MXCLI_EXEC_TIMEOUT unset, so the default limit does not apply), or raise the limit with MXCLI_EXEC_TIMEOUT, e.g. MXCLI_EXEC_TIMEOUT=30m", limit, building) +} + +// pushUntimed installs a fresh untimed-work tracker for the statement about to +// run, chained to the enclosing statement's when Execute is re-entered by +// EXECUTE SCRIPT. +func (e *Executor) pushUntimed(exclude bool) *untimedWork { + e.untimedMu.Lock() + defer e.untimedMu.Unlock() + w := newUntimedWork(e.untimed, exclude) + e.untimed = w + return w +} + +// popUntimed restores the enclosing statement's tracker. +func (e *Executor) popUntimed(w *untimedWork) { + e.untimedMu.Lock() + defer e.untimedMu.Unlock() + if e.untimed == w { + e.untimed = w.parent + } +} + // ExecuteProgram runs all statements in a program. func (e *Executor) ExecuteProgram(prog *ast.Program) error { _, err := e.ExecuteProgramReportingStop(prog) diff --git a/mdl/executor/untimed_work.go b/mdl/executor/untimed_work.go new file mode 100644 index 000000000..fe249148e --- /dev/null +++ b/mdl/executor/untimed_work.go @@ -0,0 +1,118 @@ +// SPDX-License-Identifier: Apache-2.0 + +package executor + +import ( + "context" + "sync" + "time" +) + +// untimedWork tracks the part of a statement's run that the wall-clock guard +// should not count: an implicit catalog rebuild. The guard exists to abort +// runaway loops, and a catalog build is bounded and deterministic — the reason +// REFRESH CATALOG is exempt (#651). A statement that rebuilds the catalog on +// the way to its answer (SHOW REFERENCES, SEARCH, SHOW CALLERS, SHOW IMPACT +// after the project file changed) was not, so on a large app the default 5m +// fired mid-build. The build is only saved once it finishes, so the cache +// stayed stale and every later call paid for the same doomed rebuild +// (mendixlabs/mxcli#1329). +// +// One is created per Execute. A nested statement (EXECUTE SCRIPT) gets its own, +// chained to the enclosing one, so a build inside it is excluded from the outer +// statement's clock as well. +type untimedWork struct { + parent *untimedWork + // exclude says whether this statement's clock skips untimed work. False + // under an explicit MXCLI_EXEC_TIMEOUT: an explicit cap caps everything, + // REFRESH CATALOG included, so it caps an implicit rebuild too. + exclude bool + + mu sync.Mutex + active int // builds in progress + since time.Time // when the outermost in-progress build began + total time.Duration // finished untimed time + what string // description of the build in progress, for the timeout message +} + +type untimedWorkKey struct{} + +func newUntimedWork(parent *untimedWork, exclude bool) *untimedWork { + return &untimedWork{parent: parent, exclude: exclude} +} + +// withUntimedWork returns ctx carrying w, so a handler deep in the call chain +// can mark its untimed section without the tracker being threaded through. +func withUntimedWork(ctx context.Context, w *untimedWork) context.Context { + return context.WithValue(ctx, untimedWorkKey{}, w) +} + +// beginUntimed marks the start of an implicit catalog build on the statement +// running under ctx, and returns the function that marks its end. A context +// without a tracker (a direct call from a test or a tool) is a no-op. +func beginUntimed(ctx context.Context, what string) (end func()) { + if ctx == nil { + return func() {} + } + w, _ := ctx.Value(untimedWorkKey{}).(*untimedWork) + if w == nil { + return func() {} + } + now := time.Now() + for l := w; l != nil; l = l.parent { + l.begin(now, what) + } + return func() { + now := time.Now() + for l := w; l != nil; l = l.parent { + l.end(now) + } + } +} + +func (w *untimedWork) begin(now time.Time, what string) { + w.mu.Lock() + defer w.mu.Unlock() + if w.active == 0 { + w.since = now + w.what = what + } + w.active++ +} + +func (w *untimedWork) end(now time.Time) { + w.mu.Lock() + defer w.mu.Unlock() + if w.active == 0 { + return + } + w.active-- + if w.active == 0 { + w.total += now.Sub(w.since) + w.what = "" + } +} + +// excluded is how much of the time up to now the clock must not count. +func (w *untimedWork) excluded(now time.Time) time.Duration { + w.mu.Lock() + defer w.mu.Unlock() + if !w.exclude { + return 0 + } + d := w.total + if w.active > 0 { + d += now.Sub(w.since) + } + return d +} + +// inProgress describes the untimed build running right now, or "" if none. +func (w *untimedWork) inProgress() string { + w.mu.Lock() + defer w.mu.Unlock() + if w.active == 0 { + return "" + } + return w.what +} From 707af732974fe5fcd003c5960cc022a20f4f002b Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 9 Oct 2026 12:36:16 +0000 Subject: [PATCH 5/5] fix: store, describe and alter an action button's Tooltip (mendixlabs/mxcli#1307) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `Tooltip:` on an actionbutton/linkbutton passed check and exec but was never stored: buildButtonV3 did not read it, so the codec always wrote an empty Texts$Text. describe had no reader for it, and `alter page … set (Tooltip: …)` fell through to the pluggable setter and was refused with "property \"Tooltip\" is not a property of this built-in widget". - builder: write the tooltip as a Texts$Text in the authoring language - describe: read the button's Tooltip and emit `Tooltip: '…'` - alter: a `tooltip` case that sets the stored Texts$Text (creating one when null) and refuses widgets without a Tooltip slot Verified on a fresh 11.12.5 app: exec -> describe shows the tooltip, alter sets it on a button that had none, mx check reports 0 errors. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PCo8XKGihSdss7DQsL6Fdo --- ...actionbutton-tooltip-silently-dropped.json | 1 + .../bug-tests/1307-actionbutton-tooltip.mdl | 21 ++++ mdl/backend/pagemutator/mutator.go | 27 ++++++ mdl/backend/pagemutator/tooltip_1307_test.go | 52 ++++++++++ mdl/executor/button_tooltip_1307_test.go | 96 +++++++++++++++++++ mdl/executor/cmd_pages_builder_v3_widgets.go | 13 +++ mdl/executor/cmd_pages_describe.go | 1 + mdl/executor/cmd_pages_describe_output.go | 13 ++- mdl/executor/cmd_pages_describe_parse.go | 1 + 9 files changed, 223 insertions(+), 2 deletions(-) create mode 100644 .claude/skills/fix-issue/findings/mdl-executor/2026-10-09-actionbutton-tooltip-silently-dropped.json create mode 100644 mdl-examples/bug-tests/1307-actionbutton-tooltip.mdl create mode 100644 mdl/backend/pagemutator/tooltip_1307_test.go create mode 100644 mdl/executor/button_tooltip_1307_test.go diff --git a/.claude/skills/fix-issue/findings/mdl-executor/2026-10-09-actionbutton-tooltip-silently-dropped.json b/.claude/skills/fix-issue/findings/mdl-executor/2026-10-09-actionbutton-tooltip-silently-dropped.json new file mode 100644 index 000000000..229da1798 --- /dev/null +++ b/.claude/skills/fix-issue/findings/mdl-executor/2026-10-09-actionbutton-tooltip-silently-dropped.json @@ -0,0 +1 @@ +{"area":"mdl/executor","date":"2026-10-09","symptom":"`Tooltip:` on an actionbutton/linkbutton passes check and exec but is never stored; describe omits it; `alter page … set (Tooltip: …) on btn` refused with `property \"Tooltip\" is not a property of this built-in widget`","cause":"Every hop but one carried it: grammar parsed TOOLTIP, staticWidgetKnownProps listed it (so MDL-WIDGET07 stayed quiet), pages.ActionButton had the field and widget_write.go wrote it — but buildButtonV3 never read w.Properties[\"Tooltip\"], so the codec always wrote an empty Texts$Text. describe had no reader for it, and setRawWidgetPropertyMut had no `tooltip` case so it fell to the pluggable setter's refusal","file":"mdl/executor/cmd_pages_builder_v3_widgets.go, mdl/executor/cmd_pages_describe_parse.go, mdl/backend/pagemutator/mutator.go","insight":"Being on the known-props allow-list proves only that SOME builder consumes the key, not that THIS widget's builder does — the list is a cross-widget union by design. For a silent drop, grep the struct field's writers (`\\.Tooltip =`): the codec wrote it and nothing assigned it. Tooltip is a Texts$Text (not a ClientTemplate like Caption), so the ALTER path reuses setTranslatableText on the stored empty Texts$Text; mx check on 11.12.5 accepts a translated tooltip with 0 errors","refs":["mendixlabs/mxcli#1307","mendixlabs/mxcli#1263"]} diff --git a/mdl-examples/bug-tests/1307-actionbutton-tooltip.mdl b/mdl-examples/bug-tests/1307-actionbutton-tooltip.mdl new file mode 100644 index 000000000..7a9621475 --- /dev/null +++ b/mdl-examples/bug-tests/1307-actionbutton-tooltip.mdl @@ -0,0 +1,21 @@ +-- mendixlabs/mxcli#1307: a `Tooltip:` on an action button passed check and exec +-- and was silently dropped; `alter page … set (Tooltip: …)` was refused with +-- `property "Tooltip" is not a property of this built-in widget`. +-- After the fix the tooltip is stored in the default language, `describe page` +-- prints it, and ALTER sets it on a button that had none. +mdl 1; + +create module BugTest1307; + +create page BugTest1307.TipPage (Title: 'Tips', Layout: Atlas_Core.Atlas_Default) { + actionbutton btnEdit (Caption: 'Edit', Tooltip: 'Edit this record', Action: close page) + linkbutton btnOpen (Caption: 'Open', Tooltip: 'Open this record', Action: close page) + actionbutton btnDelete (Caption: 'Delete', Action: close page) +}; + +alter page BugTest1307.TipPage { + set (Tooltip: 'Delete this record') on btnDelete +}; + +-- Expect: all three buttons show a Tooltip. +describe page BugTest1307.TipPage; diff --git a/mdl/backend/pagemutator/mutator.go b/mdl/backend/pagemutator/mutator.go index 6e2540cb6..e8f194f39 100644 --- a/mdl/backend/pagemutator/mutator.go +++ b/mdl/backend/pagemutator/mutator.go @@ -2927,6 +2927,8 @@ func setRawWidgetPropertyMut(widget bson.D, propName string, value any) error { return setWidgetContentMut(widget, value) case "label": return setWidgetLabelMut(widget, value) + case "tooltip": + return setWidgetTooltipMut(widget, value) case "buttonstyle": if s, ok := value.(string); ok { bsonnav.DSet(widget, "ButtonStyle", s) @@ -3233,6 +3235,31 @@ func setWidgetCaptionMut(widget bson.D, value any) error { return mdlerrors.NewValidation("widget has no Caption property") } +// setWidgetTooltipMut sets a button's tooltip, a Texts$Text under `Tooltip` +// (empty Items when unset). It used to fall through to the pluggable setter, +// which refused it as "not a property of this built-in widget" +// (mendixlabs/mxcli#1307). A null Tooltip gets a fresh Texts$Text; a widget +// with no Tooltip key at all has no tooltip to set. +func setWidgetTooltipMut(widget bson.D, value any) error { + if tip := bsonnav.DGetDoc(widget, "Tooltip"); tip != nil { + return setTranslatableText(tip, "Tooltip", value) + } + if !hasKey(widget, "Tooltip") { + return fmt.Errorf("a %s has no Tooltip property", + widgetTypeLabel(bsonnav.DGetString(widget, "$Type"))) + } + tip := bson.D{ + {Key: "$ID", Value: bsonutil.NewIDBsonBinary()}, + {Key: "$Type", Value: "Texts$Text"}, + {Key: "Items", Value: bson.A{int32(3)}}, + } + if err := setTranslatableText(tip, "Tooltip", value); err != nil { + return err + } + bsonnav.DSet(widget, "Tooltip", tip) + return nil +} + func setWidgetContentMut(widget bson.D, value any) error { content := bsonnav.DGetDoc(widget, "Content") if content == nil { diff --git a/mdl/backend/pagemutator/tooltip_1307_test.go b/mdl/backend/pagemutator/tooltip_1307_test.go new file mode 100644 index 000000000..a85ab75a3 --- /dev/null +++ b/mdl/backend/pagemutator/tooltip_1307_test.go @@ -0,0 +1,52 @@ +// SPDX-License-Identifier: Apache-2.0 + +package pagemutator + +import ( + "strings" + "testing" + + "go.mongodb.org/mongo-driver/bson" + + "github.com/mendixlabs/mxcli/mdl/backend/bsonnav" +) + +// mendixlabs/mxcli#1307: `alter page … set (Tooltip: '…') on btn` was refused +// with `property "Tooltip" is not a property of this built-in widget` — the +// key fell through to the pluggable setter. An action button stores its +// tooltip as a Texts$Text (empty when unset), the same shape as a tab caption. +func TestSetTooltip_ActionButton(t *testing.T) { + withAuthoringLanguage(t, "en_US") + btn := bson.D{ + {Key: "$Type", Value: "Forms$ActionButton"}, + {Key: "Name", Value: "btnEdit"}, + {Key: "Appearance", Value: bson.D{{Key: "$Type", Value: "Forms$Appearance"}}}, + {Key: "Tooltip", Value: textsText()}, + } + raw := makeRawPage(btn) + m := &Mutator{rawData: raw, widgetFinder: findBsonWidget} + if err := m.SetWidgetProperty("btnEdit", "Tooltip", "Edit this order"); err != nil { + t.Fatalf("set Tooltip: %v", err) + } + tip := bsonnav.DGetDoc(findBsonWidget(raw, "btnEdit").widget, "Tooltip") + if got := translationsOf(t, tip); got["en_US"] != "Edit this order" || len(got) != 1 { + t.Fatalf("tooltip translations = %v", got) + } +} + +// Control: a widget with no Tooltip slot is refused, not reported as altered. +func TestSetTooltip_WidgetWithoutTooltipIsRefused(t *testing.T) { + cont := bson.D{ + {Key: "$Type", Value: "Forms$DivContainer"}, + {Key: "Name", Value: "box"}, + {Key: "Appearance", Value: bson.D{{Key: "$Type", Value: "Forms$Appearance"}}}, + } + m := &Mutator{rawData: makeRawPage(cont), widgetFinder: findBsonWidget} + err := m.SetWidgetProperty("box", "Tooltip", "x") + if err == nil { + t.Fatal("set Tooltip on a container returned nil; it stored nothing") + } + if !strings.Contains(err.Error(), "Tooltip") { + t.Errorf("error does not name the property: %v", err) + } +} diff --git a/mdl/executor/button_tooltip_1307_test.go b/mdl/executor/button_tooltip_1307_test.go new file mode 100644 index 000000000..2d9b0ff38 --- /dev/null +++ b/mdl/executor/button_tooltip_1307_test.go @@ -0,0 +1,96 @@ +// SPDX-License-Identifier: Apache-2.0 + +// An action button's tooltip (mendixlabs/mxcli#1307). +// +// Reported: "A `tooltip:` on an `actionbutton` in `create page` passes `check` +// and `exec` with no warning, but the tooltip is never stored", and `describe` +// does not show it. The grammar parsed it, the validator listed `Tooltip` as a +// known property (so MDL-WIDGET07 stayed quiet), and the codec wrote whatever +// `pages.ActionButton.Tooltip` held — but buildButtonV3 never filled it, so +// the codec always wrote an empty Texts$Text. +package executor + +import ( + "strings" + "testing" + + "github.com/mendixlabs/mxcli/mdl/ast" + "github.com/mendixlabs/mxcli/mdl/visitor" + "github.com/mendixlabs/mxcli/model" + "github.com/mendixlabs/mxcli/sdk/pages" +) + +// buildButtonFromMDL runs the real parser and page builder over one button. +func buildButtonFromMDL(t *testing.T, button string) *pages.ActionButton { + t.Helper() + src := "create page Mod.P (Title: 'T') {\n " + button + "\n}" + prog, errs := visitor.Build(src) + if len(errs) > 0 { + t.Fatalf("parse %q: %v", button, errs) + } + page := prog.Statements[0].(*ast.CreatePageStmtV3) + pb := &pageBuilder{widgetScope: map[string]model.ID{}} + widget, err := pb.buildWidgetV3(page.Widgets[0]) + if err != nil { + t.Fatalf("building %q: %v", button, err) + } + return widget.(*pages.ActionButton) +} + +func TestButtonTooltip_IsStored(t *testing.T) { + for _, kw := range []string{"actionbutton", "linkbutton"} { + t.Run(kw, func(t *testing.T) { + btn := buildButtonFromMDL(t, kw+" btnEdit (Caption: 'Edit', Tooltip: 'Edit this order')") + if btn.Tooltip == nil || len(btn.Tooltip.Translations) == 0 { + t.Fatalf("Tooltip not stored: %+v", btn.Tooltip) + } + if got := btn.Tooltip.Translations[authoringLanguage(nil)]; got != "Edit this order" { + t.Errorf("Tooltip translations = %v, want the default language to hold %q", + btn.Tooltip.Translations, "Edit this order") + } + }) + } +} + +// Control: a button with no tooltip stores none, so the codec keeps writing +// the empty Texts$Text Studio Pro writes. +func TestButtonTooltip_AbsentStaysNil(t *testing.T) { + if btn := buildButtonFromMDL(t, "actionbutton b (Caption: 'Go')"); btn.Tooltip != nil { + t.Errorf("Tooltip = %+v for a button that declared none", btn.Tooltip) + } +} + +// DESCRIBE must print the stored tooltip, and its output must rebuild it. +func TestDescribeButtonTooltip_RoundTrips(t *testing.T) { + stored := map[string]any{ + "$Type": "Forms$ActionButton", + "Name": "btnEdit", + "Tooltip": map[string]any{ + "$Type": "Texts$Text", + "Items": []any{int32(3), map[string]any{ + "$Type": "Texts$Translation", + "LanguageCode": "en_US", + "Text": "Edit this order", + }}, + }, + } + out := describeButton(t, stored) + if !strings.Contains(out, "Tooltip: 'Edit this order'") { + t.Fatalf("describe did not emit the tooltip:\n%s", out) + } + + // Control: an empty Texts$Text (what every tooltip-less button stores) is silent. + empty := map[string]any{ + "$Type": "Forms$ActionButton", + "Name": "btnEdit", + "Tooltip": map[string]any{"$Type": "Texts$Text", "Items": []any{int32(3)}}, + } + if out := describeButton(t, empty); strings.Contains(out, "Tooltip") { + t.Errorf("an empty tooltip produced output:\n%s", out) + } + + btn := buildButtonFromMDL(t, "actionbutton btnEdit (Caption: 'Edit', Tooltip: 'Edit this order')") + if btn.Tooltip == nil || btn.Tooltip.Translations[authoringLanguage(nil)] != "Edit this order" { + t.Errorf("replaying DESCRIBE's tooltip did not rebuild it: %+v", btn.Tooltip) + } +} diff --git a/mdl/executor/cmd_pages_builder_v3_widgets.go b/mdl/executor/cmd_pages_builder_v3_widgets.go index 45ff64a67..8eaf48d0d 100644 --- a/mdl/executor/cmd_pages_builder_v3_widgets.go +++ b/mdl/executor/cmd_pages_builder_v3_widgets.go @@ -1131,6 +1131,19 @@ func (pb *pageBuilder) buildButtonV3(w *ast.WidgetV3) (*pages.ActionButton, erro btn.CaptionTemplate.Parameters = pb.buildClientTemplateParams(params) } + // Tooltip is a plain Texts$Text in the authoring language. The grammar and + // MDL-WIDGET07 both accepted it while nothing here read it, so it was + // dropped without a word (mendixlabs/mxcli#1307). + if tooltip := w.GetStringProp("Tooltip"); tooltip != "" { + btn.Tooltip = &model.Text{ + BaseElement: model.BaseElement{ + ID: model.ID(types.GenerateID()), + TypeName: "Texts$Text", + }, + Translations: map[string]string{pb.textLang(): tooltip}, + } + } + // Handle ButtonStyle. Normalize case (so `primary` becomes `Primary`) and // reject unknown values up front — an unrecognized style is silently // degraded to btn-default by MxBuild, which is a quiet authoring footgun. diff --git a/mdl/executor/cmd_pages_describe.go b/mdl/executor/cmd_pages_describe.go index 4a19cae80..ac22d188e 100644 --- a/mdl/executor/cmd_pages_describe.go +++ b/mdl/executor/cmd_pages_describe.go @@ -651,6 +651,7 @@ type rawWidget struct { // `Params: ( … )` (ako/mxcli#826). SnippetParams string Caption string + Tooltip string // an action button's Tooltip (mendixlabs/mxcli#1307) RenderMode string Action string ButtonStyle string diff --git a/mdl/executor/cmd_pages_describe_output.go b/mdl/executor/cmd_pages_describe_output.go index 908b2f030..962a9bcc5 100644 --- a/mdl/executor/cmd_pages_describe_output.go +++ b/mdl/executor/cmd_pages_describe_output.go @@ -504,6 +504,9 @@ func outputWidgetMDLV3(ctx *ExecContext, w rawWidget, indent int) { if len(w.Parameters) > 0 { props = append(props, fmt.Sprintf("CaptionParams: (%s)", strings.Join(formatParametersV3(w.Parameters), ", "))) } + if w.Tooltip != "" { + props = append(props, fmt.Sprintf("Tooltip: %s", mdlQuote(ctx, w.Tooltip))) + } if w.Action != "" { props = append(props, actionProp("Action", w.Action)) } @@ -1809,11 +1812,17 @@ func extractNanoflowParameters(ctx *ExecContext, action map[string]any) string { } func extractTextCaption(ctx *ExecContext, w map[string]any) string { - caption, ok := w["Caption"].(map[string]any) + return extractTextsText(ctx, w, "Caption") +} + +// extractTextsText reads the default-language translation of the Texts$Text +// stored under field — a widget's Caption, or an action button's Tooltip. +func extractTextsText(ctx *ExecContext, w map[string]any, field string) string { + text, ok := w[field].(map[string]any) if !ok { return "" } - items := getBsonArrayElements(caption["Items"]) + items := getBsonArrayElements(text["Items"]) return selectTranslationText(items, describeDefaultLanguage(ctx)) } diff --git a/mdl/executor/cmd_pages_describe_parse.go b/mdl/executor/cmd_pages_describe_parse.go index 5a9f8284c..41e1967ac 100644 --- a/mdl/executor/cmd_pages_describe_parse.go +++ b/mdl/executor/cmd_pages_describe_parse.go @@ -325,6 +325,7 @@ func parseRawWidget(ctx *ExecContext, w map[string]any, parentEntityContext ...s case "Forms$ActionButton", "Pages$ActionButton": widget.Caption = extractButtonCaption(ctx, w) widget.Parameters = extractButtonCaptionParameters(ctx, w) + widget.Tooltip = extractTextsText(ctx, w, "Tooltip") widget.ButtonStyle = extractButtonStyle(ctx, w) widget.Action = extractButtonAction(ctx, w) // RenderType "Link" is a linkbutton; the emitter uses this to choose the