From 3090a9a62e149fdd620e941364349b3e9c348433 Mon Sep 17 00:00:00 2001 From: Doug Mealing Date: Mon, 17 Aug 2026 20:44:46 -0400 Subject: [PATCH 01/44] =?UTF-8?q?docs:=20metadata=20source=20resolution=20?= =?UTF-8?q?=E2=80=94=20prior=20art,=20design,=20and=20phase-1=20plan?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Sources become a SET rather than an ordered list: super-resolution is already order-independent (#188) and the loader already discards the caller's order in _partitionOverlayLast, so declared order carries no information the loader consumes. That deletes topological sorting, cycle detection, and the diamond-dependency problem from the design. Scope is package patterns applied at OUTPUT, never to input — a partial input file list can fail to load when an extends target is missing, so input-side subsetting is wrong by construction. Prior art is open-source only, every claim carrying a public-docs URL, with licenses noted per project and hosted/commercial components excluded. Phase 1 is config-only and additive; a project with one root config and no sources declared keeps byte-identical output. Co-Authored-By: Claude Opus 5 (1M context) --- ...17-metadata-source-resolution-phase1-ts.md | 1686 +++++++++++++++++ ...08-17-metadata-source-resolution-design.md | 535 ++++++ ...17-metadata-source-resolution-prior-art.md | 384 ++++ 3 files changed, 2605 insertions(+) create mode 100644 docs/superpowers/plans/2026-08-17-metadata-source-resolution-phase1-ts.md create mode 100644 docs/superpowers/specs/2026-08-17-metadata-source-resolution-design.md create mode 100644 docs/superpowers/specs/2026-08-17-metadata-source-resolution-prior-art.md diff --git a/docs/superpowers/plans/2026-08-17-metadata-source-resolution-phase1-ts.md b/docs/superpowers/plans/2026-08-17-metadata-source-resolution-phase1-ts.md new file mode 100644 index 000000000..d509a0a5b --- /dev/null +++ b/docs/superpowers/plans/2026-08-17-metadata-source-resolution-phase1-ts.md @@ -0,0 +1,1686 @@ +# Metadata Source Resolution — Phase 1 (TypeScript) Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Make `sources` in `.metaobjects/config.json` the single authority on where metadata lives, with package-pattern scoping at output and nearest-ancestor discovery, so a consumer can point at a metadata tree elsewhere in the repo and take the slice it needs. + +**Architecture:** Sources are an unordered **set** of tagged specs resolved to a canonically-sorted file list (the loader already derives whatever order it needs, so declared order carries no information). Scope is a package-pattern include/exclude filter applied at **output**, never to input — the collection always loads in full, which makes every scope closure-complete by construction. One new `resolveCollection()` entry point replaces nine hardcoded `metaobjects/` reads. + +**Tech Stack:** TypeScript (ESM), Bun test runner, Zod for config schema, existing `@metaobjectsdev/metadata` loader and canonical serializer. + +**Spec:** `docs/superpowers/specs/2026-08-17-metadata-source-resolution-design.md` + +## Global Constraints + +- **Named constants for metamodel strings — always.** Import `PACKAGE_SEPARATOR` from `@metaobjectsdev/metadata/constants`; never inline `"::"`. +- **No `any`.** Use `unknown` and narrow. +- **Never `instanceof` a metadata node from another package** — use the exported guards (`isMetaObject`, `isMetaField`, …). Two physical copies of `metadata` in one process make `instanceof` silently false. +- **Never call `own*()` accessors** (ADR-0039). Resolving/effective accessors are the default. +- **Backward compatibility is absolute:** a project with one config at the root, no `sources`, and no `scope` must produce **byte-identical** output to today. Every task that touches a read path must prove this. +- **Public repository.** No private project names, no absolute home paths (`/home//…`) in code, tests, fixtures, or commit messages. +- **Run tests scoped:** `cd server/typescript && bun test packages/` — never a bare `bun test` at the repo root. +- Package separator is `::`. `*` matches any characters within one segment; a segment that is exactly `**` matches one or more segments. + +--- + +## File Structure + +**New — `server/typescript/packages/sdk/src/`** +- `scope.ts` — package-pattern compile + match. Pure, no I/O. The cross-port semantic core. +- `sources.ts` — `SourceSpec` union → canonically-sorted absolute file list. All filesystem I/O for source resolution. +- `discovery.ts` — nearest-ancestor config lookup. Filesystem walk only. +- `collection.ts` — `resolveCollection()`: the single authority composing the three above. + +**New — tests** +- `packages/sdk/test/scope.test.ts`, `sources.test.ts`, `discovery.test.ts`, `collection.test.ts` +- `packages/sdk/test/order-independence.test.ts` — the linchpin gate +- `packages/sdk/test/scope-conformance.test.ts` — runs the shared corpus + +**New — cross-port fixture** +- `fixtures/scope-conformance/cases.json`, `README.md` + +**Modified** +- `packages/sdk/src/config.ts` — widen `sources`, add `scope`, add `migrate.scope` +- `packages/sdk/src/memory.ts` — `loadMemory` accepts a resolved file list +- `packages/sdk/src/index.ts` — export the new surface +- `packages/cli/src/commands/{gen,docs,export,migrate}.ts` — route reads through `resolveCollection()` +- `packages/cli/src/index.ts:275` — the "is this a MetaObjects project?" probe +- `packages/cli/src/lib/detect-stack.ts` — route + nested-symlink fix +- `packages/metadata/src/errors.ts` — register new error codes + +**Deliberately unchanged** +- `packages/cli/src/commands/init.ts` — scaffolding writes the default directory. This is the one place the `"metaobjects"` literal belongs. + +--- + +## Task 1: Scope pattern engine + +**Files:** +- Create: `server/typescript/packages/sdk/src/scope.ts` +- Test: `server/typescript/packages/sdk/test/scope.test.ts` + +**Interfaces:** +- Consumes: `PACKAGE_SEPARATOR` from `@metaobjectsdev/metadata/constants` +- Produces: + - `interface Scope { readonly include?: readonly string[]; readonly exclude?: readonly string[] }` + - `interface CompiledScope { readonly include: readonly RegExp[]; readonly exclude: readonly RegExp[] }` + - `function compileScope(scope: Scope): CompiledScope` — throws `Error` whose message starts `ERR_SCOPE_PATTERN_INVALID` on an empty or malformed pattern + - `function matchesScope(fqn: string, compiled: CompiledScope): boolean` + +- [ ] **Step 1: Write the failing test** + +```ts +// server/typescript/packages/sdk/test/scope.test.ts +import { describe, test, expect } from "bun:test"; +import { compileScope, matchesScope, type Scope } from "../src/scope.js"; + +const match = (fqn: string, scope: Scope) => matchesScope(fqn, compileScope(scope)); + +describe("compileScope / matchesScope", () => { + test("empty include matches everything", () => { + expect(match("acme::commerce::Order", {})).toBe(true); + }); + + test("* matches exactly one segment", () => { + const s: Scope = { include: ["acme::*"] }; + expect(match("acme::Order", s)).toBe(true); + expect(match("acme::commerce::Order", s)).toBe(false); + }); + + test("** matches one or more segments", () => { + const s: Scope = { include: ["acme::**"] }; + expect(match("acme::Order", s)).toBe(true); + expect(match("acme::commerce::Order", s)).toBe(true); + expect(match("acme", s)).toBe(false); + expect(match("other::Order", s)).toBe(false); + }); + + test("* within a segment matches a partial name but never crosses ::", () => { + const s: Scope = { include: ["acme::Order*"] }; + expect(match("acme::OrderLine", s)).toBe(true); + expect(match("acme::Order", s)).toBe(true); + expect(match("acme::deep::OrderLine", s)).toBe(false); + }); + + test("exclude is applied after include", () => { + const s: Scope = { include: ["acme::**"], exclude: ["acme::internal::**"] }; + expect(match("acme::commerce::Order", s)).toBe(true); + expect(match("acme::internal::Secret", s)).toBe(false); + }); + + test("exclude alone narrows the implicit match-everything", () => { + const s: Scope = { exclude: ["acme::internal::**"] }; + expect(match("acme::commerce::Order", s)).toBe(true); + expect(match("acme::internal::Secret", s)).toBe(false); + }); + + test("a bare name with no package is matchable", () => { + expect(match("Order", { include: ["Order"] })).toBe(true); + expect(match("Order", { include: ["*"] })).toBe(true); + }); + + test("regex metacharacters in a pattern are literal", () => { + expect(match("acme::Order.v2", { include: ["acme::Order.v2"] })).toBe(true); + expect(match("acme::OrderXv2", { include: ["acme::Order.v2"] })).toBe(false); + }); + + test("an empty pattern is ERR_SCOPE_PATTERN_INVALID", () => { + expect(() => compileScope({ include: [""] })).toThrow(/ERR_SCOPE_PATTERN_INVALID/); + }); + + test("an empty segment is ERR_SCOPE_PATTERN_INVALID", () => { + expect(() => compileScope({ include: ["acme::::Order"] })).toThrow(/ERR_SCOPE_PATTERN_INVALID/); + }); +}); +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `cd server/typescript && bun test packages/sdk/test/scope.test.ts` +Expected: FAIL — `Cannot find module '../src/scope.js'` + +- [ ] **Step 3: Write minimal implementation** + +```ts +// server/typescript/packages/sdk/src/scope.ts +import { PACKAGE_SEPARATOR } from "@metaobjectsdev/metadata/constants"; + +/** A consumer-side output filter over fully-qualified node names. */ +export interface Scope { + /** Absent or empty means "everything". */ + readonly include?: readonly string[]; + /** Applied after `include`. */ + readonly exclude?: readonly string[]; +} + +export interface CompiledScope { + readonly include: readonly RegExp[]; + readonly exclude: readonly RegExp[]; +} + +/** One package segment: any run of characters containing no separator char. */ +const SEGMENT = "[^:]+"; +/** One or more segments, separator-joined — the `**` expansion. */ +const SEGMENTS = `${SEGMENT}(?:${PACKAGE_SEPARATOR}${SEGMENT})*`; + +function escapeLiteral(text: string): string { + return text.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); +} + +/** Compile one segment. `**` spans segments; `*` never crosses a separator. */ +function compileSegment(segment: string, pattern: string): string { + if (segment.length === 0) { + throw new Error( + `ERR_SCOPE_PATTERN_INVALID: empty segment in scope pattern "${pattern}"`, + ); + } + if (segment === "**") return `(?:${SEGMENTS})`; + // `*` inside a segment matches any characters except the separator char. + return segment.split("*").map(escapeLiteral).join("[^:]*"); +} + +export function compilePattern(pattern: string): RegExp { + if (pattern.length === 0) { + throw new Error("ERR_SCOPE_PATTERN_INVALID: scope pattern must not be empty"); + } + const body = pattern + .split(PACKAGE_SEPARATOR) + .map((segment) => compileSegment(segment, pattern)) + .join(PACKAGE_SEPARATOR); + return new RegExp(`^${body}$`); +} + +export function compileScope(scope: Scope): CompiledScope { + return { + include: (scope.include ?? []).map(compilePattern), + exclude: (scope.exclude ?? []).map(compilePattern), + }; +} + +/** True when `fqn` is inside the scope. An empty `include` means everything. */ +export function matchesScope(fqn: string, compiled: CompiledScope): boolean { + const included = + compiled.include.length === 0 || compiled.include.some((re) => re.test(fqn)); + if (!included) return false; + return !compiled.exclude.some((re) => re.test(fqn)); +} +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `cd server/typescript && bun test packages/sdk/test/scope.test.ts` +Expected: PASS — 10 tests + +- [ ] **Step 5: Typecheck** + +Run: `cd server/typescript && bun run --filter '@metaobjectsdev/sdk' typecheck` +Expected: no errors + +- [ ] **Step 6: Commit** + +```bash +git add server/typescript/packages/sdk/src/scope.ts server/typescript/packages/sdk/test/scope.test.ts +git commit -m "feat(sdk): package-pattern scope engine (* = one segment, ** = one or more)" +``` + +--- + +## Task 2: Scope-pattern conformance corpus + +Pins the semantics cross-port so `*` and `**` cannot come to mean five different things — the failure mode that produced the cross-port `LIKE`/`ILIKE` divergence. + +**Files:** +- Create: `fixtures/scope-conformance/cases.json` +- Create: `fixtures/scope-conformance/README.md` +- Test: `server/typescript/packages/sdk/test/scope-conformance.test.ts` + +**Interfaces:** +- Consumes: `compileScope` / `matchesScope` from Task 1 +- Produces: the corpus contract — `{ cases: Array<{ name: string; scope: {include?: string[]; exclude?: string[]}; expect: Array<{ fqn: string; matches: boolean }> }> }`. Every other port's runner reads this same file. + +- [ ] **Step 1: Write the corpus** + +```json +{ + "cases": [ + { + "name": "empty-scope-matches-everything", + "scope": {}, + "expect": [ + { "fqn": "acme::commerce::Order", "matches": true }, + { "fqn": "Order", "matches": true } + ] + }, + { + "name": "single-star-is-one-segment", + "scope": { "include": ["acme::*"] }, + "expect": [ + { "fqn": "acme::Order", "matches": true }, + { "fqn": "acme::commerce::Order", "matches": false }, + { "fqn": "other::Order", "matches": false } + ] + }, + { + "name": "double-star-is-one-or-more-segments", + "scope": { "include": ["acme::**"] }, + "expect": [ + { "fqn": "acme::Order", "matches": true }, + { "fqn": "acme::commerce::Order", "matches": true }, + { "fqn": "acme::commerce::internal::Secret", "matches": true }, + { "fqn": "acme", "matches": false }, + { "fqn": "acmex::Order", "matches": false } + ] + }, + { + "name": "partial-star-never-crosses-separator", + "scope": { "include": ["acme::Order*"] }, + "expect": [ + { "fqn": "acme::Order", "matches": true }, + { "fqn": "acme::OrderLine", "matches": true }, + { "fqn": "acme::deep::OrderLine", "matches": false } + ] + }, + { + "name": "exclude-applied-after-include", + "scope": { "include": ["acme::**"], "exclude": ["acme::internal::**"] }, + "expect": [ + { "fqn": "acme::commerce::Order", "matches": true }, + { "fqn": "acme::internal::Secret", "matches": false } + ] + }, + { + "name": "exclude-alone-narrows-everything", + "scope": { "exclude": ["acme::internal::**"] }, + "expect": [ + { "fqn": "other::Thing", "matches": true }, + { "fqn": "acme::internal::Secret", "matches": false } + ] + }, + { + "name": "multiple-includes-are-a-union", + "scope": { "include": ["acme::commerce::**", "acme::common::**"] }, + "expect": [ + { "fqn": "acme::commerce::Order", "matches": true }, + { "fqn": "acme::common::BaseEntity", "matches": true }, + { "fqn": "acme::billing::Invoice", "matches": false } + ] + }, + { + "name": "regex-metacharacters-are-literal", + "scope": { "include": ["acme::Order.v2"] }, + "expect": [ + { "fqn": "acme::Order.v2", "matches": true }, + { "fqn": "acme::OrderXv2", "matches": false } + ] + } + ] +} +``` + +Write `fixtures/scope-conformance/README.md` stating: the corpus is the cross-port contract for `scope` pattern semantics; every port runs it; `*` matches any characters within one segment and never crosses `::`; a segment that is exactly `**` matches one or more segments; `include` empty means everything; `exclude` is applied after `include`. + +- [ ] **Step 2: Write the failing runner test** + +```ts +// server/typescript/packages/sdk/test/scope-conformance.test.ts +import { describe, test, expect } from "bun:test"; +import { readFileSync } from "node:fs"; +import { join } from "node:path"; +import { compileScope, matchesScope, type Scope } from "../src/scope.js"; + +interface Case { + name: string; + scope: Scope; + expect: Array<{ fqn: string; matches: boolean }>; +} + +const CORPUS = join(import.meta.dir, "../../../../../fixtures/scope-conformance/cases.json"); +const cases = (JSON.parse(readFileSync(CORPUS, "utf8")) as { cases: Case[] }).cases; + +describe("scope-conformance corpus", () => { + test("corpus is non-empty (a silent zero-case run is a failed gate)", () => { + expect(cases.length).toBeGreaterThan(0); + }); + for (const c of cases) { + test(c.name, () => { + const compiled = compileScope(c.scope); + for (const e of c.expect) { + expect({ fqn: e.fqn, matches: matchesScope(e.fqn, compiled) }) + .toEqual({ fqn: e.fqn, matches: e.matches }); + } + }); + } +}); +``` + +- [ ] **Step 3: Run to verify it passes** + +Run: `cd server/typescript && bun test packages/sdk/test/scope-conformance.test.ts` +Expected: PASS — 9 tests (8 cases + the non-empty guard) + +- [ ] **Step 4: Prove the gate by breaking it** + +Temporarily change `SEGMENTS` in `scope.ts` to `".*"` (making `*` cross separators), re-run, and confirm `single-star-is-one-segment` FAILS. Then revert. + +Expected: FAIL before revert, PASS after. A gate that has never been seen red is not known to work. + +- [ ] **Step 5: Commit** + +```bash +git add fixtures/scope-conformance server/typescript/packages/sdk/test/scope-conformance.test.ts +git commit -m "test(conformance): scope-pattern corpus pins * and ** semantics cross-port" +``` + +--- + +## Task 3: Register new error codes + +**Files:** +- Modify: `server/typescript/packages/metadata/src/errors.ts:19` (the `ERROR_CODES` array) +- Test: `server/typescript/packages/metadata/test/errors.test.ts` (extend if present; create if not) + +**Interfaces:** +- Produces: `"ERR_SOURCE_UNRESOLVED"`, `"ERR_SOURCE_KIND_UNSUPPORTED"`, `"ERR_SCOPE_PATTERN_INVALID"`, `"ERR_COLLECTION_NOT_FOUND"` as members of `ERROR_CODES` + +- [ ] **Step 1: Write the failing test** + +```ts +import { describe, test, expect } from "bun:test"; +import { ERROR_CODES } from "../src/errors.js"; + +describe("phase-1 source-resolution error codes", () => { + test("are registered in the shared ledger", () => { + for (const code of [ + "ERR_SOURCE_UNRESOLVED", + "ERR_SOURCE_KIND_UNSUPPORTED", + "ERR_SCOPE_PATTERN_INVALID", + "ERR_COLLECTION_NOT_FOUND", + ]) { + expect(ERROR_CODES).toContain(code); + } + }); +}); +``` + +- [ ] **Step 2: Run to verify it fails** + +Run: `cd server/typescript && bun test packages/metadata/test/errors.test.ts` +Expected: FAIL — codes not found + +- [ ] **Step 3: Add the codes** + +Add these four string literals to the `ERROR_CODES` array in `errors.ts`, each with a comment naming the phase-1 source-resolution design as their origin, matching the file's existing comment style. + +- [ ] **Step 4: Run to verify it passes** + +Run: `cd server/typescript && bun test packages/metadata/test/errors.test.ts` +Expected: PASS + +- [ ] **Step 5: Note the cross-port debt** + +Add a line to the plan's tracking notes: Python `errors.py` (superset) and Java `ErrorCode.java` need the same four codes in the ports plan. TS `errors.ts` is exact-bidirectional, so its own gate will now expect them. + +- [ ] **Step 6: Commit** + +```bash +git add server/typescript/packages/metadata/src/errors.ts server/typescript/packages/metadata/test/errors.test.ts +git commit -m "feat(metadata): register phase-1 source-resolution error codes" +``` + +--- + +## Task 4: Source spec resolution + +**Files:** +- Create: `server/typescript/packages/sdk/src/sources.ts` +- Test: `server/typescript/packages/sdk/test/sources.test.ts` + +**Interfaces:** +- Produces: + - `type SourceSpec = { path: string } | { resource: string } | { package: string }` + - `interface ResolvedSource { readonly file: string; readonly spec: SourceSpec }` + - `function resolveSources(configDir: string, specs: readonly SourceSpec[]): Promise` — returns files sorted by absolute path (canonical, order-free); throws on an unresolvable `path`; throws `ERR_SOURCE_KIND_UNSUPPORTED` for `resource`/`package` in phase 1 + - `const DEFAULT_SOURCES: readonly SourceSpec[]` — `[{ path: "metaobjects" }]` + +- [ ] **Step 1: Write the failing test** + +```ts +// server/typescript/packages/sdk/test/sources.test.ts +import { describe, test, expect, beforeEach, afterEach } from "bun:test"; +import { mkdtempSync, rmSync, mkdirSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { resolveSources, DEFAULT_SOURCES } from "../src/sources.js"; + +let root: string; +const write = (rel: string, body = "{}") => { + const full = join(root, rel); + mkdirSync(join(full, ".."), { recursive: true }); + writeFileSync(full, body, "utf8"); + return full; +}; + +beforeEach(() => { root = mkdtempSync(join(tmpdir(), "metaobjects-sources-")); }); +afterEach(() => { rmSync(root, { recursive: true, force: true }); }); + +describe("resolveSources", () => { + test("resolves a directory recursively, metadata files only", async () => { + write("model/meta.a.json"); + write("model/nested/meta.b.yaml"); + write("model/notes.txt"); + const out = await resolveSources(root, [{ path: "model" }]); + expect(out.map((r) => r.file.replace(root + "/", ""))).toEqual([ + "model/meta.a.json", + "model/nested/meta.b.yaml", + ]); + }); + + test("resolves a single file", async () => { + write("model/meta.a.json"); + const out = await resolveSources(root, [{ path: "model/meta.a.json" }]); + expect(out).toHaveLength(1); + }); + + test("output is canonically sorted regardless of spec order", async () => { + write("b/meta.b.json"); + write("a/meta.a.json"); + const forward = await resolveSources(root, [{ path: "a" }, { path: "b" }]); + const reverse = await resolveSources(root, [{ path: "b" }, { path: "a" }]); + expect(forward.map((r) => r.file)).toEqual(reverse.map((r) => r.file)); + }); + + test("de-duplicates a file contributed by two overlapping specs", async () => { + write("model/meta.a.json"); + const out = await resolveSources(root, [{ path: "model" }, { path: "model/meta.a.json" }]); + expect(out).toHaveLength(1); + }); + + test("paths resolve against the config dir, not process.cwd()", async () => { + write("apps/ui/.keep"); + write("model/meta.a.json"); + const out = await resolveSources(join(root, "apps/ui"), [{ path: "../../model" }]); + expect(out).toHaveLength(1); + }); + + test("an unresolvable path is ERR_SOURCE_UNRESOLVED, never a silent skip", async () => { + await expect(resolveSources(root, [{ path: "missing" }])).rejects.toThrow( + /ERR_SOURCE_UNRESOLVED/, + ); + }); + + test("resource and package kinds are ERR_SOURCE_KIND_UNSUPPORTED in phase 1", async () => { + await expect(resolveSources(root, [{ resource: "acme/model" }])).rejects.toThrow( + /ERR_SOURCE_KIND_UNSUPPORTED/, + ); + await expect(resolveSources(root, [{ package: "@acme/model" }])).rejects.toThrow( + /ERR_SOURCE_KIND_UNSUPPORTED/, + ); + }); + + test("_pending is excluded at any depth", async () => { + write("model/meta.a.json"); + write("model/_pending/meta.draft.json"); + const out = await resolveSources(root, [{ path: "model" }]); + expect(out).toHaveLength(1); + }); + + test("a nested symlinked directory is followed", async () => { + write("real/meta.b.json"); + write("model/meta.a.json"); + const { symlinkSync } = await import("node:fs"); + symlinkSync(join(root, "real"), join(root, "model/linked"), "dir"); + const out = await resolveSources(root, [{ path: "model" }]); + expect(out).toHaveLength(2); + }); + + test("DEFAULT_SOURCES is the metaobjects/ directory", () => { + expect(DEFAULT_SOURCES).toEqual([{ path: "metaobjects" }]); + }); +}); +``` + +- [ ] **Step 2: Run to verify it fails** + +Run: `cd server/typescript && bun test packages/sdk/test/sources.test.ts` +Expected: FAIL — `Cannot find module '../src/sources.js'` + +- [ ] **Step 3: Write minimal implementation** + +```ts +// server/typescript/packages/sdk/src/sources.ts +import { readdir, stat } from "node:fs/promises"; +import { isAbsolute, join, resolve } from "node:path"; + +/** Tagged union of source kinds. `resource` and `package` are declared now so the + * config shape is stable; only `path` resolves in phase 1. */ +export type SourceSpec = + | { readonly path: string } + | { readonly resource: string } + | { readonly package: string }; + +export interface ResolvedSource { + /** Absolute path of one metadata file. */ + readonly file: string; + /** The spec that contributed it — provenance for diagnostics. */ + readonly spec: SourceSpec; +} + +/** Used when `sources` is absent or empty. `metaobjects/` is a DEFAULT, never a requirement. */ +export const DEFAULT_SOURCES: readonly SourceSpec[] = [{ path: "metaobjects" }]; + +const PENDING_DIR = "_pending"; + +function isMetadataFile(name: string): boolean { + return name.endsWith(".json") || name.endsWith(".yaml") || name.endsWith(".yml"); +} + +/** Recursively collect metadata files. Uses `stat` (follows symlinks) so a symlinked + * subdirectory is traversed — the loader has always followed them. */ +async function collectDir(dir: string, out: string[]): Promise { + const entries = await readdir(dir); + for (const entry of entries) { + if (entry === PENDING_DIR) continue; + const full = join(dir, entry); + const s = await stat(full); + if (s.isDirectory()) await collectDir(full, out); + else if (s.isFile() && isMetadataFile(entry)) out.push(full); + } +} + +/** + * Resolve a source SET to a canonically-sorted list of metadata files. + * + * The result is sorted by absolute path and de-duplicated, so it is a pure function + * of the source set: permuting `specs` cannot change the output. Declared order + * carries no information (the loader derives whatever order it needs). + * + * @param configDir absolute directory of the declaring config — relative `path` + * specs resolve against it, never against ambient `process.cwd()`. + */ +export async function resolveSources( + configDir: string, + specs: readonly SourceSpec[], +): Promise { + const byFile = new Map(); + + for (const spec of specs) { + if (!("path" in spec)) { + const kind = "resource" in spec ? "resource" : "package"; + throw new Error( + `ERR_SOURCE_KIND_UNSUPPORTED: source kind "${kind}" is not supported by this ` + + `toolchain yet; use a "path" source.`, + ); + } + const target = isAbsolute(spec.path) ? spec.path : resolve(configDir, spec.path); + let s; + try { + s = await stat(target); + } catch { + throw new Error( + `ERR_SOURCE_UNRESOLVED: source path "${spec.path}" does not exist ` + + `(resolved to ${target}, relative to ${configDir}).`, + ); + } + const found: string[] = []; + if (s.isDirectory()) await collectDir(target, found); + else found.push(target); + for (const file of found) if (!byFile.has(file)) byFile.set(file, spec); + } + + return [...byFile.keys()] + .sort() + .map((file) => ({ file, spec: byFile.get(file)! })); +} +``` + +- [ ] **Step 4: Run to verify it passes** + +Run: `cd server/typescript && bun test packages/sdk/test/sources.test.ts` +Expected: PASS — 10 tests + +- [ ] **Step 5: Commit** + +```bash +git add server/typescript/packages/sdk/src/sources.ts server/typescript/packages/sdk/test/sources.test.ts +git commit -m "feat(sdk): resolve a source SET to a canonically-sorted file list" +``` + +--- + +## Task 5: Config schema — `sources`, `scope`, `migrate.scope` + +**Files:** +- Modify: `server/typescript/packages/sdk/src/config.ts:64-78` +- Test: `server/typescript/packages/sdk/test/config.test.ts` + +**Interfaces:** +- Consumes: `SourceSpec` (Task 4), `Scope` (Task 1) +- Produces: `ConfigSchema` accepting `sources: SourceSpec[]`, `scope?: {include?: string[]; exclude?: string[]}`, and `migrate.scope?: string[]` + +- [ ] **Step 1: Write the failing test** + +Append to `packages/sdk/test/config.test.ts`: + +```ts +describe("ConfigSchema — phase-1 source resolution", () => { + test("accepts a path source", () => { + const p = ConfigSchema.parse({ schema_version: 1, sources: [{ path: "../model" }] }); + expect(p.sources).toEqual([{ path: "../model" }]); + }); + test("accepts resource and package source kinds", () => { + const p = ConfigSchema.parse({ + schema_version: 1, + sources: [{ resource: "acme/model" }, { package: "@acme/model" }], + }); + expect(p.sources).toHaveLength(2); + }); + test("rejects an unknown source kind", () => { + expect(() => ConfigSchema.parse({ schema_version: 1, sources: [{ nope: "x" }] })).toThrow(); + }); + test("accepts a scope block", () => { + const p = ConfigSchema.parse({ + schema_version: 1, + scope: { include: ["acme::**"], exclude: ["acme::internal::**"] }, + }); + expect(p.scope?.include).toEqual(["acme::**"]); + }); + test("scope defaults to undefined (match everything)", () => { + expect(ConfigSchema.parse({ schema_version: 1 }).scope).toBeUndefined(); + }); + test("accepts migrate.scope", () => { + const p = ConfigSchema.parse({ + schema_version: 1, + migrate: { scope: ["acme::platform::**"] }, + }); + expect(p.migrate?.scope).toEqual(["acme::platform::**"]); + }); + test("an existing config with no new keys still parses (back-compat)", () => { + const p = ConfigSchema.parse({ + schema_version: 1, pending_in_git: true, + confidence_thresholds: { pending_promote: 0.8, drift_warn: 0.7 }, + sources: [], extract: {}, + }); + expect(p.sources).toEqual([]); + expect(p.scope).toBeUndefined(); + }); +}); +``` + +- [ ] **Step 2: Run to verify it fails** + +Run: `cd server/typescript && bun test packages/sdk/test/config.test.ts` +Expected: FAIL — the `resource` source and the `scope` block are rejected + +- [ ] **Step 3: Widen the schema** + +In `config.ts`, replace the existing `sources` union and add `scope`: + +```ts +const SourceSpecSchema = z.union([ + z.object({ path: z.string().min(1) }).strict(), + z.object({ resource: z.string().min(1) }).strict(), + z.object({ package: z.string().min(1) }).strict(), +]); + +const ScopeSchema = z.object({ + include: z.array(z.string().min(1)).optional(), + exclude: z.array(z.string().min(1)).optional(), +}).strict(); +``` + +In `ConfigSchema`: replace the `sources` field with `z.array(SourceSpecSchema).default([])`, add `scope: ScopeSchema.optional()`, and add `scope: z.array(z.string().min(1))` to `MigrateBlock`'s partial shape. + +- [ ] **Step 4: Run to verify it passes** + +Run: `cd server/typescript && bun test packages/sdk/test/config.test.ts` +Expected: PASS — including the pre-existing tests, unchanged + +- [ ] **Step 5: Commit** + +```bash +git add server/typescript/packages/sdk/src/config.ts server/typescript/packages/sdk/test/config.test.ts +git commit -m "feat(sdk): config accepts a source SET, a scope block, and migrate.scope" +``` + +--- + +## Task 6: Nearest-ancestor discovery + +**Files:** +- Create: `server/typescript/packages/sdk/src/discovery.ts` +- Test: `server/typescript/packages/sdk/test/discovery.test.ts` + +**Interfaces:** +- Produces: `function findConfigDir(startDir: string): Promise` — walks up for a directory containing `.metaobjects/config.json`; stops after examining a directory containing `.git`; returns the containing directory (not the `.metaobjects` dir), or `undefined` + +- [ ] **Step 1: Write the failing test** + +```ts +// server/typescript/packages/sdk/test/discovery.test.ts +import { describe, test, expect, beforeEach, afterEach } from "bun:test"; +import { mkdtempSync, rmSync, mkdirSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { findConfigDir } from "../src/discovery.js"; + +let root: string; +const mk = (rel: string) => mkdirSync(join(root, rel), { recursive: true }); +const cfg = (rel: string) => { + mk(join(rel, ".metaobjects")); + writeFileSync(join(root, rel, ".metaobjects/config.json"), '{"schema_version":1}', "utf8"); +}; + +beforeEach(() => { root = mkdtempSync(join(tmpdir(), "metaobjects-discovery-")); mk(".git"); }); +afterEach(() => { rmSync(root, { recursive: true, force: true }); }); + +describe("findConfigDir", () => { + test("finds a config in the start directory", async () => { + cfg("apps/ui"); mk("apps/ui/src"); + expect(await findConfigDir(join(root, "apps/ui"))).toBe(join(root, "apps/ui")); + }); + test("walks up to the nearest ancestor config", async () => { + cfg("apps/ui"); mk("apps/ui/src/deep"); + expect(await findConfigDir(join(root, "apps/ui/src/deep"))).toBe(join(root, "apps/ui")); + }); + test("nearest wins over a further ancestor", async () => { + cfg("."); cfg("apps/ui"); mk("apps/ui/src"); + expect(await findConfigDir(join(root, "apps/ui/src"))).toBe(join(root, "apps/ui")); + }); + test("stops at the repository boundary — never adopts a parent checkout's config", async () => { + // A config ABOVE the .git boundary must not be found. + const outer = mkdtempSync(join(tmpdir(), "metaobjects-outer-")); + try { + mkdirSync(join(outer, "inner/.git"), { recursive: true }); + mkdirSync(join(outer, ".metaobjects"), { recursive: true }); + writeFileSync(join(outer, ".metaobjects/config.json"), '{"schema_version":1}', "utf8"); + mkdirSync(join(outer, "inner/src"), { recursive: true }); + expect(await findConfigDir(join(outer, "inner/src"))).toBeUndefined(); + } finally { + rmSync(outer, { recursive: true, force: true }); + } + }); + test("a repo-root config IS found from a subdirectory", async () => { + cfg("."); mk("apps/ui"); + expect(await findConfigDir(join(root, "apps/ui"))).toBe(root); + }); + test("returns undefined when nothing is found", async () => { + mk("apps/ui"); + expect(await findConfigDir(join(root, "apps/ui"))).toBeUndefined(); + }); +}); +``` + +- [ ] **Step 2: Run to verify it fails** + +Run: `cd server/typescript && bun test packages/sdk/test/discovery.test.ts` +Expected: FAIL — `Cannot find module '../src/discovery.js'` + +- [ ] **Step 3: Write minimal implementation** + +```ts +// server/typescript/packages/sdk/src/discovery.ts +import { stat } from "node:fs/promises"; +import { dirname, join, resolve } from "node:path"; +import { DEFAULT_METAOBJECTS_DIR } from "./memory.js"; + +const CONFIG_FILE = "config.json"; +const GIT_DIR = ".git"; + +async function exists(p: string): Promise { + try { await stat(p); return true; } catch { return false; } +} + +/** + * Walk up from `startDir` for the nearest directory holding + * `.metaobjects/config.json`. The walk STOPS after examining a directory that + * contains `.git`, so a monorepo can never silently adopt a parent checkout's + * configuration. Returns the containing directory, or undefined. + */ +export async function findConfigDir(startDir: string): Promise { + let dir = resolve(startDir); + for (;;) { + if (await exists(join(dir, DEFAULT_METAOBJECTS_DIR, CONFIG_FILE))) return dir; + // Boundary check AFTER the config check: a repo-root config is still findable. + if (await exists(join(dir, GIT_DIR))) return undefined; + const parent = dirname(dir); + if (parent === dir) return undefined; + dir = parent; + } +} +``` + +- [ ] **Step 4: Run to verify it passes** + +Run: `cd server/typescript && bun test packages/sdk/test/discovery.test.ts` +Expected: PASS — 6 tests + +- [ ] **Step 5: Commit** + +```bash +git add server/typescript/packages/sdk/src/discovery.ts server/typescript/packages/sdk/test/discovery.test.ts +git commit -m "feat(sdk): nearest-ancestor config discovery, bounded by the repo root" +``` + +--- + +## Task 7: `resolveCollection()` — the single authority + +**Files:** +- Create: `server/typescript/packages/sdk/src/collection.ts` +- Modify: `server/typescript/packages/sdk/src/index.ts` (export the new surface) +- Test: `server/typescript/packages/sdk/test/collection.test.ts` + +**Interfaces:** +- Consumes: `findConfigDir` (T6), `resolveSources`/`DEFAULT_SOURCES` (T4), `compileScope` (T1), `loadConfig` (T5) +- Produces: + - `interface Collection { readonly configDir: string; readonly files: readonly string[]; readonly sources: readonly ResolvedSource[]; readonly scope: CompiledScope; readonly migrateScope: CompiledScope | undefined }` + - `function resolveCollection(startDir: string, opts?: { explicitDir?: string }): Promise` — throws `ERR_COLLECTION_NOT_FOUND` when no config is discovered AND the default directory does not exist + +- [ ] **Step 1: Write the failing test** + +```ts +// server/typescript/packages/sdk/test/collection.test.ts +import { describe, test, expect, beforeEach, afterEach } from "bun:test"; +import { mkdtempSync, rmSync, mkdirSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { resolveCollection } from "../src/collection.js"; +import { matchesScope } from "../src/scope.js"; + +let root: string; +const write = (rel: string, body: string) => { + mkdirSync(join(root, rel, ".."), { recursive: true }); + writeFileSync(join(root, rel), body, "utf8"); +}; +const config = (dir: string, cfg: object) => + write(join(dir, ".metaobjects/config.json"), JSON.stringify({ schema_version: 1, ...cfg })); + +beforeEach(() => { root = mkdtempSync(join(tmpdir(), "metaobjects-collection-")); mkdirSync(join(root, ".git")); }); +afterEach(() => { rmSync(root, { recursive: true, force: true }); }); + +describe("resolveCollection", () => { + test("BACK-COMPAT: no sources declared falls back to metaobjects/", async () => { + write("metaobjects/meta.a.json", "{}"); + config(".", {}); + const c = await resolveCollection(root); + expect(c.files.map((f) => f.replace(root + "/", ""))).toEqual(["metaobjects/meta.a.json"]); + }); + + test("BACK-COMPAT: no config at all still finds metaobjects/ in the start dir", async () => { + write("metaobjects/meta.a.json", "{}"); + const c = await resolveCollection(root); + expect(c.files).toHaveLength(1); + }); + + test("a consumer reaches a tree elsewhere in the repo", async () => { + write("model/meta.a.json", "{}"); + config("apps/ui", { sources: [{ path: "../../model" }] }); + const c = await resolveCollection(join(root, "apps/ui")); + expect(c.configDir).toBe(join(root, "apps/ui")); + expect(c.files.map((f) => f.replace(root + "/", ""))).toEqual(["model/meta.a.json"]); + }); + + test("scope compiles and is applied by matchesScope", async () => { + write("model/meta.a.json", "{}"); + config("apps/ui", { sources: [{ path: "../../model" }], scope: { include: ["acme::**"] } }); + const c = await resolveCollection(join(root, "apps/ui")); + expect(matchesScope("acme::Order", c.scope)).toBe(true); + expect(matchesScope("other::Order", c.scope)).toBe(false); + }); + + test("migrateScope is undefined when not declared", async () => { + write("metaobjects/meta.a.json", "{}"); + config(".", {}); + expect((await resolveCollection(root)).migrateScope).toBeUndefined(); + }); + + test("migrateScope compiles when declared", async () => { + write("metaobjects/meta.a.json", "{}"); + config(".", { migrate: { scope: ["acme::platform::**"] } }); + const c = await resolveCollection(root); + expect(matchesScope("acme::platform::Job", c.migrateScope!)).toBe(true); + expect(matchesScope("arena::Match", c.migrateScope!)).toBe(false); + }); + + test("an explicit dir overrides discovery", async () => { + write("model/meta.a.json", "{}"); + config("apps/ui", { sources: [{ path: "../../model" }] }); + config("apps/api", { sources: [{ path: "../../model" }] }); + const c = await resolveCollection(join(root, "apps/ui"), { explicitDir: join(root, "apps/api") }); + expect(c.configDir).toBe(join(root, "apps/api")); + }); + + test("nothing discoverable and no default dir is ERR_COLLECTION_NOT_FOUND", async () => { + mkdirSync(join(root, "apps/ui"), { recursive: true }); + await expect(resolveCollection(join(root, "apps/ui"))).rejects.toThrow( + /ERR_COLLECTION_NOT_FOUND/, + ); + }); +}); +``` + +- [ ] **Step 2: Run to verify it fails** + +Run: `cd server/typescript && bun test packages/sdk/test/collection.test.ts` +Expected: FAIL — `Cannot find module '../src/collection.js'` + +- [ ] **Step 3: Write minimal implementation** + +```ts +// server/typescript/packages/sdk/src/collection.ts +import { stat } from "node:fs/promises"; +import { join, resolve } from "node:path"; +import { loadConfig } from "./config.js"; +import { findConfigDir } from "./discovery.js"; +import { compileScope, type CompiledScope } from "./scope.js"; +import { DEFAULT_METADATA_DIR, DEFAULT_METAOBJECTS_DIR } from "./memory.js"; +import { DEFAULT_SOURCES, resolveSources, type ResolvedSource, type SourceSpec } from "./sources.js"; + +export interface Collection { + /** Directory whose config declared this collection. */ + readonly configDir: string; + /** Canonically-sorted absolute metadata file paths. */ + readonly files: readonly string[]; + /** Same set, carrying the contributing spec for provenance. */ + readonly sources: readonly ResolvedSource[]; + /** Output filter for codegen. Empty include => everything. */ + readonly scope: CompiledScope; + /** Output filter for migrate/verify --db. Undefined => the command governs everything in scope. */ + readonly migrateScope: CompiledScope | undefined; +} + +async function isDir(p: string): Promise { + try { return (await stat(p)).isDirectory(); } catch { return false; } +} + +/** + * THE single authority on where metadata lives. Every read path routes through + * this — `metaobjects/` is the DEFAULT value of `sources`, never an assumption + * baked into a call site. + */ +export async function resolveCollection( + startDir: string, + opts?: { explicitDir?: string }, +): Promise { + const explicit = opts?.explicitDir; + const configDir = explicit !== undefined + ? resolve(explicit) + : (await findConfigDir(startDir)) ?? resolve(startDir); + + let specs: readonly SourceSpec[] = DEFAULT_SOURCES; + let scopeSpec = undefined as { include?: string[]; exclude?: string[] } | undefined; + let migrateSpec: string[] | undefined; + + if (await isDir(join(configDir, DEFAULT_METAOBJECTS_DIR))) { + try { + const cfg = await loadConfig(join(configDir, DEFAULT_METAOBJECTS_DIR)); + if (cfg.sources.length > 0) specs = cfg.sources; + scopeSpec = cfg.scope; + migrateSpec = cfg.migrate?.scope; + } catch { + // No config.json, or unreadable — fall through to the default source set. + // A malformed config surfaces from loadConfig on the paths that require it. + } + } + + // Only the DEFAULT is allowed to be absent — an explicitly declared source that + // does not resolve is an error (resolveSources throws ERR_SOURCE_UNRESOLVED). + if (specs === DEFAULT_SOURCES && !(await isDir(join(configDir, DEFAULT_METADATA_DIR)))) { + throw new Error( + `ERR_COLLECTION_NOT_FOUND: no metadata sources declared in ${configDir} and no ` + + `default "${DEFAULT_METADATA_DIR}" directory found. Declare "sources" in ` + + `${DEFAULT_METAOBJECTS_DIR}/config.json, or run 'meta init' to scaffold.`, + ); + } + + const sources = await resolveSources(configDir, specs); + return { + configDir, + files: sources.map((s) => s.file), + sources, + scope: compileScope(scopeSpec ?? {}), + migrateScope: migrateSpec === undefined ? undefined : compileScope({ include: migrateSpec }), + }; +} +``` + +- [ ] **Step 4: Run to verify it passes** + +Run: `cd server/typescript && bun test packages/sdk/test/collection.test.ts` +Expected: PASS — 8 tests + +- [ ] **Step 5: Export the surface** + +Add to `packages/sdk/src/index.ts`: `resolveCollection`, `type Collection` from `./collection.js`; `compileScope`, `matchesScope`, `type Scope`, `type CompiledScope` from `./scope.js`; `resolveSources`, `DEFAULT_SOURCES`, `type SourceSpec`, `type ResolvedSource` from `./sources.js`; `findConfigDir` from `./discovery.js`. + +- [ ] **Step 6: Typecheck and commit** + +```bash +cd server/typescript && bun run --filter '@metaobjectsdev/sdk' typecheck +git add server/typescript/packages/sdk/src/collection.ts server/typescript/packages/sdk/src/index.ts server/typescript/packages/sdk/test/collection.test.ts +git commit -m "feat(sdk): resolveCollection() — one authority for where metadata lives" +``` + +--- + +## Task 8: Order-independence gate (the linchpin) + +Without this, set semantics is a belief that decays the first time someone writes an order-sensitive code path. + +**Files:** +- Test: `server/typescript/packages/sdk/test/order-independence.test.ts` + +**Interfaces:** +- Consumes: `resolveSources` (T4), `loadMemory` (existing), the canonical serializer from `@metaobjectsdev/metadata` + +- [ ] **Step 1: Write the test** + +```ts +// server/typescript/packages/sdk/test/order-independence.test.ts +import { describe, test, expect, beforeEach, afterEach } from "bun:test"; +import { mkdtempSync, rmSync, mkdirSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { resolveSources, type SourceSpec } from "../src/sources.js"; + +let root: string; +const write = (rel: string, body: object) => { + mkdirSync(join(root, rel, ".."), { recursive: true }); + writeFileSync(join(root, rel), JSON.stringify(body), "utf8"); +}; + +beforeEach(() => { + root = mkdtempSync(join(tmpdir(), "metaobjects-order-")); + // A base declaration, an overlay onto it, and an independent third file — + // the shapes whose merge is order-sensitive if anything is. + write("a/meta.base.json", { + "metadata.root": { package: "acme", children: [ + { "object.entity": { name: "Order", children: [{ "field.string": { name: "id" } }] } }] }, + }); + write("b/meta.overlay.json", { + "metadata.root": { package: "acme", children: [ + { "object.entity": { name: "Order", overlay: true, children: [ + { "field.string": { name: "note" } }] } }] }, + }); + write("c/meta.other.json", { + "metadata.root": { package: "acme", children: [ + { "object.entity": { name: "Customer", children: [{ "field.string": { name: "id" } }] } }] }, + }); +}); +afterEach(() => { rmSync(root, { recursive: true, force: true }); }); + +function permutations(items: T[]): T[][] { + if (items.length <= 1) return [items]; + const out: T[][] = []; + for (let i = 0; i < items.length; i++) { + const rest = [...items.slice(0, i), ...items.slice(i + 1)]; + for (const p of permutations(rest)) out.push([items[i]!, ...p]); + } + return out; +} + +describe("order independence", () => { + test("resolveSources output is identical across every spec permutation", async () => { + const specs: SourceSpec[] = [{ path: "a" }, { path: "b" }, { path: "c" }]; + const results = await Promise.all( + permutations(specs).map((p) => resolveSources(root, p).then((r) => r.map((x) => x.file))), + ); + expect(results).toHaveLength(6); + for (const r of results) expect(r).toEqual(results[0]!); + }); + + test("the loaded model serializes byte-identically across every permutation", async () => { + const { MetaDataLoader, composeRegistry, coreProviders, serializeCanonical } = + await import("@metaobjectsdev/metadata"); + const { FileSource } = await import("@metaobjectsdev/metadata/core"); + const specs: SourceSpec[] = [{ path: "a" }, { path: "b" }, { path: "c" }]; + + const serialized: string[] = []; + for (const p of permutations(specs)) { + const resolved = await resolveSources(root, p); + const loader = new MetaDataLoader({ registry: composeRegistry(coreProviders) }); + const result = await loader.load(resolved.map((r) => new FileSource(r.file))); + expect(result.errors).toHaveLength(0); + serialized.push(serializeCanonical(result.root)); + } + expect(serialized).toHaveLength(6); + for (const s of serialized) expect(s).toBe(serialized[0]!); + }); +}); +``` + +- [ ] **Step 2: Run it** + +Run: `cd server/typescript && bun test packages/sdk/test/order-independence.test.ts` +Expected: PASS. + +**If the second test FAILS, stop and report before changing anything.** A failure means the loader is not in fact order-independent for this shape, which invalidates a load-bearing premise of the design — that is a finding to escalate, not a test to adjust. + +**Note for the implementer:** confirm the exact export name of the canonical serializer (`serializeCanonical` above is the expected name) by grepping `packages/metadata/src/index.ts`; use the real export and adjust the import. + +- [ ] **Step 3: Prove the gate by breaking it** + +Temporarily remove the `.sort()` from `resolveSources` in `sources.ts`, re-run, and confirm the first test FAILS. Revert. + +Expected: FAIL before revert, PASS after. + +- [ ] **Step 4: Commit** + +```bash +git add server/typescript/packages/sdk/test/order-independence.test.ts +git commit -m "test(sdk): pin order independence — permuted source sets serialize byte-identically" +``` + +--- + +## Task 9: Route `loadMemory` through a resolved collection + +**Files:** +- Modify: `server/typescript/packages/sdk/src/memory.ts:105,122-143` +- Test: `server/typescript/packages/sdk/test/memory.test.ts` + +**Interfaces:** +- Consumes: `Collection` (T7) +- Produces: `loadMemory(repoRoot: string, options?: LoadMemoryOptions & { files?: readonly string[] })` — when `files` is supplied it loads exactly those and skips all directory discovery; behavior with `files` absent is unchanged + +- [ ] **Step 1: Write the failing test** + +Append to `packages/sdk/test/memory.test.ts`: + +```ts +describe("loadMemory with an explicit file set", () => { + test("loads exactly the supplied files, ignoring any metaobjects/ dir", async () => { + const dir = mkdtempSync(join(tmpdir(), "metaobjects-memory-files-")); + try { + mkdirSync(join(dir, "model"), { recursive: true }); + mkdirSync(join(dir, "metaobjects"), { recursive: true }); + writeFileSync(join(dir, "model/meta.a.json"), JSON.stringify({ + "metadata.root": { package: "acme", children: [ + { "object.entity": { name: "Order", children: [{ "field.string": { name: "id" } }] } }] }, + }), "utf8"); + writeFileSync(join(dir, "metaobjects/meta.decoy.json"), JSON.stringify({ + "metadata.root": { package: "acme", children: [ + { "object.entity": { name: "Decoy", children: [{ "field.string": { name: "id" } }] } }] }, + }), "utf8"); + const root = await loadMemory(dir, { files: [join(dir, "model/meta.a.json")] }); + const names = root.children().map((c) => c.name); + expect(names).toContain("Order"); + expect(names).not.toContain("Decoy"); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); +}); +``` + +- [ ] **Step 2: Run to verify it fails** + +Run: `cd server/typescript && bun test packages/sdk/test/memory.test.ts` +Expected: FAIL — `Decoy` is present, because `loadMemory` still scans `metaobjects/` + +- [ ] **Step 3: Implement** + +In `memory.ts`, add `files?: readonly string[]` to `LoadMemoryOptions`, and in `loadMemory` replace the `collectMetadataPaths(repoRoot)` call with: + +```ts +const paths = options?.files !== undefined + ? [...options.files] + : await collectMetadataPaths(repoRoot); +``` + +Leave `collectMetadataPaths` and `listMetadataFiles` untouched — they remain the no-`files` fallback and the back-compat path. + +- [ ] **Step 4: Run to verify it passes** + +Run: `cd server/typescript && bun test packages/sdk/test/memory.test.ts` +Expected: PASS — including all pre-existing tests + +- [ ] **Step 5: Commit** + +```bash +git add server/typescript/packages/sdk/src/memory.ts server/typescript/packages/sdk/test/memory.test.ts +git commit -m "feat(sdk): loadMemory accepts an explicit resolved file set" +``` + +--- + +## Task 10: Route the CLI read sites + +Five of the nine hardcoded reads. `init.ts` is deliberately excluded — it writes the default. + +**Files:** +- Modify: `server/typescript/packages/cli/src/commands/gen.ts:55-72` +- Modify: `server/typescript/packages/cli/src/commands/docs.ts:292,529-530` +- Modify: `server/typescript/packages/cli/src/commands/export.ts:19` +- Modify: `server/typescript/packages/cli/src/index.ts:275` +- Test: `server/typescript/packages/cli/test/collection-routing.test.ts` (create) + +**Interfaces:** +- Consumes: `resolveCollection` (T7), `loadMemory({ files })` (T9) + +- [ ] **Step 1: Write the failing test** + +```ts +// server/typescript/packages/cli/test/collection-routing.test.ts +import { describe, test, expect, beforeEach, afterEach } from "bun:test"; +import { mkdtempSync, rmSync, mkdirSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { genCommand } from "../src/commands/gen.js"; + +let root: string; +const write = (rel: string, body: string) => { + mkdirSync(join(root, rel, ".."), { recursive: true }); + writeFileSync(join(root, rel), body, "utf8"); +}; + +beforeEach(() => { root = mkdtempSync(join(tmpdir(), "metaobjects-cli-route-")); mkdirSync(join(root, ".git")); }); +afterEach(() => { rmSync(root, { recursive: true, force: true }); }); + +describe("gen routes metadata discovery through resolveCollection", () => { + test("generates from a sources-declared tree with no metaobjects/ present", async () => { + write("model/meta.a.json", JSON.stringify({ + "metadata.root": { package: "acme", children: [ + { "object.entity": { name: "Order", children: [ + { "field.string": { name: "id" } }, + { "identity.primary": { "@fields": ["id"] } }, + { "source.rdb": { "@table": "orders", "@kind": "table" } }] } }] }, + })); + write("apps/ui/.metaobjects/config.json", JSON.stringify({ + schema_version: 1, sources: [{ path: "../../model" }], + })); + write("apps/ui/metaobjects.config.ts", [ + 'import { defineConfig } from "@metaobjectsdev/cli";', + 'import { entityFile } from "@metaobjectsdev/codegen-ts/generators";', + 'export default defineConfig({ outDir: "./src/generated", dialect: "postgres",', + ' dbImport: "../db", generators: [entityFile()] });', + ].join("\n")); + + const code = await genCommand({ cwd: join(root, "apps/ui") } as never); + expect(code).toBe(0); + }); +}); +``` + +- [ ] **Step 2: Run to verify it fails** + +Run: `cd server/typescript && bun test packages/cli/test/collection-routing.test.ts` +Expected: FAIL — exit code 2, `no metaobjects/ found` + +**Note for the implementer:** `genCommand`'s real parameter shape is in `packages/cli/src/commands/gen.ts`; adjust the call to match it rather than the `as never` placeholder above. + +- [ ] **Step 3: Route gen.ts** + +Replace the `loadMemory(projectRoot, …)` call and its error branch: + +```ts +let collection; +try { + collection = await resolveCollection(projectRoot); +} catch (err) { + log.error((err as Error).message); + return 2; +} + +let metadata; +try { + metadata = await loadMemory(collection.configDir, { + files: collection.files, + ...(forgeConfig.providers !== undefined ? { providers: forgeConfig.providers } : {}), + }); +} catch (err) { + log.error(`failed to load metadata: ${(err as Error).message}`); + return 2; +} +``` + +The `existsSync(join(projectRoot, DEFAULT_METADATA_DIR))` hint branch is deleted — `resolveCollection` now raises `ERR_COLLECTION_NOT_FOUND` with a better message, and the comment above that branch (about not swallowing genuine ParseErrors) is satisfied by construction since the two failure modes are now separate `try` blocks. + +- [ ] **Step 4: Route the remaining four sites** + +- `export.ts:19` — replace `join(projectRoot, DEFAULT_METADATA_DIR)` with `(await resolveCollection(projectRoot)).files`, passing them to the loader. +- `docs.ts:292` — replace the `existsSync` guard with a `resolveCollection` call inside a `try`, reporting its error message. +- `docs.ts:529-530` — `sourceDirs` becomes the collection's `configDir`-relative source dirs; derive `seenBasenames` from the resolved sources rather than the literal. +- `index.ts:275` — the "is this a MetaObjects project?" probe becomes `await resolveCollection(cwd).then(() => true).catch(() => false)`. + +- [ ] **Step 5: Run the full CLI suite for back-compat** + +Run: `cd server/typescript && bun test packages/cli` +Expected: PASS — every pre-existing test unchanged. A project with `metaobjects/` at the root and no `sources` must behave exactly as before. + +- [ ] **Step 6: Run the golden-output gate** + +Run: `cd server/typescript && bun test packages/codegen-ts` +Expected: PASS — generated output byte-identical. (`codegen-ts/test/golden/` lives outside the package under change and is the gate that catches accidental output drift.) + +- [ ] **Step 7: Commit** + +```bash +git add server/typescript/packages/cli/src server/typescript/packages/cli/test/collection-routing.test.ts +git commit -m "feat(cli): route gen/docs/export and the project probe through resolveCollection" +``` + +--- + +## Task 11: `detect-stack` routing and the nested-symlink fix + +Closes the divergence where `detect-stack` and the loader disagree about the same tree. + +**Files:** +- Modify: `server/typescript/packages/cli/src/lib/detect-stack.ts:24,31-69` +- Test: `server/typescript/packages/cli/test/detect-stack.test.ts` (extend if present; create if not) + +**Interfaces:** +- Consumes: `resolveCollection` (T7) +- Produces: `resolveStack(cwd, overrides)` unchanged in signature; `hasRequirementNodes` now scans the resolved collection's files + +- [ ] **Step 1: Write the failing tests** + +```ts +import { describe, test, expect, beforeEach, afterEach } from "bun:test"; +import { mkdtempSync, rmSync, mkdirSync, writeFileSync, symlinkSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { resolveStack } from "../src/lib/detect-stack.js"; + +let root: string; +const write = (rel: string, body: string) => { + mkdirSync(join(root, rel, ".."), { recursive: true }); + writeFileSync(join(root, rel), body, "utf8"); +}; +const REQ = JSON.stringify({ + "metadata.root": { package: "acme", children: [ + { "requirement.functional": { name: "FR1", "@level": 1, "@status": "live" } }] }, +}); + +beforeEach(() => { root = mkdtempSync(join(tmpdir(), "metaobjects-detect-")); mkdirSync(join(root, ".git")); }); +afterEach(() => { rmSync(root, { recursive: true, force: true }); }); + +describe("detect-stack honours sources", () => { + test("finds requirement nodes in a sources-declared tree", async () => { + write("model/meta.req.json", REQ); + write("apps/ui/.metaobjects/config.json", JSON.stringify({ + schema_version: 1, sources: [{ path: "../../model" }], + })); + const stack = await resolveStack(join(root, "apps/ui"), { servers: [], clients: [] }); + expect(stack.concerns).toContain("requirements"); + }); + + test("finds requirement nodes behind a NESTED symlinked directory", async () => { + write("real/meta.req.json", REQ); + write("metaobjects/meta.a.json", "{}"); + symlinkSync(join(root, "real"), join(root, "metaobjects/linked"), "dir"); + const stack = await resolveStack(root, { servers: [], clients: [] }); + expect(stack.concerns).toContain("requirements"); + }); +}); +``` + +- [ ] **Step 2: Run to verify they fail** + +Run: `cd server/typescript && bun test packages/cli/test/detect-stack.test.ts` +Expected: FAIL — both. The first because `sources` is ignored; the second because `Dirent.isDirectory()` is `false` for a symlinked directory, so the walk never descends. + +- [ ] **Step 3: Implement** + +Make `resolveStack` and `probe` async. Replace `hasRequirementNodes(cwd)` with a scan over `(await resolveCollection(cwd)).files` — reading each file and testing for the `REQUIREMENT_NODE_MARKER` substring — wrapped in a `try`/`catch` that returns `false`, preserving the existing "this is a cheap heuristic, never throws" contract. Delete the `METADATA_DIR` constant and the bespoke `readdirSync` walk entirely; the symlink bug disappears with the walk, because `resolveSources` uses `stat` (which follows). + +Update `resolveStack`'s callers to `await` it. + +- [ ] **Step 4: Run to verify they pass** + +Run: `cd server/typescript && bun test packages/cli` +Expected: PASS — both new tests, and every pre-existing detect-stack and agent-context test + +- [ ] **Step 5: Commit** + +```bash +git add server/typescript/packages/cli/src/lib/detect-stack.ts server/typescript/packages/cli/test/detect-stack.test.ts +git commit -m "fix(cli): detect-stack reads the resolved collection, fixing nested-symlink blindness" +``` + +--- + +## Task 12: Per-command scope for `migrate` and `verify --db` + +Without this, load-everything converts a real adopter's worst hazard — a `--from-db` migrate proposing to drop tables it does not model — from a discipline into an automation. + +**Files:** +- Modify: `server/typescript/packages/cli/src/commands/migrate.ts:260` and the expected-schema construction +- Test: `server/typescript/packages/cli/test/migrate-scope.test.ts` (create) + +**Interfaces:** +- Consumes: `Collection.migrateScope` (T7), `matchesScope` (T1) + +- [ ] **Step 1: Write the failing test** + +```ts +// server/typescript/packages/cli/test/migrate-scope.test.ts +import { describe, test, expect } from "bun:test"; +import { compileScope, matchesScope } from "@metaobjectsdev/sdk"; +import { scopeExpectedSchema } from "../src/commands/migrate.js"; + +describe("migrate scope", () => { + test("objects outside migrateScope are excluded from the expected schema", () => { + const expected = { + tables: [ + { name: "jobs", fqn: "acme::platform::Job" }, + { name: "matches", fqn: "arena::Match" }, + ], + views: [], + }; + const scoped = scopeExpectedSchema(expected as never, compileScope({ include: ["acme::platform::**"] })); + expect(scoped.tables.map((t) => t.name)).toEqual(["jobs"]); + }); + + test("an undefined scope leaves the expected schema untouched", () => { + const expected = { tables: [{ name: "jobs", fqn: "acme::platform::Job" }], views: [] }; + expect(scopeExpectedSchema(expected as never, undefined)).toEqual(expected as never); + }); + + test("matchesScope drives the decision (no second pattern implementation)", () => { + const c = compileScope({ include: ["acme::platform::**"] }); + expect(matchesScope("acme::platform::Job", c)).toBe(true); + expect(matchesScope("arena::Match", c)).toBe(false); + }); +}); +``` + +- [ ] **Step 2: Run to verify it fails** + +Run: `cd server/typescript && bun test packages/cli/test/migrate-scope.test.ts` +Expected: FAIL — `scopeExpectedSchema` is not exported + +- [ ] **Step 3: Implement** + +Export from `migrate.ts`: + +```ts +/** + * Narrow an expected schema to the objects inside `scope`. Tables and views whose + * declaring object falls outside are dropped BEFORE the diff, so the migration + * neither creates nor drops them — they belong to another owner. + */ +export function scopeExpectedSchema( + expected: ExpectedSchema, + scope: CompiledScope | undefined, +): ExpectedSchema { + if (scope === undefined) return expected; + return { + ...expected, + tables: expected.tables.filter((t) => matchesScope(t.fqn, scope)), + views: expected.views.filter((v) => matchesScope(v.fqn, scope)), + }; +} +``` + +Call it on the expected schema immediately before `diff()`, passing `collection.migrateScope`. + +**Note for the implementer:** confirm the real `ExpectedSchema` shape and whether its table/view entries already carry the declaring object's FQN. If they do not, thread it through at construction — do **not** re-derive an FQN from the SQL name, which is lossy. + +- [ ] **Step 4: Run to verify it passes** + +Run: `cd server/typescript && bun test packages/cli/test/migrate-scope.test.ts` +Expected: PASS + +- [ ] **Step 5: Run the migrate suites** + +Run: `cd server/typescript && bun test packages/migrate-ts && bun test packages/cli/test` +Expected: PASS. **A project with no `migrate.scope` must emit byte-identical migrations** — that is the back-compat guarantee for this task. + +- [ ] **Step 6: Commit** + +```bash +git add server/typescript/packages/cli/src/commands/migrate.ts server/typescript/packages/cli/test/migrate-scope.test.ts +git commit -m "feat(cli): migrate.scope narrows the expected schema so unowned tables are never touched" +``` + +--- + +## Task 13: Dogfood against the in-repo examples tree + +Proves reach and scope against a real metadata tree with zero new content. + +**Files:** +- Test: `server/typescript/packages/sdk/test/dogfood-examples.test.ts` (create) + +**Interfaces:** +- Consumes: `resolveCollection` (T7), `matchesScope` (T1) + +- [ ] **Step 1: Inspect the tree and read its declared package** + +Run: `ls examples/advanced-modeling/metaobjects && head -5 examples/advanced-modeling/metaobjects/meta.catalog.yaml` + +Record the actual `package:` value — the test below must assert against the real package, not a guess. + +- [ ] **Step 2: Write the test** + +```ts +// server/typescript/packages/sdk/test/dogfood-examples.test.ts +import { describe, test, expect, beforeEach, afterEach } from "bun:test"; +import { mkdtempSync, rmSync, mkdirSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import { resolveCollection } from "../src/collection.js"; +import { loadMemory } from "../src/memory.js"; + +const EXAMPLES = resolve(import.meta.dir, "../../../../../examples/advanced-modeling/metaobjects"); + +let consumer: string; +beforeEach(() => { + consumer = mkdtempSync(join(tmpdir(), "metaobjects-dogfood-")); + mkdirSync(join(consumer, ".git")); + mkdirSync(join(consumer, "apps/ui/.metaobjects"), { recursive: true }); + writeFileSync( + join(consumer, "apps/ui/.metaobjects/config.json"), + JSON.stringify({ schema_version: 1, sources: [{ path: EXAMPLES }] }), + "utf8", + ); +}); +afterEach(() => { rmSync(consumer, { recursive: true, force: true }); }); + +describe("dogfood: a consumer reaches the in-repo examples tree", () => { + test("resolves every metadata file in it", async () => { + const c = await resolveCollection(join(consumer, "apps/ui")); + expect(c.files.length).toBeGreaterThanOrEqual(3); + expect(c.files.every((f) => f.startsWith(EXAMPLES))).toBe(true); + }); + + test("the resolved set loads without errors", async () => { + const c = await resolveCollection(join(consumer, "apps/ui")); + const root = await loadMemory(c.configDir, { files: c.files }); + expect(root.children().length).toBeGreaterThan(0); + }); +}); +``` + +- [ ] **Step 3: Run it** + +Run: `cd server/typescript && bun test packages/sdk/test/dogfood-examples.test.ts` +Expected: PASS + +- [ ] **Step 4: Commit** + +```bash +git add server/typescript/packages/sdk/test/dogfood-examples.test.ts +git commit -m "test(sdk): dogfood reach+scope against the in-repo examples metadata tree" +``` + +--- + +## Task 14: Documentation + +**Files:** +- Modify: `server/typescript/packages/cli/README.md` +- Modify: `CLAUDE.md` (the "File organization" and "Other conventions" sections) +- Create: `docs/features/metadata-sources.md` + +- [ ] **Step 1: Write the adopter guide** + +`docs/features/metadata-sources.md` covering: `sources` as a set with `metaobjects/` as its default; the `path` source kind (relative to the declaring config, read in place, never installed); `scope` with `*`/`**` semantics and include/exclude; nearest-ancestor discovery and the `.git` boundary; `migrate.scope` and the rule that a `migrate` block belongs where the ledger lives; and a **vendoring** section stating that airgapped builds are served by copying a dependency into a directory and pointing a `path` at it — no separate mechanism needed. + +Include one worked polyglot example (generic names only — no real project names). + +- [ ] **Step 2: Update the CLI README** + +Document `sources`, `scope`, and `migrate.scope` in the config reference, next to the existing `targets` documentation. + +- [ ] **Step 3: Update CLAUDE.md** + +In "File organization", state that `metaobjects/` is the **default** value of `sources` and never a requirement. In "Other conventions", add one line: metadata location is resolved via `resolveCollection()`; no code path may hardcode the directory name except `meta init`, which scaffolds it. + +- [ ] **Step 4: Leak scan and commit** + +```bash +grep -rniE "party|/home/" docs/features/metadata-sources.md && echo LEAK || echo clean +git add docs/features/metadata-sources.md server/typescript/packages/cli/README.md CLAUDE.md +git commit -m "docs: metadata sources, scope, discovery, and the vendoring workflow" +``` + +--- + +## Task 15: Full-suite verification + +- [ ] **Step 1: Build the workspace** + +Run, from the repository root: `bun run --filter '*' build` +Expected: success + +- [ ] **Step 2: Typecheck the workspace** + +Run: `bun run --filter '*' typecheck` +Expected: no errors. (`bun test` transpiles per-file and does not typecheck, so this is the gate that catches type breakage.) + +- [ ] **Step 3: Run the server suite** + +Run: `cd server/typescript && bun test` +Expected: PASS + +- [ ] **Step 4: Run the client suites** + +Run each `client/web/packages/` suite. +Expected: PASS + +- [ ] **Step 5: Confirm no hardcoded reads remain** + +Run: `git grep -n "DEFAULT_METADATA_DIR" -- 'server/typescript/packages/cli/src/**' 'server/typescript/packages/sdk/src/**'` +Expected: hits only in `memory.ts` (the constant's definition plus the no-`files` fallback), `sources.ts` (`DEFAULT_SOURCES`), `collection.ts` (the default check), and `init.ts` (scaffolding). **Any hit in `docs.ts`, `export.ts`, `gen.ts`, `index.ts`, or `detect-stack.ts` is an unfinished task.** + +- [ ] **Step 6: Commit any fixes and push** + +```bash +git add -- # never `git add -A` +git commit -m "chore: phase-1 source resolution full-suite verification" +``` + +--- + +## Self-Review Notes + +**Spec coverage.** §4.1 set semantics → T4, T8. §4.2 source kinds → T4, T5. §4.3 scope at output → T1, T2. §4.4 scope attachment incl. per-command → T12. §4.5 precedence → *not implemented in phase 1*: local-vs-dependency precedence only becomes reachable once `package` sources exist, and phase 1 rejects them (T4). §4.6/4.6.0 one authority → T7, T10, T11. §4.6.1 discovery → T6. §4.6.2 schema ownership → T12 (the `migrate.scope` half; the "ledger marks the owner" rule is documentation, T14). §4.7 conformance → T2, T8. §4.9 naming → T5. §8 symlink divergence → T11. §8 dogfood → T13. + +**Deliberately deferred to the ports plan:** C#, Java, Kotlin and Python implementations; the Python `metadata:`-string → set widening; Java's `scope` element alongside legacy ``; the four error codes in `errors.py` and `ErrorCode.java`; port runners for the scope-conformance corpus. + +**Deliberately out of scope:** the first-party shared metadata collection (separate deliverable); named `collection` references (§6); `url` sources; everything in §10 (issues #299–#306). + +**Three places the implementer must verify against real code rather than trusting this plan:** the canonical serializer's export name (T8 Step 2), `genCommand`'s parameter shape (T10 Step 2), and whether `ExpectedSchema` entries carry a declaring FQN (T12 Step 3). Each is flagged inline. diff --git a/docs/superpowers/specs/2026-08-17-metadata-source-resolution-design.md b/docs/superpowers/specs/2026-08-17-metadata-source-resolution-design.md new file mode 100644 index 000000000..b0f629ccb --- /dev/null +++ b/docs/superpowers/specs/2026-08-17-metadata-source-resolution-design.md @@ -0,0 +1,535 @@ +# Metadata source resolution — collections, scope, and discovery (design) + +_Status: PROPOSED (awaiting review; nothing implemented)._ +_Date: 2026-08-17._ +_Prior art: `2026-08-17-metadata-source-resolution-prior-art.md`._ +_Supersedes the scope of `2026-06-11-fr-023-metadata-packages-design.md`, which becomes one +resolver inside this design rather than the whole feature._ + +## 1. Problem + +Four of five ports resolve metadata as exactly one directory per invocation. The Java port is the +exception: its Maven loader config takes an ordered list of sources, each expressible as a file, +URL, or classpath resource. Nothing in the standard ever absorbed that, and nothing in any port +lets a consumer say "the model lives over there, and I want this part of it." + +Two adopter shapes are blocked on it today, and both are polyglot monorepos. + +**A polyglot Maven + TypeScript repo.** One authored metadata tree lives in a dedicated Maven +module's `src/main/resources/metadata/`. Four Java modules consume it, each through a +hand-enumerated list of individual files in its `pom.xml` — 91, 63, 1 and 3 entries respectively. +Two TypeScript apps consume the same tree and cannot enumerate anything, because the Node CLI only +ever looks at `/metaobjects`; they reach it by **directory symlink**. Four metadata files +exist on disk that the largest list omits, and nothing distinguishes an intentional omission from +a forgotten one. + +**A two-rail repo over one database.** A Java rail and a TypeScript rail, two metadata homes, one +Postgres. Its own architecture memo records the blocker precisely: *there is no cross-project +include mechanism, so two directories sharing a common package requires symlinks or copy-sync — a +new drift class.* The chosen workaround is to consolidate into one home; the price paid in the +meantime is entities modeled twice with hand-maintained "keep in lockstep" comments, and a staging +script that copies a subset into a temporary directory to fake what the toolchain will not do. + +A third adopter — a single-app TypeScript repo on the current release — is the control. It needs +nothing, and must keep needing nothing. + +## 2. The reframing + +The obvious reading is that these repos need to *compose* several metadata collections. They do +not. **Both already have exactly one authored model.** What they cannot do is: + +1. **Reach it** — point a consumer at a tree that lives elsewhere in the repo. +2. **Scope it** — take the part of it this consumer cares about. + +Composition of multiple collections is a real need, but it is the *third* one. Building it first +(as FR-023 does) leaves both repos exactly as blocked as they are now. + +## 3. The load-order finding, and what it deletes + +The Java design assumed sources must be read in a correct sequence. That assumption no longer +holds anywhere in the codebase, and verifying it was the single largest simplification in this +design. + +- **Super-resolution is order-independent** — a pure function of the source set (#188). +- **The loader already discards the caller's order.** `MetaDataLoader._partitionOverlayLast` reads + every source, classifies each as base or overlay-only, and reorders them, with the comment + *"making the merge order-independent."* The ordered-list API takes an order it does not trust. + +Order still matters in exactly three places, and only one concerns the source list: + +| Where | Order-sensitive? | Concerns the source list? | +|---|---|---| +| Child order **within** a node — M:N reference direction, stored-proc argument binding, payload field order | Yes, load-bearing | **No.** Set by each file's own arrays at parse time. Untouched by this design. | +| Overlay precedence **across** sources | Yes | Yes — and the loader already *derives* it rather than trusting the caller. | +| Output determinism | Yes | Satisfied by a canonical sort over resolved source ids. Declared order not required. | + +**Therefore the declared source order carries no information the loader needs. Sources are a set.** + +What this deletes, rather than adds: + +- No ordered-list semantics to specify, document, or port. +- No topological sort of dependencies, and **no cycle-detection error class** — a cycle in a set + union is not a thing. (`resolveExtendsOrder` in `sdk/src/workspace.ts` exists solely for this.) +- No diamond-dependency problem: two sources both contributing a shared model dedupe by identity. +- No serialization constraint on resolution: sources can be resolved concurrently, because nothing + needs to know its position before it resolves. + +It also matters for work that is explicitly *not* in this design. Runtime metadata sources — a +database, a registry — cannot guarantee row or stream order. And because resolution is a pure +function of the set, a set can be **added to and re-resolved**, which is the only coherent basis +for incremental or hot-reloading runtime metadata. Under ordered lists, "where in the sequence does +the new thing go?" has no good answer. This design banks that property without building on it. + +## 4. Design + +### 4.1 A collection is a set of sources + +A **metadata collection** is a named set of sources plus the metadata loaded from them. It is a +**tooling-config concept and never metamodel vocabulary** — every project surveyed in the prior art +keeps "what is a collection and where does it live" outside the modeled types. Consequences: +`registry-conformance` is unaffected, ADR-0023 provenance is unaffected, and no `expected-registry` +entry changes. + +Sources are declared as a JSON/YAML array for ergonomic reasons, but the array is **specified as a +set**: position is not meaningful, and a conformance gate enforces it (§4.7). + +### 4.2 Source kinds — a tagged union + +Following the prior art's tagged-union grammar rather than a prefix-disambiguated single string. +The single-string form (local paths must begin with `./`, git needs a `git::` prefix) buys terseness +at the cost of a grammar the parser owns and ambiguity as a live failure mode. A tagged union is +self-documenting, machine-checkable, and extensible without touching a parser — consistent with +ADR-0037's stated bias toward self-documentation over economy. + +```jsonc +"sources": [ + { "path": "../model-module/src/main/resources/metadata" }, // phase 1 + { "resource": "acme/model" }, // phase 1, JVM only + { "package": "@acme/common-model" }, // phase 2 (FR-023) + { "url": "https://…/model.tar.gz" }, // phase 3 + { "collection": "model" } // phase 3 (§4.8) +] +``` + +- **`path`** — a directory or file, relative to the declaring config file (never to ambient cwd — + the `--cwd` bug class in 0.20.1 is the precedent). Read **in place, never installed**, matching + every surveyed tool's treatment of local paths. +- **`resource`** — a classpath resource root. JVM-only; the Java port already implements it as + `model:resource:` and this design keeps that mechanism, only re-spelling its declaration. Other + ports reject it with a clear "not supported on this port" error rather than silently ignoring it. +- **`package` / `url` / `collection`** — later phases; the union shape is fixed now so they slot in + without a config migration. + +A source that does not resolve is an **error**, never a silent skip. Silent skip is how the +symlink-and-hope status quo fails today. + +**Vendoring falls out for free, and should be documented as a supported workflow.** Airgapped and +audit-constrained builds need the Go `go mod vendor` pattern — resolved dependencies committed into +the repo for hermetic builds. Because a `path` source is read in place and never installed, vendoring +is simply "copy the dependency into a directory and point a `path` at it." No mechanism is required; +what is required is saying so in the docs, rather than leaving enterprise adopters to discover it. + +### 4.3 Scope — package patterns, at output only + +**The collection loads in full. Scope applies to output, never to input.** + +Input-side subsetting is not merely tedious, it is wrong by construction: a partial file list can +fail to load because an `extends` target is missing, so the author must hand-maintain a transitive +closure. That is precisely what the 91-entry list is, and precisely why four files on disk sit in +an unresolvable "deliberate or forgotten?" state. Loading the whole collection is closure-complete +by definition. + +Scope is declared as **package patterns**: + +```jsonc +"scope": { + "include": ["acme::commerce::**", "acme::common::*"], + "exclude": ["acme::commerce::internal::**"] +} +``` + +**Semantics.** Patterns match a node's fully-qualified name. `*` matches exactly one package +segment; `**` matches any depth. Absent `include` means everything; `exclude` is applied after +`include`. An unparseable pattern is an error, not a non-match. + +**Two deliberate deviations from the shipped Java spelling**, both meeting the "only for good +reasons" bar: + +1. **Explicit `include`/`exclude` arrays instead of a single list with a `!` prefix.** The `!` + sigil is fatal in the YAML authoring front-end (ADR-0006): a leading `!` is YAML's tag + indicator, so every exclude would require quoting forever. That is a footgun, not a preference. +2. **`*` is one segment, `**` is any depth**, replacing Java's `*`-crosses-everything plus an `@` + escape for single-segment matching. The current behavior carries its own TODO in + `GeneratorUtil.createRegexFromGlob` admitting `::` is not enforced as a separator. Porting a + known bug to four more languages is worse than fixing it in one. + +Java's existing `` element keeps its current semantics unchanged; the new `scope` element +is a distinct key. No shipped pom breaks. + +**Why package patterns and not a predicate function.** TypeScript's per-generator `filter` is a +JavaScript function. It cannot be written in Python's `metaobjects.config.yaml`, C#'s CLI flags, or +a `pom.xml`, and it cannot be gated by any conformance corpus. Package patterns are strings and +port to all five config surfaces unchanged. The function filter is therefore **retained as-is, +unchanged, TS-only, documented as an escape hatch — and never the thing a cross-port feature +depends on.** Nothing is deprecated and no adopter migrates. + +Field evidence supports the demotion. The one adopter on a current release uses **zero** function +filters across nine generators. The adopter that uses three is seven minor versions behind, and its +own config comments describe exactly the defects fixed centrally in 0.21.5 (#248) — write forms +emitted for projections lacking an insert schema, hooks emitted for `object.value`, CRUD code +referencing exports abstract entities do not have. A user writing `filter: (e) => !e.isAbstract` is +compensating for a generator that should already know, which is a library bug report rather than a +config feature. Kind and shape predicates belong in the generator's central guards, where #248 put +them. + +### 4.4 Where scope attaches, including the DB-facing commands + +| Attachment | Applies to | Notes | +|---|---|---| +| Collection-level `scope` | Everything the consumer emits | A default for the consumer | +| Per-generator `scope` | That generator's output | Narrows the collection default | +| **Per-command `scope`** on `migrate` / `verify --db` | Which tables/views the command governs | **Required in phase 1** | + +Narrowing only: a generator or command scope **intersects** the collection scope and can never +widen it. Predictable, and it makes the collection-level declaration a real ceiling. + +The command-level scope is not a convenience. Generator scope does not reach `migrate` or +`verify --db`, where the **loaded model is the scope** — so "load everything" would otherwise take +a real adopter's worst standing hazard (a `--from-db` migrate proposing to drop tables it does not +model) and convert it from a discipline someone can follow into an automation nobody can. That +adopter already states the rule in prose in its own memo — *migrate owns one package tree's tables; +another tool owns the other's* — which is already a package pattern. This makes it declarative and +checkable: + +```jsonc +"migrate": { "scope": ["acme::platform::**"] } +``` + +Tables outside the scope are neither created nor dropped, and `verify --db` reports them as +out-of-scope rather than as drift. + +### 4.5 Precedence — a rule, not a position + +With order gone, overlay conflicts need a declared rule. Three cases, exhaustive: + +1. **Within one source set, base vs overlay-only** — unchanged. The loader's existing derived + partition (base first, overlay-only last) already handles it. +2. **Local vs dependency** — a `path` (or `resource`) source **wins** over a `package` source. This + is FR-023's "local overlays win" intent, expressed as a property of the source kind instead of a + position in a list. +3. **Two dependencies conflicting** — an **error**, not silent last-wins. Two independently + versioned packages declaring the same node non-overlay is a genuine ambiguity, and resolving it + by whichever happened to resolve first is the class of bug this whole design exists to remove. + +### 4.6 Where the declaration lives — one port-neutral file, five CLIs + +A polyglot repo breaks any design that treats the per-port config files as interchangeable +discovery targets. A Java consumer's configuration is a `pom.xml`; `migrate` and `verify --db` are +**Node-CLI-only** (ADR-0015). So the Node CLI must operate on a model declared by a Maven module, +and would find nothing if it looked only for `metaobjects.config.ts`. + +Two different questions are being conflated, and they already have two different homes: + +| Question | Home | Read by | +|---|---|---| +| Where does metadata come from, and what is in scope? | **`.metaobjects/config.json`** (port-neutral JSON) | **all five CLIs** | +| How is code generated here? | `metaobjects.config.ts` / `metaobjects.config.yaml` / pom `` | that port only | + +This is not a new split — it is the one CLAUDE.md already documents ("`.metaobjects/config.json` +(JSON) — static project state. Parseable by non-TS tooling"). It is also already scaffolded: every +`meta init` project carries `"sources": []` in that file today, empty and inert. Phase 1 fills the +slot that already exists. + +**One gap must close in phase 1.** A JVM-rooted adopter has no `.metaobjects/config.json` at all — +only agent-context files, because it was scaffolded with `agent-docs` rather than `meta init`. The +file is therefore port-neutral in theory and TS-scaffolded in practice. **Every port's CLI must be +able to create and read it**, or the neutral file is neutral in name only. + +### 4.6.0 One authority, and `metaobjects/` is only a default + +**No adopter ever needs a directory named `metaobjects/`.** It is the default value of `sources` +when the key is absent or empty — never a requirement, and never assumed by any code path. + +The rule: **`sources` is the single authority on where metadata lives, and everything that needs to +find metadata reads it.** Today that is false in TypeScript in nine places, which is the concrete +phase-1 work item: + +| Site | Kind | Phase 1 | +|---|---|---| +| `cli/commands/docs.ts` (×3), `export.ts`, `gen.ts` | read | route through resolved `sources` | +| `cli/index.ts` — the "is this a MetaObjects project?" probe | read | route | +| `cli/lib/detect-stack.ts` — concern detection | read | route | +| `sdk/memory.ts` (×2) — the loader entry itself | read | route | +| `cli/commands/init.ts` (×2) | **write** | **keep the literal** — scaffolding the default is the one place it belongs | + +**Python is already the reference implementation of this shape**, not a laggard: its project config +reads `metadata` from the config file with the directory name as a *fallback* +(`raw.get("metadata", DEFAULT_METADATA_DIR)`). It needs widening from one string to a set, not +rearchitecting. C# takes the directory as a positional argument, which is configurable by a +different route. **TypeScript is the outlier that hardcodes.** + +`detect-stack.ts` is the load-bearing one and the least obvious. It scans for `requirement.` +markers to derive concern tokens for agent-context scaffolding; a project pointing `sources` +elsewhere gets a **silent false**, scaffolding the wrong agent docs with no error. That is the same +"two code paths disagree about where metadata is" failure as the nested-symlink divergence in §8, +from the same root cause — so routing every read through one authority closes both. + +### 4.6.1 Discovery — nearest ancestor, explicit override, no auto-discovery + +Running a CLI inside an app must find that app's configuration. + +- **Walk up from cwd** for the nearest `.metaobjects/config.json` declaring a non-empty `sources`. + Nearest wins. Per-port generator config is then read from that same directory. +- **Stop at a repository boundary** (`.git`) or the filesystem root, so a monorepo can never + silently adopt a parent checkout's configuration. +- **Explicit override wins** — the existing `--cwd` / `-C` flag and project-root positional are + unchanged and take precedence over discovery. +- **Collections are never auto-discovered.** No globbing for directories that look like metadata + homes. A collection exists only where a config names one — Go's stance rather than Cargo's, + because a polyglot repo has many directories that merely *look* like collections, and silent + membership is the hardest failure to debug. + +For Java and Kotlin, discovery is a non-issue for *codegen* and stays that way: the Maven reactor +already runs the plugin per module with that module's own configuration. It is emphatically not a +non-issue for the Node CLI operating on those same modules, which is what §4.6 exists to solve. + +**Two failure modes get explicit, useful errors rather than silence:** + +- **Invoked at a repo root that declares no `sources`** — error listing the consumers discovered + beneath it ("did you mean one of…"). This requires a downward scan, but **for the error message + only, never for resolution**, so the no-auto-discovery rule is preserved. +- **Invoked inside a collection** (a directory that is metadata, not a consumer of it) — a distinct + error saying so, rather than an empty load. + +**Existing single-directory projects are unaffected.** One config at the root, one implicit +`{ "path": "metaobjects" }` source, no scope — byte-identical output. + +### 4.6.2 Schema ownership is not codegen consumption + +A polyglot repo has **many codegen consumers and at most one schema owner per database.** In the +larger adopter, six consumers read one model over one Postgres; if each declared a `migrate` scope, +six partial migrations would result — worse than today. The two-rail adopter has the same problem +already and names it in its own memo as needing an explicit ownership rule. + +The marker already exists and does not need inventing: **whoever holds `.metaobjects/migrations/` +and the schema snapshot owns the schema.** So: + +- A `migrate` block (§4.4) is valid only in a consumer that holds a ledger. +- A second consumer running `migrate` against the same database is detectable through the ledger + rather than left to discipline. +- `verify --db` may run from any consumer, reporting out-of-scope tables as out-of-scope rather + than as drift. + +### 4.7 Conformance + +Two new corpora, plus one gate that is the linchpin of the whole design. + +1. **Order-independence gate (the linchpin).** The same source set is loaded in N permutations and + the canonical serialization must be **byte-identical** across all of them, in all five ports. + Without this, set semantics is an aspiration that decays the first time someone adds an + order-sensitive code path. With it, the property is enforced rather than believed. +2. **Scope-pattern corpus.** A matrix of patterns × fully-qualified names → expected match/no-match, + byte-matched across all five ports. This is what stops `*` and `**` from meaning five different + things — the failure mode that produced the `like`/`ILIKE` divergence. +3. **Discovery** is filesystem behavior and stays per-port, not corpus-gated. + +New error codes register in all three ledgers (TS `errors.ts` exact-bidirectional, Python +`errors.py` superset, Java `ErrorCode.java`): `ERR_SOURCE_UNRESOLVED`, +`ERR_SOURCE_KIND_UNSUPPORTED`, `ERR_SCOPE_PATTERN_INVALID`, `ERR_COLLECTION_NOT_FOUND`, +`ERR_DEPENDENCY_DECLARATION_CONFLICT`. + +### 4.8 What ships when + +**Phase 1 — the spine (releasable on its own; unblocks both adopters).** +`sources` with `path` and `resource`; `scope` with `include`/`exclude` and `*`/`**`; +nearest-ancestor discovery; load-everything; per-command scope for `migrate`/`verify --db`; the +order-independence and scope-pattern corpora. This alone turns 91 hand-maintained `` lines +into one path plus one pattern, and deletes the symlinks. + +**Phase 2 — package sources (FR-023, re-scoped).** The `package` resolver per ecosystem, the +package manifest, and per-package provenance attribution. Spike-validated for NuGet (§5). + +**Phase 3 — remote and named collections.** `url` sources with the pin-and-cache discipline every +surveyed tool has; `collection` references via an **optional** root file that names shared +collections and nothing else (§6). + +**Out of scope.** Database and other runtime sources. Ruled a runtime-metadata concern, not a +build-time one — consistent with every system surveyed, where reading schema from a live store +serves a running application and never a build. It gets its own FR. + +### 4.9 Naming + +The config key is **`sources`**, matching the Java loader element and the (currently dead) key +already present in `sdk/src/config.ts`. This collides by name with the `source.*` metamodel node +type, which was flagged as a concern worth recording. The collision is judged acceptable: the two +never appear in the same file — `source.rdb` is a node inside a metadata document, `sources` is a +key inside a tooling config — and the Java port has carried both for years without incident. + +The decisive argument is that **the key is already scaffolded into every project**: `meta init` +writes `"sources": []` into `.metaobjects/config.json`, and real adopter repos carry it today. The +slot exists, is empty, and is waiting; renaming it now would orphan it in every scaffolded project +for no semantic gain. If review disagrees, `metadataSources` is the alternative and costs nothing +but verbosity plus a scaffold migration. + +## 5. Spike results + +**Spike 1 — a code-free NuGet package can be resolved by a non-MSBuild CLI. Confirmed.** A real +`.nupkg` carrying a `metaobjects/` tree was packed and consumed from a `PackageReference` project. +Findings: `contentFiles` copies nothing useful and is the wrong mechanism; a `build/*.targets` file +correctly exposes an MSBuild property pointing into the extracted package; and — the result that +matters — `obj/project.assets.json` carries the package folder root, the library's relative path, +and a **complete file listing including the metadata files**, so a plain CLI resolves the tree by +joining two strings and reading JSON. Two warts, both minor: packing a code-free package emits +warning `NU5128` (suppressible), and `project.assets.json` only exists after a restore — which is +the same explicit-fetch precondition every surveyed tool has. + +This retires the main technical objection to per-ecosystem publishing. It does not settle the +question (§7). + +**Spike 2 — a root config declaring N collections and N consumers fails structurally.** The Buf +precedent does not transfer, and the reason is worth recording: Buf can put everything in one root +file because **Buf owns its entire config surface**. MetaObjects does not — the build tool does. A +TypeScript consumer's config holds executable generator wiring; a Java consumer's lives in its +`pom.xml`. A root config declaring consumers would have to duplicate or override both. What +survives is per-consumer declaration plus nearest-ancestor discovery, adding two keys to files that +already exist. + +## 6. Deferred: named collections + +Per-consumer declaration repeats the collection path once per consumer — six times in the larger +adopter. The fix is an **optional** root file that declares *only* where shared metadata lives, +never generator wiring and never output: + +```jsonc +// /.metaobjects/collections.json (optional) +{ "collections": { "model": { "sources": [{ "path": "model-module/src/main/resources/metadata" }] } } } +``` + +Consumers then write `{ "collection": "model" }`. Strictly additive, imports none of Shape A's +failure, and changes no semantics — so it can land whenever a repo actually feels the repetition +rather than on speculation. + +## 7. Open questions for review + +1. **Per-ecosystem publishing vs OCI.** FR-023 proposes the same code-free artifact in four + registries. No surveyed peer does this: CUE explicitly rejected it for OCI on polyglot grounds, + Buf built its own registry, Smithy stayed single-ecosystem. Spike 1 shows the mechanism works, + and the existing four-registry lockstep release machinery is an advantage none of those projects + had — but four artifacts of identical bytes means four version numbers, four resolvers, four + caches and four chances to drift. **This should be an ADR with an argued decision, not a default + inherited from FR-023.** It does not block phase 1. + + **The polyglot case sharpens this from a preference into a requirement.** *Within* one repo every + source is a `path`, so no registry is involved. But a shared model consumed **across** repos by a + polyglot consumer set must be reachable from each ecosystem: a `resource:` classpath source is + unreachable to the Node CLI, and an npm package is unreachable to Maven. So a cross-repo shared + model needs publication to **every ecosystem that consumes it**, or a single ecosystem-neutral + channel (OCI). "Publish to one registry and let others cope" is not an available option — which + is exactly the trade-off CUE resolved by leaving per-ecosystem registries behind. + + **A counterweight that cuts the other way, and belongs in the ADR.** Enterprises already run + internal mirrors of npm, Maven, PyPI and NuGet (Artifactory, Nexus, Azure Artifacts), with + scanning, approval and supply-chain policy already attached to them. Publishing to those four + means an adopting enterprise's **existing** infrastructure works unchanged; OCI generally + requires registering a new artifact type and new policy to go with it. This is an argument about + the *consumer's* infrastructure rather than the publisher's convenience, which is why neither + CUE's reasoning nor the initial framing of this section accounted for it. +2. **`sources` vs `metadataSources`** (§4.9). +3. **Scope on `verify --codegen`.** Drift detection compares generated output to metadata; if scope + narrows what is generated, drift must be evaluated within the same scope or every out-of-scope + file reads as drift. Believed straightforward; call it out so it is not discovered late. + +## 8. Risks and honest costs + +- **~~This repository cannot dogfood the feature.~~ REVISED — it can, and it should.** This repo is + itself the shape the design serves: five ports, a Maven reactor, ~20 TypeScript packages, Python, + C#, client packages. Once `sources` is the authority (§4.6.0), a first-party shared collection can + live **wherever makes sense for this repo** — no root `metaobjects/` required — and be consumed by + each port's integration tests via a `path` source plus a `scope`. That exercises reach, scope and + discovery across all five languages in this repo's own CI, which is exactly the phase-1 surface. + **But it does not close the risk**, because this repo's consumers are *ports*, not applications: + it proves the mechanics, never the product path (codegen into a running app against a database). + So an external smoke test against a real multi-consumer layout remains a phase-1 release gate — + demoted from the only gate to the second one. + +- **A layout question this repo has not had to answer before.** A code-free metadata package is + neither server-side nor client-side, so the "deployment target → language → framework" rule in + CLAUDE.md does not place it. Recommend a new top-level sibling — `spec/` holds the metamodel (the + language), so a `model/` would hold models expressed in it (the content). Small, but it should be + decided rather than defaulted. +- **Load-everything is O(collection), not O(scope).** Each consumer loads the whole collection even + when it emits a fraction. At current adopter sizes (~120 files) this is not measurable, but it is + a real asymptote and should be stated rather than discovered. +- **Discovery can surprise.** Walking up to find a config is the least surprising behavior in + developer tooling *and* a new way to pick the wrong file. The `.git` stop condition and the + explicit-override precedence are the mitigations; both need tests. +- **Two scoping mechanisms coexist in TypeScript** — package patterns and the retained function + filter. Documentation must be unambiguous that only the former is a cross-port concept, or the + next cross-port feature will be built on the one that cannot port. +- **Java carries two filter spellings** — legacy `` with its existing semantics, and the + new `scope`. Intentional, to avoid breaking shipped poms, but it is two things to explain. + +- **Toolchain version skew across ports is a polyglot hazard nothing currently checks.** A repo + whose consumers span five ports pins five toolchains, and two consumers loading the same + collection under different loader versions can legitimately disagree about it. The two-rail + adopter already names this in its own memo ("one home = one pin") after paying for a stale-pin + incident. The lockstep release policy makes agreement *possible* — all four registries share + `minor.patch` — but nothing enforces it inside a consuming repo. Recommend a warning-level check + once more than one consumer resolves the same collection; not a phase-1 blocker, but it should + not be discovered by an adopter. + +- **A pre-existing symlink inconsistency this work should absorb.** Surfaced while investigating + the adopter workaround, and verified empirically rather than reasoned about. A **top-level** + symlinked metadata directory *is* followed by both code paths. But for a **nested** symlinked + subdirectory the two disagree: `detect-stack.ts` walks with `readdirSync(…, {withFileTypes:true})` + and a `Dirent` for a symlinked directory reports `isDirectory() === false`, so it does not + descend — while `sdk/src/memory.ts` walks with `stat`, which follows, so the loader does. **The + same tree is one shape to the loader and a different shape to stack detection.** Phase 1 removes + the *need* for symlinks in the adopters that use them, but it does not fix this, and the + divergence outlives them. Worth folding into phase 1 rather than leaving as a latent trap. + (Note this corrects the framing in the handoff that motivated this work, which described the + top-level symlink as unfollowed.) + +## 9. Release shape + +Phase 1 is **additive** — new config keys, no change to any existing single-directory project's +output. It introduces a new capability adopters opt into deliberately, so it is a **MINOR** under +ADR-0035 Amendment 1's consumer-impact test rather than a patch: pre-1.0 caret ranges make a minor +a deliberate adoption, which is the correct gate for a change to how metadata is located. + +## 10. Adjacent capabilities — surveyed, deferred, filed + +A pass over the surveyed projects for enterprise capabilities MetaObjects lacks. **None is required +for phase 1**, and each was checked against the codebase before being called a gap. Filed so they +are not re-derived: + +| # | Capability | Why not phase 1 | +|---|---|---| +| #299 | **Producer-side access control** (`private`/`protected`/`public`, per dbt; `@internal` + transform, per Smithy). We have consumer-side scope only — everything in a collection is visible to everyone who loads it. | Metamodel vocabulary (MINOR); phase 1 is config-only. Becomes load-bearing when *cross-team* sharing starts, i.e. with package sources. Phase 1 must not preclude it. | +| #300 | **Breaking-change detection** against a baseline revision. Highest enterprise value in this set — it is the difference between a shared model that can evolve and one nobody dares touch. | Downstream of shared collections existing. Note we already have two-thirds: `migrate` gates schema-breaking changes behind `--allow` tokens, `verify --codegen` covers code drift; what is missing is metadata-vs-metadata across revisions. | +| #301 | **Dependency override** (Go's `replace`) for running a patched shared model. | A `path` source *is* an override while only `path` exists. Needed once `package` lands. | +| #302 | **Severity levels + suppressions** (per Smithy). | Not required by phase 1's new errors — but note this gap has already forced two design compromises (object coverage shipped as a warning because it would convict a project's first `verify`; `@verifiedBy` had to warn rather than convict on an unrecognised convention). | +| #303 | **Ownership metadata** (dbt groups carry owners). | May be adequately served by the `attr.properties` bag; run ADR-0037 before adding vocabulary. | +| #304 | **`meta fmt`** — the canonical serializer already exists in all five ports and is not exposed as a command. | Cheap, but orthogonal. | +| #305 | **Enforce `@deprecated`** — registered in all five ports, read by nothing. | Orthogonal; improves markedly once #302 lands. | +| #306 | **`meta why `** — per-node source provenance query. | Attribution is *already* tracked and surfaced in loader diagnostics, so phase 1's "which source did this come from?" need is met by error messages. A query is convenience on existing data. | + +**Deliberately not taken**, to prevent later scope creep: dbt's cross-project references depend on a +stateful metadata service (we should not build a service); Terraform's state model does not apply; +CUE's unification is a different language paradigm; and Buf's own registry is precisely what riding +existing ecosystems avoids. + +## 11. What gets deleted + +- `resolveExtendsOrder`'s topological sort and its cycle-detection error path + (`sdk/src/workspace.ts`) — meaningless under set union. +- The `package.meta.json` workspace mechanism — structurally JS-only (it recognizes a workspace + root solely by `pnpm-workspace.yaml` or `package.json` workspaces, so a Java + Python repo + silently falls through to the single hardcoded directory). It cannot be grown into this. + **Removal is not free:** the file is scaffolded by `meta init` and present in real adopter repos, + though always as an inert `{name, version, extends: []}` with an empty `extends`. Since nothing + populates `extends`, narrowing it to a no-op and removing it in a later major is the safe path — + deleting it in phase 1 would edit adopters' repos for no functional gain. +- The dead `sources` key's *emptiness* in `sdk/src/config.ts` — the key itself is kept and given + the real schema (§4.9). +- Two directory symlinks and ~158 hand-maintained file paths, in adopter repos. diff --git a/docs/superpowers/specs/2026-08-17-metadata-source-resolution-prior-art.md b/docs/superpowers/specs/2026-08-17-metadata-source-resolution-prior-art.md new file mode 100644 index 000000000..19019ffc2 --- /dev/null +++ b/docs/superpowers/specs/2026-08-17-metadata-source-resolution-prior-art.md @@ -0,0 +1,384 @@ +# Metadata source resolution — prior art + +_Status: RESEARCH (evidence for a design; contains no decisions)._ +_Date: 2026-08-17._ +_Feeds: the metadata-collection / source-resolution design (supersedes the scope of +`2026-06-11-fr-023-metadata-packages-design.md`, which is one resolver within it)._ + +## Why this document exists + +MetaObjects needs to answer four questions it has never answered uniformly across its +five ports: + +1. **What is a metadata collection?** Today a run loads exactly one directory (`metaobjects/`) + in four of five ports. A monorepo with several apps sharing a common model cannot express + itself. +2. **How does a tool know which collection it is working in?** A CLI invoked at a monorepo + root sees only the root's metadata, which is the wrong answer for an app three directories + down. +3. **How is a shared model distributed and depended upon** across repos and across languages + (npm / PyPI / NuGet / Maven — or something else)? +4. **Can a source be remote** (a URL, an OCI artifact, a database) and if so, at build time, + at runtime, or both? + +Every one of these has been solved, several times, in public, by projects with the same +shape: a declarative model, a toolchain that reads it, multiple languages downstream, and +users with monorepos. This document records what those projects actually do, so the design +argues from evidence rather than from first principles. + +## Sourcing and IP hygiene + +Rules followed while compiling this: + +- **Open-source projects only.** Every project surveyed ships its tool under an OSI-approved + license, and every behavior described is from that project's **public documentation**. +- **Behavior and public config grammar only.** No source code was read, copied, or adapted. + Descriptions are written from scratch in our own words; nothing is quoted at length. +- **Hosted/commercial components are explicitly excluded.** Several of these projects pair an + open-source CLI with a commercial hosted registry. Only the open-source CLI's *local + configuration and resolution behavior* is recorded here. The hosted services' internals, + APIs, and pricing models are out of scope and were not investigated. +- **Every claim carries a URL.** Claims marked _(background)_ are widely-known, long-standing + toolchain behavior (the Java classpath, `sys.path`, nearest-ancestor config discovery) that + was not re-verified against a citation in this pass; they are included for completeness of + the pattern and should not be treated as researched findings. +- **Licenses are noted per project.** Where a project's license changed (Terraform), that is + called out along with its OSS fork, so nothing here is mistaken for guidance to depend on a + non-OSS artifact. + +This document describes *patterns*, which are not protectable. It is a survey, not a +derivative work. + +--- + +## Part 1 — The seven recurring patterns + +### P1. The ordered root list ("include path") + +The oldest and most durable shape: a tool is given an **ordered list of roots**, and a +reference is resolved by trying each root in order until one matches. + +- **protoc** takes one or more `--proto_path` / `-I` roots. An `import` is resolved relative + to each root **in order, first match wins** — the documented idiom being to put a local + tree ahead of a vendored tree so a local override shadows a dependency. + ([protoc reference](https://protocolbuffers-protobuf-45.mintlify.app/tooling/protoc), + [Go Protobuf Tips](https://jbrandhorst.com/post/go-protobuf-tips/); protobuf is BSD-3-Clause) +- The **Java classpath**, Python's **`sys.path`**, and Ruby's **`$LOAD_PATH`** are the same + primitive: an ordered sequence of roots, resolved left to right. _(background)_ + +**Why it matters here:** MetaObjects' Java port already implements exactly this — the Maven +plugin's loader parameter takes a `sourceDir` plus an **ordered list** of sources. The other +four ports collapsed it to a single directory. This is not a new idea to invent; it is an +existing idea to propagate. + +**The subtlety worth stealing:** first-match-wins over an ordered root list, and +last-writer-wins overlay merge (MetaObjects' existing semantics), are *different* composition +rules. protoc shadows a whole file; MetaObjects merges node-by-node. A design that borrows the +ordered list must say explicitly which rule applies at which layer. + +> **CORRECTION (added on review — this pattern does NOT apply, and leading with it was a +> mis-generalization).** Every example in P1 is a **shadowing** mechanism: order encodes +> precedence and the losing file is discarded entirely. MetaObjects *merges*. Adopting an ordered +> include path would therefore import order-sensitivity that the engine has already engineered +> away — super-resolution is a pure function of the source *set*, and the loader already discards +> the caller's declared order in the one place it matters (it reads every source, classifies each +> base-vs-overlay, and reorders). P1 is the oldest pattern here and the **least** applicable. See +> P8, which is what the dependency-management prior art actually shows. + +### P2. Collection identity, and a workspace of N collections + +Once there is more than one collection, each needs a name, and something has to describe the +set. + +- **Buf v2** merged what were previously two files into one: a single `buf.yaml` at the + workspace root declares **multiple modules**, each with its own directory and its own + lint/breaking-change settings, while **external dependencies and the lock file are shared + across the whole workspace**. Critically, **dependencies *between* modules in the workspace + are not declared** — the tool infers them from the module set. One publish command covers + every module in dependency order. + ([modules and workspaces](https://buf.build/docs/cli/modules-workspaces/), + [v2 migration guide](https://buf.build/docs/migration-guides/migrate-v2-config-files/); + the Buf CLI is Apache-2.0 — [repo](https://github.com/bufbuild/buf)) +- **Cargo** defines a workspace via a `[workspace]` section in a `Cargo.toml`, which may be a + "virtual" manifest (workspace only, no package of its own) or a real one (both). Members are + listed as glob patterns, and the workspace shares one lock file and one build output + directory. ([Cargo workspaces](https://deepwiki.com/rust-lang/cargo/2.2-workspaces); + Cargo is MIT OR Apache-2.0) +- **Go workspaces** take the opposite stance on discovery: a `go.work` file lists module + directories by **explicit relative path**, and Go deliberately does **not** auto-discover + modules — you add them with an explicit command. Absent a `go.work`, the workspace is simply + the single module containing the current directory. + ([Go modules reference](https://go.dev/ref/mod); Go is BSD-3-Clause) + +**The design fork this exposes:** glob-based auto-discovery (Cargo, Buf) versus explicit +enumeration (Go). Go's rationale — no surprises, no accidental membership — is the stronger +argument in a polyglot repo where a stray directory could otherwise be swept into a build. + +### P3. Contextual discovery — nearest ancestor, plus an explicit override + +Every tool in this space eventually needs to answer "which project am I in?" and they all +converge on the same two-part answer. + +- **graphql-config** supports both shapes and says so explicitly: either multiple named + `projects` in one root config, **or** one config file per subdirectory, where the config + file's location defines a scope for its whole subtree. Their framing is the clearest + statement of the principle — a config file marks a module root the same way `package.json` + does, and editor tooling picks the config **closest in the directory hierarchy** to the file + being worked on. ([graphql-config usage](https://the-guild.dev/graphql/config/docs/user/usage); + graphql-config is MIT) +- **dbt** defaults to looking for its project file in the current working directory **and its + parents**, with an explicit `--project-dir` flag (and an environment variable) to override. + ([dbt_project.yml reference](https://docs.getdbt.com/reference/dbt_project.yml); + dbt-core is Apache-2.0) +- `tsconfig.json`, `.editorconfig`, `.gitignore`, and `package.json` all use nearest-ancestor + resolution. It is the least surprising behavior in developer tooling. _(background)_ + +**The consistent shape: walk up from cwd to find the nearest collection root; allow an +explicit flag to name one; allow a root config to enumerate several.** No surveyed project +requires the user to pass a path on every invocation, and none auto-detects without an escape +hatch. + +### P4. How a source is spelled — three grammars + +Three distinct approaches to writing down "where this dependency comes from": + +- **Prefix-disambiguated single string (Terraform).** One `source` string covers local paths, + a module registry, Git, HTTP, and object storage. The kinds are told apart by *syntax*: a + local path **must** begin with `./` or `../` (which is what distinguishes it from a registry + address), Git sources carry a `git::` prefix, registry addresses use a + `namespace/name/provider` shape. Local paths are explicitly *not* "installed" — they are used + in place. ([module sources](https://developer.hashicorp.com/terraform/language/modules/sources)) + **License note:** Terraform moved to BUSL-1.1 in 2023. Only its publicly documented + configuration grammar is described here; the MPL-2.0 fork **OpenTofu** carries the same + grammar and is the OSS artifact to reference if this pattern is adopted. +- **Tagged union (dbt).** Dependencies are declared as a list where each entry names its kind + by key — a registry package, a Git repo, or a **local path**. dbt's documentation + specifically recommends **local packages as the monorepo answer**: several projects nested in + subdirectories, combined for coordinated development and deployment. + ([dbt packages](https://docs.getdbt.com/docs/build/packages)) +- **Ecosystem coordinates (Smithy).** `smithy-build.json` declares model dependencies as + **Maven GAV coordinates** plus repository URLs, and the CLI resolves them with the actual + Apache Maven dependency resolver. Shared models are published *inside a JAR* via a dedicated + packaging plugin that adds the model files and build metadata to the jar. + ([smithy-build.json](https://smithy.io/2.0/guides/smithy-build-json.html), + [Gradle plugins](https://smithy.io/2.0/guides/gradle-plugin/index.html); Smithy is Apache-2.0) + +**Smithy is the closest precedent for MetaObjects' instinct** — a schema-first, multi-language +codegen tool that resolves its *model* dependencies through an existing language package +manager rather than inventing distribution. Note what it did **not** do: see P5. + +**The trade-off:** a single prefix-disambiguated string is terse but forces the parser to own a +grammar and makes ambiguity a real failure mode (Terraform needs the `./` rule precisely +because of it). A tagged union is verbose but self-documenting, machine-checkable, and +extensible without touching a parser — which matches how MetaObjects already treats its own +config (`ADR-0037`'s bias toward self-documentation over economy). + +### P5. Distribution channel — the three-way split, and a notable negative + +This is where the surveyed projects disagree most sharply, and the disagreement is informative. + +- **Own registry (Buf).** Built a dedicated schema registry; the CLI's `deps` name modules in + it, and every commit is content-addressed by a cryptographic manifest digest recorded in the + lock file. ([dependency management](https://buf.build/docs/bsr/module/dependency-management/)) + The hosted registry itself is a commercial service and is out of scope here; what is relevant + is that Buf chose *not* to ride existing language registries. +- **Existing language registry, one ecosystem only (Smithy).** Uses Maven — and only Maven — + even though Smithy generates code for many languages. The model artifact is a JAR regardless + of which language you generate. +- **OCI registries (CUE).** CUE's module system is built on **OCI registries** rather than + ecosystem-specific ones. The stated reasoning is directly on point for a polyglot standard: + nearly every deployment already has an OCI registry available, the protocol is HTTP-based and + simple enough to implement a custom server against, and it is an open standard — so a single + artifact serves every language instead of N per-ecosystem copies of the same bytes. + ([CUE modules](https://cuelang.org/docs/reference/modules/), + [custom module registry](https://cuelang.org/docs/tutorial/working-with-a-custom-module-registry/), + [modules design proposal](https://github.com/cue-lang/proposal/blob/main/designs/modules.v3/2939-modules.md); + CUE is Apache-2.0) + +**The negative finding, stated plainly: no surveyed project publishes the same code-free +schema artifact to four language registries.** Every one either built its own registry, picked +a single ecosystem, or moved to OCI. CUE faced precisely MetaObjects' situation — a polyglot +declarative language needing cross-language model reuse — and explicitly rejected the +per-ecosystem approach. + +This does not make the four-registry plan wrong. MetaObjects already publishes to all four +registries in lockstep, so the release machinery exists and the marginal cost of a fifth +code-free artifact per registry is lower here than it would be for a greenfield project. But +it does mean the plan should be an **argued decision** rather than an assumption, and the +argument has to address what CUE's reasoning gets right: four artifacts of identical bytes +have four version numbers, four resolvers, four caches, four lockfiles, and four opportunities +to drift. + +### P6. Pinning and reproducibility + +Every surveyed project separates **declaration** from **resolution**, and records the +resolution. + +- **Buf** pins each dependency in a lock file by content-addressed digest, not merely by + version. ([dependency management](https://buf.build/docs/bsr/module/dependency-management/)) +- **Cargo** shares one lock file across the entire workspace. + ([workspaces](https://deepwiki.com/rust-lang/cargo/2.2-workspaces)) +- **Terraform** and **dbt** each have an explicit install/fetch step separate from use; dbt + vendors resolved packages into a local directory. ([dbt packages](https://docs.getdbt.com/docs/build/packages)) +- Local-path sources are the documented exception in both Terraform and dbt: they are **not + installed**, they are read in place. + +**The pattern: remote sources get an explicit fetch step and a pinned record; local sources +skip both.** No surveyed tool silently fetches a remote dependency during a normal build. +This has a direct consequence for MetaObjects: a URL source that is read at load time, on every +`meta gen`, with no lock and no cache, is a shape nobody in this space ships. + +### P7. Build-time versus runtime is a hard boundary + +Two entirely separate worlds, and no surveyed project blurs them. + +- **Build-time** distribution (everything in P5) resolves files onto disk before codegen runs. +- **Runtime** schema access is a different product category — a schema registry service + queried over HTTP by a running application, addressing artifacts by group/id/version and + returning the schema document. **Apicurio Registry** (Apache-2.0) is the open-source + reference: a REST interface where a client fetches a specific artifact version at runtime. + ([Apicurio introduction](https://www.apicur.io/registry/docs/apicurio-registry/3.1.x/getting-started/assembly-intro-to-the-registry.html), + [artifact reference](https://www.apicur.io/registry/docs/apicurio-registry/3.3.x/getting-started/assembly-artifact-reference.html)) + +**Relevance to "sourcing from a DB":** in every surveyed system, reading schema from a live +service or store is a **runtime** capability serving a running application — not a build-time +codegen input. A database as a *codegen* source would be novel, and novelty here is a cost: +it breaks reproducible builds (the source can change between two builds of the same commit) +unless paired with the P6 pin-and-cache discipline. A database as a *runtime* source is +well-trodden and is a different feature with different requirements. + +### P8. Nobody makes the user declare load order + +_Added on review, and it reverses P1's framing._ + +Re-read for order-sensitivity rather than for structure, the dependency-management prior art is +unanimous: **the user declares a SET, and the tool derives whatever order it needs.** + +- **Buf** is the most explicit: dependencies *between* modules in a workspace are deliberately not + declared, because the tool infers them from the module set — and a single publish covers every + module in the right dependency order, computed rather than written down. + ([modules and workspaces](https://buf.build/docs/cli/modules-workspaces/)) +- **Go**, **Cargo**, **dbt** and **Terraform** all take an unordered dependency declaration and + compute the graph. Nobody hand-sorts a `go.mod`, and dbt builds its DAG from model references + rather than from list position. ([Go modules](https://go.dev/ref/mod), + [Cargo workspaces](https://deepwiki.com/rust-lang/cargo/2.2-workspaces), + [dbt packages](https://docs.getdbt.com/docs/build/packages)) +- **Smithy** hands its coordinates to the Maven resolver, which owns ordering entirely. + +The only ordered-list examples in this document (P1) are shadowing mechanisms, where order *is* the +precedence rule rather than a load sequence. + +**Implication:** a config schema that asks an author to sequence sources is asking for information +no surveyed tool requires and this engine does not consume. Precedence, where it is genuinely +needed, should be expressed as a **rule attached to a source** ("a local source wins over a +dependency") rather than as a position in a list — which also makes diamond dependencies, parallel +resolution, and incremental addition fall out for free. + +### P9. Scoping is done with patterns, not predicates + +Every surveyed tool that scopes a large model scopes it with **declarative string patterns** — +never a callback. Buf modules take path `excludes`; Smithy's build config filters models +declaratively; dbt selects with a string selector syntax. + +MetaObjects' own Java port has carried this since long before this survey: `GeneratorUtil` +implements include/exclude patterns (a `!` prefix marks an exclusion) glob-matched against a node's +fully-qualified name, with `@` matching exactly one package segment. + +**Implication:** a predicate *function* — TypeScript's per-generator `filter` — cannot be expressed +in a YAML config, an XML pom, or a CLI flag, and cannot be gated by any cross-language corpus. It +is therefore unsuitable as a cross-port primitive regardless of its ergonomics in the one port that +can express it. + +--- + +## Part 2 — Evidence table + +| Project | License | Ordered roots | Multi-collection config | Contextual discovery | Dep grammar | Distribution | Pinning | +|---|---|---|---|---|---|---|---| +| protoc | BSD-3-Clause | **yes**, first-match-wins | — | — | — | — | — | +| Buf CLI | Apache-2.0 | — | **one config, N modules**, shared deps, intra-workspace deps inferred | workspace root | module refs | own registry | digest lock | +| Cargo | MIT OR Apache-2.0 | — | `[workspace]` + member globs | walk up | coordinates | crates.io | one workspace lock | +| Go modules | BSD-3-Clause | — | `go.work`, **explicit paths, no auto-discovery** | single module containing cwd | module paths | VCS-addressed | `go.sum` | +| graphql-config | MIT | — | `projects` map **or** per-subtree config | **nearest ancestor** | — | — | — | +| dbt-core | Apache-2.0 | — | local packages = the monorepo answer | walk up + `--project-dir` | **tagged union** (registry/git/local) | package registry | vendored install step | +| Smithy | Apache-2.0 | — | — | — | **Maven GAV** | **Maven JAR** (one ecosystem) | Maven resolver | +| CUE | Apache-2.0 | — | modules | module root | module paths | **OCI registries** | module resolution | +| Terraform | BUSL-1.1 (fork: OpenTofu, MPL-2.0) | — | — | — | **prefix-disambiguated string** | multi-scheme | lock file | +| Apicurio Registry | Apache-2.0 | — | — | — | group/id/version | — | **runtime**, not build | + +--- + +## Part 3 — What the evidence says about MetaObjects specifically + +Findings, not decisions. Each is a question the design must answer explicitly. + +1. **~~The ordered-list-of-roots primitive is settled prior art~~ — REVISED.** The Java port's + loader does take an ordered source list plus a URI grammar covering file, URL and classpath + resource, and four ports collapsed that to one directory, so propagation is still the job. But + the *ordering* half should not be propagated: per P8 no surveyed tool asks the author to + sequence anything, and per P1's correction this engine already ignores the declared order. + Propagate the multi-source capability and the `resource` kind; drop the sequence. + +2. **Collection identity is tooling config in every surveyed project — never part of the + schema language itself.** Buf, Cargo, Go, dbt, and graphql-config all keep "what is a + collection and where does it live" in a config file, entirely outside the modeled types. + That is direct evidence against introducing a `collection` node into the MetaObjects + metamodel, and in favor of the config layer — which also keeps `registry-conformance` and + ADR-0023 out of it. + +3. **Contextual discovery has one converged answer: nearest ancestor, plus an explicit + override, plus optionally a root config enumerating several collections.** graphql-config + ships all three shapes and documents when each applies. MetaObjects' CLI already has the + override (`--cwd` / a project-root positional); it is missing discovery. + +4. **Go's explicit-enumeration stance deserves weight over Cargo/Buf globbing**, because a + polyglot repo has more directories that merely *look* like collections, and silent + membership is the failure mode that is hardest to debug. + +5. **Local-path sources are the recommended monorepo mechanism in the two projects that + address monorepos head-on** (dbt explicitly; Terraform by giving local paths their own rule). + Both treat local paths as read-in-place, never installed. This is the shape the blocked + adopters need, and it is the cheapest thing in this document to ship. + +6. **The four-registry distribution plan is unprecedented among surveyed peers and needs an + argued decision.** The counter-evidence (CUE's explicit rejection, Buf's own registry, + Smithy's single ecosystem) is strong enough that "publish to all four" should be recorded as + an ADR with the reasoning, or reconsidered in favor of OCI — noting that MetaObjects' + existing four-registry lockstep release machinery is a genuine advantage none of these + projects had. + +7. **Remote sources need a fetch step and a pin. No surveyed tool reads a remote source inline + during a normal build.** A URL source resolved on every `meta gen` would be a novel shape, + and the novelty is a reproducibility cost, not a feature. + +8. **A database source is a runtime pattern, not a build-time one.** Splitting it out of the + build-time design entirely is consistent with every system surveyed. + +9. **The Buf workspace precedent does not transfer — verified by spike, not by reading.** Buf can + declare N modules and their consumers in one root file because **Buf owns its entire config + surface**. MetaObjects does not: the *build tool* owns it, and a consumer's generator wiring + already lives in `metaobjects.config.ts` or a `pom.xml`. A root config enumerating consumers + would have to duplicate or override those. Buf's *module-set* idea transfers; its *single root + config* does not. Recorded because the structural precondition, not the shape, is what decides + whether a borrowed pattern works. + +10. **Scoping should be package patterns, and MetaObjects already has the reference + implementation** (P9) — in the same port that has the multi-source list, and for the same + reason: it is the only port whose adopters hit these problems at scale. + +--- + +## Sources + +All URLs are public documentation, retrieved 2026-08-17. + +- protobuf / protoc — [compiler reference](https://protocolbuffers-protobuf-45.mintlify.app/tooling/protoc) · [include-path ordering idiom](https://jbrandhorst.com/post/go-protobuf-tips/) · [files and packages](https://buf.build/docs/reference/protobuf-files-and-packages/) +- Buf CLI — [modules and workspaces](https://buf.build/docs/cli/modules-workspaces/) · [v2 config migration](https://buf.build/docs/migration-guides/migrate-v2-config-files/) · [dependency management](https://buf.build/docs/bsr/module/dependency-management/) · [repo (Apache-2.0)](https://github.com/bufbuild/buf) +- Cargo — [workspaces](https://deepwiki.com/rust-lang/cargo/2.2-workspaces) +- Go modules — [reference](https://go.dev/ref/mod) +- graphql-config — [usage](https://the-guild.dev/graphql/config/docs/user/usage) · [multi-project config](https://the-guild.dev/graphql/codegen/docs/config-reference/multiproject-config) +- dbt — [packages](https://docs.getdbt.com/docs/build/packages) · [dbt_project.yml](https://docs.getdbt.com/reference/dbt_project.yml) · [project dependencies](https://docs.getdbt.com/docs/mesh/govern/project-dependencies) +- Smithy — [smithy-build.json](https://smithy.io/2.0/guides/smithy-build-json.html) · [Gradle plugins](https://smithy.io/2.0/guides/gradle-plugin/index.html) +- CUE — [modules reference](https://cuelang.org/docs/reference/modules/) · [custom module registry](https://cuelang.org/docs/tutorial/working-with-a-custom-module-registry/) · [modules v3 design proposal](https://github.com/cue-lang/proposal/blob/main/designs/modules.v3/2939-modules.md) +- Terraform — [module sources](https://developer.hashicorp.com/terraform/language/modules/sources) +- Apicurio Registry — [introduction](https://www.apicur.io/registry/docs/apicurio-registry/3.1.x/getting-started/assembly-intro-to-the-registry.html) · [artifact reference](https://www.apicur.io/registry/docs/apicurio-registry/3.3.x/getting-started/assembly-artifact-reference.html) From fc13abc58582896f2fd03f9852aba42bd072af7c Mon Sep 17 00:00:00 2001 From: Doug Mealing Date: Mon, 17 Aug 2026 20:50:18 -0400 Subject: [PATCH 02/44] feat(metadata): register phase-1 source-resolution error codes across all ports Register ERR_SOURCE_UNRESOLVED, ERR_SOURCE_KIND_UNSUPPORTED, ERR_SCOPE_PATTERN_INVALID, and ERR_COLLECTION_NOT_FOUND across all five language ports (TS, Python, Java, C#) and the fixtures/conformance/ERROR-CODES.json ledger. These codes will be raised by later tasks when loading metadata sources from .metaobjects/config.json. Descriptions corrected to match the design: - ERR_SOURCE_UNRESOLVED: a path source on disk does not exist - ERR_SOURCE_KIND_UNSUPPORTED: source kind (resource/package) not supported by toolchain - ERR_SCOPE_PATTERN_INVALID: scope include/exclude package pattern is malformed - ERR_COLLECTION_NOT_FOUND: no metadata collection discovered (no sources config, no default metaobjects/ dir) Co-Authored-By: Claude Opus 5 (1M context) --- fixtures/conformance/ERROR-CODES.json | 4 ++++ server/csharp/MetaObjects/Errors.cs | 8 ++++++++ .../src/main/java/com/metaobjects/ErrorCode.java | 12 ++++++++++++ server/python/src/metaobjects/errors.py | 8 ++++++++ server/typescript/packages/metadata/src/errors.ts | 12 ++++++++++++ .../packages/metadata/test/errors.test.ts | 13 +++++++++++++ 6 files changed, 57 insertions(+) diff --git a/fixtures/conformance/ERROR-CODES.json b/fixtures/conformance/ERROR-CODES.json index 8bd3a0dee..c5c5f22cd 100644 --- a/fixtures/conformance/ERROR-CODES.json +++ b/fixtures/conformance/ERROR-CODES.json @@ -17,6 +17,7 @@ "ERR_PROJECTION_INHERITED_SOURCE": "FR-024 (ADR-0028): a concrete object.projection inherits a source.* through extends instead of declaring its own. A projection's extends is shape lineage, not a shared-storage hierarchy: extends only ADDS members, so the child's extra fields have no provider in the parent's view, and two objects would claim one physical view with different declared exposures. Declare the source on the concrete projection; an abstract projection base carries shape only.", "ERR_INVALID_SUBTYPE_CHILD": "A child node type/subType is not permitted under its parent.", "ERR_CHILD_NOT_ALLOWED": "FR-033: a structural child (field/identity/source/validator/\u2026 \u2014 not an attr) is placed under a parent whose registered childRules do not admit it (the structural analogue of ERR_UNKNOWN_ATTR). Strict-load only; a no-op under wildcard childRules. The detail names the parent, the child (type.subType 'name'), and which placement was rejected.", + "ERR_COLLECTION_NOT_FOUND": "Phase-1 metadata-source-resolution: no metadata collection was discovered — no config declaring sources, and no default metaobjects/ directory.", "ERR_UNKNOWN_ATTR": "An attribute name is not declared on the node's type.", "ERR_MISSING_REQUIRED_ATTR": "A required attribute is absent from the node.", "ERR_BAD_ATTR_VALUE": "An attribute value fails its declared schema (type/range).", @@ -49,8 +50,11 @@ "ERR_OBJECT_FIELD_WITHOUT_OBJECT_REF": "ADR-0013: a field.object declares no @objectRef. A field.object models a typed nested value and REQUIRES @objectRef. For a genuinely open/untyped JSON map, use the physical escape hatch @dbColumnType: jsonb on a field.string instead of a bare object.", "ERR_UNRESOLVED_OBJECT_REF": "ADR-0042: a field.object / field.map @objectRef does not resolve to any object in the loaded tree (a dangling target). The ref resolves package-locally when bare (referrer's package, else root-level) and exactly when FQN \u2014 a bare cross-package ref no longer binds elsewhere. The error names same-short-name objects in other packages so the author can qualify it.", "ERR_RESERVED_ATTR": "An @-prefixed reserved structural keyword (e.g. @name, @isArray, @children) was used as an inline attribute.", + "ERR_SCOPE_PATTERN_INVALID": "Phase-1 metadata-source-resolution: a scope include/exclude package pattern is malformed (empty pattern or empty :: segment).", "ERR_SOURCE_NO_PRIMARY": "An object declares source nodes but none has role=primary.", "ERR_SOURCE_MULTIPLE_PRIMARY": "An object declares more than one source node with role=primary.", + "ERR_SOURCE_KIND_UNSUPPORTED": "Phase-1 metadata-source-resolution: a declared source kind (resource or package) is not supported by this toolchain.", + "ERR_SOURCE_UNRESOLVED": "Phase-1 metadata-source-resolution: a path source declared in .metaobjects/config.json does not exist on disk.", "ERR_PHYSICAL_NAME_KIND_MISMATCH": "FR-016 / ADR-0018: a source.rdb declares a kind-aware physical-name alias (@view/@materializedView/@proc/@function) that does not match its @kind. The legacy @table-for-non-table case warns rather than errors.", "ERR_PHYSICAL_NAME_MULTIPLE": "FR-016 / ADR-0018: a source.rdb declares two or more kind-aware physical-name aliases at once (e.g. both @table and @view). Exactly one is permitted.", "ERR_READONLY_ASSIGNED_PRIMARY": "FR-013: a field with @readOnly: true is the target of an identity.primary with @generation: \"assigned\". The application has no path to populate the identity value (no setter; not generated; not defaulted).", diff --git a/server/csharp/MetaObjects/Errors.cs b/server/csharp/MetaObjects/Errors.cs index 2c6c8e6f5..94b9ce9da 100644 --- a/server/csharp/MetaObjects/Errors.cs +++ b/server/csharp/MetaObjects/Errors.cs @@ -128,6 +128,14 @@ public enum ErrorCode ERR_INVALID_TEMPLATE, ERR_SOURCE_NO_PRIMARY, ERR_SOURCE_MULTIPLE_PRIMARY, + // Phase-1 metadata-source-resolution: a path source declared in .metaobjects/config.json does not exist on disk. + ERR_SOURCE_UNRESOLVED, + // Phase-1 metadata-source-resolution: a declared source kind (resource or package) is not supported by this toolchain. + ERR_SOURCE_KIND_UNSUPPORTED, + // Phase-1 metadata-source-resolution: a scope include/exclude package pattern is malformed (empty pattern or empty :: segment). + ERR_SCOPE_PATTERN_INVALID, + // Phase-1 metadata-source-resolution: no metadata collection was discovered — no config declaring sources, and no default metaobjects/ directory. + ERR_COLLECTION_NOT_FOUND, // FR5c — multi-file overlay merge produced a conflicting attribute value: // two contributors set the same @attr to different non-empty values. ERR_MERGE_CONFLICT, diff --git a/server/java/metadata/src/main/java/com/metaobjects/ErrorCode.java b/server/java/metadata/src/main/java/com/metaobjects/ErrorCode.java index ef64e5921..4619a3ccf 100644 --- a/server/java/metadata/src/main/java/com/metaobjects/ErrorCode.java +++ b/server/java/metadata/src/main/java/com/metaobjects/ErrorCode.java @@ -283,6 +283,18 @@ public enum ErrorCode { /** An object declares more than one source node with role=primary. */ ERR_SOURCE_MULTIPLE_PRIMARY, + /** Phase-1 metadata-source-resolution: a {@code path} source declared in {@code .metaobjects/config.json} does not exist on disk. */ + ERR_SOURCE_UNRESOLVED, + + /** Phase-1 metadata-source-resolution: a declared source kind ({@code resource} or {@code package}) is not supported by this toolchain. */ + ERR_SOURCE_KIND_UNSUPPORTED, + + /** Phase-1 metadata-source-resolution: a {@code scope} include/exclude package pattern is malformed (empty pattern or empty {@code ::} segment). */ + ERR_SCOPE_PATTERN_INVALID, + + /** Phase-1 metadata-source-resolution: no metadata collection was discovered — no config declaring {@code sources}, and no default {@code metaobjects/} directory. */ + ERR_COLLECTION_NOT_FOUND, + /** * FR-016 / ADR-0018: a {@code source.rdb} declares a kind-aware physical-name * alias ({@code @view} / {@code @materializedView} / {@code @proc} / diff --git a/server/python/src/metaobjects/errors.py b/server/python/src/metaobjects/errors.py index d98915089..b2a8d11e8 100644 --- a/server/python/src/metaobjects/errors.py +++ b/server/python/src/metaobjects/errors.py @@ -104,6 +104,14 @@ class ErrorCode(str, Enum): # Source-v2 multi-source one-primary rule (ADR-0007). ERR_SOURCE_NO_PRIMARY = "ERR_SOURCE_NO_PRIMARY" ERR_SOURCE_MULTIPLE_PRIMARY = "ERR_SOURCE_MULTIPLE_PRIMARY" + # Phase-1 metadata-source-resolution — a path source declared in .metaobjects/config.json does not exist on disk. + ERR_SOURCE_UNRESOLVED = "ERR_SOURCE_UNRESOLVED" + # Phase-1 metadata-source-resolution — a declared source kind (resource or package) is not supported by this toolchain. + ERR_SOURCE_KIND_UNSUPPORTED = "ERR_SOURCE_KIND_UNSUPPORTED" + # Phase-1 metadata-source-resolution — a scope include/exclude package pattern is malformed (empty pattern or empty :: segment). + ERR_SCOPE_PATTERN_INVALID = "ERR_SCOPE_PATTERN_INVALID" + # Phase-1 metadata-source-resolution — no metadata collection was discovered: no config declaring sources, and no default metaobjects/ directory. + ERR_COLLECTION_NOT_FOUND = "ERR_COLLECTION_NOT_FOUND" # FR-016 / ADR-0018 — per-kind physical-name aliases on source.rdb. ERR_PHYSICAL_NAME_KIND_MISMATCH = "ERR_PHYSICAL_NAME_KIND_MISMATCH" ERR_PHYSICAL_NAME_MULTIPLE = "ERR_PHYSICAL_NAME_MULTIPLE" diff --git a/server/typescript/packages/metadata/src/errors.ts b/server/typescript/packages/metadata/src/errors.ts index 70beb35a7..7fa3c6eed 100644 --- a/server/typescript/packages/metadata/src/errors.ts +++ b/server/typescript/packages/metadata/src/errors.ts @@ -208,6 +208,18 @@ export const ERROR_CODES = [ // an integer array. An array-of-enum stays string-backed: drop @intValueMap, // or make the field scalar. "ERR_ENUM_INT_VALUE_MAP_ARRAY", + // Phase-1 metadata-source-resolution — a path source declared in + // .metaobjects/config.json does not exist on disk. + "ERR_SOURCE_UNRESOLVED", + // Phase-1 metadata-source-resolution — a declared source kind (resource or + // package) is not supported by this toolchain. + "ERR_SOURCE_KIND_UNSUPPORTED", + // Phase-1 metadata-source-resolution — a scope include/exclude package + // pattern is malformed (empty pattern or empty :: segment). + "ERR_SCOPE_PATTERN_INVALID", + // Phase-1 metadata-source-resolution — no metadata collection was discovered: + // no config declaring sources, and no default metaobjects/ directory. + "ERR_COLLECTION_NOT_FOUND", "ERR_UNKNOWN", ] as const; diff --git a/server/typescript/packages/metadata/test/errors.test.ts b/server/typescript/packages/metadata/test/errors.test.ts index 2eba627e1..9ccf57b47 100644 --- a/server/typescript/packages/metadata/test/errors.test.ts +++ b/server/typescript/packages/metadata/test/errors.test.ts @@ -28,3 +28,16 @@ test("MetaModelError carries a stable ERR_PROVIDER_* code", () => { expect(thrown).toBeInstanceOf(MetaModelError); expect((thrown as MetaModelError).code).toBe("ERR_PROVIDER_DUPLICATE_ID"); }); + +// Phase-1 metadata-source-resolution design: register error codes that will be +// raised when loading sources from .metaobjects/config.json. +test("phase-1 source-resolution error codes are registered in the shared ledger", () => { + for (const code of [ + "ERR_SOURCE_UNRESOLVED", + "ERR_SOURCE_KIND_UNSUPPORTED", + "ERR_SCOPE_PATTERN_INVALID", + "ERR_COLLECTION_NOT_FOUND", + ]) { + expect(ERROR_CODES).toContain(code); + } +}); From 2ff8a1d2c1909c2971df44f2da5c05a3d35b0ec9 Mon Sep 17 00:00:00 2001 From: Doug Mealing Date: Mon, 17 Aug 2026 21:07:59 -0400 Subject: [PATCH 03/44] feat(sdk): package-pattern scope engine (* = one segment, ** = one or more) Phase-1 metadata-source-resolution, task 1. A pure, no-I/O module deciding whether a fully-qualified node name falls inside a consumer's declared include/exclude scope, ahead of the source-resolution and discovery work that builds on it. Raises ERR_SCOPE_PATTERN_INVALID via ParseError + codeSource (this repo's loader-error convention), not a bare Error with the code embedded in the message text. Co-Authored-By: Claude Opus 5 (1M context) --- server/typescript/packages/sdk/src/scope.ts | 80 ++++++++++++++++++ .../packages/sdk/test/scope.test.ts | 81 +++++++++++++++++++ 2 files changed, 161 insertions(+) create mode 100644 server/typescript/packages/sdk/src/scope.ts create mode 100644 server/typescript/packages/sdk/test/scope.test.ts diff --git a/server/typescript/packages/sdk/src/scope.ts b/server/typescript/packages/sdk/src/scope.ts new file mode 100644 index 000000000..b9a7f5dca --- /dev/null +++ b/server/typescript/packages/sdk/src/scope.ts @@ -0,0 +1,80 @@ +// server/typescript/packages/sdk/src/scope.ts +// +// Phase-1 metadata-source-resolution — the scope pattern engine. +// +// A pure, no-I/O module deciding whether a fully-qualified node name falls +// inside a consumer's declared `include`/`exclude` scope. Source resolution +// and discovery (later phase-1 tasks) build on this; a cross-language +// conformance corpus pins its semantics, so exact pattern behavior matters. + +// NOTE: PACKAGE_SEPARATOR is NOT re-exported from the browser-safe +// `@metaobjectsdev/metadata/constants` barrel (that barrel only re-exports +// the per-concern `*-constants.ts` modules; `PACKAGE_SEPARATOR` lives in +// `shared/structural.ts`, exported from the package root). This package +// (`@metaobjectsdev/sdk`) is server-side, not a `client/web/**` browser +// package, so importing metamodel values from the root — the same thing +// `memory.ts` and `forge-types.ts` in this package already do — is correct. +import { PACKAGE_SEPARATOR, ParseError, codeSource } from "@metaobjectsdev/metadata"; + +/** A consumer-side output filter over fully-qualified node names. */ +export interface Scope { + /** Absent or empty means "everything". */ + readonly include?: readonly string[]; + /** Applied after `include`. */ + readonly exclude?: readonly string[]; +} + +export interface CompiledScope { + readonly include: readonly RegExp[]; + readonly exclude: readonly RegExp[]; +} + +/** One package segment: any run of characters containing no separator char. */ +const SEGMENT = "[^:]+"; +/** One or more segments, separator-joined — the `**` expansion. */ +const SEGMENTS = `${SEGMENT}(?:${PACKAGE_SEPARATOR}${SEGMENT})*`; + +function escapeLiteral(text: string): string { + return text.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); +} + +/** Compile one segment. `**` spans segments; `*` never crosses a separator. */ +function compileSegment(segment: string, pattern: string): string { + if (segment.length === 0) { + throw new ParseError(`empty segment in scope pattern "${pattern}"`, { + code: "ERR_SCOPE_PATTERN_INVALID", + source: codeSource("compileSegment"), + }); + } + if (segment === "**") return `(?:${SEGMENTS})`; + // `*` inside a segment matches any characters except the separator char. + return segment.split("*").map(escapeLiteral).join("[^:]*"); +} + +export function compilePattern(pattern: string): RegExp { + if (pattern.length === 0) { + throw new ParseError(`scope pattern must not be empty`, { + code: "ERR_SCOPE_PATTERN_INVALID", + source: codeSource("compilePattern"), + }); + } + const body = pattern + .split(PACKAGE_SEPARATOR) + .map((segment) => compileSegment(segment, pattern)) + .join(PACKAGE_SEPARATOR); + return new RegExp(`^${body}$`); +} + +export function compileScope(scope: Scope): CompiledScope { + return { + include: (scope.include ?? []).map(compilePattern), + exclude: (scope.exclude ?? []).map(compilePattern), + }; +} + +/** True when `fqn` is inside the scope. An empty `include` means everything. */ +export function matchesScope(fqn: string, compiled: CompiledScope): boolean { + const included = compiled.include.length === 0 || compiled.include.some((re) => re.test(fqn)); + if (!included) return false; + return !compiled.exclude.some((re) => re.test(fqn)); +} diff --git a/server/typescript/packages/sdk/test/scope.test.ts b/server/typescript/packages/sdk/test/scope.test.ts new file mode 100644 index 000000000..9b62e6ed4 --- /dev/null +++ b/server/typescript/packages/sdk/test/scope.test.ts @@ -0,0 +1,81 @@ +import { describe, test, expect } from "bun:test"; +import { compileScope, matchesScope, type Scope } from "../src/scope.js"; + +const match = (fqn: string, scope: Scope) => matchesScope(fqn, compileScope(scope)); + +/** Pull the stable ERR_ code off a caught error, if it carries one. */ +function errorCode(err: unknown): string { + const code = (err as { code?: unknown }).code; + return typeof code === "string" ? code : "ERR_UNKNOWN"; +} + +describe("compileScope / matchesScope", () => { + test("empty include matches everything", () => { + expect(match("acme::commerce::Order", {})).toBe(true); + }); + + test("* matches exactly one segment", () => { + const s: Scope = { include: ["acme::*"] }; + expect(match("acme::Order", s)).toBe(true); + expect(match("acme::commerce::Order", s)).toBe(false); + }); + + test("** matches one or more segments", () => { + const s: Scope = { include: ["acme::**"] }; + expect(match("acme::Order", s)).toBe(true); + expect(match("acme::commerce::Order", s)).toBe(true); + expect(match("acme", s)).toBe(false); + expect(match("other::Order", s)).toBe(false); + }); + + test("* within a segment matches a partial name but never crosses ::", () => { + const s: Scope = { include: ["acme::Order*"] }; + expect(match("acme::OrderLine", s)).toBe(true); + expect(match("acme::Order", s)).toBe(true); + expect(match("acme::deep::OrderLine", s)).toBe(false); + }); + + test("exclude is applied after include", () => { + const s: Scope = { include: ["acme::**"], exclude: ["acme::internal::**"] }; + expect(match("acme::commerce::Order", s)).toBe(true); + expect(match("acme::internal::Secret", s)).toBe(false); + }); + + test("exclude alone narrows the implicit match-everything", () => { + const s: Scope = { exclude: ["acme::internal::**"] }; + expect(match("acme::commerce::Order", s)).toBe(true); + expect(match("acme::internal::Secret", s)).toBe(false); + }); + + test("a bare name with no package is matchable", () => { + expect(match("Order", { include: ["Order"] })).toBe(true); + expect(match("Order", { include: ["*"] })).toBe(true); + }); + + test("regex metacharacters in a pattern are literal", () => { + expect(match("acme::Order.v2", { include: ["acme::Order.v2"] })).toBe(true); + expect(match("acme::OrderXv2", { include: ["acme::Order.v2"] })).toBe(false); + }); + + test("an empty pattern is ERR_SCOPE_PATTERN_INVALID", () => { + let caught: unknown; + try { + compileScope({ include: [""] }); + } catch (err) { + caught = err; + } + expect(caught).toBeDefined(); + expect(errorCode(caught)).toBe("ERR_SCOPE_PATTERN_INVALID"); + }); + + test("an empty segment is ERR_SCOPE_PATTERN_INVALID", () => { + let caught: unknown; + try { + compileScope({ include: ["acme::::Order"] }); + } catch (err) { + caught = err; + } + expect(caught).toBeDefined(); + expect(errorCode(caught)).toBe("ERR_SCOPE_PATTERN_INVALID"); + }); +}); From daf69acfadf65d7086681a0b5cfdb718225390d6 Mon Sep 17 00:00:00 2001 From: Doug Mealing Date: Mon, 17 Aug 2026 21:17:22 -0400 Subject: [PATCH 04/44] test(conformance): scope-pattern corpus pins * and ** semantics cross-port Pins the scope-engine pattern semantics (compileScope/matchesScope, task 1) as a shared cross-port corpus so `*`/`**` cannot drift into five different meanings per language port -- the same failure class as the LIKE/ILIKE divergence fixed in 0.21.6. Adds a mid-pattern `**` case (acme::**::Order) the task-1 review flagged as untested; verified against the real implementation that a zero-segment gap is correctly rejected (acme::Order does not match), consistent with the "one or more segments" rule. Co-Authored-By: Claude Opus 5 (1M context) --- fixtures/scope-conformance/README.md | 52 ++++++++++++ fixtures/scope-conformance/cases.json | 84 +++++++++++++++++++ .../sdk/test/scope-conformance.test.ts | 29 +++++++ 3 files changed, 165 insertions(+) create mode 100644 fixtures/scope-conformance/README.md create mode 100644 fixtures/scope-conformance/cases.json create mode 100644 server/typescript/packages/sdk/test/scope-conformance.test.ts diff --git a/fixtures/scope-conformance/README.md b/fixtures/scope-conformance/README.md new file mode 100644 index 000000000..3e2b44053 --- /dev/null +++ b/fixtures/scope-conformance/README.md @@ -0,0 +1,52 @@ +# scope-conformance + +Pins the pattern semantics of a consumer's `include`/`exclude` **scope** — the +filter over fully-qualified node names (`pkg::Sub::Name`) that decides which +metadata a source is authoritative for (phase 1 of metadata-source +resolution). `*` and `**` are easy to reinvent slightly differently per port; +this is the same failure mode that produced the cross-port `LIKE`/`ILIKE` +divergence fixed in 0.21.6. Every port's runner reads the single committed +`cases.json` — there is no per-port fixture and no ledger. + +## Shape + +``` +cases.json # { cases: [{ name, scope: {include?, exclude?}, expect: [{fqn, matches}] }] } +README.md +``` + +## Semantics + +- **Separator** is `::` (the package separator). A fully-qualified name is a + `::`-joined sequence of one or more segments. +- **`*`** inside a segment matches any run of characters, but **never crosses + a `::`** — it is scoped to a single segment. `acme::Order*` matches + `acme::OrderLine` but not `acme::deep::OrderLine`. +- **A segment that is exactly `**`** matches **one or more** whole segments. + `acme::**` matches `acme::Order` and `acme::a::b::Secret` but not the bare + `acme` (zero segments) — `**` never matches "nothing". `**` may also appear + mid-pattern (`acme::**::Order`), where it still requires at least one + segment between the fixed literals — `acme::Order` does **not** match + `acme::**::Order` (see `double-star-in-the-middle`). +- All other pattern characters (including regex metacharacters like `.`) are + **literal** — a pattern is never a general regex. +- **`include`** absent or empty means "everything is included". Otherwise a + name matches if **any** `include` pattern matches it (union). +- **`exclude`** is applied **after** `include` — a name excluded is excluded + regardless of which `include` pattern admitted it. `exclude` with no + `include` narrows the "everything" default. + +## Behavioral contract + +Each port's runner reads `cases.json`, and for every case: compiles `scope` +with its native pattern compiler, then for every `expect` entry asserts +`isInScope(fqn, compiledScope) === matches`. All ports assert the same +booleans — single-source, byte-identical expectations. + +## Reference implementation + +`server/typescript/packages/sdk/src/scope.ts` (`compileScope` / `matchesScope` +/ `compilePattern`) is the TypeScript reference this corpus was authored +against; other ports are free to implement the same semantics however is +idiomatic (e.g. a native regex engine, or a hand-rolled segment matcher) as +long as every case in this file passes. diff --git a/fixtures/scope-conformance/cases.json b/fixtures/scope-conformance/cases.json new file mode 100644 index 000000000..20858dd8a --- /dev/null +++ b/fixtures/scope-conformance/cases.json @@ -0,0 +1,84 @@ +{ + "cases": [ + { + "name": "empty-scope-matches-everything", + "scope": {}, + "expect": [ + { "fqn": "acme::commerce::Order", "matches": true }, + { "fqn": "Order", "matches": true } + ] + }, + { + "name": "single-star-is-one-segment", + "scope": { "include": ["acme::*"] }, + "expect": [ + { "fqn": "acme::Order", "matches": true }, + { "fqn": "acme::commerce::Order", "matches": false }, + { "fqn": "other::Order", "matches": false } + ] + }, + { + "name": "double-star-is-one-or-more-segments", + "scope": { "include": ["acme::**"] }, + "expect": [ + { "fqn": "acme::Order", "matches": true }, + { "fqn": "acme::commerce::Order", "matches": true }, + { "fqn": "acme::commerce::internal::Secret", "matches": true }, + { "fqn": "acme", "matches": false }, + { "fqn": "acmex::Order", "matches": false } + ] + }, + { + "name": "double-star-in-the-middle", + "scope": { "include": ["acme::**::Order"] }, + "expect": [ + { "fqn": "acme::commerce::Order", "matches": true }, + { "fqn": "acme::a::b::Order", "matches": true }, + { "fqn": "acme::Order", "matches": false }, + { "fqn": "acme::commerce::Invoice", "matches": false } + ] + }, + { + "name": "partial-star-never-crosses-separator", + "scope": { "include": ["acme::Order*"] }, + "expect": [ + { "fqn": "acme::Order", "matches": true }, + { "fqn": "acme::OrderLine", "matches": true }, + { "fqn": "acme::deep::OrderLine", "matches": false } + ] + }, + { + "name": "exclude-applied-after-include", + "scope": { "include": ["acme::**"], "exclude": ["acme::internal::**"] }, + "expect": [ + { "fqn": "acme::commerce::Order", "matches": true }, + { "fqn": "acme::internal::Secret", "matches": false } + ] + }, + { + "name": "exclude-alone-narrows-everything", + "scope": { "exclude": ["acme::internal::**"] }, + "expect": [ + { "fqn": "other::Thing", "matches": true }, + { "fqn": "acme::internal::Secret", "matches": false } + ] + }, + { + "name": "multiple-includes-are-a-union", + "scope": { "include": ["acme::commerce::**", "acme::common::**"] }, + "expect": [ + { "fqn": "acme::commerce::Order", "matches": true }, + { "fqn": "acme::common::BaseEntity", "matches": true }, + { "fqn": "acme::billing::Invoice", "matches": false } + ] + }, + { + "name": "regex-metacharacters-are-literal", + "scope": { "include": ["acme::Order.v2"] }, + "expect": [ + { "fqn": "acme::Order.v2", "matches": true }, + { "fqn": "acme::OrderXv2", "matches": false } + ] + } + ] +} diff --git a/server/typescript/packages/sdk/test/scope-conformance.test.ts b/server/typescript/packages/sdk/test/scope-conformance.test.ts new file mode 100644 index 000000000..c7334a555 --- /dev/null +++ b/server/typescript/packages/sdk/test/scope-conformance.test.ts @@ -0,0 +1,29 @@ +// server/typescript/packages/sdk/test/scope-conformance.test.ts +import { describe, test, expect } from "bun:test"; +import { readFileSync } from "node:fs"; +import { join } from "node:path"; +import { compileScope, matchesScope, type Scope } from "../src/scope.js"; + +interface Case { + name: string; + scope: Scope; + expect: Array<{ fqn: string; matches: boolean }>; +} + +const CORPUS = join(import.meta.dir, "../../../../../fixtures/scope-conformance/cases.json"); +const cases = (JSON.parse(readFileSync(CORPUS, "utf8")) as { cases: Case[] }).cases; + +describe("scope-conformance corpus", () => { + test("corpus is non-empty (a silent zero-case run is a failed gate)", () => { + expect(cases.length).toBeGreaterThan(0); + }); + for (const c of cases) { + test(c.name, () => { + const compiled = compileScope(c.scope); + for (const e of c.expect) { + expect({ fqn: e.fqn, matches: matchesScope(e.fqn, compiled) }) + .toEqual({ fqn: e.fqn, matches: e.matches }); + } + }); + } +}); From b650ae1c767b8c48da7752d50ec960021c0e5f9c Mon Sep 17 00:00:00 2001 From: Doug Mealing Date: Mon, 17 Aug 2026 21:26:42 -0400 Subject: [PATCH 05/44] feat(sdk): resolve a source SET to a canonically-sorted file list MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit resolveSources turns a declared source SET into a sorted, de-duplicated list of metadata file paths — sorted by absolute path so permuting the input specs cannot change the result. Only `path` specs resolve in phase 1 (`resource`/`package` throw ERR_SOURCE_KIND_UNSUPPORTED); an unresolvable path throws ERR_SOURCE_UNRESOLVED rather than silently contributing nothing. File collection uses stat (not lstat or Dirent.isDirectory()) so a symlinked subdirectory is traversed, matching DirectorySource in @metaobjectsdev/metadata. Co-Authored-By: Claude Opus 5 (1M context) --- server/typescript/packages/sdk/src/sources.ts | 107 ++++++++++++++++ .../packages/sdk/test/sources.test.ts | 114 ++++++++++++++++++ 2 files changed, 221 insertions(+) create mode 100644 server/typescript/packages/sdk/src/sources.ts create mode 100644 server/typescript/packages/sdk/test/sources.test.ts diff --git a/server/typescript/packages/sdk/src/sources.ts b/server/typescript/packages/sdk/src/sources.ts new file mode 100644 index 000000000..f4bac2437 --- /dev/null +++ b/server/typescript/packages/sdk/src/sources.ts @@ -0,0 +1,107 @@ +// server/typescript/packages/sdk/src/sources.ts +// +// Phase-1 metadata-source-resolution — source spec resolution. +// +// Turns a declared source SET (`.metaobjects/config.json`'s `sources`) into a +// canonically-sorted, de-duplicated list of metadata file paths. The result is +// a pure function of the source SET, never of declaration order: permuting +// `specs` cannot change the output. A later phase-1 task pins this with a +// permutation test, so the sort-by-absolute-path + de-dup step is load-bearing. +import { readdir, stat } from "node:fs/promises"; +import { isAbsolute, join, resolve } from "node:path"; +import { ParseError, codeSource } from "@metaobjectsdev/metadata"; + +/** Tagged union of source kinds. `resource` and `package` are declared now so + * the config shape is stable across phases; only `path` resolves in phase 1 — + * `resource`/`package` throw `ERR_SOURCE_KIND_UNSUPPORTED`. */ +export type SourceSpec = + | { readonly path: string } + | { readonly resource: string } + | { readonly package: string }; + +export interface ResolvedSource { + /** Absolute path of one metadata file. */ + readonly file: string; + /** The spec that contributed it — provenance for diagnostics. */ + readonly spec: SourceSpec; +} + +/** Used when `sources` is absent or empty in `.metaobjects/config.json`. A + * DEFAULT, never a requirement — a project that declares `sources` explicitly + * need not include `metaobjects/` at all. */ +export const DEFAULT_SOURCES: readonly SourceSpec[] = [{ path: "metaobjects" }]; + +const PENDING_DIR = "_pending"; + +function isMetadataFile(name: string): boolean { + return name.endsWith(".json") || name.endsWith(".yaml") || name.endsWith(".yml"); +} + +/** Recursively collect metadata files under `dir`, excluding `_pending/` at any + * depth. Uses `stat` (follows symlinks) rather than `lstat` or + * `Dirent.isDirectory()` — `DirectorySource` in `@metaobjectsdev/metadata` has + * always followed symlinks this way, so a symlinked subdirectory must be + * traversed here too, or this walk and the loader's would silently disagree + * about the same tree. */ +async function collectDir(dir: string, out: string[]): Promise { + const entries = await readdir(dir); + for (const entry of entries) { + if (entry === PENDING_DIR) continue; + const full = join(dir, entry); + const s = await stat(full); + if (s.isDirectory()) await collectDir(full, out); + else if (s.isFile() && isMetadataFile(entry)) out.push(full); + } +} + +/** + * Resolve a declared source SET to a canonically-sorted list of metadata files. + * + * The result is sorted by absolute path and de-duplicated by file, so it is a + * pure function of the SET of `specs` — permuting `specs` cannot change the + * output. Declared order carries no information (the loader derives whatever + * order it needs from the files themselves). + * + * Only `path` specs resolve in phase 1: a directory is walked recursively, a + * file is taken as-is. An unresolvable `path` throws `ERR_SOURCE_UNRESOLVED` + * rather than silently contributing nothing; `resource`/`package` specs throw + * `ERR_SOURCE_KIND_UNSUPPORTED`. + * + * @param configDir absolute directory of the declaring config (the parent of + * `.metaobjects/`) — relative `path` specs resolve against it, never against + * ambient `process.cwd()`. + */ +export async function resolveSources( + configDir: string, + specs: readonly SourceSpec[], +): Promise { + const byFile = new Map(); + + for (const spec of specs) { + if (!("path" in spec)) { + const kind = "resource" in spec ? "resource" : "package"; + throw new ParseError( + `source kind "${kind}" is not supported by this toolchain yet; use a "path" source`, + { code: "ERR_SOURCE_KIND_UNSUPPORTED", source: codeSource("resolveSources") }, + ); + } + + const target = isAbsolute(spec.path) ? spec.path : resolve(configDir, spec.path); + let stats; + try { + stats = await stat(target); + } catch { + throw new ParseError( + `source path "${spec.path}" does not exist (resolved to ${target}, relative to ${configDir})`, + { code: "ERR_SOURCE_UNRESOLVED", source: codeSource("resolveSources") }, + ); + } + + const found: string[] = []; + if (stats.isDirectory()) await collectDir(target, found); + else found.push(target); + for (const file of found) if (!byFile.has(file)) byFile.set(file, spec); + } + + return [...byFile.keys()].sort().map((file) => ({ file, spec: byFile.get(file)! })); +} diff --git a/server/typescript/packages/sdk/test/sources.test.ts b/server/typescript/packages/sdk/test/sources.test.ts new file mode 100644 index 000000000..c213b1845 --- /dev/null +++ b/server/typescript/packages/sdk/test/sources.test.ts @@ -0,0 +1,114 @@ +import { describe, test, expect, beforeEach, afterEach } from "bun:test"; +import { mkdtempSync, rmSync, mkdirSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { resolveSources, DEFAULT_SOURCES } from "../src/sources.js"; + +let root: string; +const write = (rel: string, body = "{}") => { + const full = join(root, rel); + mkdirSync(join(full, ".."), { recursive: true }); + writeFileSync(full, body, "utf8"); + return full; +}; + +beforeEach(() => { root = mkdtempSync(join(tmpdir(), "metaobjects-sources-")); }); +afterEach(() => { rmSync(root, { recursive: true, force: true }); }); + +/** Pull the stable ERR_ code off a caught error, if it carries one. Mirrors + * scope.test.ts's `errorCode` — property-based, never message-matching: a + * cross-package `instanceof ParseError` is silently false when two physical + * copies of `@metaobjectsdev/metadata` are loaded (a globally-installed or + * linked CLI alongside a project-local dependency), so `.code` is the only + * reliable read. */ +function errorCode(err: unknown): string { + const code = (err as { code?: unknown }).code; + return typeof code === "string" ? code : "ERR_UNKNOWN"; +} + +/** Await `promise`, expecting it to reject — returns the rejection's stable + * code. The async counterpart of `errorCode` above, needed because + * `resolveSources` is async where `compileScope` (scope.test.ts) is not. */ +async function rejectedCode(promise: Promise): Promise { + try { + await promise; + } catch (err) { + return errorCode(err); + } + throw new Error("expected the promise to reject, but it resolved"); +} + +describe("resolveSources", () => { + test("resolves a directory recursively, metadata files only", async () => { + write("model/meta.a.json"); + write("model/nested/meta.b.yaml"); + write("model/notes.txt"); + const out = await resolveSources(root, [{ path: "model" }]); + expect(out.map((r) => r.file.replace(root + "/", ""))).toEqual([ + "model/meta.a.json", + "model/nested/meta.b.yaml", + ]); + }); + + test("resolves a single file", async () => { + write("model/meta.a.json"); + const out = await resolveSources(root, [{ path: "model/meta.a.json" }]); + expect(out).toHaveLength(1); + }); + + test("output is canonically sorted regardless of spec order", async () => { + write("b/meta.b.json"); + write("a/meta.a.json"); + const forward = await resolveSources(root, [{ path: "a" }, { path: "b" }]); + const reverse = await resolveSources(root, [{ path: "b" }, { path: "a" }]); + expect(forward.map((r) => r.file)).toEqual(reverse.map((r) => r.file)); + }); + + test("de-duplicates a file contributed by two overlapping specs", async () => { + write("model/meta.a.json"); + const out = await resolveSources(root, [{ path: "model" }, { path: "model/meta.a.json" }]); + expect(out).toHaveLength(1); + }); + + test("paths resolve against the config dir, not process.cwd()", async () => { + write("apps/ui/.keep"); + write("model/meta.a.json"); + const out = await resolveSources(join(root, "apps/ui"), [{ path: "../../model" }]); + expect(out).toHaveLength(1); + }); + + test("an unresolvable path is ERR_SOURCE_UNRESOLVED, never a silent skip", async () => { + expect(await rejectedCode(resolveSources(root, [{ path: "missing" }]))).toBe( + "ERR_SOURCE_UNRESOLVED", + ); + }); + + test("resource and package kinds are ERR_SOURCE_KIND_UNSUPPORTED in phase 1", async () => { + expect(await rejectedCode(resolveSources(root, [{ resource: "acme/model" }]))).toBe( + "ERR_SOURCE_KIND_UNSUPPORTED", + ); + expect(await rejectedCode(resolveSources(root, [{ package: "@acme/model" }]))).toBe( + "ERR_SOURCE_KIND_UNSUPPORTED", + ); + }); + + test("_pending is excluded at any depth", async () => { + write("model/meta.a.json"); + write("model/_pending/meta.draft.json"); + const out = await resolveSources(root, [{ path: "model" }]); + expect(out).toHaveLength(1); + }); + + test("a nested symlinked directory is followed", async () => { + write("real/meta.b.json"); + write("model/meta.a.json"); + const { symlinkSync } = await import("node:fs"); + symlinkSync(join(root, "real"), join(root, "model/linked"), "dir"); + const out = await resolveSources(root, [{ path: "model" }]); + expect(out).toHaveLength(2); + }); + + test("DEFAULT_SOURCES is the metaobjects/ directory", () => { + expect(DEFAULT_SOURCES).toEqual([{ path: "metaobjects" }]); + }); +}); From cb08e940623ebcc5a30afa63fe59abbe85bac6ba Mon Sep 17 00:00:00 2001 From: Doug Mealing Date: Mon, 17 Aug 2026 21:34:19 -0400 Subject: [PATCH 06/44] fix(sdk): make the overlapping-file spec attribution order-independent MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two overlapping specs contributing the same file previously kept "first spec processed wins," so `.file` was permutation-safe but the attributed `.spec` on that entry was not — a later phase-1 task's order-independence gate deep-equals the full ResolvedSource[] and would have failed on any overlapping-source project. The tie-break is now content-only (smaller JSON.stringify(spec) wins), so the full result is a pure function of the source set regardless of processing order. Also makes the metadata-file extension match case-insensitive (extname().toLowerCase()), matching DirectorySource in @metaobjectsdev/metadata exactly — a meta.JSON file was previously picked up by the loader and silently skipped here. Co-Authored-By: Claude Opus 5 (1M context) --- server/typescript/packages/sdk/src/sources.ts | 43 ++++++++++++++----- .../packages/sdk/test/sources.test.ts | 16 +++++++ 2 files changed, 48 insertions(+), 11 deletions(-) diff --git a/server/typescript/packages/sdk/src/sources.ts b/server/typescript/packages/sdk/src/sources.ts index f4bac2437..e4c3a4c21 100644 --- a/server/typescript/packages/sdk/src/sources.ts +++ b/server/typescript/packages/sdk/src/sources.ts @@ -3,12 +3,15 @@ // Phase-1 metadata-source-resolution — source spec resolution. // // Turns a declared source SET (`.metaobjects/config.json`'s `sources`) into a -// canonically-sorted, de-duplicated list of metadata file paths. The result is -// a pure function of the source SET, never of declaration order: permuting -// `specs` cannot change the output. A later phase-1 task pins this with a -// permutation test, so the sort-by-absolute-path + de-dup step is load-bearing. +// canonically-sorted, de-duplicated list of metadata file paths. The FULL +// result — including which spec each entry attributes to — is a pure +// function of the source SET, never of declaration order: permuting `specs` +// cannot change the output, even when two specs overlap on the same file. A +// later phase-1 task pins this with a permutation test (the design's +// linchpin), so both the sort-by-absolute-path step and the content-based +// overlap tie-break below are load-bearing. import { readdir, stat } from "node:fs/promises"; -import { isAbsolute, join, resolve } from "node:path"; +import { extname, isAbsolute, join, resolve } from "node:path"; import { ParseError, codeSource } from "@metaobjectsdev/metadata"; /** Tagged union of source kinds. `resource` and `package` are declared now so @@ -33,8 +36,14 @@ export const DEFAULT_SOURCES: readonly SourceSpec[] = [{ path: "metaobjects" }]; const PENDING_DIR = "_pending"; +/** Recognized metadata file extensions, matched case-insensitively — mirrors + * `DirectorySource` in `@metaobjectsdev/metadata`, which checks + * `extname().toLowerCase()`. Without this, `meta.JSON` would be picked up by + * the loader and silently skipped here. */ +const METADATA_EXTENSIONS = new Set([".json", ".yaml", ".yml"]); + function isMetadataFile(name: string): boolean { - return name.endsWith(".json") || name.endsWith(".yaml") || name.endsWith(".yml"); + return METADATA_EXTENSIONS.has(extname(name).toLowerCase()); } /** Recursively collect metadata files under `dir`, excluding `_pending/` at any @@ -57,10 +66,15 @@ async function collectDir(dir: string, out: string[]): Promise { /** * Resolve a declared source SET to a canonically-sorted list of metadata files. * - * The result is sorted by absolute path and de-duplicated by file, so it is a - * pure function of the SET of `specs` — permuting `specs` cannot change the - * output. Declared order carries no information (the loader derives whatever - * order it needs from the files themselves). + * The full result — each entry's `.file` AND its `.spec` — is a pure function + * of the SET of `specs`: permuting `specs` cannot change the output. Two parts + * make that hold: entries are sorted by absolute path (so file ORDER carries no + * declaration-order information), and when two specs overlap on the same file, + * the one attributed is chosen by comparing `JSON.stringify(spec)` — a + * content-only tie-break, so which spec "wins" never depends on which was + * processed first. Declared order carries no information anywhere in this + * function (the loader derives whatever order it needs from the files + * themselves). * * Only `path` specs resolve in phase 1: a directory is walked recursively, a * file is taken as-is. An unresolvable `path` throws `ERR_SOURCE_UNRESOLVED` @@ -100,7 +114,14 @@ export async function resolveSources( const found: string[] = []; if (stats.isDirectory()) await collectDir(target, found); else found.push(target); - for (const file of found) if (!byFile.has(file)) byFile.set(file, spec); + for (const file of found) { + const existing = byFile.get(file); + // Content-only tie-break: never "first spec processed wins", or the + // attributed `.spec` would depend on declaration order. + if (existing === undefined || JSON.stringify(spec) < JSON.stringify(existing)) { + byFile.set(file, spec); + } + } } return [...byFile.keys()].sort().map((file) => ({ file, spec: byFile.get(file)! })); diff --git a/server/typescript/packages/sdk/test/sources.test.ts b/server/typescript/packages/sdk/test/sources.test.ts index c213b1845..45bb30ab6 100644 --- a/server/typescript/packages/sdk/test/sources.test.ts +++ b/server/typescript/packages/sdk/test/sources.test.ts @@ -70,6 +70,22 @@ describe("resolveSources", () => { expect(out).toHaveLength(1); }); + test("the spec attributed to an overlapping file is order-independent, not just the file list", async () => { + write("model/meta.a.json"); + const forward = await resolveSources(root, [{ path: "model" }, { path: "model/meta.a.json" }]); + const reverse = await resolveSources(root, [{ path: "model/meta.a.json" }, { path: "model" }]); + // Deep-equal on the FULL ResolvedSource[] — .spec included, not just .file. + // A first-spec-wins tie-break would pass the two `de-duplicates` / + // `canonically sorted` tests above yet fail here, because which spec is + // attributed would flip between forward and reverse. + expect(forward).toEqual(reverse); + // Pin the actual deterministic winner: content-only comparison picks + // whichever spec's JSON.stringify sorts first, regardless of which was + // declared (or processed) first. + expect(forward).toHaveLength(1); + expect(forward[0]?.spec).toEqual({ path: "model" }); + }); + test("paths resolve against the config dir, not process.cwd()", async () => { write("apps/ui/.keep"); write("model/meta.a.json"); From de9296031145c838ef0d751d20ad3295e36c9d9c Mon Sep 17 00:00:00 2001 From: Doug Mealing Date: Mon, 17 Aug 2026 21:40:51 -0400 Subject: [PATCH 07/44] feat(sdk): config accepts a source SET, a scope block, and migrate.scope MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Widens ConfigSchema.sources from the dead 2-arm kind-discriminated union to the 3-arm path/resource/package shape from SourceSpec (sources.ts), adds an optional top-level scope block mirroring Scope (scope.ts), and adds migrate.scope for restricting a migrate run to a subset of loaded metadata. SourceSpecSchema deliberately skips .strict() on its arms: an existing project's config.json may still carry the pre-phase-1 { kind: "path", path: "..." } shape, and .strict() would reject the extra kind key and break that config on the next run. Zod's default strip-unknown-keys behavior parses it under the modern { path: "..." } shape instead, keeping the pre-existing kind-shaped tests in config.test.ts green untouched. Adds a compile-time parity assertion (a direct assignment, not a conditional type — the latter silently resolves to never on drift instead of erroring) so SourceSpecSchema's inferred type and the hand-written SourceSpec can't drift unnoticed. Co-Authored-By: Claude Opus 5 (1M context) --- server/typescript/packages/sdk/src/config.ts | 52 ++++++++++++++++--- .../packages/sdk/test/config.test.ts | 43 +++++++++++++++ 2 files changed, 87 insertions(+), 8 deletions(-) diff --git a/server/typescript/packages/sdk/src/config.ts b/server/typescript/packages/sdk/src/config.ts index eaf548c0c..f5269cd15 100644 --- a/server/typescript/packages/sdk/src/config.ts +++ b/server/typescript/packages/sdk/src/config.ts @@ -1,6 +1,7 @@ import { z } from "zod"; import { readFile, writeFile } from "node:fs/promises"; import { join } from "node:path"; +import type { SourceSpec } from "./sources.js"; const DialectEnum = z.enum(["sqlite", "postgres", "d1"]); @@ -50,8 +51,47 @@ const MigrateBlock = z.object({ onAmbiguous: OnAmbiguousEnum, allow: z.array(AllowTokenEnum), d1: D1Block, + /** Restricts a `meta migrate` run to a subset of the loaded metadata, by the + * same package-glob pattern grammar as top-level `scope` (see `scope.ts`). + * Include-only — there's no `migrate.scope.exclude`, since a migration run + * is scoped to what it's touching, not filtered down from "everything". */ + scope: z.array(z.string().min(1)), }).partial(); +/** + * Mirrors the hand-written `SourceSpec` union in `./sources.ts` — a + * declared source kind, one of `path` (resolves today), `resource`, or + * `package` (both reserved, throw `ERR_SOURCE_KIND_UNSUPPORTED` until a + * later phase). Deliberately NOT `.strict()`: an existing project's + * `.metaobjects/config.json` may still carry the pre-phase-1 `{ kind: + * "path", path: "..." }` shape (the dead 2-arm discriminated union this + * replaces) — `.strict()` would reject the extra `kind` key outright and + * break that project's config on the next `meta` run. Zod's default + * (strip-unknown-keys) parses it as the modern `{ path: "..." }` shape + * instead, which is what back-compat requires. + */ +const SourceSpecSchema = z.union([ + z.object({ path: z.string().min(1) }), + z.object({ resource: z.string().min(1) }), + z.object({ package: z.string().min(1) }), +]); + +// Compile-time parity: if SourceSpecSchema and the hand-written SourceSpec +// (./sources.ts) ever drift, this assignment stops compiling. A conditional +// type (`z.infer<...> extends SourceSpec ? true : never`) would silently +// resolve to `never` instead of erroring — this form fails for real. +const _sourceSpecParity: SourceSpec = {} as z.infer; +void _sourceSpecParity; + +/** Mirrors the hand-written `Scope` interface in `./scope.ts`. An absent or + * empty `include` means "everything" — see `matchesScope`. */ +const ScopeSchema = z + .object({ + include: z.array(z.string().min(1)).optional(), + exclude: z.array(z.string().min(1)).optional(), + }) + .strict(); + export const ConfigSchema = z.object({ schema_version: z.literal(1), pending_in_git: z.boolean().default(true), @@ -61,14 +101,10 @@ export const ConfigSchema = z.object({ drift_warn: z.number().min(0).max(1).default(0.7), }) .default({}), - sources: z - .array( - z.union([ - z.object({ kind: z.literal("path"), path: z.string() }), - z.object({ kind: z.literal("package"), package: z.string() }), - ]), - ) - .default([]), + sources: z.array(SourceSpecSchema).default([]), + /** Output filter applied across every command — see `./scope.ts`. Absent + * means "everything" (no filtering), matching `Scope`'s own contract. */ + scope: ScopeSchema.optional(), extract: z .object({ metaignore: z.string().optional(), diff --git a/server/typescript/packages/sdk/test/config.test.ts b/server/typescript/packages/sdk/test/config.test.ts index 804e3ed3d..6fd60d4a8 100644 --- a/server/typescript/packages/sdk/test/config.test.ts +++ b/server/typescript/packages/sdk/test/config.test.ts @@ -100,3 +100,46 @@ describe("ConfigSchema — migrate block", () => { expect(parsed.migrate?.databaseUrl).toBe("postgres://localhost/db"); }); }); + +describe("ConfigSchema — phase-1 source resolution", () => { + test("accepts a path source", () => { + const p = ConfigSchema.parse({ schema_version: 1, sources: [{ path: "../model" }] }); + expect(p.sources).toEqual([{ path: "../model" }]); + }); + test("accepts resource and package source kinds", () => { + const p = ConfigSchema.parse({ + schema_version: 1, + sources: [{ resource: "acme/model" }, { package: "@acme/model" }], + }); + expect(p.sources).toHaveLength(2); + }); + test("rejects an unknown source kind", () => { + expect(() => ConfigSchema.parse({ schema_version: 1, sources: [{ nope: "x" }] })).toThrow(); + }); + test("accepts a scope block", () => { + const p = ConfigSchema.parse({ + schema_version: 1, + scope: { include: ["acme::**"], exclude: ["acme::internal::**"] }, + }); + expect(p.scope?.include).toEqual(["acme::**"]); + }); + test("scope defaults to undefined (match everything)", () => { + expect(ConfigSchema.parse({ schema_version: 1 }).scope).toBeUndefined(); + }); + test("accepts migrate.scope", () => { + const p = ConfigSchema.parse({ + schema_version: 1, + migrate: { scope: ["acme::platform::**"] }, + }); + expect(p.migrate?.scope).toEqual(["acme::platform::**"]); + }); + test("an existing config with no new keys still parses (back-compat)", () => { + const p = ConfigSchema.parse({ + schema_version: 1, pending_in_git: true, + confidence_thresholds: { pending_promote: 0.8, drift_warn: 0.7 }, + sources: [], extract: {}, + }); + expect(p.sources).toEqual([]); + expect(p.scope).toBeUndefined(); + }); +}); From e32febb6f2531345d53a053e1874165cf0b3ea4d Mon Sep 17 00:00:00 2001 From: Doug Mealing Date: Mon, 17 Aug 2026 21:43:45 -0400 Subject: [PATCH 08/44] fix(sdk): restore .strict() on SourceSpecSchema arms; fail closed on typos MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ruling reversal on the prior commit: the pre-phase-1 { kind: "path", path: "..." } shape never shipped to an adopter (meta init has only ever scaffolded "sources": [], and nothing under src/ ever read it) — it existed only in this package's own tests. There is no live config to be lenient for, so the back-compat concern that motivated dropping .strict() doesn't hold, and this project is fail-closed on undeclared keys everywhere else (ADR-0023). Restores .strict() on all three SourceSpecSchema arms and updates the four legacy { kind, ... } fixture shapes (sdk/test/config.test.ts, cli/test/init.test.ts) to the modern shape, same assertion intent. Adds a test pinning the strictness itself: an unrecognized sibling key on a source is now a hard parse error rather than being silently stripped. Co-Authored-By: Claude Opus 5 (1M context) --- .../typescript/packages/cli/test/init.test.ts | 4 ++-- server/typescript/packages/sdk/src/config.ts | 24 +++++++++++-------- .../packages/sdk/test/config.test.ts | 13 ++++++++-- 3 files changed, 27 insertions(+), 14 deletions(-) diff --git a/server/typescript/packages/cli/test/init.test.ts b/server/typescript/packages/cli/test/init.test.ts index 9a0d559ee..11ce1f68f 100644 --- a/server/typescript/packages/cli/test/init.test.ts +++ b/server/typescript/packages/cli/test/init.test.ts @@ -152,7 +152,7 @@ describe("init() --force config preservation", () => { schema_version: 1 as const, pending_in_git: false, // changed from default confidence_thresholds: { pending_promote: 0.95, drift_warn: 0.8 }, - sources: [{ kind: "package" as const, package: "@acme/entities" }], + sources: [{ package: "@acme/entities" }], extract: {}, }; await saveConfig(join(cwd, ".metaobjects"), ConfigSchema.parse(customConfig)); @@ -165,7 +165,7 @@ describe("init() --force config preservation", () => { const reloaded = JSON.parse(readFileSync(join(cwd, ".metaobjects", "config.json"), "utf8")); expect(reloaded.pending_in_git).toBe(false); expect(reloaded.confidence_thresholds.pending_promote).toBe(0.95); - expect(reloaded.sources).toEqual([{ kind: "package", package: "@acme/entities" }]); + expect(reloaded.sources).toEqual([{ package: "@acme/entities" }]); }); test("writes fresh defaults when existing config is invalid (and warns)", async () => { diff --git a/server/typescript/packages/sdk/src/config.ts b/server/typescript/packages/sdk/src/config.ts index f5269cd15..f66ff7a82 100644 --- a/server/typescript/packages/sdk/src/config.ts +++ b/server/typescript/packages/sdk/src/config.ts @@ -62,18 +62,22 @@ const MigrateBlock = z.object({ * Mirrors the hand-written `SourceSpec` union in `./sources.ts` — a * declared source kind, one of `path` (resolves today), `resource`, or * `package` (both reserved, throw `ERR_SOURCE_KIND_UNSUPPORTED` until a - * later phase). Deliberately NOT `.strict()`: an existing project's - * `.metaobjects/config.json` may still carry the pre-phase-1 `{ kind: - * "path", path: "..." }` shape (the dead 2-arm discriminated union this - * replaces) — `.strict()` would reject the extra `kind` key outright and - * break that project's config on the next `meta` run. Zod's default - * (strip-unknown-keys) parses it as the modern `{ path: "..." }` shape - * instead, which is what back-compat requires. + * later phase). `.strict()` on every arm: this project is fail-closed on + * undeclared keys everywhere else (ADR-0023 makes an unregistered metadata + * attribute a hard error for the same reason) — a config schema that + * silently strips an unknown key would let `{ path: "model", pathh: "typo" + * }` parse clean and resolve one source instead of erroring on the typo. + * The pre-phase-1 `{ kind: "path", path: "..." }` shape (the dead 2-arm + * discriminated union this replaces) never shipped to an adopter — `meta + * init` has only ever scaffolded `"sources": []`, and nothing under `src/` + * ever read the old shape — so there is no live config to be lenient for; + * it only ever existed in this package's own tests, updated alongside this + * schema. */ const SourceSpecSchema = z.union([ - z.object({ path: z.string().min(1) }), - z.object({ resource: z.string().min(1) }), - z.object({ package: z.string().min(1) }), + z.object({ path: z.string().min(1) }).strict(), + z.object({ resource: z.string().min(1) }).strict(), + z.object({ package: z.string().min(1) }).strict(), ]); // Compile-time parity: if SourceSpecSchema and the hand-written SourceSpec diff --git a/server/typescript/packages/sdk/test/config.test.ts b/server/typescript/packages/sdk/test/config.test.ts index 6fd60d4a8..f3b89d56b 100644 --- a/server/typescript/packages/sdk/test/config.test.ts +++ b/server/typescript/packages/sdk/test/config.test.ts @@ -23,14 +23,14 @@ describe("ConfigSchema", () => { test("accepts a path source", () => { const parsed = ConfigSchema.parse({ schema_version: 1, - sources: [{ kind: "path", path: "../shared/.meta" }], + sources: [{ path: "../shared/.meta" }], }); expect(parsed.sources).toHaveLength(1); }); test("accepts a package source", () => { const parsed = ConfigSchema.parse({ schema_version: 1, - sources: [{ kind: "package", package: "@acme/entities" }], + sources: [{ package: "@acme/entities" }], }); expect(parsed.sources).toHaveLength(1); }); @@ -116,6 +116,15 @@ describe("ConfigSchema — phase-1 source resolution", () => { test("rejects an unknown source kind", () => { expect(() => ConfigSchema.parse({ schema_version: 1, sources: [{ nope: "x" }] })).toThrow(); }); + test("rejects a source with an unrecognized extra key (fail-closed, not stripped)", () => { + // A typo'd sibling key must not silently vanish and leave a + // valid-looking single-key source behind — .strict() on every + // SourceSpecSchema arm means an unknown key is a hard parse error, + // matching this project's fail-closed posture elsewhere (ADR-0023). + expect(() => + ConfigSchema.parse({ schema_version: 1, sources: [{ path: "model", pathh: "typo" }] }), + ).toThrow(); + }); test("accepts a scope block", () => { const p = ConfigSchema.parse({ schema_version: 1, From 9be8d35a19d53d78c1f069fb3604314819a5f67b Mon Sep 17 00:00:00 2001 From: Doug Mealing Date: Mon, 17 Aug 2026 21:48:20 -0400 Subject: [PATCH 09/44] feat(sdk): nearest-ancestor config discovery, bounded by the repo root Walk up from a starting directory for the nearest .metaobjects/config.json, stopping after examining a directory containing .git so a monorepo can never silently adopt a parent checkout's configuration. The config check runs before the .git check within each directory, so a repo-root config (sharing its directory with .git) stays reachable from any subdirectory. Co-Authored-By: Claude Opus 5 (1M context) --- .../typescript/packages/sdk/src/discovery.ts | 50 ++++++++++++++++++ .../packages/sdk/test/discovery.test.ts | 51 +++++++++++++++++++ 2 files changed, 101 insertions(+) create mode 100644 server/typescript/packages/sdk/src/discovery.ts create mode 100644 server/typescript/packages/sdk/test/discovery.test.ts diff --git a/server/typescript/packages/sdk/src/discovery.ts b/server/typescript/packages/sdk/src/discovery.ts new file mode 100644 index 000000000..7e2b818be --- /dev/null +++ b/server/typescript/packages/sdk/src/discovery.ts @@ -0,0 +1,50 @@ +// server/typescript/packages/sdk/src/discovery.ts +// +// Phase-1 metadata-source-resolution — nearest-ancestor config discovery. +// +// Walks up from a starting directory to find the nearest `.metaobjects/` +// carrying `config.json`. This is what makes a CLI *contextual*: run it +// inside an app in a monorepo and it finds that app's config rather than the +// repo root's. Two properties are load-bearing: nearest wins (a config in a +// subdirectory beats one in an ancestor — the walk returns on the FIRST +// directory found), and the walk stops at a repository boundary (`.git`), so +// a monorepo checkout can never silently adopt a *parent checkout's* +// configuration. The config check runs BEFORE the `.git` check within each +// directory — reversed, a config at the repo root (where `.git` also lives) +// would be unreachable from any subdirectory, since the boundary would stop +// the walk one directory too early. +import { stat } from "node:fs/promises"; +import { dirname, join, resolve } from "node:path"; +import { DEFAULT_METAOBJECTS_DIR } from "./memory.js"; + +const CONFIG_FILE = "config.json"; +const GIT_DIR = ".git"; + +async function exists(path: string): Promise { + try { + await stat(path); + return true; + } catch { + return false; + } +} + +/** + * Walk up from `startDir` for the nearest directory holding + * `.metaobjects/config.json`. The walk stops after examining a directory + * that contains `.git`, so a monorepo can never silently adopt a parent + * checkout's configuration. Returns the containing directory (not the + * `.metaobjects` directory itself), or undefined when nothing is found. + */ +export async function findConfigDir(startDir: string): Promise { + let dir = resolve(startDir); + for (;;) { + if (await exists(join(dir, DEFAULT_METAOBJECTS_DIR, CONFIG_FILE))) return dir; + // Boundary check AFTER the config check: a repo-root config (sharing its + // directory with `.git`) is still findable from any subdirectory. + if (await exists(join(dir, GIT_DIR))) return undefined; + const parent = dirname(dir); + if (parent === dir) return undefined; + dir = parent; + } +} diff --git a/server/typescript/packages/sdk/test/discovery.test.ts b/server/typescript/packages/sdk/test/discovery.test.ts new file mode 100644 index 000000000..64689e3e0 --- /dev/null +++ b/server/typescript/packages/sdk/test/discovery.test.ts @@ -0,0 +1,51 @@ +import { describe, test, expect, beforeEach, afterEach } from "bun:test"; +import { mkdtempSync, rmSync, mkdirSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { findConfigDir } from "../src/discovery.js"; + +let root: string; +const mk = (rel: string) => mkdirSync(join(root, rel), { recursive: true }); +const cfg = (rel: string) => { + mk(join(rel, ".metaobjects")); + writeFileSync(join(root, rel, ".metaobjects/config.json"), '{"schema_version":1}', "utf8"); +}; + +beforeEach(() => { root = mkdtempSync(join(tmpdir(), "metaobjects-discovery-")); mk(".git"); }); +afterEach(() => { rmSync(root, { recursive: true, force: true }); }); + +describe("findConfigDir", () => { + test("finds a config in the start directory", async () => { + cfg("apps/ui"); mk("apps/ui/src"); + expect(await findConfigDir(join(root, "apps/ui"))).toBe(join(root, "apps/ui")); + }); + test("walks up to the nearest ancestor config", async () => { + cfg("apps/ui"); mk("apps/ui/src/deep"); + expect(await findConfigDir(join(root, "apps/ui/src/deep"))).toBe(join(root, "apps/ui")); + }); + test("nearest wins over a further ancestor", async () => { + cfg("."); cfg("apps/ui"); mk("apps/ui/src"); + expect(await findConfigDir(join(root, "apps/ui/src"))).toBe(join(root, "apps/ui")); + }); + test("stops at the repository boundary — never adopts a parent checkout's config", async () => { + // A config ABOVE the .git boundary must not be found. + const outer = mkdtempSync(join(tmpdir(), "metaobjects-outer-")); + try { + mkdirSync(join(outer, "inner/.git"), { recursive: true }); + mkdirSync(join(outer, ".metaobjects"), { recursive: true }); + writeFileSync(join(outer, ".metaobjects/config.json"), '{"schema_version":1}', "utf8"); + mkdirSync(join(outer, "inner/src"), { recursive: true }); + expect(await findConfigDir(join(outer, "inner/src"))).toBeUndefined(); + } finally { + rmSync(outer, { recursive: true, force: true }); + } + }); + test("a repo-root config IS found from a subdirectory", async () => { + cfg("."); mk("apps/ui"); + expect(await findConfigDir(join(root, "apps/ui"))).toBe(root); + }); + test("returns undefined when nothing is found", async () => { + mk("apps/ui"); + expect(await findConfigDir(join(root, "apps/ui"))).toBeUndefined(); + }); +}); From 4300b7bd787245defb64573d7be9c861bddd1953 Mon Sep 17 00:00:00 2001 From: Doug Mealing Date: Mon, 17 Aug 2026 21:56:20 -0400 Subject: [PATCH 10/44] =?UTF-8?q?feat(sdk):=20resolveCollection()=20?= =?UTF-8?q?=E2=80=94=20one=20authority=20for=20where=20metadata=20lives?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Composes discovery, config, source resolution and scope into the single function every metadata read path routes through. `metaobjects/` is the DEFAULT value of `sources`, never a requirement — no call site downstream of this may assume the directory name. Uses ParseError + a `code` property (this codebase's error convention), not a message-prefixed plain Error, and wires scope/sources/discovery into the package barrel alongside it. Co-Authored-By: Claude Opus 5 (1M context) --- .../typescript/packages/sdk/src/collection.ts | 123 ++++++++++++++++++ server/typescript/packages/sdk/src/index.ts | 15 +++ .../packages/sdk/test/collection.test.ts | 98 ++++++++++++++ 3 files changed, 236 insertions(+) create mode 100644 server/typescript/packages/sdk/src/collection.ts create mode 100644 server/typescript/packages/sdk/test/collection.test.ts diff --git a/server/typescript/packages/sdk/src/collection.ts b/server/typescript/packages/sdk/src/collection.ts new file mode 100644 index 000000000..6d8c6c061 --- /dev/null +++ b/server/typescript/packages/sdk/src/collection.ts @@ -0,0 +1,123 @@ +// server/typescript/packages/sdk/src/collection.ts +// +// Phase-1 metadata-source-resolution — the single authority. +// +// `resolveCollection()` composes discovery (`discovery.ts`), config +// (`config.ts`), source resolution (`sources.ts`) and the scope engine +// (`scope.ts`) into one function that decides where a project's metadata +// lives. `metaobjects/` is the DEFAULT value of `sources`, never a +// requirement — a project that declares nothing still resolves exactly as +// today (`DEFAULT_SOURCES` in `sources.ts`); a project that declares +// `sources` can point anywhere. No other call site may assume the directory +// name — this is where that assumption is allowed to live, exactly once. +import { stat } from "node:fs/promises"; +import { join, resolve } from "node:path"; +import { ParseError, codeSource } from "@metaobjectsdev/metadata"; +import { loadConfig, type Config } from "./config.js"; +import { findConfigDir } from "./discovery.js"; +import { compileScope, type CompiledScope, type Scope } from "./scope.js"; +import { DEFAULT_METADATA_DIR, DEFAULT_METAOBJECTS_DIR } from "./memory.js"; +import { DEFAULT_SOURCES, resolveSources, type ResolvedSource, type SourceSpec } from "./sources.js"; + +export interface Collection { + /** Directory whose config declared this collection (or the resolved start + * directory, when nothing was discovered and the default applies). */ + readonly configDir: string; + /** Canonically-sorted absolute metadata file paths — see `resolveSources`. */ + readonly files: readonly string[]; + /** Same set, carrying the contributing spec for provenance. */ + readonly sources: readonly ResolvedSource[]; + /** Output filter for codegen. Empty include => everything. */ + readonly scope: CompiledScope; + /** Output filter for migrate/verify --db. Undefined => the command governs + * everything in scope. */ + readonly migrateScope: CompiledScope | undefined; +} + +async function isDir(p: string): Promise { + try { + return (await stat(p)).isDirectory(); + } catch { + return false; + } +} + +/** Narrow the zod-inferred `Config["scope"]` (whose `.optional()` fields are + * typed `T | undefined` even when present) down to `Scope`'s + * exactOptionalPropertyTypes-safe shape — a key is omitted entirely rather + * than assigned `undefined`. */ +function toScope(spec: Config["scope"]): Scope { + return { + ...(spec?.include !== undefined && { include: spec.include }), + ...(spec?.exclude !== undefined && { exclude: spec.exclude }), + }; +} + +/** + * THE single authority on where metadata lives. Every read path routes + * through this — `metaobjects/` is the DEFAULT value of `sources`, never an + * assumption baked into a call site. + * + * Resolution order: an explicit `opts.explicitDir` wins outright; otherwise + * `findConfigDir` walks up from `startDir` for the nearest + * `.metaobjects/config.json`, falling back to `startDir` itself when none is + * found. When the resolved directory carries a config, its declared + * `sources`/`scope`/`migrate.scope` govern; an absent or malformed config + * (no `.metaobjects/config.json`, or one `loadConfig` cannot read) falls + * through to `DEFAULT_SOURCES` — the same `metaobjects/` directory the + * pre-source-resolution toolchain always read. Throws + * `ERR_COLLECTION_NOT_FOUND` only when BOTH have failed: no `sources` were + * declared AND the default `metaobjects/` directory does not exist either. + * + * A declared source that fails to resolve is a different, louder failure — + * `resolveSources` throws `ERR_SOURCE_UNRESOLVED` for that case; only the + * DEFAULT is allowed to be silently absent. + */ +export async function resolveCollection( + startDir: string, + opts?: { explicitDir?: string }, +): Promise { + const explicit = opts?.explicitDir; + const configDir = + explicit !== undefined ? resolve(explicit) : ((await findConfigDir(startDir)) ?? resolve(startDir)); + + let specs: readonly SourceSpec[] = DEFAULT_SOURCES; + let scopeSpec: Config["scope"]; + let migrateSpec: string[] | undefined; + + if (await isDir(join(configDir, DEFAULT_METAOBJECTS_DIR))) { + try { + const cfg = await loadConfig(join(configDir, DEFAULT_METAOBJECTS_DIR)); + if (cfg.sources.length > 0) specs = cfg.sources; + scopeSpec = cfg.scope; + migrateSpec = cfg.migrate?.scope; + } catch { + // `.metaobjects/` exists but `config.json` is absent or unreadable — + // fall through to the default source set, matching the no-config + // case below. A genuinely MALFORMED config.json (present, readable, + // but failing ConfigSchema.parse) takes the same path here, since + // loadConfig's current signature gives no way to tell "absent" apart + // from "malformed" without re-implementing its read+parse. See the + // task report for why that's flagged rather than fixed in place. + } + } + + // Only the DEFAULT is allowed to be absent — an explicitly declared source + // that does not resolve is `resolveSources`'s ERR_SOURCE_UNRESOLVED, not this. + if (specs === DEFAULT_SOURCES && !(await isDir(join(configDir, DEFAULT_METADATA_DIR)))) { + throw new ParseError( + `no metadata sources declared in ${configDir} and no default "${DEFAULT_METADATA_DIR}" directory found. ` + + `Declare "sources" in ${DEFAULT_METAOBJECTS_DIR}/config.json, or run 'meta init' to scaffold.`, + { code: "ERR_COLLECTION_NOT_FOUND", source: codeSource("resolveCollection") }, + ); + } + + const sources = await resolveSources(configDir, specs); + return { + configDir, + files: sources.map((s) => s.file), + sources, + scope: compileScope(toScope(scopeSpec)), + migrateScope: migrateSpec === undefined ? undefined : compileScope({ include: migrateSpec }), + }; +} diff --git a/server/typescript/packages/sdk/src/index.ts b/server/typescript/packages/sdk/src/index.ts index 3c87b14d9..d4051db1e 100644 --- a/server/typescript/packages/sdk/src/index.ts +++ b/server/typescript/packages/sdk/src/index.ts @@ -87,6 +87,21 @@ export { } from "./memory.js"; export type { LoadMemoryOptions } from "./memory.js"; +// Scope — output filter over fully-qualified node names +export { compileScope, matchesScope } from "./scope.js"; +export type { Scope, CompiledScope } from "./scope.js"; + +// Source resolution — a declared source SET to a canonically-sorted file list +export { resolveSources, DEFAULT_SOURCES } from "./sources.js"; +export type { SourceSpec, ResolvedSource } from "./sources.js"; + +// Discovery — nearest-ancestor `.metaobjects/config.json`, bounded by the repo root +export { findConfigDir } from "./discovery.js"; + +// Collection — the single authority on where a project's metadata lives +export { resolveCollection } from "./collection.js"; +export type { Collection } from "./collection.js"; + // Workspace discovery — finds peer metadata packages in a monorepo export { discoverWorkspace, resolveExtendsOrder, packageLabel } from "./workspace.js"; export type { Workspace, WorkspacePackage } from "./workspace.js"; diff --git a/server/typescript/packages/sdk/test/collection.test.ts b/server/typescript/packages/sdk/test/collection.test.ts new file mode 100644 index 000000000..7ec6661d0 --- /dev/null +++ b/server/typescript/packages/sdk/test/collection.test.ts @@ -0,0 +1,98 @@ +import { describe, test, expect, beforeEach, afterEach } from "bun:test"; +import { mkdtempSync, rmSync, mkdirSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { resolveCollection } from "../src/collection.js"; +import { matchesScope } from "../src/scope.js"; + +let root: string; +const write = (rel: string, body: string) => { + mkdirSync(join(root, rel, ".."), { recursive: true }); + writeFileSync(join(root, rel), body, "utf8"); +}; +const config = (dir: string, cfg: object) => + write(join(dir, ".metaobjects/config.json"), JSON.stringify({ schema_version: 1, ...cfg })); + +/** Pull the stable ERR_ code off a caught error, if it carries one. Mirrors + * scope.test.ts / sources.test.ts's `errorCode` — property-based, never + * message-matching or `instanceof`: a cross-package `instanceof ParseError` + * is silently false when two physical copies of `@metaobjectsdev/metadata` + * are loaded, so `.code` is the only reliable read. */ +function errorCode(err: unknown): string { + const code = (err as { code?: unknown }).code; + return typeof code === "string" ? code : "ERR_UNKNOWN"; +} + +/** Await `promise`, expecting it to reject — returns the rejection's stable + * code. Mirrors sources.test.ts's `rejectedCode`. */ +async function rejectedCode(promise: Promise): Promise { + try { + await promise; + } catch (err) { + return errorCode(err); + } + throw new Error("expected the promise to reject, but it resolved"); +} + +beforeEach(() => { root = mkdtempSync(join(tmpdir(), "metaobjects-collection-")); mkdirSync(join(root, ".git")); }); +afterEach(() => { rmSync(root, { recursive: true, force: true }); }); + +describe("resolveCollection", () => { + test("BACK-COMPAT: no sources declared falls back to metaobjects/", async () => { + write("metaobjects/meta.a.json", "{}"); + config(".", {}); + const c = await resolveCollection(root); + expect(c.files.map((f) => f.replace(root + "/", ""))).toEqual(["metaobjects/meta.a.json"]); + }); + + test("BACK-COMPAT: no config at all still finds metaobjects/ in the start dir", async () => { + write("metaobjects/meta.a.json", "{}"); + const c = await resolveCollection(root); + expect(c.files).toHaveLength(1); + }); + + test("a consumer reaches a tree elsewhere in the repo", async () => { + write("model/meta.a.json", "{}"); + config("apps/ui", { sources: [{ path: "../../model" }] }); + const c = await resolveCollection(join(root, "apps/ui")); + expect(c.configDir).toBe(join(root, "apps/ui")); + expect(c.files.map((f) => f.replace(root + "/", ""))).toEqual(["model/meta.a.json"]); + }); + + test("scope compiles and is applied by matchesScope", async () => { + write("model/meta.a.json", "{}"); + config("apps/ui", { sources: [{ path: "../../model" }], scope: { include: ["acme::**"] } }); + const c = await resolveCollection(join(root, "apps/ui")); + expect(matchesScope("acme::Order", c.scope)).toBe(true); + expect(matchesScope("other::Order", c.scope)).toBe(false); + }); + + test("migrateScope is undefined when not declared", async () => { + write("metaobjects/meta.a.json", "{}"); + config(".", {}); + expect((await resolveCollection(root)).migrateScope).toBeUndefined(); + }); + + test("migrateScope compiles when declared", async () => { + write("metaobjects/meta.a.json", "{}"); + config(".", { migrate: { scope: ["acme::platform::**"] } }); + const c = await resolveCollection(root); + expect(matchesScope("acme::platform::Job", c.migrateScope!)).toBe(true); + expect(matchesScope("arena::Match", c.migrateScope!)).toBe(false); + }); + + test("an explicit dir overrides discovery", async () => { + write("model/meta.a.json", "{}"); + config("apps/ui", { sources: [{ path: "../../model" }] }); + config("apps/api", { sources: [{ path: "../../model" }] }); + const c = await resolveCollection(join(root, "apps/ui"), { explicitDir: join(root, "apps/api") }); + expect(c.configDir).toBe(join(root, "apps/api")); + }); + + test("nothing discoverable and no default dir is ERR_COLLECTION_NOT_FOUND", async () => { + mkdirSync(join(root, "apps/ui"), { recursive: true }); + expect(await rejectedCode(resolveCollection(join(root, "apps/ui")))).toBe( + "ERR_COLLECTION_NOT_FOUND", + ); + }); +}); From 6c6f8c0c29253045e5fbefc4b41c6d61d33765ae Mon Sep 17 00:00:00 2001 From: Doug Mealing Date: Mon, 17 Aug 2026 21:59:14 -0400 Subject: [PATCH 11/44] fix(sdk): resolveCollection propagates a malformed config, never swallows it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A config.json that EXISTS but fails to load (malformed JSON, a ConfigSchema violation) was being caught and silently treated as "no config declared", falling through to DEFAULT_SOURCES. That let a typo'd config quietly generate from a possibly-stale metaobjects/ with no diagnostic — a worse failure than the one this design exists to remove, since resolveCollection is the single authority every command routes through. Fixed by checking for the config FILE (fileExists), not the .metaobjects directory (isDir told us nothing about whether config.json was actually inside it) — an absent file still falls through silently (the ordinary "no config" case), but a present-and-broken one now propagates uncaught. Co-Authored-By: Claude Opus 5 (1M context) --- .../typescript/packages/sdk/src/collection.ts | 58 ++++++++++++------- .../packages/sdk/test/collection.test.ts | 18 ++++++ 2 files changed, 55 insertions(+), 21 deletions(-) diff --git a/server/typescript/packages/sdk/src/collection.ts b/server/typescript/packages/sdk/src/collection.ts index 6d8c6c061..8823493b0 100644 --- a/server/typescript/packages/sdk/src/collection.ts +++ b/server/typescript/packages/sdk/src/collection.ts @@ -42,6 +42,20 @@ async function isDir(p: string): Promise { } } +/** `config.json`'s own basename — kept local rather than imported, matching + * `discovery.ts`'s own private `CONFIG_FILE` constant (the string is not + * exported from `config.js`). */ +const CONFIG_FILE = "config.json"; + +async function fileExists(p: string): Promise { + try { + await stat(p); + return true; + } catch { + return false; + } +} + /** Narrow the zod-inferred `Config["scope"]` (whose `.optional()` fields are * typed `T | undefined` even when present) down to `Scope`'s * exactOptionalPropertyTypes-safe shape — a key is omitted entirely rather @@ -62,12 +76,15 @@ function toScope(spec: Config["scope"]): Scope { * `findConfigDir` walks up from `startDir` for the nearest * `.metaobjects/config.json`, falling back to `startDir` itself when none is * found. When the resolved directory carries a config, its declared - * `sources`/`scope`/`migrate.scope` govern; an absent or malformed config - * (no `.metaobjects/config.json`, or one `loadConfig` cannot read) falls - * through to `DEFAULT_SOURCES` — the same `metaobjects/` directory the - * pre-source-resolution toolchain always read. Throws - * `ERR_COLLECTION_NOT_FOUND` only when BOTH have failed: no `sources` were - * declared AND the default `metaobjects/` directory does not exist either. + * `sources`/`scope`/`migrate.scope` govern. Only a genuinely ABSENT + * `config.json` falls through to `DEFAULT_SOURCES` — the same `metaobjects/` + * directory the pre-source-resolution toolchain always read; a config.json + * that EXISTS but fails to load (malformed JSON, schema violation) is the + * author's error and propagates rather than silently degrading — a source + * that fails to resolve must never look like one that was never declared. + * Throws `ERR_COLLECTION_NOT_FOUND` only when BOTH have failed: no + * `sources` were declared AND the default `metaobjects/` directory does not + * exist either. * * A declared source that fails to resolve is a different, louder failure — * `resolveSources` throws `ERR_SOURCE_UNRESOLVED` for that case; only the @@ -85,21 +102,20 @@ export async function resolveCollection( let scopeSpec: Config["scope"]; let migrateSpec: string[] | undefined; - if (await isDir(join(configDir, DEFAULT_METAOBJECTS_DIR))) { - try { - const cfg = await loadConfig(join(configDir, DEFAULT_METAOBJECTS_DIR)); - if (cfg.sources.length > 0) specs = cfg.sources; - scopeSpec = cfg.scope; - migrateSpec = cfg.migrate?.scope; - } catch { - // `.metaobjects/` exists but `config.json` is absent or unreadable — - // fall through to the default source set, matching the no-config - // case below. A genuinely MALFORMED config.json (present, readable, - // but failing ConfigSchema.parse) takes the same path here, since - // loadConfig's current signature gives no way to tell "absent" apart - // from "malformed" without re-implementing its read+parse. See the - // task report for why that's flagged rather than fixed in place. - } + // Check the FILE, not the `.metaobjects/` directory: a directory that + // exists but holds no `config.json` is the ordinary "no config" case and + // must fall through silently, same as no `.metaobjects/` at all. + if (await fileExists(join(configDir, DEFAULT_METAOBJECTS_DIR, CONFIG_FILE))) { + // No try/catch here: a config.json that EXISTS but fails to load + // (malformed JSON, a ConfigSchema violation) propagates. Swallowing it + // would make a typo'd config behave identically to no config at all — + // silently generating from a possibly-stale `metaobjects/` with no + // diagnostic, which is a worse failure than the one this design exists + // to remove. + const cfg = await loadConfig(join(configDir, DEFAULT_METAOBJECTS_DIR)); + if (cfg.sources.length > 0) specs = cfg.sources; + scopeSpec = cfg.scope; + migrateSpec = cfg.migrate?.scope; } // Only the DEFAULT is allowed to be absent — an explicitly declared source diff --git a/server/typescript/packages/sdk/test/collection.test.ts b/server/typescript/packages/sdk/test/collection.test.ts index 7ec6661d0..2bdea5cc1 100644 --- a/server/typescript/packages/sdk/test/collection.test.ts +++ b/server/typescript/packages/sdk/test/collection.test.ts @@ -95,4 +95,22 @@ describe("resolveCollection", () => { "ERR_COLLECTION_NOT_FOUND", ); }); + + test("a malformed config.json rejects rather than silently falling back to metaobjects/", async () => { + write("metaobjects/meta.a.json", "{}"); + // Truncated JSON — config.json EXISTS but cannot be parsed. Must surface + // as a real load failure, never a silent DEFAULT_SOURCES fallback: a + // typo'd config that quietly generates from a possibly-stale + // `metaobjects/` with no diagnostic is worse than the status quo this + // design set out to fix. + write(".metaobjects/config.json", '{ "schema_version": 1, '); + await expect(resolveCollection(root)).rejects.toThrow(SyntaxError); + }); + + test("a .metaobjects/ directory with no config.json still falls back to metaobjects/", async () => { + write("metaobjects/meta.a.json", "{}"); + mkdirSync(join(root, ".metaobjects"), { recursive: true }); // dir exists, file does not + const c = await resolveCollection(root); + expect(c.files.map((f) => f.replace(root + "/", ""))).toEqual(["metaobjects/meta.a.json"]); + }); }); From 4651484cb27c051c4b5c653313b9ca17220a6ed9 Mon Sep 17 00:00:00 2001 From: Doug Mealing Date: Mon, 17 Aug 2026 22:11:12 -0400 Subject: [PATCH 12/44] =?UTF-8?q?test(sdk):=20pin=20order=20independence?= =?UTF-8?q?=20=E2=80=94=20permuted=20source=20sets=20serialize=20byte-iden?= =?UTF-8?q?tically?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The linchpin gate for the source-resolution design: resolution is a pure function of the declared source SET, so declaration order carries no information. Loads a base + an overlay onto it + an independent third file across all six permutations of the three specs, and asserts: 1. resolveSources() output is deep-equal (full ResolvedSource[], .spec included, not just .file) across every permutation. 2. The loaded model's own-mode canonicalSerialize() output is byte-identical across every permutation. Also pins that the permutations() helper genuinely produces 6 distinct orderings, not 6 copies of the same one. Verified the gate is real by temporarily removing resolveSources' .sort() (sources.ts unmodified in this commit) — both assertions failed with a diff naming the exact diverging permutation, confirming the test actually depends on order-independent behavior rather than passing vacuously. Co-Authored-By: Claude Opus 5 (1M context) --- .../sdk/test/order-independence.test.ts | 124 ++++++++++++++++++ 1 file changed, 124 insertions(+) create mode 100644 server/typescript/packages/sdk/test/order-independence.test.ts diff --git a/server/typescript/packages/sdk/test/order-independence.test.ts b/server/typescript/packages/sdk/test/order-independence.test.ts new file mode 100644 index 000000000..d7c81d49f --- /dev/null +++ b/server/typescript/packages/sdk/test/order-independence.test.ts @@ -0,0 +1,124 @@ +// server/typescript/packages/sdk/test/order-independence.test.ts +// +// The order-independence gate — the linchpin of the whole design. The +// premise everything else rests on: resolution is a pure function of the +// declared source SET, so the order sources are declared in carries no +// information. That is why the design has no ordered-list semantics, no +// topological sort, no cycle detection, and no diamond-dependency problem. +// This file is what turns that premise from a belief into an enforced +// property, at two tiers: +// 1. resolveSources() itself (T4 already documents this contract; this +// re-asserts it across all six permutations of three specs, on the +// FULL ResolvedSource[] — .spec included, not just .file). +// 2. The loaded MODEL — resolveSources() feeding MetaDataLoader, its +// canonical (own-mode) serialization byte-identical across the same +// six permutations. +// +// The fixture shape — a base declaration, an overlay onto it, and an +// independent third file — is deliberately the one whose merge is most +// order-sensitive if anything is: an overlay must find its base regardless +// of which file the loader saw first. +import { describe, test, expect, beforeEach, afterEach } from "bun:test"; +import { mkdtempSync, rmSync, mkdirSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { resolveSources, type SourceSpec } from "../src/sources.js"; + +let root: string; +const write = (rel: string, body: object) => { + mkdirSync(join(root, rel, ".."), { recursive: true }); + writeFileSync(join(root, rel), JSON.stringify(body), "utf8"); +}; + +beforeEach(() => { + root = mkdtempSync(join(tmpdir(), "metaobjects-order-")); + // A base declaration, an overlay onto it, and an independent third file — + // the shapes whose merge is order-sensitive if anything is. + write("a/meta.base.json", { + "metadata.root": { package: "acme", children: [ + { "object.entity": { name: "Order", children: [{ "field.string": { name: "id" } }] } }] }, + }); + write("b/meta.overlay.json", { + "metadata.root": { package: "acme", children: [ + { "object.entity": { name: "Order", overlay: true, children: [ + { "field.string": { name: "note" } }] } }] }, + }); + write("c/meta.other.json", { + "metadata.root": { package: "acme", children: [ + { "object.entity": { name: "Customer", children: [{ "field.string": { name: "id" } }] } }] }, + }); +}); +afterEach(() => { rmSync(root, { recursive: true, force: true }); }); + +function permutations(items: T[]): T[][] { + if (items.length <= 1) return [items]; + const out: T[][] = []; + for (let i = 0; i < items.length; i++) { + const rest = [...items.slice(0, i), ...items.slice(i + 1)]; + for (const p of permutations(rest)) out.push([items[i]!, ...p]); + } + return out; +} + +describe("permutations helper", () => { + test("produces 6 distinct orderings of 3 items", () => { + // Sanity-check the helper itself, not just its output length: 6 entries + // that were secretly duplicates would let both gates below pass + // vacuously without ever exercising a real reordering. + const specs: SourceSpec[] = [{ path: "a" }, { path: "b" }, { path: "c" }]; + const perms = permutations(specs); + expect(perms).toHaveLength(6); + const distinct = new Set(perms.map((p) => p.map((s) => (s as { path: string }).path).join(","))); + expect(distinct.size).toBe(6); + }); +}); + +describe("order independence", () => { + test("resolveSources output is identical across every spec permutation", async () => { + const specs: SourceSpec[] = [{ path: "a" }, { path: "b" }, { path: "c" }]; + const perms = permutations(specs); + expect(perms).toHaveLength(6); + + const results = await Promise.all(perms.map((p) => resolveSources(root, p))); + expect(results).toHaveLength(6); + + // Deep-equal on the FULL ResolvedSource[] — .spec included, not just + // .file. T4's de-dup tie-break is already content-based (compares + // JSON.stringify(spec)), so the full structure is order-free too; a + // .file-only assertion would leave this gate narrower than the property + // it exists to prove. + const expected = results[0]!; + for (let i = 1; i < results.length; i++) { + expect( + results[i], + `permutation ${i} (${JSON.stringify(perms[i])}) diverged from permutation 0 (${JSON.stringify(perms[0])})`, + ).toEqual(expected); + } + }); + + test("the loaded model serializes byte-identically across every permutation", async () => { + const { MetaDataLoader, composeRegistry, coreProviders, canonicalSerialize } = + await import("@metaobjectsdev/metadata"); + const { FileSource } = await import("@metaobjectsdev/metadata/core"); + const specs: SourceSpec[] = [{ path: "a" }, { path: "b" }, { path: "c" }]; + const perms = permutations(specs); + expect(perms).toHaveLength(6); + + const serialized: string[] = []; + for (const p of perms) { + const resolved = await resolveSources(root, p); + const loader = new MetaDataLoader({ registry: composeRegistry(coreProviders) }); + const result = await loader.load(resolved.map((r) => new FileSource(r.file))); + expect(result.errors).toHaveLength(0); + serialized.push(canonicalSerialize(result.root)); + } + expect(serialized).toHaveLength(6); + + for (let i = 1; i < serialized.length; i++) { + expect( + serialized[i], + `permutation ${i} (${JSON.stringify(perms[i])}) serialized differently than permutation 0 (${JSON.stringify(perms[0])})`, + ).toBe(serialized[0]!); + } + }); +}); From 814719eb4cb1fbc3834b55ea364ffd28790c68d8 Mon Sep 17 00:00:00 2001 From: Doug Mealing Date: Mon, 17 Aug 2026 22:32:07 -0400 Subject: [PATCH 13/44] =?UTF-8?q?fix(sdk):=20order-independence=20gate=20?= =?UTF-8?q?=E2=80=94=20assert=20content=20resolution,=20not=20sibling=20or?= =?UTF-8?q?der?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Second correction to the order-independence gate (task 8). Review found the first fix overcorrected: comparing whole-root canonicalSerialize() output across permuted FileSource[] input failed on unmodified code, but only because MetaRoot's top-level children array follows raw parse order — Order vs Customer swapping position depending on which file the loader saw first. That was never a design claim: canonicalSerialize()'s own contract promises alphabetical @-attr keys and a trailing newline, nothing about sibling ordering, and production never hands the loader a permuted list — resolveSources() sorts by absolute path first (test 1). Test 2 now compares CONTENT, not the whole tree: a name-keyed Map of each top-level object's own canonicalSerialize(), built per permutation and deep-equal'd across all six — insensitive to sibling order by construction, but still catches any real content divergence. It also directly asserts the overlay's `note` field landed on Order in every permutation, since that's the specific fact _partitionOverlayLast is responsible for. File header rewritten to document the three-layer truth this gate now encodes: resolveSources canonicalizes file order (test 1); the loader resolves content order-independently including overlay-before-base (test 2); sibling order of unrelated top-level nodes follows input order and is deliberately not asserted, because production never permutes. Break-and-revert re-confirmed against the rewritten test: commenting out _partitionOverlayLast's call in meta-data-loader.ts makes permutation [b, a, c] (overlay before base) throw ERR_OVERLAY_NO_TARGET, caught by the per-permutation result.errors assertion; reverted immediately (meta-data-loader.ts is byte-identical to HEAD in this commit — the diff touches only the test file). Co-Authored-By: Claude Opus 5 (1M context) --- .../sdk/test/order-independence.test.ts | 120 ++++++++++++++---- 1 file changed, 92 insertions(+), 28 deletions(-) diff --git a/server/typescript/packages/sdk/test/order-independence.test.ts b/server/typescript/packages/sdk/test/order-independence.test.ts index d7c81d49f..c6e9ce45f 100644 --- a/server/typescript/packages/sdk/test/order-independence.test.ts +++ b/server/typescript/packages/sdk/test/order-independence.test.ts @@ -5,19 +5,41 @@ // declared source SET, so the order sources are declared in carries no // information. That is why the design has no ordered-list semantics, no // topological sort, no cycle detection, and no diamond-dependency problem. -// This file is what turns that premise from a belief into an enforced -// property, at two tiers: -// 1. resolveSources() itself (T4 already documents this contract; this -// re-asserts it across all six permutations of three specs, on the -// FULL ResolvedSource[] — .spec included, not just .file). -// 2. The loaded MODEL — resolveSources() feeding MetaDataLoader, its -// canonical (own-mode) serialization byte-identical across the same -// six permutations. // -// The fixture shape — a base declaration, an overlay onto it, and an -// independent third file — is deliberately the one whose merge is most -// order-sensitive if anything is: an overlay must find its base regardless -// of which file the loader saw first. +// The premise splits into THREE layers, and this file is the design's +// documentation of record on how each one is satisfied — deliberately not +// collapsed into one over-broad assertion, because two earlier drafts of +// this gate got that collapse wrong in opposite directions: +// 1. `resolveSources` CANONICALIZES file order — it sorts its output by +// absolute path (sources.ts:127), so every permutation of a declared +// source SET collapses to the same file list before the loader ever +// runs. Test 1 pins this directly. +// 2. The LOADER resolves CONTENT order-independently, given whatever file +// list it's handed — including an overlay arriving before its base. +// `_partitionOverlayLast` is the mechanism (stable-partitions +// overlay-only sources to the end before the parse loop runs); test 2 +// proves it by permuting FILE PATHS directly into `FileSource[]`, +// bypassing `resolveSources` entirely (routing through it would erase +// all order variation before the loader ever saw it, and reach +// overlay-before-base in zero of the six permutations — an earlier +// draft of this test did exactly that and passed vacuously). Test 2 +// compares CONTENT — each top-level object's own serialization, keyed +// by name — not the whole tree, for the reason in point 3. +// 3. SIBLING ORDER of unrelated top-level nodes (e.g. which of two +// unrelated entities appears first in `MetaRoot`'s `children` array) +// follows raw input order and is DELIBERATELY NOT asserted here. It is +// not a design claim: `canonicalSerialize`'s own contract +// (serializer-json.ts:159-167) promises exactly two normalizations — +// alphabetical `@`-attr keys and a trailing newline — and says nothing +// about sibling ordering; `serializeNodeInner` emits `ownChildren()` in +// whatever order the tree holds them. It also doesn't need to be a +// claim: production never hands the loader a permuted list — layer 1 +// sorts first. A prior draft of test 2 asserted whole-tree +// `canonicalSerialize` equality across all six permutations and failed +// on unmodified code for exactly this reason (Order vs Customer swap), +// even though content resolution was correct in every case — that was +// the amended test inventing a bar the design never set, not a real +// defect. import { describe, test, expect, beforeEach, afterEach } from "bun:test"; import { mkdtempSync, rmSync, mkdirSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; @@ -65,10 +87,10 @@ describe("permutations helper", () => { // Sanity-check the helper itself, not just its output length: 6 entries // that were secretly duplicates would let both gates below pass // vacuously without ever exercising a real reordering. - const specs: SourceSpec[] = [{ path: "a" }, { path: "b" }, { path: "c" }]; - const perms = permutations(specs); + const items = ["a", "b", "c"]; + const perms = permutations(items); expect(perms).toHaveLength(6); - const distinct = new Set(perms.map((p) => p.map((s) => (s as { path: string }).path).join(","))); + const distinct = new Set(perms.map((p) => p.join(","))); expect(distinct.size).toBe(6); }); }); @@ -96,29 +118,71 @@ describe("order independence", () => { } }); - test("the loaded model serializes byte-identically across every permutation", async () => { + test("the loader resolves content order-independently given a permuted file list, including overlay-before-base", async () => { const { MetaDataLoader, composeRegistry, coreProviders, canonicalSerialize } = await import("@metaobjectsdev/metadata"); const { FileSource } = await import("@metaobjectsdev/metadata/core"); - const specs: SourceSpec[] = [{ path: "a" }, { path: "b" }, { path: "c" }]; - const perms = permutations(specs); + + // Permute the FILE PATHS directly — deliberately bypassing + // resolveSources(), whose own sort would erase all order variation + // before the loader ever saw it (see the file header). Building + // FileSource[] straight from these paths is what actually reaches an + // overlay-before-base ordering. + const basePath = join(root, "a/meta.base.json"); + const overlayPath = join(root, "b/meta.overlay.json"); + const otherPath = join(root, "c/meta.other.json"); + const perms = permutations([basePath, overlayPath, otherPath]); expect(perms).toHaveLength(6); - const serialized: string[] = []; + // Confirm the permutation actually reaches the shape this test exists + // to cover: half of the six orderings must place the overlay-only file + // before its base, or this gate would be no stronger than test 1 above. + const overlayBeforeBase = perms.filter( + (p) => p.indexOf(overlayPath) < p.indexOf(basePath), + ).length; + expect(overlayBeforeBase).toBe(3); + + const label = (p: string[]): string => + JSON.stringify(p.map((f) => f.replace(root + "/", ""))); + + // Per permutation: a name-keyed map of each top-level object's OWN + // canonical serialization. Keying by NAME rather than comparing the + // whole root (or relying on array position) makes the comparison + // insensitive to sibling order by construction — see point 3 in the + // file header — while still catching any real content difference, + // which is the property this test exists to prove. + const perObject: Map[] = []; for (const p of perms) { - const resolved = await resolveSources(root, p); const loader = new MetaDataLoader({ registry: composeRegistry(coreProviders) }); - const result = await loader.load(resolved.map((r) => new FileSource(r.file))); - expect(result.errors).toHaveLength(0); - serialized.push(canonicalSerialize(result.root)); + const result = await loader.load(p.map((file) => new FileSource(file))); + expect(result.errors, `permutation ${label(p)} errored`).toHaveLength(0); + + const byName = new Map(); + for (const child of result.root.ownChildren()) { + byName.set(child.name, canonicalSerialize(child)); + } + perObject.push(byName); + + // The overlay's contribution must have actually landed on Order in + // EVERY permutation — this is the assertion that disabling + // `_partitionOverlayLast` breaks (3 of 6 permutations throw + // ERR_OVERLAY_NO_TARGET without it, dropping this field entirely; see + // the break-and-revert evidence in the task report). The empty-errors + // check above already catches the hard-failure case; this confirms + // the MERGE actually happened, not merely that nothing errored. + expect( + byName.get("Order"), + `permutation ${label(p)} — Order is missing the overlay's note field`, + ).toContain('"name": "note"'); } - expect(serialized).toHaveLength(6); + expect(perObject).toHaveLength(6); - for (let i = 1; i < serialized.length; i++) { + const expected = perObject[0]!; + for (let i = 1; i < perObject.length; i++) { expect( - serialized[i], - `permutation ${i} (${JSON.stringify(perms[i])}) serialized differently than permutation 0 (${JSON.stringify(perms[0])})`, - ).toBe(serialized[0]!); + perObject[i], + `permutation ${i} (${label(perms[i]!)}) resolved different CONTENT than permutation 0 (${label(perms[0]!)})`, + ).toEqual(expected); } }); }); From b3082e1303c243ef78f5726a537284843937006a Mon Sep 17 00:00:00 2001 From: Doug Mealing Date: Mon, 17 Aug 2026 23:03:37 -0400 Subject: [PATCH 14/44] =?UTF-8?q?fix(sdk):=20correctness=20fixes=20from=20?= =?UTF-8?q?code=20review=20=E2=80=94=20symlinks,=20scope=20patterns,=20err?= =?UTF-8?q?or-code=20order,=20strict=20config,=20parity=20guard?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Five verified defects in the phase-1 metadata-source-resolution module set: - sources.ts collectDir: a dangling symlink (or TOCTOU removal/EACCES) inside a walked directory crashed resolveSources with a raw Node ENOENT carrying no ERR_ code. Now caught and skipped, matching DirectorySource in @metaobjectsdev/metadata, which this walk already claims to mirror. A declared path spec that itself doesn't resolve still throws ERR_SOURCE_UNRESOLVED — unchanged, tested. - scope.ts compilePattern: an odd colon run (e.g. "acme:::Order") survived the split on the two-char "::" separator with a leftover ":" inside a segment, compiling to a regex no legal fully-qualified name could ever match — a typo'd include pattern silently scoped out everything. Now rejected as ERR_SCOPE_PATTERN_INVALID. - sources.ts resolveSources: kind validation and path resolution were interleaved in one loop, so an unsupported source kind was reported only if no earlier spec's path failed to resolve first — the error code depended on declaration order, contradicting the module's own pure-function-of-the-SET invariant. Kind is now validated for every spec up front, before any filesystem I/O. - config.ts ConfigSchema: strict() on the source-spec arms and the scope block but not the enclosing object, so a misspelled top-level key (e.g. "scopes") was silently stripped and the collection resolved as "everything in scope" — the exact fail-open .strict() elsewhere exists to prevent, one level up. Now strict() at the top level too; audited every call site (cli/src/commands/init.ts, cli/src/lib/config.ts) — none passes an extra key, and the cli suite stays at the same 556/3/0 baseline. - config.ts _sourceSpecParity: a one-directional assignment only proved the Zod schema's inferred type assignable to the hand-written SourceSpec, so an arm added to SourceSpec without a matching schema arm still compiled clean. Now bidirectional (two assignments, opposite directions); each half was deliberately broken and reverted to confirm it fires (see the quality-pass report for both red tsc outputs). Also carries two changes that are compile-coupled to this file set via a new sources.ts -> memory.ts import edge (DEFAULT_SOURCES no longer hardcodes "metaobjects" as a second encoding of memory.ts's DEFAULT_METADATA_DIR; the package's two metadata-file walkers now share one case-insensitive isMetadataFile owned by memory.ts, aligning sources.ts's already-fixed DirectorySource-matching behavior into memory.ts's loadMemory walk too — a real behavior change, pinned by a new test) — full rationale, including a concrete crash repro for why the ownership direction differs from the originally-requested one, is in the quality-pass report (not committed; listed in .git/info/exclude). Verified: bun test packages/sdk 219/0 (was 214/0), bun test packages/cli 556/3/0 (unchanged), sdk + cli typecheck clean, all confirmed in isolation (stashed the remaining reuse/efficiency changes before running these). Co-Authored-By: Claude Opus 5 (1M context) --- server/typescript/packages/sdk/src/config.ts | 35 +++++++--- server/typescript/packages/sdk/src/memory.ts | 23 +++++-- server/typescript/packages/sdk/src/scope.ts | 15 ++++ server/typescript/packages/sdk/src/sources.ts | 68 +++++++++++++------ .../packages/sdk/test/memory.test.ts | 31 +++++++++ .../packages/sdk/test/scope.test.ts | 34 ++++++++-- .../packages/sdk/test/sources.test.ts | 59 +++++++++------- .../packages/sdk/test/support/error-code.ts | 28 ++++++++ 8 files changed, 229 insertions(+), 64 deletions(-) create mode 100644 server/typescript/packages/sdk/test/support/error-code.ts diff --git a/server/typescript/packages/sdk/src/config.ts b/server/typescript/packages/sdk/src/config.ts index f66ff7a82..8dae20f27 100644 --- a/server/typescript/packages/sdk/src/config.ts +++ b/server/typescript/packages/sdk/src/config.ts @@ -80,12 +80,22 @@ const SourceSpecSchema = z.union([ z.object({ package: z.string().min(1) }).strict(), ]); -// Compile-time parity: if SourceSpecSchema and the hand-written SourceSpec -// (./sources.ts) ever drift, this assignment stops compiling. A conditional -// type (`z.infer<...> extends SourceSpec ? true : never`) would silently -// resolve to `never` instead of erroring — this form fails for real. -const _sourceSpecParity: SourceSpec = {} as z.infer; -void _sourceSpecParity; +// Compile-time parity, BOTH directions: if SourceSpecSchema and the +// hand-written SourceSpec (./sources.ts) ever drift, one of these two +// assignments stops compiling. Each direction alone catches only HALF the +// drift — `z.infer<...>` assignable to `SourceSpec` catches an arm added to +// the schema but missing from SourceSpec, while `SourceSpec` assignable to +// `z.infer<...>` catches the opposite: an arm added to the hand-written +// SourceSpec that the schema never gained. A single one-directional +// assignment (the prior form of this guard) let a SourceSpec-only addition +// compile clean — proven by deliberately breaking each direction in +// isolation; see the quality-pass report for both failing `tsc` outputs. A +// conditional type (`X extends Y ? true : never`) would silently resolve to +// `never` instead of erroring — this direct-assignment form fails for real. +const _sourceSpecParityInferToSpec: SourceSpec = {} as z.infer; +const _sourceSpecParitySpecToInfer: z.infer = {} as SourceSpec; +void _sourceSpecParityInferToSpec; +void _sourceSpecParitySpecToInfer; /** Mirrors the hand-written `Scope` interface in `./scope.ts`. An absent or * empty `include` means "everything" — see `matchesScope`. */ @@ -96,6 +106,11 @@ const ScopeSchema = z }) .strict(); +// .strict() at the TOP level too, not just the source-spec arms and the +// scope block: without it, a misspelled top-level key (e.g. "scopes" for +// "scope") is silently stripped by zod and the collection resolves as +// "everything in scope" — the exact silent fail-open .strict() on the +// nested arms exists to prevent, one level up. export const ConfigSchema = z.object({ schema_version: z.literal(1), pending_in_git: z.boolean().default(true), @@ -115,13 +130,17 @@ export const ConfigSchema = z.object({ }) .default({}), migrate: MigrateBlock.optional(), -}); +}).strict(); export type Config = z.infer; export const DEFAULT_CONFIG: Config = ConfigSchema.parse({ schema_version: 1 }); -const CONFIG_FILE = "config.json"; +/** `config.json`'s basename — the single owner. `discovery.ts` and + * `collection.ts` import this rather than each keeping their own copy of + * the literal (a duplication that had drifted into three separate + * declarations of the same string). */ +export const CONFIG_FILE = "config.json"; export async function loadConfig(metaRoot: string): Promise { const raw = await readFile(join(metaRoot, CONFIG_FILE), "utf8"); diff --git a/server/typescript/packages/sdk/src/memory.ts b/server/typescript/packages/sdk/src/memory.ts index 860c4d5b4..1f4866821 100644 --- a/server/typescript/packages/sdk/src/memory.ts +++ b/server/typescript/packages/sdk/src/memory.ts @@ -1,4 +1,4 @@ -import { join } from "node:path"; +import { extname, join } from "node:path"; import { readdir, stat } from "node:fs/promises"; import { composeRegistry, @@ -24,6 +24,20 @@ export const DEFAULT_METADATA_DIR = "metaobjects"; */ export const DEFAULT_METAOBJECTS_DIR = ".metaobjects"; +/** Recognized metadata file extensions, matched case-insensitively — mirrors + * `DirectorySource` in `@metaobjectsdev/metadata`, which checks + * `extname().toLowerCase()`. The single definition every metadata-file + * walker in this package uses: `sources.ts`'s `resolveSources` imports + * `isMetadataFile` from here rather than keeping its own copy, so the + * package's two walkers cannot silently disagree about whether e.g. + * `meta.JSON` counts (a real drift this fixes — this walk used to be + * case-sensitive while `sources.ts`'s was already case-insensitive). */ +export const METADATA_EXTENSIONS = new Set([".json", ".yaml", ".yml"]); + +export function isMetadataFile(name: string): boolean { + return METADATA_EXTENSIONS.has(extname(name).toLowerCase()); +} + /** * Options for {@link loadMemory}. Consumers can supply additional * {@link MetaDataTypeProvider}s to extend the metamodel with their own @@ -143,7 +157,8 @@ async function collectMetadataPaths(repoRoot: string): Promise { } /** - * Recursively list metadata files (*.json, *.yaml, *.yml) under a directory, + * Recursively list metadata files (*.json, *.yaml, *.yml, matched + * case-insensitively — see `isMetadataFile` above) under a directory, * excluding _pending/ at any level. Subdirectories (e.g. projections/) are * walked depth-first. Files within a directory are sorted alphabetically for * deterministic load order; subdirectories are visited after files at the @@ -183,7 +198,3 @@ async function listMetadataFiles(dir: string): Promise { } return paths; } - -function isMetadataFile(name: string): boolean { - return name.endsWith(".json") || name.endsWith(".yaml") || name.endsWith(".yml"); -} diff --git a/server/typescript/packages/sdk/src/scope.ts b/server/typescript/packages/sdk/src/scope.ts index b9a7f5dca..26d7da398 100644 --- a/server/typescript/packages/sdk/src/scope.ts +++ b/server/typescript/packages/sdk/src/scope.ts @@ -46,6 +46,21 @@ function compileSegment(segment: string, pattern: string): string { source: codeSource("compileSegment"), }); } + // A segment surviving the split on the two-character PACKAGE_SEPARATOR + // ("::") can still contain a lone ":" when the pattern has an odd colon + // run — e.g. "acme:::Order".split("::") => ["acme", ":Order"]. SEGMENT + // ([^:]+) already excludes ":" from a well-formed segment, so a leftover + // ":" here means the separator was malformed, not that ":" is meant + // literally. Left unchecked, escapeLiteral treats it as a literal + // character and compiles a regex requiring three colons in a row — which + // no legal "::"-joined fully-qualified name can ever contain, so the + // pattern silently matches nothing instead of failing loud. + if (segment.includes(":")) { + throw new ParseError( + `scope pattern "${pattern}" has a malformed separator (an odd run of ":") — segments are joined by "::", never a single ":"`, + { code: "ERR_SCOPE_PATTERN_INVALID", source: codeSource("compileSegment") }, + ); + } if (segment === "**") return `(?:${SEGMENTS})`; // `*` inside a segment matches any characters except the separator char. return segment.split("*").map(escapeLiteral).join("[^:]*"); diff --git a/server/typescript/packages/sdk/src/sources.ts b/server/typescript/packages/sdk/src/sources.ts index e4c3a4c21..c94c47340 100644 --- a/server/typescript/packages/sdk/src/sources.ts +++ b/server/typescript/packages/sdk/src/sources.ts @@ -11,8 +11,9 @@ // linchpin), so both the sort-by-absolute-path step and the content-based // overlap tie-break below are load-bearing. import { readdir, stat } from "node:fs/promises"; -import { extname, isAbsolute, join, resolve } from "node:path"; +import { isAbsolute, join, resolve } from "node:path"; import { ParseError, codeSource } from "@metaobjectsdev/metadata"; +import { DEFAULT_METADATA_DIR, isMetadataFile } from "./memory.js"; /** Tagged union of source kinds. `resource` and `package` are declared now so * the config shape is stable across phases; only `path` resolves in phase 1 — @@ -31,21 +32,18 @@ export interface ResolvedSource { /** Used when `sources` is absent or empty in `.metaobjects/config.json`. A * DEFAULT, never a requirement — a project that declares `sources` explicitly - * need not include `metaobjects/` at all. */ -export const DEFAULT_SOURCES: readonly SourceSpec[] = [{ path: "metaobjects" }]; + * need not include `metaobjects/` at all. Built from `DEFAULT_METADATA_DIR` + * (`memory.ts`'s own default-directory constant) rather than restating the + * literal "metaobjects" here: a second independent encoding of the same + * default would let `resolveCollection`'s "does the default dir exist" + * check (`collection.ts`) desync from what `resolveSources` actually + * resolves the moment the default ever changed — silently reproducing the + * "two code paths disagree about where metadata lives" class of bug this + * whole mechanism exists to eliminate. */ +export const DEFAULT_SOURCES: readonly SourceSpec[] = [{ path: DEFAULT_METADATA_DIR }]; const PENDING_DIR = "_pending"; -/** Recognized metadata file extensions, matched case-insensitively — mirrors - * `DirectorySource` in `@metaobjectsdev/metadata`, which checks - * `extname().toLowerCase()`. Without this, `meta.JSON` would be picked up by - * the loader and silently skipped here. */ -const METADATA_EXTENSIONS = new Set([".json", ".yaml", ".yml"]); - -function isMetadataFile(name: string): boolean { - return METADATA_EXTENSIONS.has(extname(name).toLowerCase()); -} - /** Recursively collect metadata files under `dir`, excluding `_pending/` at any * depth. Uses `stat` (follows symlinks) rather than `lstat` or * `Dirent.isDirectory()` — `DirectorySource` in `@metaobjectsdev/metadata` has @@ -57,12 +55,37 @@ async function collectDir(dir: string, out: string[]): Promise { for (const entry of entries) { if (entry === PENDING_DIR) continue; const full = join(dir, entry); - const s = await stat(full); + let s; + try { + s = await stat(full); + } catch { + // A dangling symlink, a TOCTOU removal between readdir and stat, or an + // inaccessible (EACCES) entry — skip it, matching DirectorySource in + // @metaobjectsdev/metadata (directory-source.ts), which this walk + // otherwise mirrors. An uncaught stat() here would crash + // resolveSources with a raw Node ENOENT on a tree the loader reads + // fine. + continue; + } if (s.isDirectory()) await collectDir(full, out); else if (s.isFile() && isMetadataFile(entry)) out.push(full); } } +/** Narrows `spec` to its `path` arm, throwing `ERR_SOURCE_KIND_UNSUPPORTED` + * for `resource`/`package` — phase 1 resolves `path` only. Called in two + * separate passes by {@link resolveSources} (see the comment there): an + * unsupported kind must be reported regardless of where it sits in the + * declared list. */ +function assertPathSpec(spec: SourceSpec): asserts spec is { readonly path: string } { + if ("path" in spec) return; + const kind = "resource" in spec ? "resource" : "package"; + throw new ParseError( + `source kind "${kind}" is not supported by this toolchain yet; use a "path" source`, + { code: "ERR_SOURCE_KIND_UNSUPPORTED", source: codeSource("resolveSources") }, + ); +} + /** * Resolve a declared source SET to a canonically-sorted list of metadata files. * @@ -89,16 +112,19 @@ export async function resolveSources( configDir: string, specs: readonly SourceSpec[], ): Promise { + // Validate every spec's KIND up front, before any filesystem I/O. Without + // this separate pass, kind-validation and path resolution were + // interleaved in one loop, so which error code came back depended on + // DECLARATION ORDER: an unsupported-kind spec placed after an + // unresolvable path spec never got reached (the path spec's + // ERR_SOURCE_UNRESOLVED fired first) — contradicting this module's own + // "pure function of the SET" invariant (see the file header). + for (const spec of specs) assertPathSpec(spec); + const byFile = new Map(); for (const spec of specs) { - if (!("path" in spec)) { - const kind = "resource" in spec ? "resource" : "package"; - throw new ParseError( - `source kind "${kind}" is not supported by this toolchain yet; use a "path" source`, - { code: "ERR_SOURCE_KIND_UNSUPPORTED", source: codeSource("resolveSources") }, - ); - } + assertPathSpec(spec); // already validated above; narrows `spec.path` for TS below. const target = isAbsolute(spec.path) ? spec.path : resolve(configDir, spec.path); let stats; diff --git a/server/typescript/packages/sdk/test/memory.test.ts b/server/typescript/packages/sdk/test/memory.test.ts index 435601dd1..bfe3e0b40 100644 --- a/server/typescript/packages/sdk/test/memory.test.ts +++ b/server/typescript/packages/sdk/test/memory.test.ts @@ -70,6 +70,37 @@ describe("loadMemory", () => { } }); + // C4 — memory.ts's own `isMetadataFile` used to match extensions + // case-SENSITIVELY while sources.ts's (already fixed to mirror + // DirectorySource in @metaobjectsdev/metadata) matched case-insensitively. + // Two metadata-file walkers in one package disagreeing about whether + // `meta.JSON` counts is exactly the drift this package's design exists to + // prevent; memory.ts now imports the shared, case-insensitive + // implementation. This is an intentional BEHAVIOR CHANGE — a file named + // `*.JSON` (previously silently skipped by loadMemory) is now collected. + test("collects a metadata file with an uppercase extension (meta.JSON), case-insensitively", async () => { + const root = makeMetaRoot(); + try { + writeFileSync( + join(root, "metaobjects", "shouty.JSON"), + JSON.stringify({ + metadata: { + package: "test", + children: [ + { object: { name: "Shouty", subType: "entity", children: [] } }, + ], + }, + }), + ); + + const meta = await loadMemory(root); + const shouty = meta.findObject("Shouty"); + expect(shouty).toBeDefined(); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + test("loads decision children when metadata files contain them", async () => { const root = makeMetaRoot(); try { diff --git a/server/typescript/packages/sdk/test/scope.test.ts b/server/typescript/packages/sdk/test/scope.test.ts index 9b62e6ed4..50967ed4c 100644 --- a/server/typescript/packages/sdk/test/scope.test.ts +++ b/server/typescript/packages/sdk/test/scope.test.ts @@ -1,14 +1,9 @@ import { describe, test, expect } from "bun:test"; import { compileScope, matchesScope, type Scope } from "../src/scope.js"; +import { errorCode } from "./support/error-code.js"; const match = (fqn: string, scope: Scope) => matchesScope(fqn, compileScope(scope)); -/** Pull the stable ERR_ code off a caught error, if it carries one. */ -function errorCode(err: unknown): string { - const code = (err as { code?: unknown }).code; - return typeof code === "string" ? code : "ERR_UNKNOWN"; -} - describe("compileScope / matchesScope", () => { test("empty include matches everything", () => { expect(match("acme::commerce::Order", {})).toBe(true); @@ -78,4 +73,31 @@ describe("compileScope / matchesScope", () => { expect(caught).toBeDefined(); expect(errorCode(caught)).toBe("ERR_SCOPE_PATTERN_INVALID"); }); + + test("an odd colon run (malformed separator) is ERR_SCOPE_PATTERN_INVALID, not a silently-unmatchable pattern", () => { + // "acme:::Order".split("::") => ["acme", ":Order"] — the leftover ":" + // used to compile as a literal character into `^acme:::Order$`, a + // regex no legal "::"-joined name can ever match. A typo'd include + // pattern therefore silently scoped out EVERYTHING instead of failing + // to load. + let caught: unknown; + try { + compileScope({ include: ["acme:::Order"] }); + } catch (err) { + caught = err; + } + expect(caught).toBeDefined(); + expect(errorCode(caught)).toBe("ERR_SCOPE_PATTERN_INVALID"); + }); + + test("a single stray colon (not the :: separator) is also ERR_SCOPE_PATTERN_INVALID", () => { + let caught: unknown; + try { + compileScope({ include: ["acme:Order"] }); + } catch (err) { + caught = err; + } + expect(caught).toBeDefined(); + expect(errorCode(caught)).toBe("ERR_SCOPE_PATTERN_INVALID"); + }); }); diff --git a/server/typescript/packages/sdk/test/sources.test.ts b/server/typescript/packages/sdk/test/sources.test.ts index 45bb30ab6..07f503686 100644 --- a/server/typescript/packages/sdk/test/sources.test.ts +++ b/server/typescript/packages/sdk/test/sources.test.ts @@ -3,6 +3,7 @@ import { mkdtempSync, rmSync, mkdirSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { resolveSources, DEFAULT_SOURCES } from "../src/sources.js"; +import { rejectedCode } from "./support/error-code.js"; let root: string; const write = (rel: string, body = "{}") => { @@ -15,29 +16,6 @@ const write = (rel: string, body = "{}") => { beforeEach(() => { root = mkdtempSync(join(tmpdir(), "metaobjects-sources-")); }); afterEach(() => { rmSync(root, { recursive: true, force: true }); }); -/** Pull the stable ERR_ code off a caught error, if it carries one. Mirrors - * scope.test.ts's `errorCode` — property-based, never message-matching: a - * cross-package `instanceof ParseError` is silently false when two physical - * copies of `@metaobjectsdev/metadata` are loaded (a globally-installed or - * linked CLI alongside a project-local dependency), so `.code` is the only - * reliable read. */ -function errorCode(err: unknown): string { - const code = (err as { code?: unknown }).code; - return typeof code === "string" ? code : "ERR_UNKNOWN"; -} - -/** Await `promise`, expecting it to reject — returns the rejection's stable - * code. The async counterpart of `errorCode` above, needed because - * `resolveSources` is async where `compileScope` (scope.test.ts) is not. */ -async function rejectedCode(promise: Promise): Promise { - try { - await promise; - } catch (err) { - return errorCode(err); - } - throw new Error("expected the promise to reject, but it resolved"); -} - describe("resolveSources", () => { test("resolves a directory recursively, metadata files only", async () => { write("model/meta.a.json"); @@ -108,6 +86,28 @@ describe("resolveSources", () => { ); }); + test("an unsupported kind is reported regardless of declaration order relative to an unresolvable path", async () => { + // Kind validation used to be interleaved with per-spec filesystem I/O in + // one loop, so an unsupported-kind spec placed AFTER an unresolvable + // path spec never got reached — the path spec's ERR_SOURCE_UNRESOLVED + // fired first, and the reported code silently depended on which spec + // was declared first. Both orderings must report the SAME code. + const unsupportedFirst: Parameters[1] = [ + { resource: "acme/model" }, + { path: "missing" }, + ]; + const unresolvedFirst: Parameters[1] = [ + { path: "missing" }, + { resource: "acme/model" }, + ]; + expect(await rejectedCode(resolveSources(root, unsupportedFirst))).toBe( + "ERR_SOURCE_KIND_UNSUPPORTED", + ); + expect(await rejectedCode(resolveSources(root, unresolvedFirst))).toBe( + "ERR_SOURCE_KIND_UNSUPPORTED", + ); + }); + test("_pending is excluded at any depth", async () => { write("model/meta.a.json"); write("model/_pending/meta.draft.json"); @@ -124,6 +124,19 @@ describe("resolveSources", () => { expect(out).toHaveLength(2); }); + test("a dangling symlink inside a source directory is skipped, not a raw ENOENT crash", async () => { + // DirectorySource in @metaobjectsdev/metadata catches and skips exactly + // this case (directory-source.ts). Before the fix, the bare `stat()` in + // collectDir had no try/catch, so a dangling symlink crashed + // resolveSources with a raw Node ENOENT carrying no ERR_ code — on a + // tree the loader itself reads fine. + write("model/meta.a.json"); + const { symlinkSync } = await import("node:fs"); + symlinkSync(join(root, "model/does-not-exist"), join(root, "model/dangling.json")); + const out = await resolveSources(root, [{ path: "model" }]); + expect(out.map((r) => r.file.replace(root + "/", ""))).toEqual(["model/meta.a.json"]); + }); + test("DEFAULT_SOURCES is the metaobjects/ directory", () => { expect(DEFAULT_SOURCES).toEqual([{ path: "metaobjects" }]); }); diff --git a/server/typescript/packages/sdk/test/support/error-code.ts b/server/typescript/packages/sdk/test/support/error-code.ts new file mode 100644 index 000000000..1328fc5be --- /dev/null +++ b/server/typescript/packages/sdk/test/support/error-code.ts @@ -0,0 +1,28 @@ +// server/typescript/packages/sdk/test/support/error-code.ts +// +// Shared by scope.test.ts, sources.test.ts and collection.test.ts. Each used +// to define its own copy of these two helpers, cross-referencing the others +// in a comment as the only sync mechanism — one copy, imported by all three. +// +// Property-based, never message-matching or `instanceof`: a cross-package +// `instanceof ParseError` is silently false when two physical copies of +// `@metaobjectsdev/metadata` are loaded (a globally-installed or linked CLI +// alongside a project-local dependency), so `.code` is the only reliable +// read. + +/** Pull the stable ERR_ code off a caught error, if it carries one. */ +export function errorCode(err: unknown): string { + const code = (err as { code?: unknown }).code; + return typeof code === "string" ? code : "ERR_UNKNOWN"; +} + +/** Await `promise`, expecting it to reject — returns the rejection's stable + * code. */ +export async function rejectedCode(promise: Promise): Promise { + try { + await promise; + } catch (err) { + return errorCode(err); + } + throw new Error("expected the promise to reject, but it resolved"); +} From f496b1dfac294d5eb165eed05221ac23b0c57891 Mon Sep 17 00:00:00 2001 From: Doug Mealing Date: Mon, 17 Aug 2026 23:03:54 -0400 Subject: [PATCH 15/44] refactor(sdk): dedupe fileExists/CONFIG_FILE, thread config-existence boolean MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reuse and one small efficiency fix flagged by the quality pass, confined to discovery.ts and collection.ts: - collection.ts's fileExists was byte-identical to discovery.ts's exists. discovery.ts's exists is now exported and imported instead of duplicated; collection.ts's isDir stays local (a genuinely different predicate). - CONFIG_FILE ("config.json") was declared three times: config.ts (private), discovery.ts, and collection.ts (with a comment justifying the duplication because config.ts didn't export it). config.ts now exports it as the single owner; discovery.ts and collection.ts import it, and collection.ts's justifying comment is gone along with the duplicate. - collection.ts re-stat'd .metaobjects/config.json on the discovered-dir path even though findConfigDir (discovery.ts) had already proved its presence/absence to return its result. hasConfig is now derived directly from findConfigDir's return value on that path; the stat only still runs on the explicitDir path, where findConfigDir never executes and nothing else has proven the file's existence. Deliberately left alone: collection.ts's isDir pre-flight before the ERR_COLLECTION_NOT_FOUND throw is a second stat that exists purely to produce a clearer diagnostic than the raw ERR_SOURCE_UNRESOLVED resolveSources would otherwise throw — trading that for one syscall is a bad trade, so it's untouched (now with a comment saying so explicitly). Verified: bun test packages/sdk 219/0, sdk typecheck clean. Co-Authored-By: Claude Opus 5 (1M context) --- .../typescript/packages/sdk/src/collection.ts | 51 +++++++++++-------- .../typescript/packages/sdk/src/discovery.ts | 6 ++- .../packages/sdk/test/collection.test.ts | 22 +------- 3 files changed, 34 insertions(+), 45 deletions(-) diff --git a/server/typescript/packages/sdk/src/collection.ts b/server/typescript/packages/sdk/src/collection.ts index 8823493b0..b928e5c0e 100644 --- a/server/typescript/packages/sdk/src/collection.ts +++ b/server/typescript/packages/sdk/src/collection.ts @@ -13,8 +13,8 @@ import { stat } from "node:fs/promises"; import { join, resolve } from "node:path"; import { ParseError, codeSource } from "@metaobjectsdev/metadata"; -import { loadConfig, type Config } from "./config.js"; -import { findConfigDir } from "./discovery.js"; +import { CONFIG_FILE, loadConfig, type Config } from "./config.js"; +import { exists, findConfigDir } from "./discovery.js"; import { compileScope, type CompiledScope, type Scope } from "./scope.js"; import { DEFAULT_METADATA_DIR, DEFAULT_METAOBJECTS_DIR } from "./memory.js"; import { DEFAULT_SOURCES, resolveSources, type ResolvedSource, type SourceSpec } from "./sources.js"; @@ -34,6 +34,12 @@ export interface Collection { readonly migrateScope: CompiledScope | undefined; } +// Deliberately NOT deduped with `exists` (imported from `./discovery.js`) +// even though both wrap a bare stat/catch: this predicate exists to produce +// the friendlier `ERR_COLLECTION_NOT_FOUND` diagnostic below rather than the +// raw `ERR_SOURCE_UNRESOLVED` `resolveSources` would throw on a genuinely +// missing default directory — trading that clearer error for one syscall is +// a bad trade, so the redundant `stat` here is intentional, not an oversight. async function isDir(p: string): Promise { try { return (await stat(p)).isDirectory(); @@ -42,20 +48,6 @@ async function isDir(p: string): Promise { } } -/** `config.json`'s own basename — kept local rather than imported, matching - * `discovery.ts`'s own private `CONFIG_FILE` constant (the string is not - * exported from `config.js`). */ -const CONFIG_FILE = "config.json"; - -async function fileExists(p: string): Promise { - try { - await stat(p); - return true; - } catch { - return false; - } -} - /** Narrow the zod-inferred `Config["scope"]` (whose `.optional()` fields are * typed `T | undefined` even when present) down to `Scope`'s * exactOptionalPropertyTypes-safe shape — a key is omitted entirely rather @@ -95,17 +87,32 @@ export async function resolveCollection( opts?: { explicitDir?: string }, ): Promise { const explicit = opts?.explicitDir; - const configDir = - explicit !== undefined ? resolve(explicit) : ((await findConfigDir(startDir)) ?? resolve(startDir)); + + // Whether `configDir` carries a `config.json` — threaded through rather + // than re-`stat`'d below. On the non-explicit path, `findConfigDir` + // already proved this: it returns a directory ONLY after confirming + // `.metaobjects/config.json` exists there (discovery.ts's own `exists` + // check), and returns undefined only after confirming the same file is + // absent at every directory it examined, `resolve(startDir)` included. A + // second `stat` of the identical file would just re-prove what discovery + // already established. The check is only load-bearing on the + // `explicitDir` path, where `findConfigDir` never runs at all. + let configDir: string; + let hasConfig: boolean; + if (explicit !== undefined) { + configDir = resolve(explicit); + hasConfig = await exists(join(configDir, DEFAULT_METAOBJECTS_DIR, CONFIG_FILE)); + } else { + const found = await findConfigDir(startDir); + configDir = found ?? resolve(startDir); + hasConfig = found !== undefined; + } let specs: readonly SourceSpec[] = DEFAULT_SOURCES; let scopeSpec: Config["scope"]; let migrateSpec: string[] | undefined; - // Check the FILE, not the `.metaobjects/` directory: a directory that - // exists but holds no `config.json` is the ordinary "no config" case and - // must fall through silently, same as no `.metaobjects/` at all. - if (await fileExists(join(configDir, DEFAULT_METAOBJECTS_DIR, CONFIG_FILE))) { + if (hasConfig) { // No try/catch here: a config.json that EXISTS but fails to load // (malformed JSON, a ConfigSchema violation) propagates. Swallowing it // would make a typo'd config behave identically to no config at all — diff --git a/server/typescript/packages/sdk/src/discovery.ts b/server/typescript/packages/sdk/src/discovery.ts index 7e2b818be..09c5f0909 100644 --- a/server/typescript/packages/sdk/src/discovery.ts +++ b/server/typescript/packages/sdk/src/discovery.ts @@ -15,12 +15,14 @@ // the walk one directory too early. import { stat } from "node:fs/promises"; import { dirname, join, resolve } from "node:path"; +import { CONFIG_FILE } from "./config.js"; import { DEFAULT_METAOBJECTS_DIR } from "./memory.js"; -const CONFIG_FILE = "config.json"; const GIT_DIR = ".git"; -async function exists(path: string): Promise { +/** Exported for reuse — `collection.ts` had its own byte-identical copy + * (`fileExists`); one definition, imported. */ +export async function exists(path: string): Promise { try { await stat(path); return true; diff --git a/server/typescript/packages/sdk/test/collection.test.ts b/server/typescript/packages/sdk/test/collection.test.ts index 2bdea5cc1..a88eec76e 100644 --- a/server/typescript/packages/sdk/test/collection.test.ts +++ b/server/typescript/packages/sdk/test/collection.test.ts @@ -4,6 +4,7 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import { resolveCollection } from "../src/collection.js"; import { matchesScope } from "../src/scope.js"; +import { rejectedCode } from "./support/error-code.js"; let root: string; const write = (rel: string, body: string) => { @@ -13,27 +14,6 @@ const write = (rel: string, body: string) => { const config = (dir: string, cfg: object) => write(join(dir, ".metaobjects/config.json"), JSON.stringify({ schema_version: 1, ...cfg })); -/** Pull the stable ERR_ code off a caught error, if it carries one. Mirrors - * scope.test.ts / sources.test.ts's `errorCode` — property-based, never - * message-matching or `instanceof`: a cross-package `instanceof ParseError` - * is silently false when two physical copies of `@metaobjectsdev/metadata` - * are loaded, so `.code` is the only reliable read. */ -function errorCode(err: unknown): string { - const code = (err as { code?: unknown }).code; - return typeof code === "string" ? code : "ERR_UNKNOWN"; -} - -/** Await `promise`, expecting it to reject — returns the rejection's stable - * code. Mirrors sources.test.ts's `rejectedCode`. */ -async function rejectedCode(promise: Promise): Promise { - try { - await promise; - } catch (err) { - return errorCode(err); - } - throw new Error("expected the promise to reject, but it resolved"); -} - beforeEach(() => { root = mkdtempSync(join(tmpdir(), "metaobjects-collection-")); mkdirSync(join(root, ".git")); }); afterEach(() => { rmSync(root, { recursive: true, force: true }); }); From 9200627c8a3ddc9b7db25287db665cd92f05b97e Mon Sep 17 00:00:00 2001 From: Doug Mealing Date: Mon, 17 Aug 2026 23:09:44 -0400 Subject: [PATCH 16/44] feat(sdk): loadMemory accepts an explicit resolved file set MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add `files?: readonly string[]` to LoadMemoryOptions. When supplied, loadMemory loads exactly those files and skips all directory discovery (no metaobjects/ scan, no workspace extends: walk). Absent, behavior is unchanged — collectMetadataPaths/listMetadataFiles stay the fallback and back-compat path. Later tasks route the CLI's read sites through resolveCollection() into this option. Co-Authored-By: Claude Opus 5 (1M context) --- server/typescript/packages/sdk/src/memory.ts | 19 +++++++- .../packages/sdk/test/memory.test.ts | 48 +++++++++++++++++++ 2 files changed, 65 insertions(+), 2 deletions(-) diff --git a/server/typescript/packages/sdk/src/memory.ts b/server/typescript/packages/sdk/src/memory.ts index 1f4866821..2257fe419 100644 --- a/server/typescript/packages/sdk/src/memory.ts +++ b/server/typescript/packages/sdk/src/memory.ts @@ -64,6 +64,14 @@ export interface LoadMemoryOptions { * the `meta verify` command opts in to `true` (strict-by-default, #96). */ strict?: boolean; + /** + * An already-resolved, absolute metadata file list — e.g. from + * `resolveCollection(...).files` in `collection.ts`. When supplied, + * `loadMemory` loads exactly these files and performs no directory + * discovery at all (no `metaobjects/` scan, no workspace `extends:` + * walk). Omit to keep the default discovery behavior unchanged. + */ + files?: readonly string[]; } /** Default provider bundle threaded by {@link loadMemory} when no options @@ -80,6 +88,8 @@ export const defaultLoadMemoryProviders: readonly MetaDataTypeProvider[] = [ * MetaData. If `/.meta/package.meta.json` declares `extends:` deps * and a workspace can be discovered (pnpm-workspace.yaml or package.json * workspaces), peer packages are loaded too in topological dep-first order. + * Pass {@link LoadMemoryOptions.files} to bypass this discovery entirely and + * load an already-resolved file list instead (e.g. from `resolveCollection`). * * Excludes `_pending/`. Registers metaobjects core types plus Meta Forge's * descriptive top-level types (decision, principle, etc.) so mixed content @@ -87,7 +97,8 @@ export const defaultLoadMemoryProviders: readonly MetaDataTypeProvider[] = [ * {@link LoadMemoryOptions.providers}) are composed AFTER the defaults so * they may depend on core/forge ids. * - * Throws if `metaobjects/` doesn't exist (callers should run `meta init`). + * Throws if `metaobjects/` doesn't exist (callers should run `meta init`), + * unless `options.files` is supplied. * * @param repoRoot The project's working-directory root (e.g. process.cwd()). * `loadMemory` resolves `metaobjects/` and (if workspace-aware) the @@ -116,7 +127,11 @@ export async function loadMemory( // Collect all metadata file paths to load. Order matters for the parser's // deferred-resolution pass (it parses in array order, then resolves supers // against the merged tree afterwards) — dep packages first, current last. - const paths = await collectMetadataPaths(repoRoot); + // An explicit `files` list (already resolved, e.g. by `resolveCollection`) + // wins outright and skips discovery entirely. + const paths = options?.files !== undefined + ? [...options.files] + : await collectMetadataPaths(repoRoot); const loader = new MetaDataLoader({ registry, diff --git a/server/typescript/packages/sdk/test/memory.test.ts b/server/typescript/packages/sdk/test/memory.test.ts index bfe3e0b40..bc5f9da17 100644 --- a/server/typescript/packages/sdk/test/memory.test.ts +++ b/server/typescript/packages/sdk/test/memory.test.ts @@ -361,3 +361,51 @@ describe("loadMemory — cross-package loading via workspace", () => { } }); }); + +describe("loadMemory with an explicit file set", () => { + test("loads exactly the supplied files, ignoring any metaobjects/ dir", async () => { + const dir = mkdtempSync(join(tmpdir(), "metaobjects-memory-files-")); + try { + mkdirSync(join(dir, "model"), { recursive: true }); + mkdirSync(join(dir, "metaobjects"), { recursive: true }); + writeFileSync(join(dir, "model/meta.a.json"), JSON.stringify({ + "metadata.root": { package: "acme", children: [ + { "object.entity": { name: "Order", children: [{ "field.string": { name: "id" } }] } }] }, + }), "utf8"); + writeFileSync(join(dir, "metaobjects/meta.decoy.json"), JSON.stringify({ + "metadata.root": { package: "acme", children: [ + { "object.entity": { name: "Decoy", children: [{ "field.string": { name: "id" } }] } }] }, + }), "utf8"); + const root = await loadMemory(dir, { files: [join(dir, "model/meta.a.json")] }); + const names = root.children().map((c) => c.name); + expect(names).toContain("Order"); + expect(names).not.toContain("Decoy"); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + test("with no `files` option, a project with a metaobjects/ tree still loads exactly as before", async () => { + const root = makeMetaRoot(); + try { + writeFileSync( + join(root, "metaobjects", "domain.json"), + JSON.stringify({ + "metadata.root": { + package: "acme", + children: [ + { "object.entity": { name: "Order", children: [{ "field.string": { name: "id" } }] } }, + ], + }, + }), + "utf8", + ); + + const meta = await loadMemory(root); + const names = meta.children().map((c) => c.name); + expect(names).toEqual(["Order"]); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +}); From 44a8bc588ddf5bc9602b0d686e138849cc0ad2d0 Mon Sep 17 00:00:00 2001 From: Doug Mealing Date: Mon, 17 Aug 2026 23:24:09 -0400 Subject: [PATCH 17/44] feat(cli): route gen/docs/export and the project probe through resolveCollection MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `sources` in .metaobjects/config.json becomes the real authority on where metadata lives; `metaobjects/` becomes merely its default. Five hardcoded reads now route through `resolveCollection()` (from Task 7) and pass its resolved file list to `loadMemory({ files })` (Task 9): - gen.ts — the existsSync(metaobjects/) hint branch is deleted; discovery and load stay two separate try blocks so a genuine ParseError is never misreported as "no metaobjects/ found". - docs.ts — the markdown-surface load, and the standalone HTML site's own loader (which wants directories, not a file list; new collectionSourceDirs helper derives them from the collection's distinct source specs). - export.ts — resolveCollection failures fold into the same exit-1 path loadAndExportJson's directory failures already used (export has never used exit 2 for a metadata problem, only for a bad CLI flag), so the existing back-compat contract holds exactly. - index.ts — the no-args "is this a MetaObjects project?" probe, previously the one site that inlined the "metaobjects" string literal instead of importing the constant. init.ts is untouched (it's the scaffolder writing the default) and so is detect-stack.ts (separate task). New collection-routing.test.ts proves a project with `sources` declared elsewhere generates successfully with no metaobjects/ directory anywhere. Rebuilt packages/sdk's stale dist/ (missing resolveCollection and loadMemory's `files` option) so the dist-based gen-split-tree regression gate could resolve them. cli: 557 pass / 3 skip / 0 fail (556 pre-existing + 1 new). codegen-ts golden-output gate: 1241 pass / 0 fail, unchanged. Co-Authored-By: Claude Opus 5 (1M context) --- .../packages/cli/src/commands/docs.ts | 82 +++++++++++++----- .../packages/cli/src/commands/export.ts | 36 ++++++-- .../packages/cli/src/commands/gen.ts | 35 ++++---- server/typescript/packages/cli/src/index.ts | 7 +- .../cli/test/collection-routing.test.ts | 85 +++++++++++++++++++ 5 files changed, 199 insertions(+), 46 deletions(-) create mode 100644 server/typescript/packages/cli/test/collection-routing.test.ts diff --git a/server/typescript/packages/cli/src/commands/docs.ts b/server/typescript/packages/cli/src/commands/docs.ts index 84451ba9b..cde8bbded 100644 --- a/server/typescript/packages/cli/src/commands/docs.ts +++ b/server/typescript/packages/cli/src/commands/docs.ts @@ -11,11 +11,11 @@ // output is therefore guaranteed — it is byte-for-byte the same generator the // `meta gen` pipeline runs (gated by the docs conformance fixture). -import { resolve as resolvePath, basename } from "node:path"; +import { resolve as resolvePath, basename, isAbsolute } from "node:path"; import { mkdir, writeFile } from "node:fs/promises"; import { log } from "../lib/log.js"; import { loadMetaobjectsConfig } from "../lib/load-metaobjects-config.js"; -import { loadMemory, DEFAULT_METADATA_DIR } from "@metaobjectsdev/sdk"; +import { loadMemory, resolveCollection, type Collection } from "@metaobjectsdev/sdk"; import { existsSync } from "node:fs"; import { join } from "node:path"; import { @@ -280,20 +280,28 @@ export async function docsCommand(args: string[], cwd: string): Promise return emitSite(metaRoot, outDir, configProviders, promptsDir); } + // Discovery and load are two separate failure modes, kept in separate try + // blocks deliberately — same reasoning as `meta gen` (gen.ts): a broad + // catch around both would swallow a genuine ParseError as "no metaobjects/ + // found", masking the real failure. + let collection; + try { + collection = await resolveCollection(metaRoot); + } catch (err) { + log.error(`docs: ${(err as Error).message}`); + return 2; + } + // Load metadata standalone — same loader path as migrate/gen. Threads any // consumer providers from the config so custom types resolve. let root; try { - root = await loadMemory(metaRoot, { + root = await loadMemory(collection.configDir, { + files: collection.files, ...(configProviders !== undefined ? { providers: configProviders } : {}), }); } catch (err) { - const msg = (err as Error).message; - if (!existsSync(join(metaRoot, DEFAULT_METADATA_DIR))) { - log.error(`docs: no metaobjects/ found in ${metaRoot}; run 'meta init' to scaffold`); - } else { - log.error(`docs: failed to load metadata: ${msg}`); - } + log.error(`docs: failed to load metadata: ${(err as Error).message}`); return 2; } @@ -504,14 +512,36 @@ async function scaffoldSiteCommand(metaRoot: string): Promise { return 0; } +/** + * Distinct directories declared by the collection's source specs, resolved + * absolute against `collection.configDir`. The site's own loader + * (`@metaobjectsdev/docs-site`) wants whole directories — each becomes a + * symlinked, basename-keyed source group — unlike the per-file list + * `resolveCollection` produces for the sdk `loadMemory` path. + */ +function collectionSourceDirs(collection: Collection): string[] { + const dirs = new Map(); + for (const { spec } of collection.sources) { + if (!("path" in spec)) continue; // phase 1 resolves "path" specs only + const key = JSON.stringify(spec); + if (dirs.has(key)) continue; + dirs.set( + key, + isAbsolute(spec.path) ? spec.path : resolvePath(collection.configDir, spec.path), + ); + } + return [...dirs.values()]; +} + /** * Emit the browsable HTML documentation site via `@metaobjectsdev/docs-site`. - * The site loads the model with its OWN loader from the metadata source dir - * (`/metaobjects`), so this is independent of the sdk loadMemory path - * used for the markdown surfaces. Writes under `/site` so it can coexist - * with the markdown output. Scaffold-and-own: when the consumer has copied - * templates/assets into `/codegen/docs-site/` (via `--scaffold-site`), - * those win over the bundled defaults. + * The site loads the model with its OWN loader from the resolved metadata + * source directories (see `resolveCollection`/`collectionSourceDirs`), so + * this is independent of the sdk loadMemory path used for the markdown + * surfaces. Writes under `/site` so it can coexist with the markdown + * output. Scaffold-and-own: when the consumer has copied templates/assets + * into `/codegen/docs-site/` (via `--scaffold-site`), those win + * over the bundled defaults. */ async function emitSite( metaRoot: string, @@ -520,14 +550,22 @@ async function emitSite( promptsDir?: string, ): Promise { const siteOutDir = resolvePath(outDir, "site"); - // metaobjects/ is REQUIRED (the site loads the model from it) and always first. - // Prompt `.mustache` source is additionally searched in the conventional - // /templates/ and any explicit --prompts dir (for a project whose templates - // live elsewhere, e.g. data/templates/) — else the site can't show the prompt TEXT - // and prints a "source missing" note. Only existing dirs are added, and dirs are + // The resolved metadata source dir(s) are REQUIRED (the site loads the + // model from them) and always first. Prompt `.mustache` source is + // additionally searched in the conventional /templates/ and any + // explicit --prompts dir (for a project whose templates live elsewhere, + // e.g. data/templates/) — else the site can't show the prompt TEXT and + // prints a "source missing" note. Only existing dirs are added, and dirs are // deduped by BASENAME (the site keys source groups by basename, and rejects a dup). - const sourceDirs = [join(metaRoot, DEFAULT_METADATA_DIR)]; - const seenBasenames = new Set([basename(join(metaRoot, DEFAULT_METADATA_DIR))]); + let collection; + try { + collection = await resolveCollection(metaRoot); + } catch (err) { + log.error(`docs: ${(err as Error).message}`); + return 2; + } + const sourceDirs = collectionSourceDirs(collection); + const seenBasenames = new Set(sourceDirs.map((d) => basename(d))); if (promptsDir !== undefined && !existsSync(promptsDir)) { log.warn(`docs: --prompts dir does not exist: ${promptsDir}`); } diff --git a/server/typescript/packages/cli/src/commands/export.ts b/server/typescript/packages/cli/src/commands/export.ts index 46109099e..e12d65f11 100644 --- a/server/typescript/packages/cli/src/commands/export.ts +++ b/server/typescript/packages/cli/src/commands/export.ts @@ -1,10 +1,10 @@ -import { resolve, join } from "node:path"; +import { resolve } from "node:path"; import { writeFile } from "node:fs/promises"; import { parseExportArgs } from "../lib/args.js"; import { log } from "../lib/log.js"; -import { loadAndExportJson } from "@metaobjectsdev/metadata/core"; -import { TypeRegistry, registerCoreTypes } from "@metaobjectsdev/metadata"; -import { DEFAULT_METADATA_DIR, registerForgeTypes } from "@metaobjectsdev/sdk"; +import { FileSource } from "@metaobjectsdev/metadata/core"; +import { TypeRegistry, registerCoreTypes, MetaDataLoader, canonicalSerialize } from "@metaobjectsdev/metadata"; +import { registerForgeTypes, resolveCollection } from "@metaobjectsdev/sdk"; export async function exportCommand(args: string[], cwd: string): Promise { let flags; @@ -16,7 +16,6 @@ export async function exportCommand(args: string[], cwd: string): Promise new FileSource(f)), + ); + const result = { + json: canonicalSerialize(loadResult.root), + errors: loadResult.errors, + warnings: loadResult.warnings.map((w) => w.message), + }; for (const w of result.warnings) { log.warn(w); diff --git a/server/typescript/packages/cli/src/commands/gen.ts b/server/typescript/packages/cli/src/commands/gen.ts index f260b426e..2b3ea5517 100644 --- a/server/typescript/packages/cli/src/commands/gen.ts +++ b/server/typescript/packages/cli/src/commands/gen.ts @@ -1,5 +1,4 @@ -import { relative, join } from "node:path"; -import { existsSync } from "node:fs"; +import { relative } from "node:path"; import { parseGenArgs } from "../lib/args.js"; import { resolveGenConfig } from "../lib/config.js"; import { loadMetaobjectsConfig } from "../lib/load-metaobjects-config.js"; @@ -9,7 +8,7 @@ import type { OutputFormat } from "../lib/format.js"; import { log } from "../lib/log.js"; import { warnIfAgentContextStale } from "../lib/agent-context-staleness.js"; import { scanSourceForAntiPatterns } from "../lib/anti-patterns.js"; -import { loadMemory, DEFAULT_METADATA_DIR } from "@metaobjectsdev/sdk"; +import { loadMemory, resolveCollection } from "@metaobjectsdev/sdk"; import { runGen, listGenerators } from "@metaobjectsdev/codegen-ts"; import type { WriteStatus } from "@metaobjectsdev/codegen-ts"; @@ -50,24 +49,28 @@ export async function genCommand(args: string[], cwd: string, fmt: OutputFormat return 2; } + // Discovery and load are two separate failure modes, kept in separate try + // blocks deliberately: a broad catch around both previously swallowed + // genuine ParseErrors (e.g. `origin.@via "X.y" ...: no such relationship + // "y" on X`) as "no metaobjects/ found", masking the real failure. + // `resolveCollection` raises `ERR_COLLECTION_NOT_FOUND` with its own + // message when nothing is discovered and no default directory exists. + let collection; + try { + collection = await resolveCollection(projectRoot); + } catch (err) { + log.error((err as Error).message); + return 2; + } + let metadata; try { - metadata = await loadMemory(projectRoot, { + metadata = await loadMemory(collection.configDir, { + files: collection.files, ...(forgeConfig.providers !== undefined ? { providers: forgeConfig.providers } : {}), }); } catch (err) { - const msg = (err as Error).message; - // Only emit the scaffold hint for the ACTUAL missing-metadata-dir - // condition — checked explicitly here. A broad substring match on - // "no such" / "cannot read" wrongly swallowed genuine ParseErrors (e.g. - // `origin.@via "X.y" ...: no such relationship "y" on X`) as "no - // metaobjects/ found", masking the real failure. Real parse/validation - // errors propagate with their actual message. - if (!existsSync(join(projectRoot, DEFAULT_METADATA_DIR))) { - log.error(`no metaobjects/ found in ${projectRoot}; run 'meta init' to scaffold`); - } else { - log.error(`failed to load metadata: ${msg}`); - } + log.error(`failed to load metadata: ${(err as Error).message}`); return 2; } diff --git a/server/typescript/packages/cli/src/index.ts b/server/typescript/packages/cli/src/index.ts index dac469d0d..b6180fff4 100644 --- a/server/typescript/packages/cli/src/index.ts +++ b/server/typescript/packages/cli/src/index.ts @@ -2,6 +2,7 @@ import { resolve } from "node:path"; import { log } from "./lib/log.js"; import { cliVersion } from "./lib/version.js"; import { resolveFormat, isValidFormat, VALID_FORMATS } from "./lib/format.js"; +import { resolveCollection } from "@metaobjectsdev/sdk"; export { defineConfig } from "@metaobjectsdev/codegen-ts"; export type { MetaobjectsGenConfig } from "@metaobjectsdev/codegen-ts"; @@ -271,8 +272,10 @@ export async function run(argv: string[]): Promise { case undefined: { // Content-first no-args view: concise status + next-step help[] rather than // dumping the full manual (full manual is still available via `meta --help`). - const metaobjectsExists = await import("node:fs/promises") - .then(({ stat }) => stat(resolve(cwd, "metaobjects")).then(() => true).catch(() => false)); + // "Is this a MetaObjects project?" routes through resolveCollection — the + // single authority on where metadata lives — rather than assuming the + // default `metaobjects/` directory name. + const metaobjectsExists = await resolveCollection(cwd).then(() => true).catch(() => false); const statusLine = metaobjectsExists ? `meta — MetaObjects CLI (v${VERSION}) · metaobjects/ found` : `meta — MetaObjects CLI (v${VERSION}) · no metaobjects/ here`; diff --git a/server/typescript/packages/cli/test/collection-routing.test.ts b/server/typescript/packages/cli/test/collection-routing.test.ts new file mode 100644 index 000000000..ff73d82b4 --- /dev/null +++ b/server/typescript/packages/cli/test/collection-routing.test.ts @@ -0,0 +1,85 @@ +import { describe, test, expect } from "bun:test"; +import { mkdtempSync, mkdirSync, rmSync, writeFileSync } from "node:fs"; +import { join, resolve } from "node:path"; +import { genCommand } from "../src/commands/gen.js"; + +// Place temp dirs inside the monorepo so metaobjects.config.ts's +// `@metaobjectsdev/*` imports resolve the same way the existing +// integration/gen-sqlite.test.ts fixtures do. +const WORKSPACE_TMP = resolve(import.meta.dirname, "fixtures/__tmp__"); + +function genOutDir(root: string): string { + return join(root, "generated", "db"); +} + +describe("gen routes metadata discovery through resolveCollection", () => { + test("generates from a sources-declared tree with no metaobjects/ present anywhere", async () => { + mkdirSync(WORKSPACE_TMP, { recursive: true }); + const root = mkdtempSync(join(WORKSPACE_TMP, "collection-routing-")); + try { + mkdirSync(join(root, ".git")); + + // Metadata lives OUTSIDE the app directory entirely — under `model/`, + // not `metaobjects/` — and nowhere under `apps/ui`. + mkdirSync(join(root, "model"), { recursive: true }); + writeFileSync( + join(root, "model", "meta.a.json"), + JSON.stringify({ + "metadata.root": { + package: "acme", + children: [ + { + "object.entity": { + name: "Order", + children: [ + { "source.rdb": { "@table": "orders" } }, + { "field.long": { name: "id", "@column": "id" } }, + { + "identity.primary": { + name: "pk", + "@fields": ["id"], + "@generation": "increment", + }, + }, + ], + }, + }, + ], + }, + }), + ); + + // The app's config declares its own metadata source — a relative path + // outside the app dir — instead of relying on a `metaobjects/` default. + mkdirSync(join(root, "apps", "ui", ".metaobjects"), { recursive: true }); + writeFileSync( + join(root, "apps", "ui", ".metaobjects", "config.json"), + JSON.stringify({ + schema_version: 1, + sources: [{ path: "../../model" }], + }), + ); + + const appRoot = join(root, "apps", "ui"); + writeFileSync( + join(appRoot, "metaobjects.config.ts"), + ` +import { defineConfig } from "@metaobjectsdev/codegen-ts"; +import { entityFile } from "@metaobjectsdev/codegen-ts/generators"; +export default defineConfig({ + outDir: ${JSON.stringify(genOutDir(appRoot))}, + dialect: "sqlite", + dbImport: "~/db", + extStyle: "none", + generators: [entityFile()], +}); +`, + ); + + const code = await genCommand([], appRoot); + expect(code).toBe(0); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +}); From 833e4300a61d46b0add81c9b5b200fb4e08c520e Mon Sep 17 00:00:00 2001 From: Doug Mealing Date: Mon, 17 Aug 2026 23:27:16 -0400 Subject: [PATCH 18/44] test(cli): add sdk to ensureFreshDist gate The gen-split-tree test gate runs the built CLI under node, requiring all runtime imports to have dist/ at least as new as src/. The helper ensureFreshDist() checked codegen-ts and cli but not sdk, leaving a stale-dist hole: any change to sdk/src/ was invisible to the gate until dist/ was rebuilt by hand. Add sdk to the rebuild loop following the exact pattern of codegen-ts: resolve the package root via createRequire, then check srcDir and distFile with the stale-rebuild behavior already wired. Co-Authored-By: Claude Opus 5 (1M context) --- .../packages/cli/test/gen-split-tree-single-import.test.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/server/typescript/packages/cli/test/gen-split-tree-single-import.test.ts b/server/typescript/packages/cli/test/gen-split-tree-single-import.test.ts index 845f35140..04785deb5 100644 --- a/server/typescript/packages/cli/test/gen-split-tree-single-import.test.ts +++ b/server/typescript/packages/cli/test/gen-split-tree-single-import.test.ts @@ -99,9 +99,13 @@ function ensureFreshDist(): void { const codegenTsRoot = dirname( createRequire(import.meta.url).resolve("@metaobjectsdev/codegen-ts/package.json"), ); + const sdkRoot = dirname( + createRequire(import.meta.url).resolve("@metaobjectsdev/sdk/package.json"), + ); for (const { name, pkgRoot, srcDir, distFile } of [ { name: "codegen-ts", pkgRoot: codegenTsRoot, srcDir: join(codegenTsRoot, "src"), distFile: join(codegenTsRoot, "dist", "index.js") }, { name: "cli", pkgRoot: CLI_ROOT, srcDir: join(CLI_ROOT, "src"), distFile: META_BIN }, + { name: "sdk", pkgRoot: sdkRoot, srcDir: join(sdkRoot, "src"), distFile: join(sdkRoot, "dist", "index.js") }, ]) { const stale = (): boolean => !existsSync(distFile) || newestSrcMtime(srcDir) > statSync(distFile).mtimeMs; From 218a29f4c26657bbe4434a398e8258377785ee2a Mon Sep 17 00:00:00 2001 From: Doug Mealing Date: Mon, 17 Aug 2026 23:33:33 -0400 Subject: [PATCH 19/44] fix(cli): detect-stack reads the resolved collection, fixing nested-symlink blindness MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit hasRequirementNodes now scans (await resolveCollection(cwd)).files instead of a bespoke readdirSync walk hardcoded to metaobjects/. This fixes two bugs at once: a project that declares `sources` elsewhere had its requirement.* nodes silently invisible to agent-context scaffolding, and the old walk used Dirent.isDirectory() (false for a symlinked directory) so it never descended into a nested symlinked subdirectory — while the loader (stat-based) does. Both the bespoke walk and the METADATA_DIR constant are deleted. resolveStack and probe are now async; init.ts's stackForAgentContext caller chain is updated to await. meta init still succeeds on an empty directory — resolveCollection's ERR_COLLECTION_NOT_FOUND (no metadata yet) is caught and treated as "no requirement nodes", preserving the heuristic's never-throws contract. Co-Authored-By: Claude Opus 5 (1M context) --- .../packages/cli/src/commands/init.ts | 4 +- .../packages/cli/src/lib/detect-stack.ts | 53 +++++-------- .../cli/test/unit/detect-stack.test.ts | 74 ++++++++++++++----- 3 files changed, 78 insertions(+), 53 deletions(-) diff --git a/server/typescript/packages/cli/src/commands/init.ts b/server/typescript/packages/cli/src/commands/init.ts index 00c9f4f79..73f655a19 100644 --- a/server/typescript/packages/cli/src/commands/init.ts +++ b/server/typescript/packages/cli/src/commands/init.ts @@ -192,7 +192,7 @@ function warnIfMonorepoSubdir(opts: InitOptions, result: InitResult): void { * string[]s (or nothing) straight through is safe — no need to special-case an empty * or absent prior. */ -function stackForAgentContext(opts: InitOptions, prior: Manifest | undefined): Stack { +async function stackForAgentContext(opts: InitOptions, prior: Manifest | undefined): Promise { const hasOverride = (opts.servers?.length ?? 0) > 0 || (opts.clients?.length ?? 0) > 0; const overrides = hasOverride ? { servers: opts.servers ?? [], clients: opts.clients ?? [] } @@ -203,7 +203,7 @@ function stackForAgentContext(opts: InitOptions, prior: Manifest | undefined): S async function writeAgentContext(opts: InitOptions, result: InitResult): Promise { warnIfMonorepoSubdir(opts, result); const prior = await readManifest(opts.cwd); - const stack = stackForAgentContext(opts, prior); + const stack = await stackForAgentContext(opts, prior); let assembled = assemble({ contentRoot: resolveAgentContextRoot(), stack }); if (opts.noSkills) assembled = assembled.filter((f) => !f.path.startsWith(".claude/skills/")); diff --git a/server/typescript/packages/cli/src/lib/detect-stack.ts b/server/typescript/packages/cli/src/lib/detect-stack.ts index 074a44853..2ff6c5eaa 100644 --- a/server/typescript/packages/cli/src/lib/detect-stack.ts +++ b/server/typescript/packages/cli/src/lib/detect-stack.ts @@ -1,6 +1,6 @@ import { existsSync, readFileSync, readdirSync } from "node:fs"; -import type { Dirent } from "node:fs"; import { join } from "node:path"; +import { resolveCollection } from "@metaobjectsdev/sdk"; import { detectStack, detectConcerns, makeStack, type ServerLang, type ClientFramework, type Stack, type ProjectProbe, @@ -21,62 +21,49 @@ function depNames(cwd: string): Set { return out; } -const METADATA_DIR = "metaobjects"; -const METADATA_FILE_PATTERN = /\.(json|ya?ml)$/i; // Cheap substring probe, not a metamodel load: matches both canonical JSON's // quoted `"requirement.functional"` key and sigil-free YAML's bare // `requirement.functional:` authoring form. const REQUIREMENT_NODE_MARKER = "requirement."; -/** Recursively scans `metaobjects/` for any `.json`/`.yaml`/`.yml` file containing a - * `requirement.*` node marker. Defensive throughout: a missing/unreadable directory - * or file is treated as "not found", never thrown — this is a cheap heuristic, not - * a metamodel load. */ -function hasRequirementNodes(cwd: string): boolean { - const root = join(cwd, METADATA_DIR); - if (!existsSync(root)) return false; - const pending: string[] = [root]; - while (pending.length > 0) { - const dir = pending.pop()!; - let entries: Dirent[]; - try { - entries = readdirSync(dir, { withFileTypes: true }); - } catch { - continue; // unreadable directory — skip it, keep scanning siblings - } - for (const entry of entries) { - const full = join(dir, entry.name); - if (entry.isDirectory()) { - pending.push(full); - } else if (METADATA_FILE_PATTERN.test(entry.name)) { - try { - if (readFileSync(full, "utf8").includes(REQUIREMENT_NODE_MARKER)) return true; - } catch { /* unreadable file — treat as no match */ } - } +/** Scans the project's resolved metadata collection (`resolveCollection` — the + * single authority on where metadata lives, honouring declared `sources` rather + * than assuming `metaobjects/`) for any file containing a `requirement.*` node + * marker. Defensive throughout: no declared sources and no default directory, an + * unresolvable source, or an unreadable file are all treated as "not found", + * never thrown — this is a cheap heuristic, not a metamodel load. */ +async function hasRequirementNodes(cwd: string): Promise { + try { + const { files } = await resolveCollection(cwd); + for (const file of files) { + if (readFileSync(file, "utf8").includes(REQUIREMENT_NODE_MARKER)) return true; } + return false; + } catch { + return false; } - return false; } -function probe(cwd: string): ProjectProbe { +async function probe(cwd: string): Promise { const deps = depNames(cwd); const names = existsSync(cwd) ? readdirSync(cwd) : []; + const requirementNodes = await hasRequirementNodes(cwd); return { hasDep: (name) => deps.has(name), hasFileMatching: (re) => names.some((n) => re.test(n)), - hasRequirementNodes: () => hasRequirementNodes(cwd), + hasRequirementNodes: () => requirementNodes, }; } /** Resolve the stack: explicit --server/--client overrides take precedence; otherwise detect. * Concern tokens (e.g. requirements) are always OBSERVED from project state, independent of * any --server/--client override — a concern is not a stack axis. */ -export function resolveStack(cwd: string, overrides: { servers: string[]; clients: string[] }): Stack { +export async function resolveStack(cwd: string, overrides: { servers: string[]; clients: string[] }): Promise { const validServers = SERVER_LANGS as readonly string[]; const validClients = CLIENT_FRAMEWORKS as readonly string[]; const oServers = overrides.servers.filter((s): s is ServerLang => validServers.includes(s)); const oClients = overrides.clients.filter((c): c is ClientFramework => validClients.includes(c)); - const p = probe(cwd); + const p = await probe(cwd); const concerns = detectConcerns(p); if (oServers.length > 0 || oClients.length > 0) return makeStack(oServers, oClients, concerns); const detected = detectStack(p); diff --git a/server/typescript/packages/cli/test/unit/detect-stack.test.ts b/server/typescript/packages/cli/test/unit/detect-stack.test.ts index 4c2077531..aa26bddde 100644 --- a/server/typescript/packages/cli/test/unit/detect-stack.test.ts +++ b/server/typescript/packages/cli/test/unit/detect-stack.test.ts @@ -1,43 +1,50 @@ import { test, expect, describe } from "bun:test"; -import { mkdtempSync, mkdirSync, rmSync, writeFileSync, chmodSync } from "node:fs"; +import { mkdtempSync, mkdirSync, rmSync, writeFileSync, chmodSync, symlinkSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { resolveStack } from "../../src/lib/detect-stack.js"; function tmp(): string { return mkdtempSync(join(tmpdir(), "detect-")); } +const REQ = JSON.stringify({ + "metadata.root": { + package: "acme", + children: [{ "requirement.functional": { name: "FR1", "@level": 1, "@status": "live" } }], + }, +}); + describe("resolveStack", () => { - test("explicit --server/--client overrides win over detection", () => { + test("explicit --server/--client overrides win over detection", async () => { const dir = tmp(); try { writeFileSync(join(dir, "package.json"), JSON.stringify({ dependencies: { "@metaobjectsdev/react": "1" } })); - const s = resolveStack(dir, { servers: ["java", "kotlin"], clients: ["tanstack"] }); + const s = await resolveStack(dir, { servers: ["java", "kotlin"], clients: ["tanstack"] }); expect(s.servers).toEqual(["java", "kotlin"]); expect(s.clients).toEqual(["tanstack"]); } finally { rmSync(dir, { recursive: true, force: true }); } }); - test("detects a TS server + react/tanstack from package.json deps", () => { + test("detects a TS server + react/tanstack from package.json deps", async () => { const dir = tmp(); try { writeFileSync(join(dir, "package.json"), JSON.stringify({ dependencies: { "@metaobjectsdev/cli": "1", "@metaobjectsdev/react": "1", "@metaobjectsdev/tanstack": "1" } })); - const s = resolveStack(dir, { servers: [], clients: [] }); + const s = await resolveStack(dir, { servers: [], clients: [] }); expect(s.servers).toEqual(["typescript"]); expect(s.clients.sort()).toEqual(["react", "tanstack"]); } finally { rmSync(dir, { recursive: true, force: true }); } }); - test("detects a Java (Maven) server from pom.xml", () => { + test("detects a Java (Maven) server from pom.xml", async () => { const dir = tmp(); try { writeFileSync(join(dir, "pom.xml"), ""); - const s = resolveStack(dir, { servers: [], clients: [] }); + const s = await resolveStack(dir, { servers: [], clients: [] }); expect(s.servers).toEqual(["java"]); } finally { rmSync(dir, { recursive: true, force: true }); } }); describe("requirements concern (observed, not a config flag)", () => { - test("detects a requirement.* node in a nested metadata file", () => { + test("detects a requirement.* node in a nested metadata file", async () => { const dir = tmp(); try { const nested = join(dir, "metaobjects", "caps"); @@ -51,13 +58,13 @@ describe("resolveStack", () => { }, }), ); - const s = resolveStack(dir, { servers: [], clients: [] }); + const s = await resolveStack(dir, { servers: [], clients: [] }); expect(s.tokens.has("requirements")).toBe(true); expect(s.concerns).toEqual(["requirements"]); } finally { rmSync(dir, { recursive: true, force: true }); } }); - test("no requirements token for a project with no requirement.* nodes", () => { + test("no requirements token for a project with no requirement.* nodes", async () => { const dir = tmp(); try { mkdirSync(join(dir, "metaobjects"), { recursive: true }); @@ -65,21 +72,21 @@ describe("resolveStack", () => { join(dir, "metaobjects", "meta.users.json"), JSON.stringify({ "metadata.root": { children: [{ "object.entity": { name: "User" } }] } }), ); - const s = resolveStack(dir, { servers: [], clients: [] }); + const s = await resolveStack(dir, { servers: [], clients: [] }); expect(s.tokens.has("requirements")).toBe(false); expect(s.concerns).toEqual([]); } finally { rmSync(dir, { recursive: true, force: true }); } }); - test("no metaobjects/ directory at all — treated as no requirements", () => { + test("no metaobjects/ directory at all — treated as no requirements", async () => { const dir = tmp(); try { - const s = resolveStack(dir, { servers: [], clients: [] }); + const s = await resolveStack(dir, { servers: [], clients: [] }); expect(s.tokens.has("requirements")).toBe(false); } finally { rmSync(dir, { recursive: true, force: true }); } }); - test("survives an unreadable metadata subdirectory (defensive, no throw)", () => { + test("survives an unreadable metadata subdirectory (defensive, no throw)", async () => { const dir = tmp(); try { const locked = join(dir, "metaobjects", "locked"); @@ -87,8 +94,7 @@ describe("resolveStack", () => { writeFileSync(join(locked, "meta.caps.json"), JSON.stringify({ "requirement.functional": { name: "X" } })); chmodSync(locked, 0o000); try { - expect(() => resolveStack(dir, { servers: [], clients: [] })).not.toThrow(); - const s = resolveStack(dir, { servers: [], clients: [] }); + const s = await resolveStack(dir, { servers: [], clients: [] }); expect(s.tokens.has("requirements")).toBe(false); } finally { chmodSync(locked, 0o755); // restore so recursive cleanup below can descend into it @@ -96,16 +102,48 @@ describe("resolveStack", () => { } finally { rmSync(dir, { recursive: true, force: true }); } }); - test("concerns are observed independent of explicit --server/--client overrides", () => { + test("concerns are observed independent of explicit --server/--client overrides", async () => { const dir = tmp(); try { const nested = join(dir, "metaobjects"); mkdirSync(nested, { recursive: true }); writeFileSync(join(nested, "meta.caps.json"), JSON.stringify({ "requirement.architectural": { name: "X" } })); - const s = resolveStack(dir, { servers: ["java"], clients: [] }); + const s = await resolveStack(dir, { servers: ["java"], clients: [] }); expect(s.servers).toEqual(["java"]); expect(s.tokens.has("requirements")).toBe(true); } finally { rmSync(dir, { recursive: true, force: true }); } }); + + test("finds requirement nodes in a sources-declared tree (no metaobjects/ at the start dir)", async () => { + const dir = tmp(); + try { + mkdirSync(join(dir, ".git")); + mkdirSync(join(dir, "model"), { recursive: true }); + writeFileSync(join(dir, "model", "meta.req.json"), REQ); + mkdirSync(join(dir, "apps", "ui", ".metaobjects"), { recursive: true }); + writeFileSync( + join(dir, "apps", "ui", ".metaobjects", "config.json"), + JSON.stringify({ schema_version: 1, sources: [{ path: "../../model" }] }), + ); + const s = await resolveStack(join(dir, "apps", "ui"), { servers: [], clients: [] }); + expect(s.tokens.has("requirements")).toBe(true); + expect(s.concerns).toContain("requirements"); + } finally { rmSync(dir, { recursive: true, force: true }); } + }); + + test("finds requirement nodes behind a NESTED symlinked directory", async () => { + const dir = tmp(); + try { + mkdirSync(join(dir, ".git")); + mkdirSync(join(dir, "real"), { recursive: true }); + writeFileSync(join(dir, "real", "meta.req.json"), REQ); + mkdirSync(join(dir, "metaobjects"), { recursive: true }); + writeFileSync(join(dir, "metaobjects", "meta.a.json"), "{}"); + symlinkSync(join(dir, "real"), join(dir, "metaobjects", "linked"), "dir"); + const s = await resolveStack(dir, { servers: [], clients: [] }); + expect(s.tokens.has("requirements")).toBe(true); + expect(s.concerns).toContain("requirements"); + } finally { rmSync(dir, { recursive: true, force: true }); } + }); }); }); From ad4695c0882c6377b4f1dd24fd13e3731a8b1f4f Mon Sep 17 00:00:00 2001 From: Doug Mealing Date: Mon, 17 Aug 2026 23:36:26 -0400 Subject: [PATCH 20/44] fix(metadata): add as const to error code array in typecheck test Array literal inferring element type as string instead of literal union, causing TS2769 when passed to ERROR_CODES.toContain(). Preserving literal types with as const keeps the type-safe comparison. Co-Authored-By: Claude Opus 5 (1M context) --- server/typescript/packages/metadata/test/errors.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/typescript/packages/metadata/test/errors.test.ts b/server/typescript/packages/metadata/test/errors.test.ts index 9ccf57b47..9b1d087f6 100644 --- a/server/typescript/packages/metadata/test/errors.test.ts +++ b/server/typescript/packages/metadata/test/errors.test.ts @@ -37,7 +37,7 @@ test("phase-1 source-resolution error codes are registered in the shared ledger" "ERR_SOURCE_KIND_UNSUPPORTED", "ERR_SCOPE_PATTERN_INVALID", "ERR_COLLECTION_NOT_FOUND", - ]) { + ] as const) { expect(ERROR_CODES).toContain(code); } }); From 8375772c3dd1f780b0d3a61e9a32701e1917ce2d Mon Sep 17 00:00:00 2001 From: Doug Mealing Date: Tue, 18 Aug 2026 06:22:49 -0400 Subject: [PATCH 21/44] feat(cli): migrate.scope narrows both sides so unowned tables are never touched MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `meta migrate` and `meta verify --db` now route through `resolveCollection()` and honour the per-command `migrate.scope` it carries. A consumer sharing a database with another owner declares: "migrate": { "scope": ["acme::platform::**"] } and every table or view whose DECLARING object falls outside is neither created, altered, dropped, nor reported as drift. The suppression is two-sided, deliberately. Dropping out-of-scope tables from the expected schema alone is strictly worse than not having the feature: `diff` proposes a DROP for everything present in `actual` and absent from `expected`, so expected-only filtering converts every out-of-scope table that EXISTS in the database into a proposed `DROP TABLE` — the precise hazard the feature exists to remove. The out-of-scope names therefore also ride `diff`'s `unmanagedNames` seam (merged with `collectUnmanagedNames`, never replacing it), the same mechanism `@unmanaged` uses to stop an external table being dropped. The scope decision is made on the declaring object's `resolutionKey()`, threaded out of the pass that already holds it (`buildExpectedSchemaWithProvenance`; `buildExpectedSchema` is now a thin wrapper over it). It is NOT re-derived from the SQL name, which is lossy, and NOT collected by a second metadata walk, which would have to duplicate Pass 1's skip rules — abstract / TPH subtype / no writable source / `@unmanaged` — and would drift from them. A TPH table is attributed to its discriminator BASE, so an out-of-scope subtype can never suppress the base's table. Provenance never reaches disk: `SNAPSHOT_FORMAT_VERSION` stays 3 and an unscoped project's snapshot bytes are unchanged (`canonicalize()` spreads the descriptor, so a descriptor field would land in the committed snapshot and owe a format bump that hard-fails older readers). View provenance travels on `ExpectedViewInput.fqn` and is recorded in the map only. Covered paths: the online Kysely diff, the offline snapshot diff (`planOffline`), the D1 diff, and `computeDriftFromActual` — the single choke point both `verify --db` paths share. `verify --db` prints one line naming what it excluded (silence would misreport an unchecked table as a checked one), and its committed- snapshot check (#292) filters out-of-scope objects from BOTH sides, since a scoped migrate writes a scoped snapshot. `migrate baseline` is deliberately unscoped: the `--from-db` arm captures whatever the database holds, and an offline baseline that recorded less would disagree with it. Unchanged by design: a table NO loaded object declares is still a proposed drop — scope only silences objects that were loaded and fell outside it. A project with no `migrate.scope` gets a byte-identical migration, a byte-identical snapshot and an unchanged drift verdict (verified by regenerating a fixed project's snapshot + up/down.sql before and after: all three md5s identical). One qualified-name definition now serves all three keyers (`diff`'s identity maps, `collectUnmanagedNames`, the out-of-scope set) — a second spelling would silently un-suppress an object and propose its drop. migrate-ts: 753 pass / 22 skip / 0 fail. cli: 568 pass / 3 skip / 0 fail. codegen-ts golden gate: 1241 pass / 0 fail. Workspace typecheck: 18/18 exit 0. Co-Authored-By: Claude Opus 5 (1M context) --- .../packages/cli/src/commands/migrate.ts | 118 +++++++++--- .../packages/cli/src/commands/verify.ts | 73 +++++-- .../packages/cli/src/lib/migrate-scope.ts | 24 +++ .../test/integration/verify-db-scope.test.ts | 165 ++++++++++++++++ .../packages/cli/test/migrate-scope.test.ts | 155 +++++++++++++++ .../src/projection/build-projection-views.ts | 10 + .../packages/migrate-ts/src/diff/index.ts | 13 +- .../packages/migrate-ts/src/drift/drift.ts | 53 +++-- .../migrate-ts/src/expected-schema.ts | 59 +++++- .../packages/migrate-ts/src/index.ts | 10 +- .../packages/migrate-ts/src/qualified-name.ts | 22 +++ .../packages/migrate-ts/src/scope.ts | 76 ++++++++ .../packages/migrate-ts/src/snapshot/plan.ts | 32 +++- .../packages/migrate-ts/src/unmanaged.ts | 5 +- .../migrate-ts/test/drift/drift-scope.test.ts | 79 ++++++++ .../test/expected-schema-scope.test.ts | 181 ++++++++++++++++++ 16 files changed, 1000 insertions(+), 75 deletions(-) create mode 100644 server/typescript/packages/cli/src/lib/migrate-scope.ts create mode 100644 server/typescript/packages/cli/test/integration/verify-db-scope.test.ts create mode 100644 server/typescript/packages/cli/test/migrate-scope.test.ts create mode 100644 server/typescript/packages/migrate-ts/src/qualified-name.ts create mode 100644 server/typescript/packages/migrate-ts/src/scope.ts create mode 100644 server/typescript/packages/migrate-ts/test/drift/drift-scope.test.ts create mode 100644 server/typescript/packages/migrate-ts/test/expected-schema-scope.test.ts diff --git a/server/typescript/packages/cli/src/commands/migrate.ts b/server/typescript/packages/cli/src/commands/migrate.ts index 41c0a9dad..de7665906 100644 --- a/server/typescript/packages/cli/src/commands/migrate.ts +++ b/server/typescript/packages/cli/src/commands/migrate.ts @@ -10,10 +10,12 @@ import type { OutputFormat } from "../lib/format.js"; import { toonEncode } from "../lib/format.js"; import { buildKyselyFromUrl, redactUrl } from "../lib/kysely.js"; import { log } from "../lib/log.js"; -import { loadMemory } from "@metaobjectsdev/sdk"; +import { loadMemory, resolveCollection } from "@metaobjectsdev/sdk"; import { loadMetaobjectsConfig } from "../lib/load-metaobjects-config.js"; +import { toObjectScope } from "../lib/migrate-scope.js"; import { - buildExpectedSchema, + buildExpectedSchemaWithProvenance, + scopeExpectedSchema, introspect, diff, collectUnmanagedNames, @@ -122,6 +124,19 @@ function resolveFormatOutDir(config: ResolvedMigrateConfig, metaRoot: string): s return resolvePath(metaRoot, config.outDir); } +/** + * Say what a declared `migrate.scope` left out. An excluded object produces neither + * a create nor a drop, so without this line "no changes" and "no changes to the half + * of the model this run governs" read identically. + */ +function logOutOfScope(names: readonly string[]): void { + if (names.length === 0) return; + log.info( + `meta migrate — ${names.length} object(s) out-of-scope (outside migrate.scope, ` + + `governed elsewhere): ${names.join(", ")}`, + ); +} + function emitStructuredError(error: string, hint: string, fmt: OutputFormat): void { const payload = { error, hint }; if (fmt === "json") { @@ -389,18 +404,27 @@ export async function migrateCommand( postgresConfigProviders = undefined; } + // Discovery and load are two separate failure modes, kept in separate try blocks + // (the `meta gen` pattern): a broad catch around both reports a genuine ParseError + // as "no metadata found", masking the real failure. `resolveCollection` raises + // ERR_COLLECTION_NOT_FOUND with its own message — the same exit 2 the hand-rolled + // ENOENT sniff used to produce. + let collection; + try { + collection = await resolveCollection(metaRoot); + } catch (err) { + log.error((err as Error).message); + return 2; + } + let metadata; try { - metadata = await loadMemory(metaRoot, { + metadata = await loadMemory(collection.configDir, { + files: collection.files, ...(postgresConfigProviders !== undefined ? { providers: postgresConfigProviders } : {}), }); } catch (err) { - const msg = (err as Error).message; - if (msg.includes("ENOENT") || msg.includes("no such") || msg.includes("cannot read")) { - log.error(`no metaobjects/ found in ${metaRoot}; run 'meta init' to scaffold`); - } else { - log.error(`failed to load metadata: ${msg}`); - } + log.error(`failed to load metadata: ${(err as Error).message}`); return 2; } @@ -435,11 +459,20 @@ export async function migrateCommand( // view DDL (create/drop/replace + dependency-recreate) and emit() renders it — // there is no separate view-migration emitter. const expectedViews = buildProjectionViews(metadata, { dialect: kysely.dialect, columnNamingStrategy }); - const expected = buildExpectedSchema(metadata, { - dialect: kysely.dialect, - columnNamingStrategy, - views: expectedViews, - }); + // Per-command scope: objects outside `migrate.scope` are another owner's. They + // leave the expected schema here and are suppressed on the actual side below — + // dropping them from `expected` ALONE would propose DROP TABLE for every one of + // them that exists in the database. + const scoped = scopeExpectedSchema( + buildExpectedSchemaWithProvenance(metadata, { + dialect: kysely.dialect, + columnNamingStrategy, + views: expectedViews, + }), + toObjectScope(collection.migrateScope), + ); + const expected = scoped.snapshot; + logOutOfScope(scoped.outOfScope); let actual; try { actual = await introspect(kysely.db, kysely.dialect); @@ -464,8 +497,9 @@ export async function migrateCommand( // the constraint and breaks referencing FKs at apply. refusePrimaryKeyChange: true, // #208 §7 — declared-@unmanaged objects are external: exclude them from the - // actual side so migrate proposes neither create nor drop for them. - unmanagedNames: collectUnmanagedNames(metadata), + // actual side so migrate proposes neither create nor drop for them. Objects + // outside `migrate.scope` ride the same seam, for the same reason. + unmanagedNames: [...collectUnmanagedNames(metadata), ...scoped.outOfScope], onAmbiguous: async (a) => { collectedAmbiguous.push(a); return onAmbiguousResolution; @@ -745,8 +779,15 @@ export async function runBaseline( } catch { // config absent — no custom providers, default snake_case } + // `baseline` records a STARTING POINT, so it is deliberately NOT scoped: the + // `--from-db` arm captures whatever the database holds (there is no provenance + // for an introspected table), and an offline baseline that recorded less would + // disagree with it. An out-of-scope table sitting in the snapshot is harmless — + // every later run suppresses it on both sides. try { - metadata = await loadMemory(metaRoot, { + const collection = await resolveCollection(metaRoot); + metadata = await loadMemory(collection.configDir, { + files: collection.files, ...(baselineConfigProviders !== undefined ? { providers: baselineConfigProviders } : {}), }); } catch (err) { @@ -880,8 +921,11 @@ export async function runOfflineGenerate( } let metadata; + let collection; try { - metadata = await loadMemory(metaRoot, { + collection = await resolveCollection(metaRoot); + metadata = await loadMemory(collection.configDir, { + files: collection.files, ...(offlineConfigProviders !== undefined ? { providers: offlineConfigProviders } : {}), }); } catch (err) { @@ -917,6 +961,7 @@ export async function runOfflineGenerate( const onAmbiguousResolution = mapOnAmbiguous(config.onAmbiguous); const offlineViews = buildProjectionViews(metadata, { dialect: config.dialect, columnNamingStrategy: offlineStrategy }); + const offlineScope = toObjectScope(collection.migrateScope); let plan; try { @@ -926,6 +971,8 @@ export async function runOfflineGenerate( snapshot, columnNamingStrategy: offlineStrategy, views: offlineViews, + // Per-command scope — narrows BOTH sides of the offline diff (see planOffline). + ...(offlineScope !== undefined ? { inScope: offlineScope } : {}), allow: tokensToAllowOptions(config.allow), onAmbiguous: async (a) => { collectedAmbiguous.push(a); @@ -947,6 +994,7 @@ export async function runOfflineGenerate( } const { diff: diffResult, nextSnapshot } = plan; + logOutOfScope(plan.outOfScope); if (diffResult.blocked.length > 0) { log.error(`migrate: ${diffResult.blocked.length} destructive change(s) blocked; re-run with --allow `); @@ -1101,18 +1149,25 @@ async function runD1Migrate( d1ConfigProviders = undefined; } + // Discovery and load are separate failure modes (the `meta gen` pattern); + // `resolveCollection`'s own ERR_COLLECTION_NOT_FOUND replaces the hand-rolled + // ENOENT sniff, with the same exit 2. + let collection; + try { + collection = await resolveCollection(metaRoot); + } catch (err) { + log.error((err as Error).message); + return 2; + } + let metadata; try { - metadata = await loadMemory(metaRoot, { + metadata = await loadMemory(collection.configDir, { + files: collection.files, ...(d1ConfigProviders !== undefined ? { providers: d1ConfigProviders } : {}), }); } catch (err) { - const msg = (err as Error).message; - if (msg.includes("ENOENT") || msg.includes("no such") || msg.includes("cannot read")) { - log.error(`no metaobjects/ found in ${metaRoot}; run 'meta init' to scaffold`); - } else { - log.error(`migrate: failed to load metadata: ${msg}`); - } + log.error(`migrate: failed to load metadata: ${(err as Error).message}`); return 2; } @@ -1125,7 +1180,13 @@ async function runD1Migrate( // metaobjects.config.ts absent or invalid — use default snake_case } const expectedViews = buildProjectionViews(metadata, { dialect: "d1", columnNamingStrategy }); - const expected = buildExpectedSchema(metadata, { dialect: "d1", columnNamingStrategy, views: expectedViews }); + // Per-command scope — both-sided, exactly as on the Kysely path above. + const scoped = scopeExpectedSchema( + buildExpectedSchemaWithProvenance(metadata, { dialect: "d1", columnNamingStrategy, views: expectedViews }), + toObjectScope(collection.migrateScope), + ); + const expected = scoped.snapshot; + logOutOfScope(scoped.outOfScope); let actual; try { actual = await introspectD1({ @@ -1157,8 +1218,9 @@ async function runD1Migrate( // has no expressible migration; refuse loudly instead of emitting SQL that drops // the constraint and breaks referencing FKs at apply (same failure as the online path). refusePrimaryKeyChange: true, - // #208 §7 — declared-@unmanaged objects are external (see the online path above). - unmanagedNames: collectUnmanagedNames(metadata), + // #208 §7 — declared-@unmanaged objects are external (see the online path above), + // and so are objects outside `migrate.scope`. + unmanagedNames: [...collectUnmanagedNames(metadata), ...scoped.outOfScope], onAmbiguous: async (a) => { collectedAmbiguous.push(a); return onAmbiguousResolution; diff --git a/server/typescript/packages/cli/src/commands/verify.ts b/server/typescript/packages/cli/src/commands/verify.ts index bf55fb5d8..338316463 100644 --- a/server/typescript/packages/cli/src/commands/verify.ts +++ b/server/typescript/packages/cli/src/commands/verify.ts @@ -33,6 +33,7 @@ import { computeDrift, computeDriftFromActual, collectUnmanagedNames, + qualifiedDbName, introspect, diff, readSnapshot, @@ -45,9 +46,10 @@ import { type Change, type D1Binding, type D1Runner, - type DiffResult, + type DriftResult, } from "@metaobjectsdev/migrate-ts"; -import { loadMemory } from "@metaobjectsdev/sdk"; +import { loadMemory, resolveCollection } from "@metaobjectsdev/sdk"; +import { toObjectScope } from "../lib/migrate-scope.js"; import { TYPE_TEMPLATE, TEMPLATE_SUBTYPE_PROMPT, @@ -140,20 +142,28 @@ export async function verifyCommand( } const configProviders = forgeConfig?.providers; + // Where the metadata lives is `resolveCollection`'s decision, not a hardcoded + // directory. It also carries the per-command `migrate.scope` the schema gate below + // honours — `verify --db` and `migrate` govern the identical object set. + let collection; + try { + collection = await resolveCollection(cwd); + } catch (err) { + log.error((err as Error).message); + return 2; + } + // ADR-0023 strict-by-default (#96): verify loads strict unless --lax is passed, // so an undeclared/typo'd own @attr fails verify (matching Java's Maven goal). let root: Awaited>; try { - root = await loadMemory(cwd, { + root = await loadMemory(collection.configDir, { + files: collection.files, ...(configProviders !== undefined ? { providers: configProviders } : {}), strict: !flags.lax, }); } catch (err) { const msg = (err as Error).message; - if (msg.includes("ENOENT") || msg.includes("no such") || msg.includes("cannot read")) { - log.error(`no metaobjects/ found in ${cwd}; run 'meta init' to scaffold`); - return 2; - } log.error(`failed to load metadata: ${msg}`); // Strict-load rejection (ADR-0023): give the author the three exits — register // the attr on a provider, stash it in the `attr.properties` bag, or pass --lax. @@ -169,6 +179,12 @@ export async function verifyCommand( return 1; } + // The schema gate governs exactly the objects `meta migrate` governs — ONE + // declaration (`migrate.scope`), not a second key: a drift gate that fails on + // tables migrate deliberately does not own is incoherent. Undefined ⇒ everything + // loaded, which is every project that declares no scope. + const schemaScope = toObjectScope(collection.migrateScope); + const promptsDir = join(cwd, flags.prompts ?? DEFAULT_PROMPTS_DIR); const provider = new FileProvider(promptsDir); @@ -428,7 +444,11 @@ export async function verifyCommand( // `actual` this drift comparison uses, and re-introspecting for it would both // cost a second round trip and open a window where the two could disagree. actual = await introspect(kysely.db, kysely.dialect); - driftResult = await computeDriftFromActual(actual, kysely.dialect, root, { allow, views: expectedViews }); + driftResult = await computeDriftFromActual(actual, kysely.dialect, root, { + allow, + views: expectedViews, + ...(schemaScope !== undefined ? { inScope: schemaScope } : {}), + }); } catch (err) { log.error(`verify: failed to introspect ${kysely.displayUrl}: ${(err as Error).message}`); return 1; @@ -436,7 +456,7 @@ export async function verifyCommand( const snapshotDrift = driftResult.changes.length === 0 - ? await checkCommittedSnapshot(actual, kysely.dialect, kysely.displayUrl) + ? await checkCommittedSnapshot(actual, kysely.dialect, kysely.displayUrl, driftResult.outOfScope) : []; return reportSchemaDrift(driftResult, [...ledgerDrift, ...snapshotDrift], kysely.displayUrl); @@ -510,7 +530,11 @@ export async function verifyCommand( const expectedViews = buildProjectionViews(root, { dialect: "d1", columnNamingStrategy: viewStrategy }); let driftResult; try { - driftResult = await computeDriftFromActual(actual, "d1", root, { allow, views: expectedViews }); + driftResult = await computeDriftFromActual(actual, "d1", root, { + allow, + views: expectedViews, + ...(schemaScope !== undefined ? { inScope: schemaScope } : {}), + }); } catch (err) { log.error(`verify: ${(err as Error).message}`); return 1; @@ -554,6 +578,7 @@ export async function verifyCommand( actual: SchemaSnapshot, dialect: Dialect, displayUrl: string, + outOfScope: readonly string[], ): Promise { if (dialect === "d1") return []; // d1 keeps migrations Wrangler-native; no offline snapshot // Resolve the migrations dir through migrate's OWN precedence (flag > config > @@ -570,11 +595,24 @@ export async function verifyCommand( } if (snapshot === null) return []; + // Out-of-scope objects leave BOTH sides of this comparison. `unmanagedNames` + // suppresses the actual side only, which is right for the metadata↔DB diff (the + // expected side is already scoped) but not here: the committed snapshot is the + // expected side, and a snapshot written before the scope was declared still + // carries the other owner's tables — leaving them in would report a phantom + // "snapshot disagrees" for an object this consumer does not manage. + const excluded = new Set(outOfScope); + const scopedSnapshot: SchemaSnapshot = excluded.size === 0 ? snapshot : { + ...snapshot, + tables: snapshot.tables.filter((t) => !excluded.has(qualifiedDbName(t))), + views: snapshot.views.filter((v) => !excluded.has(qualifiedDbName(v))), + }; + const result = await diff({ - expected: snapshot, + expected: scopedSnapshot, actual, allow: {}, - unmanagedNames: collectUnmanagedNames(root), + unmanagedNames: [...collectUnmanagedNames(root), ...outOfScope], }); if (result.changes.length === 0) return []; @@ -586,7 +624,7 @@ export async function verifyCommand( ]; } - function reportSchemaDrift(driftResult: DiffResult, ledgerDrift: string[], displayUrl: string): number { + function reportSchemaDrift(driftResult: DriftResult, ledgerDrift: string[], displayUrl: string): number { // #208 §8 — make declared-external objects visible: they are excluded from the // drift comparison (computeDrift/computeDriftFromActual thread them out), so // annotate them as external (declared) rather than let them vanish silently. @@ -597,6 +635,15 @@ export async function verifyCommand( ); } + // Same reasoning for the per-command scope: an object `migrate.scope` excluded + // was NOT checked, and silence would misreport it as checked-and-clean. + if (driftResult.outOfScope.length > 0) { + log.info( + `meta verify — ${driftResult.outOfScope.length} object(s) out-of-scope ` + + `(outside migrate.scope, governed elsewhere): ${driftResult.outOfScope.join(", ")}`, + ); + } + const changes = driftResult.changes; if (changes.length === 0 && ledgerDrift.length === 0) { log.info(`meta verify — schema in sync with ${displayUrl}.`); diff --git a/server/typescript/packages/cli/src/lib/migrate-scope.ts b/server/typescript/packages/cli/src/lib/migrate-scope.ts new file mode 100644 index 000000000..ab84f2741 --- /dev/null +++ b/server/typescript/packages/cli/src/lib/migrate-scope.ts @@ -0,0 +1,24 @@ +// The one adapter between a declared `migrate.scope` and migrate-ts's scope seam. +// +// `resolveCollection` compiles `.metaobjects/config.json`'s `migrate.scope` into a +// `CompiledScope`; migrate-ts takes a plain predicate over an object's +// fully-qualified name so it never carries a second implementation of the pattern +// grammar. `matchesScope` (@metaobjectsdev/sdk) is THE pattern engine — there is no +// other, and adding one would let `migrate` and `gen` disagree about what +// `acme::platform::**` means. +// +// Both `meta migrate` and `meta verify --db` import this: the two commands govern +// the identical object set, so they share the one declaration rather than each +// growing a key of its own. + +import { matchesScope, type CompiledScope } from "@metaobjectsdev/sdk"; +import type { ObjectScopePredicate } from "@metaobjectsdev/migrate-ts"; + +/** + * Adapt a compiled `migrate.scope` to migrate-ts's predicate seam. Undefined in, + * undefined out — a project that declared no scope governs everything it loaded, + * and the undefined predicate is what keeps its expected schema untouched. + */ +export function toObjectScope(scope: CompiledScope | undefined): ObjectScopePredicate | undefined { + return scope === undefined ? undefined : (fqn: string): boolean => matchesScope(fqn, scope); +} diff --git a/server/typescript/packages/cli/test/integration/verify-db-scope.test.ts b/server/typescript/packages/cli/test/integration/verify-db-scope.test.ts new file mode 100644 index 000000000..7cdbf2135 --- /dev/null +++ b/server/typescript/packages/cli/test/integration/verify-db-scope.test.ts @@ -0,0 +1,165 @@ +/** + * `meta verify --db` honours `migrate.scope` (real sqlite, whole CLI pipeline). + * + * `migrate` and `verify --db` govern the identical object set — a drift gate + * that fails on tables `migrate` deliberately does not own is incoherent — so + * the two share ONE declaration (`migrate.scope`) rather than a second key. + * An out-of-scope object is reported as out-of-scope, never as drift: silence + * alone would misreport an unchecked table as a checked one. + */ +import { describe, test, expect, beforeEach, afterEach } from "bun:test"; +import { mkdtempSync, mkdirSync, rmSync, writeFileSync, readFileSync, readdirSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { createClient } from "@libsql/client"; +import { run } from "../../src/index.js"; + +const PLATFORM = JSON.stringify({ + "metadata.root": { + package: "acme::platform", + children: [{ + "object.entity": { + name: "Job", + children: [ + { "source.rdb": { name: "src", "@table": "jobs" } }, + { "field.long": { name: "id" } }, + { "field.string": { name: "title" } }, + { "identity.primary": { name: "pk", "@fields": ["id"] } }, + ], + }, + }], + }, +}); + +/** Another owner's package, sharing the database. `venue` models a column the + * other owner has not migrated yet — drift for THEM, never for this consumer. */ +const ARENA = (venue: boolean): string => JSON.stringify({ + "metadata.root": { + package: "arena", + children: [{ + "object.entity": { + name: "Match", + children: [ + { "source.rdb": { name: "src", "@table": "matches" } }, + { "field.long": { name: "id" } }, + ...(venue ? [{ "field.string": { name: "venue" } }] : []), + { "identity.primary": { name: "pk", "@fields": ["id"] } }, + ], + }, + }], + }, +}); + +function scaffold(): { repo: string; dbUrl: string } { + const repo = mkdtempSync(join(tmpdir(), "metaobjects-verify-scope-")); + mkdirSync(join(repo, "metaobjects"), { recursive: true }); + writeFileSync(join(repo, "metaobjects", "meta.platform.json"), PLATFORM, "utf8"); + writeFileSync(join(repo, "metaobjects", "meta.arena.json"), ARENA(false), "utf8"); + return { repo, dbUrl: `file:${join(repo, "local.db")}` }; +} + +function declareScope(repo: string, scope: string[]): void { + mkdirSync(join(repo, ".metaobjects"), { recursive: true }); + writeFileSync( + join(repo, ".metaobjects", "config.json"), + JSON.stringify({ schema_version: 1, migrate: { scope } }), + "utf8", + ); +} + +/** Materialize the current metadata schema into the DB via the real migrate path. */ +async function materialize(repo: string, dbUrl: string): Promise { + const exit = await run(["migrate", "--from-db", "--cwd", repo, "--db", dbUrl, "--dialect", "sqlite", "--slug", "initial"]); + expect(exit).toBe(0); + const migrationsRoot = join(repo, ".metaobjects", "migrations"); + const dir = readdirSync(migrationsRoot).find((s) => s.endsWith("-initial"))!; + const sql = readFileSync(join(migrationsRoot, dir, "up.sql"), "utf8"); + const client = createClient({ url: dbUrl }); + for (const stmt of sql.split(";").map((s) => s.trim()).filter((s) => s.length > 0)) { + await client.execute(stmt); + } + client.close(); +} + +let out: string[]; +let err: string[]; +let origLog: typeof console.log; +let origErr: typeof console.error; + +beforeEach(() => { + out = []; + err = []; + origLog = console.log; + origErr = console.error; + console.log = (...a: unknown[]) => { out.push(a.map(String).join(" ")); }; + console.error = (...a: unknown[]) => { err.push(a.map(String).join(" ")); }; +}); +afterEach(() => { + console.log = origLog; + console.error = origErr; +}); + +describe("meta verify --db — migrate.scope", () => { + test("an out-of-scope object's divergence is reported as out-of-scope, not as drift", async () => { + const { repo, dbUrl } = scaffold(); + try { + await materialize(repo, dbUrl); + expect(await run(["verify", "--cwd", repo, "--db", dbUrl, "--dialect", "sqlite"])).toBe(0); + + declareScope(repo, ["acme::platform::**"]); + // The other owner's model gains a column its own migration has not applied. + writeFileSync(join(repo, "metaobjects", "meta.arena.json"), ARENA(true), "utf8"); + out = []; + err = []; + + expect(await run(["verify", "--cwd", repo, "--db", dbUrl, "--dialect", "sqlite"])).toBe(0); + const all = [...out, ...err].join("\n"); + expect(all).toContain("out-of-scope"); + expect(all).toContain("matches"); + expect(all).not.toContain("venue"); + } finally { + rmSync(repo, { recursive: true, force: true }); + } + }); + + test("in-scope drift still fails the gate under a scope", async () => { + const { repo, dbUrl } = scaffold(); + try { + await materialize(repo, dbUrl); + declareScope(repo, ["acme::platform::**"]); + // This consumer's OWN model gains a column the database lacks. + writeFileSync( + join(repo, "metaobjects", "meta.platform.json"), + PLATFORM.replace( + `{"field.string":{"name":"title"}}`, + `{"field.string":{"name":"title"}},{"field.string":{"name":"owner"}}`, + ), + "utf8", + ); + out = []; + err = []; + + expect(await run(["verify", "--cwd", repo, "--db", dbUrl, "--dialect", "sqlite"])).toBe(1); + expect([...out, ...err].join("\n")).toContain("owner"); + } finally { + rmSync(repo, { recursive: true, force: true }); + } + }); + + test("with no migrate.scope declared, the same divergence IS drift (unchanged)", async () => { + const { repo, dbUrl } = scaffold(); + try { + await materialize(repo, dbUrl); + writeFileSync(join(repo, "metaobjects", "meta.arena.json"), ARENA(true), "utf8"); + out = []; + err = []; + + expect(await run(["verify", "--cwd", repo, "--db", dbUrl, "--dialect", "sqlite"])).toBe(1); + const all = [...out, ...err].join("\n"); + expect(all).toContain("venue"); + expect(all).not.toContain("out-of-scope"); + } finally { + rmSync(repo, { recursive: true, force: true }); + } + }); +}); diff --git a/server/typescript/packages/cli/test/migrate-scope.test.ts b/server/typescript/packages/cli/test/migrate-scope.test.ts new file mode 100644 index 000000000..5c977e3a1 --- /dev/null +++ b/server/typescript/packages/cli/test/migrate-scope.test.ts @@ -0,0 +1,155 @@ +/** + * `migrate.scope` — a `meta migrate` run governs only the objects it declares. + * + * Without this, "load everything" turns a real adopter's worst standing hazard — + * a migrate proposing to DROP tables it does not model — from a discipline into + * an automation. The suppression is BOTH-sided: the out-of-scope tables leave + * the expected schema AND are excluded from the actual side, so the run neither + * creates nor drops them. + * + * Boundary (deliberately unchanged): a table that NO loaded object declares is + * still a proposed drop. Scope only silences tables whose declaring object was + * loaded and fell outside it. + */ +import { describe, test, expect, afterAll } from "bun:test"; +import { mkdtemp, rm, mkdir, writeFile, readdir, readFile, unlink } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { compileScope } from "@metaobjectsdev/sdk"; +import { runBaseline, runOfflineGenerate } from "../src/commands/migrate.js"; +import { toObjectScope } from "../src/lib/migrate-scope.js"; + +const dirs: string[] = []; +afterAll(async () => { for (const d of dirs) await rm(d, { recursive: true, force: true }); }); + +const PLATFORM = (extraField: boolean): string => JSON.stringify({ + "metadata.root": { + package: "acme::platform", + children: [{ + "object.entity": { + name: "Job", + children: [ + { "source.rdb": { name: "src", "@table": "jobs" } }, + { "field.long": { name: "id" } }, + { "field.string": { name: "title", "@maxLength": 80 } }, + ...(extraField ? [{ "field.string": { name: "note" } }] : []), + { "identity.primary": { name: "pk", "@fields": ["id"], "@generation": "increment" } }, + ], + }, + }], + }, +}); + +/** Another owner's package, in the same database. */ +const ARENA = (extraField: boolean): string => JSON.stringify({ + "metadata.root": { + package: "arena", + children: [{ + "object.entity": { + name: "Match", + children: [ + { "source.rdb": { name: "src", "@table": "matches" } }, + { "field.long": { name: "id" } }, + ...(extraField ? [{ "field.string": { name: "venue" } }] : []), + { "identity.primary": { name: "pk", "@fields": ["id"], "@generation": "increment" } }, + ], + }, + }], + }, +}); + +async function project(): Promise { + const root = await mkdtemp(join(tmpdir(), "migrate-scope-")); + dirs.push(root); + await mkdir(join(root, "metaobjects"), { recursive: true }); + await writeFile(join(root, "metaobjects", "meta.platform.json"), PLATFORM(false), "utf8"); + await writeFile(join(root, "metaobjects", "meta.arena.json"), ARENA(false), "utf8"); + return root; +} + +async function declareScope(root: string, scope: string[]): Promise { + await mkdir(join(root, ".metaobjects"), { recursive: true }); + await writeFile( + join(root, ".metaobjects", "config.json"), + JSON.stringify({ schema_version: 1, migrate: { scope } }), + "utf8", + ); +} + +const cfg = () => + ({ dialect: "sqlite", outDir: "./.metaobjects/migrations", onAmbiguous: "abort", + allow: [], slug: "auto", dryRun: false } as never); + +const migrationDirs = async (root: string): Promise => + (await readdir(join(root, ".metaobjects/migrations"))).filter((e) => !e.startsWith(".")); + +describe("toObjectScope", () => { + test("matchesScope drives the decision — no second pattern implementation", () => { + const inScope = toObjectScope(compileScope({ include: ["acme::platform::**"] }))!; + expect(inScope("acme::platform::Job")).toBe(true); + expect(inScope("acme::platform::billing::Invoice")).toBe(true); + expect(inScope("arena::Match")).toBe(false); + }); + + test("no declared scope → no predicate (the command governs everything loaded)", () => { + expect(toObjectScope(undefined)).toBeUndefined(); + }); +}); + +describe("meta migrate — migrate.scope", () => { + test("an out-of-scope table is neither altered nor dropped", async () => { + const root = await project(); + // Baseline BEFORE the scope is declared: the reference snapshot records both + // owners' tables, exactly as a `--from-db` baseline of the shared database would. + expect(await runBaseline(cfg(), root)).toBe(0); + await declareScope(root, ["acme::platform::**"]); + // The other owner evolves ITS model. Three outcomes are distinguishable here: + // no scoping at all migrates the foreign column; scoping the EXPECTED side alone + // proposes DROP TABLE "matches" (blocked → exit 1); correct both-sided + // suppression produces silence. + await writeFile(join(root, "metaobjects", "meta.arena.json"), ARENA(true), "utf8"); + + expect(await runOfflineGenerate(cfg(), root)).toBe(0); + expect(await migrationDirs(root)).toHaveLength(0); + }); + + test("in-scope changes still migrate under a scope", async () => { + const root = await project(); + await runBaseline(cfg(), root); + await declareScope(root, ["acme::platform::**"]); + await writeFile(join(root, "metaobjects", "meta.platform.json"), PLATFORM(true), "utf8"); + + expect(await runOfflineGenerate(cfg(), root)).toBe(0); + const [dir] = await migrationDirs(root); + expect(dir).toBeDefined(); + const up = await readFile(join(root, ".metaobjects/migrations", dir!, "up.sql"), "utf8"); + expect(up).toBe(`ALTER TABLE "jobs" ADD COLUMN "note" TEXT;\n`); + }); + + test("back-compat: with NO migrate.scope the emitted SQL is unchanged", async () => { + const root = await project(); + await runBaseline(cfg(), root); + await writeFile(join(root, "metaobjects", "meta.platform.json"), PLATFORM(true), "utf8"); + + expect(await runOfflineGenerate(cfg(), root)).toBe(0); + const [dir] = await migrationDirs(root); + const up = await readFile(join(root, ".metaobjects/migrations", dir!, "up.sql"), "utf8"); + const down = await readFile(join(root, ".metaobjects/migrations", dir!, "down.sql"), "utf8"); + // Byte-for-byte what this project emitted before per-command scope existed. + expect(up).toBe(`ALTER TABLE "jobs" ADD COLUMN "note" TEXT;\n`); + expect(down).toBe(`ALTER TABLE "jobs" DROP COLUMN "note";\n`); + }); + + test("a table NO loaded object declares is still proposed for drop, scope or not", async () => { + const root = await project(); + await runBaseline(cfg(), root); + await declareScope(root, ["acme::platform::**"]); + // The arena model leaves the collection entirely — nothing declares `matches` + // any more, so migrate is back to its unchanged behaviour: propose the drop + // (blocked here, since `allow` is empty → exit 1). + await unlink(join(root, "metaobjects", "meta.arena.json")); + + expect(await runOfflineGenerate(cfg(), root)).toBe(1); + expect(await migrationDirs(root)).toHaveLength(0); + }); +}); diff --git a/server/typescript/packages/codegen-ts/src/projection/build-projection-views.ts b/server/typescript/packages/codegen-ts/src/projection/build-projection-views.ts index dcba4d83a..91de41102 100644 --- a/server/typescript/packages/codegen-ts/src/projection/build-projection-views.ts +++ b/server/typescript/packages/codegen-ts/src/projection/build-projection-views.ts @@ -53,6 +53,14 @@ export interface ExpectedView { name: string; schema?: string; sql: string; + /** + * `resolutionKey()` of the object that declared this view — the projection, or the + * write-through entity hosting its own read view. migrate-ts records it as + * PROVENANCE (never onto the view descriptor, never into the committed snapshot) so + * a per-command `migrate.scope` can decide ownership on the declaring FQN rather + * than on the physical view name, which no naming strategy can reverse. + */ + fqn: string; /** * Physical tables this view reads (base + every joined table). The migrate-ts * diff uses this to recreate the view when one of its source tables undergoes a @@ -199,6 +207,7 @@ function emitViewFor( sql: body, dependsOn, columns, + fqn: host.resolutionKey(), ...(schema !== undefined ? { schema } : {}), }); } @@ -240,6 +249,7 @@ function emitSqlView( name: source.physicalName, // FR-016 four-step physical name sql: source.sqlBody!, // verbatim — never parsed, never re-wrapped dependsOn, + fqn: host.resolutionKey(), // columns OMITTED → "unknown" → gated drop+create fail-safe. ...(schema !== undefined ? { schema } : {}), }); diff --git a/server/typescript/packages/migrate-ts/src/diff/index.ts b/server/typescript/packages/migrate-ts/src/diff/index.ts index 4da7ea67f..f5b3d7a1b 100644 --- a/server/typescript/packages/migrate-ts/src/diff/index.ts +++ b/server/typescript/packages/migrate-ts/src/diff/index.ts @@ -15,6 +15,7 @@ import { viewReplaceIsLegal } from "../view-column-types.js"; import { checkExprEquals, normalizeCheckExpr } from "../check-expr-compare.js"; import { isPgAutoSequenceDefault } from "../pg-identity-default.js"; import { DEFAULT_DB_SCHEMA_POSTGRES } from "@metaobjectsdev/metadata"; +import { qualifiedDbName } from "../qualified-name.js"; export interface DiffArgs { expected: SchemaSnapshot; @@ -90,10 +91,12 @@ const DEFAULT_IGNORE_TABLES: string[] = [ * * For SQLite (no schema concept), every table has schema=undefined, so this maps * all tables to the same "public." prefix — harmless and preserves existing behavior. + * + * `qualifiedDbName` is THE definition (qualified-name.ts): the act-side exclusion + * sets — declared-`@unmanaged` and out-of-scope — are matched against these keys, so + * a second spelling here would silently un-suppress an object and propose its drop. */ -function tableIdentity(table: { name: string; schema?: string }): string { - return (table.schema ?? DEFAULT_DB_SCHEMA_POSTGRES) + "." + table.name; -} +const tableIdentity = qualifiedDbName; /** * Build the optional-schema spread used when constructing Change records. @@ -590,9 +593,7 @@ function diffTableChecks( } } -function viewIdentity(v: { name: string; schema?: string }): string { - return (v.schema ?? DEFAULT_DB_SCHEMA_POSTGRES) + "." + v.name; -} +const viewIdentity = qualifiedDbName; /** * Decide, per view, whether the DB matches the model. diff --git a/server/typescript/packages/migrate-ts/src/drift/drift.ts b/server/typescript/packages/migrate-ts/src/drift/drift.ts index f5369ee83..bf8dda28c 100644 --- a/server/typescript/packages/migrate-ts/src/drift/drift.ts +++ b/server/typescript/packages/migrate-ts/src/drift/drift.ts @@ -16,10 +16,11 @@ import type { Kysely } from "kysely"; import type { MetaRoot } from "@metaobjectsdev/metadata"; import type { ColumnNamingStrategy } from "@metaobjectsdev/metadata"; -import { buildExpectedSchema } from "../expected-schema.js"; +import { buildExpectedSchemaWithProvenance } from "../expected-schema.js"; import { introspect } from "../introspect/index.js"; import { diff } from "../diff/index.js"; import { collectUnmanagedNames } from "../unmanaged.js"; +import { scopeExpectedSchema, type ObjectScopePredicate } from "../scope.js"; import type { AllowOptions, Dialect, DiffResult, SchemaSnapshot } from "../types.js"; export interface ComputeDriftOptions { @@ -46,6 +47,26 @@ export interface ComputeDriftOptions { * itself; pass these so view drift is detected. Defaults to none. */ views?: readonly import("../expected-schema.js").ExpectedViewInput[]; + /** + * Per-command scope (`migrate.scope`): objects whose declaring FQN this predicate + * rejects are governed by somebody else. They leave the expected side AND are + * suppressed on the actual side, so their divergence is neither drift nor a + * proposed drop — `verify` reports them as out-of-scope instead (see + * `DriftResult.outOfScope`). Omit to govern everything loaded (unchanged behavior). + * + * `verify --db` and `migrate` deliberately share ONE declaration: a drift gate + * failing on tables migrate does not own is incoherent. + */ + inScope?: ObjectScopePredicate; +} + +export interface DriftResult extends DiffResult { + /** + * Qualified physical names excluded by `inScope` — empty when no scope was + * given. The caller REPORTS these: an object silently dropped from the + * comparison is indistinguishable from one that was checked and found clean. + */ + outOfScope: readonly string[]; } /** @@ -65,24 +86,30 @@ export async function computeDriftFromActual( dialect: Dialect, metadata: MetaRoot, opts?: ComputeDriftOptions, -): Promise { - const expected = buildExpectedSchema(metadata, { - dialect, - ...(opts?.columnNamingStrategy !== undefined - ? { columnNamingStrategy: opts.columnNamingStrategy } - : {}), - ...(opts?.views !== undefined ? { views: opts.views } : {}), - }); - return diff({ - expected, +): Promise { + const scoped = scopeExpectedSchema( + buildExpectedSchemaWithProvenance(metadata, { + dialect, + ...(opts?.columnNamingStrategy !== undefined + ? { columnNamingStrategy: opts.columnNamingStrategy } + : {}), + ...(opts?.views !== undefined ? { views: opts.views } : {}), + }), + opts?.inScope, + ); + const result = await diff({ + expected: scoped.snapshot, actual, dialect, allow: opts?.allow ?? {}, // #208 §7 — a declared-@unmanaged object is external, so it is not drift: exclude it // from the actual side (same as `meta migrate`) rather than surface a false drop-*. - unmanagedNames: collectUnmanagedNames(metadata), + // Out-of-scope objects join it: dropping them from `expected` alone would turn each + // one that EXISTS in the database into a spurious drop-* drift. + unmanagedNames: [...collectUnmanagedNames(metadata), ...scoped.outOfScope], ...(opts?.ignoreTables !== undefined ? { ignoreTables: opts.ignoreTables } : {}), }); + return { ...result, outOfScope: scoped.outOfScope }; } /** @@ -97,7 +124,7 @@ export async function computeDrift( dialect: Dialect, metadata: MetaRoot, opts?: ComputeDriftOptions, -): Promise { +): Promise { const actual = await introspect(db, dialect); return computeDriftFromActual(actual, dialect, metadata, opts); } diff --git a/server/typescript/packages/migrate-ts/src/expected-schema.ts b/server/typescript/packages/migrate-ts/src/expected-schema.ts index fe25f1cbb..8c32e675a 100644 --- a/server/typescript/packages/migrate-ts/src/expected-schema.ts +++ b/server/typescript/packages/migrate-ts/src/expected-schema.ts @@ -61,6 +61,7 @@ import type { Dialect, SchemaSnapshot, TableDescriptor, ColumnDescriptor, IndexDescriptor, FkDescriptor, CheckDescriptor, ViewDescriptor, } from "./types.js"; +import { qualifiedDbName } from "./qualified-name.js"; import { viewFingerprint } from "./view-fingerprint.js"; import { resolveViewColumns, type ExpectedViewColumnInput } from "./view-column-types.js"; import { @@ -112,12 +113,55 @@ export interface ExpectedViewInput { sql?: string; dependsOn?: readonly string[]; columns?: readonly ExpectedViewColumnInput[]; + /** + * `resolutionKey()` of the object that declared this view — its PROVENANCE. + * Recorded in the provenance map and deliberately NEVER copied onto the + * `ViewDescriptor`: descriptors are serialized into the committed snapshot, and + * a descriptor that gains a field owes a `SNAPSHOT_FORMAT_VERSION` bump, which + * hard-fails every older reader. Optional — a caller that supplies no FQN gets a + * view with no provenance, which `scopeExpectedSchema` keeps (never guesses). + */ + fqn?: string; } +/** + * Qualified physical name (`qualifiedDbName`) → the `resolutionKey()` of the + * metadata object that declared it. The ONLY sound basis for a per-command scope + * decision: a SQL name cannot be reversed into an FQN (naming strategies, `@table` + * overrides and TPH folding are all lossy), and a second metadata walk would have + * to re-implement Pass 1's skip rules — abstract, TPH subtype, no writable source, + * `@unmanaged` — and would drift from them. + */ +export type SchemaProvenance = ReadonlyMap; + +export interface ExpectedSchemaWithProvenance { + snapshot: SchemaSnapshot; + provenance: SchemaProvenance; +} + +/** + * The expected schema as every existing caller wants it. Thin wrapper over + * {@link buildExpectedSchemaWithProvenance}; byte-identical output. + */ export function buildExpectedSchema( root: MetaData, opts?: BuildExpectedSchemaOptions, ): SchemaSnapshot { + return buildExpectedSchemaWithProvenance(root, opts).snapshot; +} + +/** + * The expected schema PLUS the declaring FQN of every table and view in it. + * + * Provenance is threaded out of the passes that already hold the declaring node — + * Pass 2 has each table's entity, Pass 4 each view's input — so there is exactly + * one walk and one set of skip rules. Callers that filter by scope + * (`scopeExpectedSchema`) consume it; callers that don't use the wrapper above. + */ +export function buildExpectedSchemaWithProvenance( + root: MetaData, + opts?: BuildExpectedSchemaOptions, +): ExpectedSchemaWithProvenance { // D1 is SQLite at the SQL level; normalize it so downstream dialect checks // don't need to handle "d1" separately. const dialect = opts?.dialect === "d1" ? "sqlite" : opts?.dialect; @@ -207,6 +251,12 @@ export function buildExpectedSchema( return byBareHit === AMBIGUOUS ? undefined : byBareHit; }; + // Provenance: qualified physical name → declaring object's FQN. Recorded as the + // descriptors are built, never re-derived from a SQL name (lossy) and never by a + // second walk (it would have to duplicate Pass 1's skip rules and would drift from + // them — a TPH subtype, for one, shares its base's table and declares none of its own). + const provenance = new Map(); + // Pass 2: build full descriptors with FK resolution. // Schema is resolved here (not stored in Pass 1) to avoid exactOptionalPropertyTypes // issues with `string | undefined` vs `schema?: string`. @@ -214,6 +264,7 @@ export function buildExpectedSchema( const t = buildTable(entity, tableName, resolveTargetTable, root as MetaRoot, strategy, dialect); const schema = resolveTableSchema(entity); if (schema !== undefined) t.schema = schema; + provenance.set(qualifiedDbName(t), entity.resolutionKey()); return t; }); @@ -288,13 +339,17 @@ export function buildExpectedSchema( // whether a view change can use a non-destructive CREATE OR REPLACE. const views: ViewDescriptor[] = (opts?.views ?? []).map((v) => { const columns = resolveViewColumns(v.columns, tables); - return { + const descriptor: ViewDescriptor = { name: v.name, ...(v.schema !== undefined ? { schema: v.schema } : {}), ...(v.sql !== undefined ? { sql: v.sql, fingerprint: viewFingerprint(v.sql) } : {}), ...(v.dependsOn !== undefined ? { dependsOn: v.dependsOn } : {}), ...(columns !== undefined ? { columns } : {}), }; + // The declaring FQN goes to the provenance map ONLY — never onto the descriptor, + // which is what the committed snapshot serializes (see ExpectedViewInput.fqn). + if (v.fqn !== undefined) provenance.set(qualifiedDbName(descriptor), v.fqn); + return descriptor; }); // Collision guard: two DISTINCT metadata objects that resolve to the same generated @@ -324,7 +379,7 @@ export function buildExpectedSchema( ); } - return { tables, views }; + return { snapshot: { tables, views }, provenance }; } /** diff --git a/server/typescript/packages/migrate-ts/src/index.ts b/server/typescript/packages/migrate-ts/src/index.ts index d09c47d6f..c1770220b 100644 --- a/server/typescript/packages/migrate-ts/src/index.ts +++ b/server/typescript/packages/migrate-ts/src/index.ts @@ -8,11 +8,17 @@ // See docs/specs/2026-05-11-v0.2-sp4-migrate-ts-design.md. // Pipeline functions -export { buildExpectedSchema } from "./expected-schema.js"; +export { buildExpectedSchema, buildExpectedSchemaWithProvenance } from "./expected-schema.js"; +export type { ExpectedSchemaWithProvenance, SchemaProvenance } from "./expected-schema.js"; export { introspect, introspectPostgres, introspectSqlite } from "./introspect/index.js"; export { diff } from "./diff/index.js"; export { collectUnmanagedNames } from "./unmanaged.js"; -export { computeDrift, computeDriftFromActual, type ComputeDriftOptions } from "./drift/drift.js"; +// Per-command scope (`migrate.scope`) — see scope.ts for why the suppression is +// two-sided and why the pattern engine stays in @metaobjectsdev/sdk. +export { scopeExpectedSchema } from "./scope.js"; +export type { ObjectScopePredicate, ScopedExpectedSchema } from "./scope.js"; +export { qualifiedDbName } from "./qualified-name.js"; +export { computeDrift, computeDriftFromActual, type ComputeDriftOptions, type DriftResult } from "./drift/drift.js"; export { classifyDrift, driftAgainstSnapshot } from "./drift/classify.js"; export type { DriftClassification } from "./drift/classify.js"; export { emit } from "./emit/index.js"; diff --git a/server/typescript/packages/migrate-ts/src/qualified-name.ts b/server/typescript/packages/migrate-ts/src/qualified-name.ts new file mode 100644 index 000000000..b702b5742 --- /dev/null +++ b/server/typescript/packages/migrate-ts/src/qualified-name.ts @@ -0,0 +1,22 @@ +// The ONE qualified-physical-name form: `.`, with an absent schema +// normalized to the Postgres default. +// +// Three things must key DB objects identically or the diff silently disagrees with +// itself: `diff`'s table/view identity maps, the declared-`@unmanaged` exclusion set +// (`collectUnmanagedNames`), and the out-of-scope exclusion set (`scopeExpectedSchema`). +// The last two are ACT-side suppressions matched against the first, so a name built a +// second way — a different default schema, a different separator — reads as "not +// suppressed" and the object it names comes back as a proposed DROP. One function. +// +// SQLite has no schema concept, so every SQLite object normalizes to the same prefix. +// That is harmless: it is a constant, and the un-prefixed names were already unique. + +import { DEFAULT_DB_SCHEMA_POSTGRES } from "@metaobjectsdev/metadata"; + +/** `.`; an absent schema is the Postgres default (`public`). The + * parameter accepts an EXPLICIT `undefined` schema (not only an omitted key) so a + * caller holding a `string | undefined` can pass it straight through under + * `exactOptionalPropertyTypes` — the two spell the same thing here. */ +export function qualifiedDbName(obj: { name: string; schema?: string | undefined }): string { + return `${obj.schema ?? DEFAULT_DB_SCHEMA_POSTGRES}.${obj.name}`; +} diff --git a/server/typescript/packages/migrate-ts/src/scope.ts b/server/typescript/packages/migrate-ts/src/scope.ts new file mode 100644 index 000000000..797911e30 --- /dev/null +++ b/server/typescript/packages/migrate-ts/src/scope.ts @@ -0,0 +1,76 @@ +// Per-command scope — narrowing a migrate/verify run to the objects it governs. +// +// A consumer sharing a database with another owner declares +// `"migrate": { "scope": ["acme::platform::**"] }`. Tables and views outside that +// scope are neither created nor dropped, which takes TWO suppressions: +// +// 1. drop them from the EXPECTED side, so nothing is created or altered; +// 2. suppress the same names on the ACTUAL side (via `diff`'s `unmanagedNames`, +// the seam `@unmanaged` already uses), so nothing is dropped. +// +// Doing only (1) is strictly worse than doing nothing: every out-of-scope table that +// EXISTS in the database becomes a proposed `DROP TABLE` — the precise hazard this +// feature exists to remove. `scopeExpectedSchema` therefore returns both halves and +// callers must thread `outOfScope` into the diff. + +import type { ExpectedSchemaWithProvenance } from "./expected-schema.js"; +import { qualifiedDbName } from "./qualified-name.js"; +import type { SchemaSnapshot } from "./types.js"; + +/** + * Decides whether an object's fully-qualified name (`resolutionKey()`) is governed + * by this run. Supplied by the caller as a PREDICATE so migrate-ts never carries a + * second implementation of the scope-pattern grammar — `matchesScope` in + * `@metaobjectsdev/sdk` is the only one, and the CLI adapts a compiled scope to + * this seam. + */ +export type ObjectScopePredicate = (fqn: string) => boolean; + +export interface ScopedExpectedSchema { + /** The expected schema narrowed to the governed objects. */ + snapshot: SchemaSnapshot; + /** + * Qualified physical names (`.`) of the tables and views removed + * above. MUST be threaded into `diff`'s `unmanagedNames` (merged with + * `collectUnmanagedNames`, never replacing it) so the actual side is suppressed + * too — see the module header. + */ + outOfScope: string[]; +} + +/** + * Narrow an expected schema to the objects inside `inScope`. + * + * An undefined predicate returns the input untouched — the SAME snapshot object, + * not an equal copy — so a project that declares no `migrate.scope` reaches the + * diff, the emitter and the committed snapshot through an unchanged value. + * + * A table or view with NO recorded provenance is KEPT. Scope decides on the + * declaring object's FQN, and an object whose FQN is unknown was never proven to be + * anyone else's; dropping it would silently un-manage it (and, worse, suppressing + * its name on the actual side would hide real drift). + */ +export function scopeExpectedSchema( + built: ExpectedSchemaWithProvenance, + inScope: ObjectScopePredicate | undefined, +): ScopedExpectedSchema { + if (inScope === undefined) return { snapshot: built.snapshot, outOfScope: [] }; + + const outOfScope: string[] = []; + const governed = (obj: T): boolean => { + const qualified = qualifiedDbName(obj); + const fqn = built.provenance.get(qualified); + if (fqn === undefined || inScope(fqn)) return true; + outOfScope.push(qualified); + return false; + }; + + return { + snapshot: { + ...built.snapshot, + tables: built.snapshot.tables.filter(governed), + views: built.snapshot.views.filter(governed), + }, + outOfScope, + }; +} diff --git a/server/typescript/packages/migrate-ts/src/snapshot/plan.ts b/server/typescript/packages/migrate-ts/src/snapshot/plan.ts index f681d52e7..f05a9f67d 100644 --- a/server/typescript/packages/migrate-ts/src/snapshot/plan.ts +++ b/server/typescript/packages/migrate-ts/src/snapshot/plan.ts @@ -1,8 +1,9 @@ // src/snapshot/plan.ts import type { ColumnNamingStrategy, MetaData } from "@metaobjectsdev/metadata"; -import { buildExpectedSchema } from "../expected-schema.js"; +import { buildExpectedSchema, buildExpectedSchemaWithProvenance } from "../expected-schema.js"; import { diff, type DiffArgs } from "../diff/index.js"; import { collectUnmanagedNames } from "../unmanaged.js"; +import { scopeExpectedSchema, type ObjectScopePredicate } from "../scope.js"; import type { Dialect, DiffResult, SchemaSnapshot } from "../types.js"; import type { ExpectedViewInput } from "../expected-schema.js"; @@ -14,6 +15,13 @@ export interface PlanOfflineArgs extends Pick { - const nextSnapshot = buildExpectedSchema(args.metadata, { - dialect: args.dialect, - ...(args.columnNamingStrategy ? { columnNamingStrategy: args.columnNamingStrategy } : {}), - ...(args.views !== undefined ? { views: args.views } : {}), - }); + const scoped = scopeExpectedSchema( + buildExpectedSchemaWithProvenance(args.metadata, { + dialect: args.dialect, + ...(args.columnNamingStrategy ? { columnNamingStrategy: args.columnNamingStrategy } : {}), + ...(args.views !== undefined ? { views: args.views } : {}), + }), + args.inScope, + ); + const nextSnapshot = scoped.snapshot; const result = await diff({ expected: nextSnapshot, actual: args.snapshot, @@ -45,12 +59,14 @@ export async function planOffline(args: PlanOfflineArgs): Promise { + return (await new MetaDataLoader().load([new InMemoryStringSource(META)])).root; +} + +const platformOnly = (fqn: string): boolean => fqn.startsWith("acme::platform::"); + +describe("computeDriftFromActual — migrate.scope", () => { + test("an out-of-scope table present in the DB is NOT drift", async () => { + const root = await load(); + // The live database holds both tables; only `jobs` is this consumer's. + const actual = buildExpectedSchema(root, { dialect: "sqlite" }); + + const result = await computeDriftFromActual(actual, "sqlite", root, { inScope: platformOnly }); + expect(result.changes).toEqual([]); + expect(result.outOfScope).toEqual(["public.matches"]); + }); + + test("in-scope drift is still reported", async () => { + const root = await load(); + const actual = buildExpectedSchema(root, { dialect: "sqlite" }); + actual.tables = actual.tables.filter((t) => t.name !== "jobs"); + + const result = await computeDriftFromActual(actual, "sqlite", root, { inScope: platformOnly }); + expect(result.changes.map((c) => c.kind)).toContain("create-table"); + }); + + test("no scope → unchanged: every table is compared, nothing is out of scope", async () => { + const root = await load(); + const actual = buildExpectedSchema(root, { dialect: "sqlite" }); + actual.tables = actual.tables.filter((t) => t.name !== "matches"); + + const result = await computeDriftFromActual(actual, "sqlite", root); + expect(result.changes.map((c) => c.kind)).toEqual(["create-table"]); + expect(result.outOfScope).toEqual([]); + }); +}); diff --git a/server/typescript/packages/migrate-ts/test/expected-schema-scope.test.ts b/server/typescript/packages/migrate-ts/test/expected-schema-scope.test.ts new file mode 100644 index 000000000..88a359bf2 --- /dev/null +++ b/server/typescript/packages/migrate-ts/test/expected-schema-scope.test.ts @@ -0,0 +1,181 @@ +/** + * Per-command scope (`migrate.scope`) — the expected-schema half. + * + * A consumer that owns one package tree's tables in a database another owner + * also writes to declares `migrate": { "scope": [...] }`. Tables outside that + * scope are neither created nor dropped, which takes TWO suppressions, not one: + * dropping them from the EXPECTED side alone would turn every out-of-scope + * table that exists in the database into a proposed DROP TABLE — the exact + * hazard the feature exists to remove. + * + * The scope decision is made on the DECLARING OBJECT's fully-qualified name, + * threaded out of the same Pass 1 walk that builds the tables (never re-derived + * from a SQL name, which is lossy, and never a second walk, which would drift + * from Pass 1's skip rules). + */ +import { describe, test, expect } from "bun:test"; +import { MetaDataLoader, InMemoryStringSource, type MetaRoot } from "@metaobjectsdev/metadata"; +import { buildExpectedSchema, buildExpectedSchemaWithProvenance } from "../src/expected-schema.js"; +import { scopeExpectedSchema } from "../src/scope.js"; +import { diff } from "../src/diff/index.js"; +import { serializeSnapshot, SNAPSHOT_FORMAT_VERSION } from "../src/snapshot/serialize.js"; +import type { SchemaSnapshot } from "../src/types.js"; + +const PLATFORM = JSON.stringify({ + "metadata.root": { + package: "acme::platform", + children: [ + { + "object.entity": { + name: "Job", + children: [ + { "source.rdb": { name: "src", "@table": "jobs" } }, + { "field.long": { name: "id" } }, + { "field.string": { name: "title" } }, + { "identity.primary": { name: "pk", "@fields": ["id"], "@generation": "increment" } }, + ], + }, + }, + ], + }, +}); + +const ARENA = JSON.stringify({ + "metadata.root": { + package: "arena", + children: [ + { + "object.entity": { + name: "Match", + children: [ + { "source.rdb": { name: "src", "@table": "matches" } }, + { "field.long": { name: "id" } }, + { "identity.primary": { name: "pk", "@fields": ["id"], "@generation": "increment" } }, + ], + }, + }, + ], + }, +}); + +async function loadBoth(): Promise { + const loaded = await new MetaDataLoader().load([ + new InMemoryStringSource(PLATFORM), + new InMemoryStringSource(ARENA), + ]); + return loaded.root; +} + +/** The `migrate.scope: ["acme::platform::**"]` decision, without importing the + * pattern engine — migrate-ts takes a predicate precisely so it never carries a + * second implementation of one (`matchesScope` in the sdk is the only one). */ +const platformOnly = (fqn: string): boolean => fqn.startsWith("acme::platform::"); + +describe("scopeExpectedSchema", () => { + test("drops tables whose declaring object falls outside the scope", async () => { + const root = await loadBoth(); + const built = buildExpectedSchemaWithProvenance(root, { dialect: "sqlite" }); + expect(built.snapshot.tables.map((t) => t.name).sort()).toEqual(["jobs", "matches"]); + + const scoped = scopeExpectedSchema(built, platformOnly); + expect(scoped.snapshot.tables.map((t) => t.name)).toEqual(["jobs"]); + }); + + test("names the dropped tables so the ACTUAL side can be suppressed too", async () => { + const root = await loadBoth(); + const scoped = scopeExpectedSchema( + buildExpectedSchemaWithProvenance(root, { dialect: "sqlite" }), + platformOnly, + ); + // Qualified exactly as diff keys its tables (`.`, schema + // defaulting to the Postgres default), so the names feed `unmanagedNames`. + expect(scoped.outOfScope).toEqual(["public.matches"]); + }); + + test("an undefined scope leaves the expected schema untouched", async () => { + const root = await loadBoth(); + const built = buildExpectedSchemaWithProvenance(root, { dialect: "sqlite" }); + const scoped = scopeExpectedSchema(built, undefined); + // Same object, not merely an equal one: an unscoped project must reach the + // diff through byte-identical input. + expect(scoped.snapshot).toBe(built.snapshot); + expect(scoped.outOfScope).toEqual([]); + }); + + test("both sides: an out-of-scope table present in `actual` produces NO drop-table", async () => { + const root = await loadBoth(); + const scoped = scopeExpectedSchema( + buildExpectedSchemaWithProvenance(root, { dialect: "sqlite" }), + platformOnly, + ); + // The database holds BOTH tables — `matches` is another owner's, created by + // another tool. It is absent from the scoped expected side. + const actual: SchemaSnapshot = buildExpectedSchema(root, { dialect: "sqlite" }); + + const result = await diff({ + expected: scoped.snapshot, + actual, + dialect: "sqlite", + unmanagedNames: scoped.outOfScope, + }); + expect(result.changes).toEqual([]); + }); + + test("without the actual-side suppression the same diff WOULD drop it (the hazard is real)", async () => { + const root = await loadBoth(); + const scoped = scopeExpectedSchema( + buildExpectedSchemaWithProvenance(root, { dialect: "sqlite" }), + platformOnly, + ); + const actual: SchemaSnapshot = buildExpectedSchema(root, { dialect: "sqlite" }); + + const result = await diff({ expected: scoped.snapshot, actual, dialect: "sqlite" }); + expect(result.changes.map((c) => c.kind)).toEqual(["drop-table"]); + }); + + test("a table whose provenance is unknown is kept — scope never guesses", async () => { + const root = await loadBoth(); + const built = buildExpectedSchemaWithProvenance(root, { dialect: "sqlite" }); + const withStranger: typeof built = { + snapshot: { + ...built.snapshot, + tables: [ + ...built.snapshot.tables, + { name: "stranger", columns: [], indexes: [], foreignKeys: [], checks: [], primaryKey: [] }, + ], + }, + provenance: built.provenance, + }; + const scoped = scopeExpectedSchema(withStranger, platformOnly); + expect(scoped.snapshot.tables.map((t) => t.name)).toEqual(["jobs", "stranger"]); + expect(scoped.outOfScope).toEqual(["public.matches"]); + }); +}); + +describe("provenance never reaches the committed snapshot", () => { + test("a view's declaring FQN is recorded but never serialized; formatVersion stays 3", async () => { + const root = await loadBoth(); + const view = { name: "v_jobs", sql: "SELECT id FROM jobs", dependsOn: ["jobs"] }; + + const withFqn = buildExpectedSchemaWithProvenance(root, { + dialect: "sqlite", + views: [{ ...view, fqn: "acme::platform::JobSummary" }], + }); + const withoutFqn = buildExpectedSchema(root, { dialect: "sqlite", views: [view] }); + + // The provenance thread is invisible on disk: carrying it changes no bytes. + expect(serializeSnapshot(withFqn.snapshot)).toBe(serializeSnapshot(withoutFqn)); + expect(serializeSnapshot(withFqn.snapshot)).not.toContain("fqn"); + expect(SNAPSHOT_FORMAT_VERSION).toBe(3); + + // ...and it IS recorded, keyed like every other qualified name. + expect(withFqn.provenance.get("public.v_jobs")).toBe("acme::platform::JobSummary"); + }); + + test("a table's provenance is its declaring entity's resolutionKey()", async () => { + const root = await loadBoth(); + const { provenance } = buildExpectedSchemaWithProvenance(root, { dialect: "sqlite" }); + expect(provenance.get("public.jobs")).toBe("acme::platform::Job"); + expect(provenance.get("public.matches")).toBe("arena::Match"); + }); +}); From dfb31c521536869ab6972f723ebe074d42836bda Mon Sep 17 00:00:00 2001 From: Doug Mealing Date: Tue, 18 Aug 2026 06:48:07 -0400 Subject: [PATCH 22/44] feat(codegen): collection scope filters generated output, and verify --codegen honours it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `runGen` gains an optional `scope?: (fqn: string) => boolean` predicate applied at its single entity-selection choke point, intersecting with `entityFilter` and matched against `obj.resolutionKey()` — a plain predicate, not the sdk-owned pattern strings, since codegen-ts must not depend on @metaobjectsdev/sdk. `meta gen` now always passes `(fqn) => matchesScope(fqn, collection.scope)`; an unconfigured project's compiled scope has an empty include/exclude, which `matchesScope` treats as "everything", so this is a no-op for the common case. `verify --codegen` threads the identical predicate into `computeCodegenDrift`'s regeneration pass (design §7 open question 3) — otherwise a scoped `meta gen` and an unscoped regen disagree about which files should exist, and every out-of-scope entity reads as false drift. `runGen`'s "No entities to generate" warning now distinguishes a scope that admitted nothing from an empty root or an unmatched entityFilter, rather than misattributing the cause. The scope===undefined path is byte-identical to the pre-scope two-way branch, proven by the golden-output gate (unmoved, 1241 pre-existing assertions untouched) plus a new content-comparison test. Dangling references (an in-scope object referencing an out-of-scope FK target / @objectRef / projection base) are documented, not warned on: detecting them correctly needs a general reference-walker across every reference kind, which is new machinery this task's seam doesn't fit: the closest existing map (relation-resolver's targetEntity) is bare-name only (the #228 limitation), and the actual failure mode is a loud compiler error on the generated code's unresolved import, not a silent one. codegen-ts: 1245 pass / 0 fail (1241 baseline + 4 new, golden gate unmoved). cli: 571 pass / 3 skip / 0 fail (568 baseline + 3 new). Workspace typecheck: 18/18 exit 0 — caught a real implicit-any in verify.ts's pre-existing `let collection;` once it was read from a nested function, fixed with an explicit type annotation. Co-Authored-By: Claude Opus 5 (1M context) --- .../packages/cli/src/commands/gen.ts | 8 +- .../packages/cli/src/commands/verify.ts | 18 ++- .../packages/cli/src/lib/codegen-drift.ts | 9 ++ .../cli/test/integration/gen-scope.test.ts | 86 +++++++++++ .../integration/verify-codegen-scope.test.ts | 91 +++++++++++ .../packages/codegen-ts/src/runner.ts | 66 +++++++- .../packages/codegen-ts/test/run-gen.test.ts | 141 ++++++++++++++++++ 7 files changed, 409 insertions(+), 10 deletions(-) create mode 100644 server/typescript/packages/cli/test/integration/gen-scope.test.ts create mode 100644 server/typescript/packages/cli/test/integration/verify-codegen-scope.test.ts diff --git a/server/typescript/packages/cli/src/commands/gen.ts b/server/typescript/packages/cli/src/commands/gen.ts index 2b3ea5517..592a19556 100644 --- a/server/typescript/packages/cli/src/commands/gen.ts +++ b/server/typescript/packages/cli/src/commands/gen.ts @@ -8,7 +8,7 @@ import type { OutputFormat } from "../lib/format.js"; import { log } from "../lib/log.js"; import { warnIfAgentContextStale } from "../lib/agent-context-staleness.js"; import { scanSourceForAntiPatterns } from "../lib/anti-patterns.js"; -import { loadMemory, resolveCollection } from "@metaobjectsdev/sdk"; +import { loadMemory, resolveCollection, matchesScope } from "@metaobjectsdev/sdk"; import { runGen, listGenerators } from "@metaobjectsdev/codegen-ts"; import type { WriteStatus } from "@metaobjectsdev/codegen-ts"; @@ -84,6 +84,12 @@ export async function genCommand(args: string[], cwd: string, fmt: OutputFormat // --dry-run must actually preview. This was previously passed only to the // display object below, so a "preview" run wrote every file. dryRun: cliConfig.dryRun, + // Collection-level `scope` (Task 12b) — the output filter over + // GENERATED entities, never over what the collection loads. Always + // passed: an unconfigured project's `collection.scope` compiles to an + // empty include/exclude, and `matchesScope` treats that as "everything" + // — so this is a no-op for the common case, not a behavior change. + scope: (fqn) => matchesScope(fqn, collection.scope), ...(cliConfig.entities.length > 0 ? { entityFilter: cliConfig.entities } : {}), }); } catch (err) { diff --git a/server/typescript/packages/cli/src/commands/verify.ts b/server/typescript/packages/cli/src/commands/verify.ts index 338316463..14f745214 100644 --- a/server/typescript/packages/cli/src/commands/verify.ts +++ b/server/typescript/packages/cli/src/commands/verify.ts @@ -48,7 +48,7 @@ import { type D1Runner, type DriftResult, } from "@metaobjectsdev/migrate-ts"; -import { loadMemory, resolveCollection } from "@metaobjectsdev/sdk"; +import { loadMemory, resolveCollection, matchesScope } from "@metaobjectsdev/sdk"; import { toObjectScope } from "../lib/migrate-scope.js"; import { TYPE_TEMPLATE, @@ -144,8 +144,14 @@ export async function verifyCommand( // Where the metadata lives is `resolveCollection`'s decision, not a hardcoded // directory. It also carries the per-command `migrate.scope` the schema gate below - // honours — `verify --db` and `migrate` govern the identical object set. - let collection; + // honours — `verify --db` and `migrate` govern the identical object set — and the + // top-level `scope` `runCodegenVerify` (a nested function below) threads into + // `computeCodegenDrift`. Explicitly typed (unlike the `let collection;` pattern + // elsewhere in this codebase): a nested function body is OUTSIDE the control-flow + // narrowing TS performs on a same-scope `let x;` reassignment, so a bare + // `let collection;` type-checked clean until this task added exactly that nested + // reference — the reader who removes the annotation next reintroduces TS7034. + let collection: Awaited>; try { collection = await resolveCollection(cwd); } catch (err) { @@ -676,9 +682,13 @@ export async function verifyCommand( return 2; } + // The identical predicate `meta gen` applies (Task 12b / design §7 open + // question 3) — a `gen` that committed under a narrowed scope and a + // `verify --codegen` that regenerates unscoped would disagree about which + // files should exist, reporting every out-of-scope entity as drift. let result; try { - result = await computeCodegenDrift(forgeConfig, root, cwd); + result = await computeCodegenDrift(forgeConfig, root, cwd, (fqn) => matchesScope(fqn, collection.scope)); } catch (err) { log.error(`verify --codegen: regeneration failed: ${(err as Error).message}`); return 1; diff --git a/server/typescript/packages/cli/src/lib/codegen-drift.ts b/server/typescript/packages/cli/src/lib/codegen-drift.ts index 7ce47bb58..70d921fa6 100644 --- a/server/typescript/packages/cli/src/lib/codegen-drift.ts +++ b/server/typescript/packages/cli/src/lib/codegen-drift.ts @@ -81,11 +81,19 @@ function listFiles(dir: string): string[] { * @param config the loaded metaobjects config (provides outDir/targets). * @param metadata the loaded MetaRoot (same object `meta gen` would use). * @param projectRoot absolute project root (committed outDirs are keyed off it). + * @param scope the SAME output-scope predicate `meta gen` used to produce the + * committed output (Task 12b / design §7 open question 3). A `verify --codegen` + * that regenerates unscoped while the committed output was scoped would read + * every out-of-scope entity as drift — regen would try to emit it, but it was + * never committed because the `meta gen` that produced the committed tree + * never emitted it either. Undefined ⇒ everything is in scope (byte-identical + * to a project with no `scope` declared). */ export async function computeCodegenDrift( config: MetaobjectsGenConfig, metadata: MetaData, projectRoot: string, + scope?: (fqn: string) => boolean, ): Promise { const root = isAbsolute(projectRoot) ? projectRoot : resolve(projectRoot); @@ -157,6 +165,7 @@ export async function computeCodegenDrift( genStateDir: join(tempRoot, ".gen-state"), mergeStrategy: "overwrite", baseline: "fresh", + ...(scope !== undefined ? { scope } : {}), }); // Diff each committed outDir against its temp mirror. diff --git a/server/typescript/packages/cli/test/integration/gen-scope.test.ts b/server/typescript/packages/cli/test/integration/gen-scope.test.ts new file mode 100644 index 000000000..bdf10406e --- /dev/null +++ b/server/typescript/packages/cli/test/integration/gen-scope.test.ts @@ -0,0 +1,86 @@ +/** + * Task 12b: `meta gen` honours the collection-level `scope` declared in + * `.metaobjects/config.json`, filtering GENERATED output — never input. The + * collection still loads the whole model; only the emitted file set narrows. + */ +import { describe, test, expect } from "bun:test"; +import { cpSync, mkdtempSync, mkdirSync, rmSync, readdirSync, writeFileSync } from "node:fs"; +import { join, resolve } from "node:path"; +import { run } from "../../src/index.js"; + +const FIXTURES = resolve(import.meta.dirname, "../fixtures"); +// Place temp dirs inside the monorepo so jiti can resolve @metaobjectsdev/* +// when it loads metaobjects.config.ts (same rationale as gen-sqlite.test.ts). +const WORKSPACE_TMP = resolve(import.meta.dirname, "../fixtures/__tmp__"); + +function genOutDir(root: string): string { + return join(root, "generated", "db"); +} + +/** trainer-website-meta declares User/Post/Tag, all in package "trainerWebsite". */ +function setupRepo(): string { + mkdirSync(WORKSPACE_TMP, { recursive: true }); + const root = mkdtempSync(join(WORKSPACE_TMP, "forge-gen-scope-")); + cpSync(join(FIXTURES, "trainer-website-meta"), root, { recursive: true }); + writeFileSync( + join(root, "metaobjects.config.ts"), + ` +import { defineConfig } from "@metaobjectsdev/codegen-ts"; +import { entityFile } from "@metaobjectsdev/codegen-ts/generators"; +export default defineConfig({ + outDir: ${JSON.stringify(genOutDir(root))}, + dialect: "sqlite", + dbImport: "~/db", + extStyle: "none", + generators: [entityFile()], +}); +`, + ); + return root; +} + +function declareScope(repo: string, include: string[]): void { + mkdirSync(join(repo, ".metaobjects"), { recursive: true }); + writeFileSync( + join(repo, ".metaobjects", "config.json"), + JSON.stringify({ schema_version: 1, scope: { include } }), + "utf8", + ); +} + +describe("meta gen — collection scope", () => { + test("a declared scope emits only the in-scope entity's files", async () => { + const root = setupRepo(); + try { + declareScope(root, ["trainerWebsite::Post"]); + + const exit = await run(["gen", "--cwd", root]); + expect(exit).toBe(0); + + const outDir = genOutDir(root); + const files = readdirSync(outDir); + expect(files).toContain("Post.ts"); + expect(files).not.toContain("User.ts"); + expect(files).not.toContain("Tag.ts"); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + test("no declared scope emits every entity (byte-identical to today)", async () => { + const root = setupRepo(); + try { + // No .metaobjects/config.json at all — the default, unscoped path. + const exit = await run(["gen", "--cwd", root]); + expect(exit).toBe(0); + + const outDir = genOutDir(root); + const files = readdirSync(outDir); + expect(files).toContain("Post.ts"); + expect(files).toContain("User.ts"); + expect(files).toContain("Tag.ts"); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +}); diff --git a/server/typescript/packages/cli/test/integration/verify-codegen-scope.test.ts b/server/typescript/packages/cli/test/integration/verify-codegen-scope.test.ts new file mode 100644 index 000000000..fba8ecb76 --- /dev/null +++ b/server/typescript/packages/cli/test/integration/verify-codegen-scope.test.ts @@ -0,0 +1,91 @@ +/** + * Task 12b, requirement 3 / design §7 open question 3: `verify --codegen` must + * regenerate under the SAME `collection.scope` `meta gen` used to produce the + * committed output — otherwise every out-of-scope entity reads as drift (regen + * would try to emit it; it was never committed because `meta gen` never emitted + * it either). + */ +import { describe, test, expect, beforeEach, afterEach } from "bun:test"; +import { cpSync, mkdtempSync, mkdirSync, rmSync, writeFileSync } from "node:fs"; +import { join, resolve } from "node:path"; +import { run } from "../../src/index.js"; + +const FIXTURES = resolve(import.meta.dirname, "../fixtures"); +const WORKSPACE_TMP = resolve(import.meta.dirname, "../fixtures/__tmp__"); + +function genOutDir(root: string): string { + return join(root, "generated", "db"); +} + +/** trainer-website-meta declares User/Post/Tag, all in package "trainerWebsite". */ +function setupRepo(): string { + mkdirSync(WORKSPACE_TMP, { recursive: true }); + const root = mkdtempSync(join(WORKSPACE_TMP, "forge-verify-codegen-scope-")); + cpSync(join(FIXTURES, "trainer-website-meta"), root, { recursive: true }); + writeFileSync( + join(root, "metaobjects.config.ts"), + ` +import { defineConfig } from "@metaobjectsdev/codegen-ts"; +import { entityFile } from "@metaobjectsdev/codegen-ts/generators"; +export default defineConfig({ + outDir: ${JSON.stringify(genOutDir(root))}, + dialect: "sqlite", + dbImport: "~/db", + extStyle: "none", + generators: [entityFile()], +}); +`, + ); + return root; +} + +function declareScope(repo: string, include: string[]): void { + mkdirSync(join(repo, ".metaobjects"), { recursive: true }); + writeFileSync( + join(repo, ".metaobjects", "config.json"), + JSON.stringify({ schema_version: 1, scope: { include } }), + "utf8", + ); +} + +let out: string[]; +let err: string[]; +let origLog: typeof console.log; +let origErr: typeof console.error; + +beforeEach(() => { + out = []; + err = []; + origLog = console.log; + origErr = console.error; + console.log = (...a: unknown[]) => { out.push(a.map(String).join(" ")); }; + console.error = (...a: unknown[]) => { err.push(a.map(String).join(" ")); }; +}); +afterEach(() => { + console.log = origLog; + console.error = origErr; +}); + +describe("meta verify --codegen — collection scope", () => { + test("reports no drift for out-of-scope entities the scoped gen never committed", async () => { + const root = setupRepo(); + try { + declareScope(root, ["trainerWebsite::Post"]); + + // `meta gen` under the scope commits ONLY Post.ts. + expect(await run(["gen", "--cwd", root])).toBe(0); + + // `verify --codegen` must regenerate under the identical scope — if it + // regenerated unscoped, User.ts/Tag.ts would appear in the fresh tree + // but not the committed one, reading as drift on entities this scope + // deliberately excludes. + const exit = await run(["verify", "--cwd", root, "--codegen"]); + const all = [...out, ...err].join("\n"); + expect(exit).toBe(0); + expect(all).not.toContain("User.ts"); + expect(all).not.toContain("Tag.ts"); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +}); diff --git a/server/typescript/packages/codegen-ts/src/runner.ts b/server/typescript/packages/codegen-ts/src/runner.ts index 73e645f8c..bc7ad62f5 100644 --- a/server/typescript/packages/codegen-ts/src/runner.ts +++ b/server/typescript/packages/codegen-ts/src/runner.ts @@ -60,6 +60,41 @@ export interface RunGenOpts { * `--dry-run`, and watching it reappear. */ dryRun?: boolean; + /** + * Output scope — an object is generated only when this predicate returns true + * for its fully-qualified name (`obj.resolutionKey()`, `::`). + * Intersects with `entityFilter`: both must pass. Absent ⇒ every object is + * in scope (byte-identical to a project with no `scope` declared). + * + * The collection metadata always loads in FULL regardless of this predicate — + * scope filters OUTPUT, never input (design §4.3). So an in-scope object may + * reference an out-of-scope one (an FK target, a relationship `@objectRef`, a + * projection's base) and resolve perfectly at load time, while the code + * emitted FOR the in-scope object still imports/names a symbol that was never + * generated. This is left silent by design, not auto-widened: the adopter + * declared the scope precisely because something else (another consumer, + * another codegen run) owns those objects, and the reference is real. Warning + * on it correctly would require walking every reference kind (identity.reference, + * every relationship.* @objectRef, projection extends bases, field.object + * @objectRef) FQN-resolved against the SAME scope — genuinely new machinery, + * not a fit for the existing `warnings: string[]` channel at this seam. If an + * adopter hits it, the failure is a plain compiler error in the generated + * code (an unresolved import) — loud, at build time, not silent at runtime. + * + * Deliberately a PLAIN PREDICATE, not the `include`/`exclude` pattern strings + * `@metaobjectsdev/sdk`'s `scope.ts` compiles. `codegen-ts` must not depend on + * `@metaobjectsdev/sdk` — the dependency runs the other way (`cli` depends on + * both) — so it cannot import `matchesScope`/`CompiledScope` itself. The + * design's "package patterns, never a predicate function" rule (§4.3 of the + * metadata-source-resolution design doc) governs CONFIG SURFACES that must + * port identically to a `pom.xml` / `metaobjects.config.yaml` in every + * language port; it says nothing about internal plumbing between two + * TypeScript packages in this one repo. Do not "fix" this into a config + * shape — `cli`'s `gen`/`verify` commands are the only callers, and they + * already hold a compiled `CompiledScope` from `resolveCollection()` and + * adapt it to this predicate with `(fqn) => matchesScope(fqn, collection.scope)`. + */ + scope?: (fqn: string) => boolean; } export interface RunGenResult { @@ -136,16 +171,37 @@ export async function runGen(opts: RunGenOpts): Promise { } const root = opts.metadata; - // 1. Resolve entities (filter + safety check). + // 1. Resolve entities (entityFilter + scope + safety check). This is the + // single choke point for entity selection — scope INTERSECTS entityFilter + // (an object must pass both), matched against the object's + // fully-qualified name (resolutionKey(), never the bare name — two + // packages may declare the same short name). const allObjects = root.objects(); const entityFilter = opts.entityFilter; - const filtered = entityFilter + const afterEntityFilter = entityFilter ? allObjects.filter((o) => entityFilter.includes(o.name)) : allObjects; + const scope = opts.scope; + const filtered = scope + ? afterEntityFilter.filter((o) => scope(o.resolutionKey())) + : afterEntityFilter; if (filtered.length === 0) { - const reason = opts.entityFilter - ? "no object children match the provided entityFilter" - : "root has no object children"; + // Name the REAL cause. When `scope` is absent, this is byte-identical to + // the pre-scope two-way branch (kept as its own arm, rather than folded + // into the scope-aware logic below, so an unscoped project's warning text + // — including its quirky edge case: an empty root with entityFilter set + // still blames entityFilter — is untouched). Only when `scope` is + // present does a THIRD reason become reachable: "root has no object + // children" for a scoped-out model, or "...entityFilter" for a scope + // that admitted everything entityFilter then excluded, are both false + // statements that send the reader to the wrong file. + const reason = scope === undefined + ? (entityFilter ? "no object children match the provided entityFilter" : "root has no object children") + : (allObjects.length === 0 + ? "root has no object children" + : afterEntityFilter.length === 0 + ? "no object children match the provided entityFilter" + : "no object children match the configured scope"); warnings.push(`No entities to generate — ${reason}.`); return { files: [], warnings, conflicts: [] }; } diff --git a/server/typescript/packages/codegen-ts/test/run-gen.test.ts b/server/typescript/packages/codegen-ts/test/run-gen.test.ts index 3116d7b1a..6dad7c008 100644 --- a/server/typescript/packages/codegen-ts/test/run-gen.test.ts +++ b/server/typescript/packages/codegen-ts/test/run-gen.test.ts @@ -333,3 +333,144 @@ describe("runGen — entityFilter", () => { expect(existsSync(join(tmp, "index.ts"))).toBe(false); }); }); + +// --------------------------------------------------------------------------- +// Task 12b: collection-level `scope` filters generated output. +// --------------------------------------------------------------------------- +describe("runGen — scope", () => { + test("a scope predicate emits artifacts for in-scope entities only (file list + contents)", async () => { + const loader = new MetaDataLoader(); + const result = await loader.load([new FileSource(join(FIXTURE_DIR, "two-entities-fk.json"))]); + expect(result.errors).toEqual([]); + + const out = await runGen({ + config: defineConfig({ + outDir: tmp, + extStyle: "none", + dbImport: "~/server/db", + dialect: "postgres", + generators: [entityFile(), queriesFile(), barrel()], + }), + metadata: result.root, + // "demo::Post" only — User is in the same package but declares an + // `identity.reference`/`relationship.association` TO Post's declared + // scope, so this fixture doubles as evidence the predicate is matched + // against the FQN (`demo::Post`), never the bare "Post". + scope: (fqn) => fqn === "demo::Post", + }); + + expect(out.warnings).toEqual([]); + expect(existsSync(join(tmp, "Post.ts"))).toBe(true); + expect(existsSync(join(tmp, "Post.queries.ts"))).toBe(true); + expect(existsSync(join(tmp, "User.ts"))).toBe(false); + expect(existsSync(join(tmp, "User.queries.ts"))).toBe(false); + + const postContent = readFileSync(join(tmp, "Post.ts"), "utf-8"); + expect(postContent).toContain("pgTable"); + expect(postContent).toContain("authorId"); + + // Barrel content, not just the file's existence — only Post is re-exported. + const barrelContent = readFileSync(join(tmp, "index.ts"), "utf-8"); + expect(barrelContent).toContain('export * from "./Post"'); + expect(barrelContent).not.toContain("User"); + }); + + test("no scope option produces byte-identical output to an always-matching scope predicate", async () => { + const loader = new MetaDataLoader(); + const result = await loader.load([new FileSource(join(FIXTURE_DIR, "two-entities-fk.json"))]); + expect(result.errors).toEqual([]); + + const noScopeDir = join(tmp, "no-scope"); + const alwaysTrueDir = join(tmp, "always-true"); + + const baseConfig = { + extStyle: "none" as const, + dbImport: "~/server/db", + dialect: "postgres" as const, + generators: [entityFile(), queriesFile(), routesFile(), barrel()], + }; + + // The real-world "no scope declared" path (`meta gen`) still ALWAYS passes a + // predicate — `matchesScope(fqn, collection.scope)` with an empty compiled + // scope, which matches everything. So the byte-identical guarantee that + // matters is exactly this: omitting `scope` entirely vs. a predicate that + // matches every entity must produce identical output, not merely "close". + const outA = await runGen({ + config: defineConfig({ ...baseConfig, outDir: noScopeDir }), + metadata: result.root, + }); + const outB = await runGen({ + config: defineConfig({ ...baseConfig, outDir: alwaysTrueDir }), + metadata: result.root, + scope: () => true, + }); + + expect(outB.warnings).toEqual(outA.warnings); + + const filesA = readdirSync(noScopeDir).sort(); + const filesB = readdirSync(alwaysTrueDir).sort(); + expect(filesB).toEqual(filesA); + for (const f of filesA) { + expect(readFileSync(join(alwaysTrueDir, f), "utf-8")).toEqual( + readFileSync(join(noScopeDir, f), "utf-8"), + ); + } + }); + + test("scope intersects entityFilter — an entity passing only one of the two is not emitted", async () => { + const loader = new MetaDataLoader(); + const result = await loader.load([new FileSource(join(FIXTURE_DIR, "two-entities-fk.json"))]); + expect(result.errors).toEqual([]); + + // User passes entityFilter but is excluded by scope (which admits only + // Post); Post passes scope but is excluded by entityFilter. Neither alone + // is enough — intersection means NEITHER is emitted. + const out = await runGen({ + config: defineConfig({ + outDir: tmp, + extStyle: "none", + dbImport: "~/server/db", + dialect: "postgres", + generators: [entityFile(), queriesFile(), barrel()], + }), + metadata: result.root, + entityFilter: ["User"], + scope: (fqn) => fqn === "demo::Post", + }); + + expect(out.files).toHaveLength(0); + expect(existsSync(join(tmp, "User.ts"))).toBe(false); + expect(existsSync(join(tmp, "Post.ts"))).toBe(false); + // Attributed to the real cause (scope), not entityFilter — User genuinely + // matched entityFilter and was then excluded by scope. + expect(out.warnings.some((w) => w.includes("scope"))).toBe(true); + }); + + test("a scope matching nothing warns with a reason that names scope", async () => { + const loader = new MetaDataLoader(); + const result = await loader.load([new FileSource(join(FIXTURE_DIR, "two-entities-fk.json"))]); + expect(result.errors).toEqual([]); + + const out = await runGen({ + config: defineConfig({ + outDir: tmp, + extStyle: "none", + dbImport: "~/server/db", + dialect: "postgres", + generators: [entityFile(), queriesFile(), barrel()], + }), + metadata: result.root, + scope: () => false, + }); + + expect(out.files).toHaveLength(0); + // The reason must name scope specifically — "root has no object children" + // would be a false statement (the root has two) that sends the reader to + // the wrong file. + const scopeWarning = out.warnings.find((w) => w.includes("No entities to generate")); + expect(scopeWarning).toBeDefined(); + expect(scopeWarning).toContain("scope"); + expect(scopeWarning).not.toContain("entityFilter"); + expect(scopeWarning).not.toContain("root has no object children"); + }); +}); From 9a95a58357f6053fcf844f12131261aacd1d3463 Mon Sep 17 00:00:00 2001 From: Doug Mealing Date: Tue, 18 Aug 2026 06:59:48 -0400 Subject: [PATCH 23/44] test(sdk): dogfood reach+scope against the in-repo examples metadata tree MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit resolveCollection + loadMemory + matchesScope run against examples/advanced-modeling/metaobjects/ (acme::learn, 3 files, 9 objects) — zero new metadata authored. Three angles: a synthetic consumer reaching the tree via an absolute-path source; the examples project's own committed config (sources: [] falling back to metaobjects/, the exact shape every `meta init` scaffold produces); and scope patterns (**, single-segment *, Program*-exclude) evaluated over the real resolutionKey() FQNs the loader produced, not hardcoded object names. --- .../sdk/test/dogfood-examples.test.ts | 118 ++++++++++++++++++ 1 file changed, 118 insertions(+) create mode 100644 server/typescript/packages/sdk/test/dogfood-examples.test.ts diff --git a/server/typescript/packages/sdk/test/dogfood-examples.test.ts b/server/typescript/packages/sdk/test/dogfood-examples.test.ts new file mode 100644 index 000000000..54428374f --- /dev/null +++ b/server/typescript/packages/sdk/test/dogfood-examples.test.ts @@ -0,0 +1,118 @@ +// server/typescript/packages/sdk/test/dogfood-examples.test.ts +// +// Dogfoods `resolveCollection` (reach) + `matchesScope` (scope) against a +// real metadata tree already committed in this repo — zero new metadata +// authored. Three angles, per the task-13 addendum: +// 1. a synthetic consumer elsewhere in a repo reaching the examples tree +// via an absolute-path `sources` entry (the general cross-tree case); +// 2. the examples project's OWN committed config, whose `sources: []` is +// the exact shape every `meta init` scaffold produces — nothing else on +// this branch pins that empty-array-falls-back-to-metaobjects/ path +// against a real committed config; +// 3. scope patterns evaluated over the FQNs the loader actually produced +// for that tree, not string literals invented for the test. +import { describe, test, expect, beforeEach, afterEach } from "bun:test"; +import { mkdtempSync, rmSync, mkdirSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { basename, join, resolve } from "node:path"; +import { resolveCollection } from "../src/collection.js"; +import { loadMemory } from "../src/memory.js"; +import { compileScope, matchesScope } from "../src/scope.js"; + +// Located relative to this test file (never a hardcoded absolute home path — +// this repo is public) so the test runs unchanged on any checkout. +const EXAMPLES_PROJECT = resolve(import.meta.dir, "../../../../../examples/advanced-modeling"); +const EXAMPLES = join(EXAMPLES_PROJECT, "metaobjects"); + +// The tree holds exactly these three files today (verified at HEAD by the +// controller before dispatching this task) — asserted as a floor ("contains +// all three"), never as an exact `length`, since the example tree is +// documentation and may legitimately grow. +const KNOWN_BASENAMES = ["meta.catalog.yaml", "meta.content.yaml", "meta.prompts.yaml"]; + +/** Shared shape for both dogfood file-set assertions (F22): every resolved + * path sits under `under`, the known files are all present, and the set is + * sorted — `resolveSources` canonicalizes by sorting absolute paths, so + * pinning that here is what makes load order irrelevant to a consumer. */ +function assertKnownFileSet(files: readonly string[], under: string): void { + expect(files.every((f) => f.startsWith(under))).toBe(true); + const names = files.map((f) => basename(f)); + for (const known of KNOWN_BASENAMES) expect(names).toContain(known); + expect([...files]).toEqual([...files].sort()); +} + +describe("dogfood: a consumer reaches the in-repo examples tree", () => { + let consumer: string; + beforeEach(() => { + consumer = mkdtempSync(join(tmpdir(), "metaobjects-dogfood-")); + mkdirSync(join(consumer, ".git")); + mkdirSync(join(consumer, "apps/ui/.metaobjects"), { recursive: true }); + writeFileSync( + join(consumer, "apps/ui/.metaobjects/config.json"), + JSON.stringify({ schema_version: 1, sources: [{ path: EXAMPLES }] }), + "utf8", + ); + }); + afterEach(() => { + rmSync(consumer, { recursive: true, force: true }); + }); + + test("resolves every metadata file in it", async () => { + const c = await resolveCollection(join(consumer, "apps/ui")); + assertKnownFileSet(c.files, EXAMPLES); + }); + + test("the resolved set loads without errors", async () => { + const c = await resolveCollection(join(consumer, "apps/ui")); + const root = await loadMemory(c.configDir, { files: c.files }); + expect(root.children().length).toBeGreaterThan(0); + }); +}); + +describe("dogfood: the examples project's own committed config (sources: [])", () => { + test("falls back to metaobjects/ under the project root, exactly like every adopter's default config", async () => { + const c = await resolveCollection(EXAMPLES_PROJECT); + expect(c.configDir).toBe(EXAMPLES_PROJECT); + assertKnownFileSet(c.files, EXAMPLES); + }); +}); + +describe("dogfood: scope evaluated over real loaded FQNs", () => { + test("acme::learn::** matches every loaded object; acme::* (one segment) matches none; excluding Program* narrows without emptying", async () => { + const c = await resolveCollection(EXAMPLES_PROJECT); + const root = await loadMemory(c.configDir, { files: c.files }); + // Derived from the loaded root, not a hardcoded object-name list — this + // must keep working the moment someone edits the example tree. + const fqns = root.children().map((child) => child.resolutionKey()); + expect(fqns.length).toBeGreaterThan(0); + + const broad = compileScope({ include: ["acme::learn::**"] }); + expect(fqns.every((f) => matchesScope(f, broad))).toBe(true); + + // The discriminating case: `*` never crosses `::`, and every object here + // sits two segments below `acme` — a port that treated `*` as "any + // characters" would pass the assertion above and fail this one. + const tooNarrow = compileScope({ include: ["acme::*"] }); + expect(fqns.every((f) => !matchesScope(f, tooNarrow))).toBe(true); + + const excludingProgram = compileScope({ + include: ["acme::learn::**"], + exclude: ["acme::learn::Program*"], + }); + const actualIncluded = fqns.filter((f) => matchesScope(f, excludingProgram)); + const actualExcluded = fqns.filter((f) => !matchesScope(f, excludingProgram)); + + // Expected sets computed from the same FQN list (never a hardcoded name + // list), by the same rule the pattern encodes: the last `::`-segment + // starts with "Program". + const expectedExcluded = fqns.filter((f) => f.split("::").at(-1)!.startsWith("Program")); + const expectedIncluded = fqns.filter((f) => !f.split("::").at(-1)!.startsWith("Program")); + expect([...actualExcluded].sort()).toEqual([...expectedExcluded].sort()); + expect([...actualIncluded].sort()).toEqual([...expectedIncluded].sort()); + + // Non-vacuous on both sides of the exclude — the example tree does carry + // Program-prefixed and non-Program-prefixed objects today. + expect(expectedExcluded.length).toBeGreaterThan(0); + expect(expectedIncluded.length).toBeGreaterThan(0); + }); +}); From 0fc08d989a30068607fdb017fc8c6807847ec0ef Mon Sep 17 00:00:00 2001 From: Doug Mealing Date: Tue, 18 Aug 2026 06:59:53 -0400 Subject: [PATCH 24/44] test(scope-conformance): pin case-sensitive scope matching MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Nine existing cases never exercised case — a port could implement case-insensitive matching and pass the corpus clean, the same shape as the cross-port LIKE/ILIKE divergence this corpus's own README cites as its reason for existing. Adds matching-is-case-sensitive plus the corresponding README Semantics bullet. No product-code change: compilePattern already builds its RegExp with no `i` flag, verified against the case before adding it to the corpus. --- fixtures/scope-conformance/README.md | 2 ++ fixtures/scope-conformance/cases.json | 9 +++++++++ 2 files changed, 11 insertions(+) diff --git a/fixtures/scope-conformance/README.md b/fixtures/scope-conformance/README.md index 3e2b44053..fff022c86 100644 --- a/fixtures/scope-conformance/README.md +++ b/fixtures/scope-conformance/README.md @@ -35,6 +35,8 @@ README.md - **`exclude`** is applied **after** `include` — a name excluded is excluded regardless of which `include` pattern admitted it. `exclude` with no `include` narrows the "everything" default. +- **Matching is case-sensitive** — a pattern and a name must agree in case + (`acme::Order` does not match `acme::order` or `ACME::Order`). ## Behavioral contract diff --git a/fixtures/scope-conformance/cases.json b/fixtures/scope-conformance/cases.json index 20858dd8a..39ad0a81b 100644 --- a/fixtures/scope-conformance/cases.json +++ b/fixtures/scope-conformance/cases.json @@ -79,6 +79,15 @@ { "fqn": "acme::Order.v2", "matches": true }, { "fqn": "acme::OrderXv2", "matches": false } ] + }, + { + "name": "matching-is-case-sensitive", + "scope": { "include": ["acme::Order"] }, + "expect": [ + { "fqn": "acme::Order", "matches": true }, + { "fqn": "acme::order", "matches": false }, + { "fqn": "ACME::Order", "matches": false } + ] } ] } From ceeae33c724f97ab82c321cea8a42d04f25b5738 Mon Sep 17 00:00:00 2001 From: Doug Mealing Date: Tue, 18 Aug 2026 07:10:47 -0400 Subject: [PATCH 25/44] docs: metadata sources, scope, discovery, and the vendoring workflow MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adopter guide for phase-1 metadata source resolution, describing what shipped rather than what was planned: - docs/features/metadata-sources.md (new) — `sources` as a set with `metaobjects/` as its DEFAULT value (so `"sources": []`, which `meta init` has always scaffolded, changes nothing); the `path` kind, read in place and never installed; the `*`/`**` pattern grammar; nearest-ancestor discovery and the `.git` boundary; `migrate.scope` and why a `migrate` block belongs where the ledger lives; vendoring as "copy a directory and point a `path` at it"; a worked polyglot example (a Maven module owning the model, two Node consumers, one schema owner). - cli/README.md — `sources`, `scope` and `migrate.scope` documented in the config reference beside `targets`. - CLAUDE.md — two additive lines: `metaobjects/` is a default, never a requirement; metadata location resolves via `resolveCollection()`, and only `meta init` may hardcode the directory name. - docs/CONFORMANCE.md — `fixtures/scope-conformance/` (10 cases) added to the corpus matrix and given a detail subsection; total 19 -> 20. TS is the only port with a runner today; the other four are deferred to the ports plan. - design doc §3 — the sharpened three-layer order-independence statement (resolveSources canonicalizes; the loader resolves content order-free; sibling order of unrelated top-level nodes is NOT a contract), naming the gate that pins layers 1 and 2. §4.7's whole-tree byte-identity bar is marked corrected so it stops contradicting §3. Deliberately documents only shipped behaviour: `resource`/`package` sources, per-generator declarative scope, and the non-TS CLIs are listed as deferred. --- AGENTS.md | 3 + docs/CONFORMANCE.md | 29 +- docs/README.md | 2 + docs/features/metadata-sources.md | 469 ++++++++++++++++++ ...08-17-metadata-source-resolution-design.md | 24 + server/typescript/packages/cli/README.md | 78 +++ 6 files changed, 602 insertions(+), 3 deletions(-) create mode 100644 docs/features/metadata-sources.md diff --git a/AGENTS.md b/AGENTS.md index b16e43fe1..3edc033f3 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -200,6 +200,7 @@ import { EntityFetcherProvider, EntityGrid } from "@metaobjectsdev/tanstack"; - **Codegen substrate**: ts-poet for greenfield emit, ts-morph for in-place edits, Biome for format pass, `git merge-file --diff3` for hand-edit-preserving regen. - **Runtime substrate**: Kysely for TS (user-provided connection, async-only). - **Migration substrate**: Postgres + SQLite for TS v0.3. +- **Metadata location**: resolved via `resolveCollection()` (`@metaobjectsdev/sdk`) — the single authority. No code path may hardcode the `metaobjects/` directory name except `meta init`, which scaffolds it. See [docs/features/metadata-sources.md](docs/features/metadata-sources.md). ## Explicitly out of scope @@ -224,6 +225,8 @@ import { EntityFetcherProvider, EntityGrid } from "@metaobjectsdev/tanstack"; **Default convention**: one file per domain concept under `metaobjects/`. Multiple objects per file when they share a domain. Projections (`source.dbView`) live inline with their base entity. +`metaobjects/` is the **default value** of `sources` in `.metaobjects/config.json` — never a requirement. A project declaring `sources` explicitly can point anywhere (and need not have such a directory at all); `"sources": []`, which is what `meta init` scaffolds, takes the default. + ``` project-root/ ├── metaobjects/ # VISIBLE — entity declarations diff --git a/docs/CONFORMANCE.md b/docs/CONFORMANCE.md index 09ae7efaa..42c8e2c4e 100644 --- a/docs/CONFORMANCE.md +++ b/docs/CONFORMANCE.md @@ -1,6 +1,6 @@ # Conformance coverage -The MetaObjects standard ships **19 shared conformance corpora** under +The MetaObjects standard ships **20 shared conformance corpora** under [`fixtures/`](../fixtures/). Every port runs every corpus that is *applicable to it* and asserts the same expected behaviour against the same fixtures. **This page is the inverse index**: fixture → feature doc + per-port pass status, and it is the @@ -42,6 +42,7 @@ regenerate with `ls -d fixtures//*/ | wc -l`. | [`fixtures/template-output-render-conformance/`](../fixtures/template-output-render-conformance/) | 5 | ✓ | ✓ | ✓ | ✓ | ✓ | | [`fixtures/generator-registry-conformance/`](../fixtures/generator-registry-conformance/) | 1 canonical manifest | ✓ | ✓ | ✓ | ✓ | ✓ | | [`fixtures/provider-composition-conformance/`](../fixtures/provider-composition-conformance/) | 9 (5 error-shape + 4 compose-load) | ✓ | ✓ | — (JVM registry via Java) | ✓ | ✓ | +| [`fixtures/scope-conformance/`](../fixtures/scope-conformance/) | 10 cases | ✓ (reference implementation) | — | — | — | — | | [`fixtures/agent-context-conformance/`](../fixtures/agent-context-conformance/) | 4 | ✓ (the emitter is TS-owned) | — | — | — | — | | [`fixtures/metamodel-docs/`](../fixtures/metamodel-docs/) | 1 | ✓ (docs emit is TS-owned) | — | — | — | — | @@ -138,10 +139,32 @@ inheritance), `m2m/` (3), `jsonb/` (2, typed value-object columns) and Kotlin, C#, Python — run it in BOTH lanes: a hand-rolled reference server and the port's own GENERATED API artifact booted over HTTP. +### `fixtures/scope-conformance/` (10 cases) + +All 10 cases → [features/metadata-sources.md](features/metadata-sources.md) (the +`scope` pattern grammar). The corpus is file-shaped: one committed `cases.json`, +read directly by every port's runner, with no per-port fixture and no ledger. + +It pins the semantics of a consumer's `include`/`exclude` scope over +fully-qualified names — **`*` matches within one `::` segment and never crosses +it; a segment that is exactly `**` matches one or more whole segments (so +`acme::**` does not match the bare `acme`); every other character is literal, +regex metacharacters included; an absent or empty `include` means everything; +multiple `include` patterns are a union and `exclude` is applied after it; and +matching is case-sensitive.** These are exactly the rules four independent +implementations would otherwise each get slightly wrong — the failure mode that +produced the cross-port `LIKE`/`ILIKE` divergence fixed in 0.21.6. + +**TypeScript is the only port with a runner today.** The reference implementation is +[`server/typescript/packages/sdk/src/scope.ts`](../server/typescript/packages/sdk/src/scope.ts) +(`compilePattern` / `compileScope` / `matchesScope`), and the corpus was authored +against it. Java, Kotlin, C# and Python are deferred to the phase-1 ports plan; the +corpus exists now precisely so those four land on one grammar rather than four. + ## Orphaned fixtures (tested but not yet documented) -The fixtures in the six corpora mapped above (metamodel 255 + yaml 15 + verify 31 -+ render 15 + persistence 33 + api-contract 41) each map to a feature doc. None +The fixtures in the seven corpora mapped above (metamodel 255 + yaml 15 + verify 31 ++ render 15 + persistence 33 + api-contract 41 + scope 10) each map to a feature doc. None are orphaned today. The remaining corpora in the totals table gate tooling contracts (registry manifests, provider composition, agent context, docs emit) rather than user-facing metamodel behaviour, so they have no feature-doc row. diff --git a/docs/README.md b/docs/README.md index bf86035a8..7903aad34 100644 --- a/docs/README.md +++ b/docs/README.md @@ -37,6 +37,7 @@ docs/ │ ├── downstream-metadata-decisions.md # guidance for adopters extending the metamodel │ ├── generated-mutations.md # generated POST/PATCH mutation surface │ ├── image-upload.md # view.image form control (TS-web) +│ ├── metadata-sources.md # where metadata comes from: sources, scope, discovery │ └── own-your-codegen.md # scaffold-and-own generator ownership (ADR-0034) └── ports/ # one file per language/framework port ├── typescript.md @@ -58,6 +59,7 @@ this tree is documentation, not the source of truth. | Understand what `object.entity`, `source.rdb`, `template.prompt` mean | [`features/`](features/) | | Compare what TS vs Java vs Kotlin vs C# vs Python emit for the same metadata | any [`features/*.md`](features/) — every feature shows all five ports side-by-side | | Author metadata in YAML instead of JSON | [`features/yaml-authoring.md`](features/yaml-authoring.md) | +| Point the toolchain at metadata that lives somewhere other than `metaobjects/`, or scope what a project generates and migrates | [`features/metadata-sources.md`](features/metadata-sources.md) | | Record what the system is supposed to do, and stop agents reviving retired features | [`features/requirements.md`](features/requirements.md) | | Wire prompt construction (FR-004) | [`features/templates-and-payloads.md`](features/templates-and-payloads.md) | | Share a metadata shape across multiple instances (abstracts, `extends:`) | [`features/abstracts-and-inheritance.md`](features/abstracts-and-inheritance.md) | diff --git a/docs/features/metadata-sources.md b/docs/features/metadata-sources.md new file mode 100644 index 000000000..3ee6bdec6 --- /dev/null +++ b/docs/features/metadata-sources.md @@ -0,0 +1,469 @@ +# Metadata sources, scope, and discovery + +**Where does my metadata come from?** From the `sources` set in +`.metaobjects/config.json`. When that key is absent or empty, `sources` takes its +default value — the `metaobjects/` directory sitting beside the `.metaobjects/` +folder that holds the config. Nothing else in the toolchain assumes that directory +name. + +**How do I point it somewhere else?** Declare it: + +```json +{ + "schema_version": 1, + "sources": [ + { "path": "../model/src/main/resources/metadata" }, + { "path": "metaobjects" } + ] +} +``` + +`meta gen`, `meta migrate`, `meta verify`, `meta docs` and `meta export` all read +exactly that set. A `path` is read **in place and never installed** or copied. + +**Nothing breaks if you do nothing.** `meta init` has always scaffolded +`"sources": []`, so every existing project takes the default and resolves the same +files it always did. + +**Port support.** `sources` / `scope` / `migrate.scope` are read by the **Node +`meta` CLI** today. The Java, Kotlin, C# and Python CLIs still take their metadata +location their own way (a Maven `` element, a positional directory, a +`metadata` config key). The cross-port pattern grammar is already pinned by +[`fixtures/scope-conformance/`](../../fixtures/scope-conformance/); wiring the other +four CLIs to the same config file is the phase-1 ports plan. + +--- + +## `sources` — a set, not an ordered list + +`sources` is an array for authoring convenience, but it is **specified as a set**. +Reordering it cannot change what resolves: `resolveSources` sorts the resolved +absolute paths, and when two entries overlap on the same file the one recorded as +its provenance is chosen by comparing the entry's content, never by which was +declared first. + +That is a real guarantee rather than a stylistic claim, because the loader does not +need an order either — it derives overlay precedence from the files themselves +(see [Order independence](#order-independence-is-three-layers) below). + +Consequences worth knowing: + +- Two entries may overlap. A file reached by two `path` entries is loaded once. +- There is no cycle detection, because a set union cannot have a cycle. +- A `path` that does not exist is an **error** (`ERR_SOURCE_UNRESOLVED`), never a + silent skip. Only the *default* is allowed to be absent, and only then to produce + the friendlier `ERR_COLLECTION_NOT_FOUND`. + +### The `path` kind + +```jsonc +"sources": [ + { "path": "metaobjects" }, // a directory, relative to this config + { "path": "../shared-model/metadata" }, // a sibling module — read in place + { "path": "vendor/model/meta.catalog.json" } // a single file +] +``` + +- A relative `path` resolves **against the directory holding the declaring + `.metaobjects/` folder**, never against the ambient working directory. Moving + where you run the command from cannot change what resolves. +- An absolute `path` is taken as-is. +- A directory is walked **recursively**. A file counts as metadata when its + extension is `.json`, `.yaml` or `.yml`, matched case-insensitively. +- A `_pending/` directory is skipped at any depth (it holds proposed, unpromoted + records). +- Symlinked subdirectories are followed, matching the loader's own directory walk. + +### `resource` and `package` are declared but not resolved + +```jsonc +"sources": [ + { "resource": "acme/model" }, // reserved — JVM classpath resource root + { "package": "@acme/common-model" } // reserved — a published metadata package +] +``` + +Both parse (the config shape is fixed now so a later phase slots in without a config +migration) and both throw `ERR_SOURCE_KIND_UNSUPPORTED` when the toolchain tries to +resolve them. A misspelled key such as `{ "pathh": "model" }` is a **load error**, +not a silently ignored extra: `.metaobjects/config.json` is validated strictly, top +level and every nested block. + +--- + +## Discovery — nearest ancestor, stopping at the repository boundary + +Running a CLI inside an app should find that app's configuration, not the repo +root's. + +1. Start at the working directory (`--cwd` / `-C` moves the starting point). +2. Walk **up**, looking for `.metaobjects/config.json`. The **first one found wins** + — a config in a subdirectory beats one in an ancestor. +3. **Stop after examining a directory containing `.git`.** A checkout can never + silently adopt a parent checkout's configuration. The config check runs before + the boundary check within each directory, so a config at the repository root — + sharing its directory with `.git` — is still reachable from any subdirectory. +4. If nothing is found, the starting directory is used with the default `sources`. + +Collections are **never auto-discovered**. Nothing globs the tree for directories +that merely look like metadata homes; a collection exists only where a config names +one. + +**A config that exists but fails to load propagates its error.** Malformed JSON or a +schema violation is the author's mistake, and it fails loudly — it does not fall +through to the default and quietly generate from a stale `metaobjects/`. + +**`ERR_COLLECTION_NOT_FOUND`** is raised only when *both* have failed: no `sources` +were declared anywhere up the walk, **and** no default `metaobjects/` directory +exists either. + +--- + +## `scope` — an output filter over fully-qualified names + +```jsonc +"scope": { + "include": ["acme::blog::**", "acme::common::*"], + "exclude": ["acme::blog::internal::**"] +} +``` + +**The collection always loads in full. Scope filters output, never input.** This is +deliberate and not an optimization left on the table: a partial file list can fail to +load outright when an `extends` target is in a file that was filtered away, so an +author would have to hand-maintain a transitive closure. Loading everything is +closure-complete by definition, and the filter is applied where the toolchain emits. + +### Pattern grammar + +Patterns match a node's fully-qualified name — `::`, e.g. +`acme::blog::Author`. An object declared with no package has a bare name as its FQN. + +| Rule | Example | +|---|---| +| `::` separates segments | `acme::blog::Author` is three segments | +| `*` matches any run of characters **within one segment**, never crossing `::` | `acme::blog::Author*` matches `acme::blog::AuthorDraft`, not `acme::blog::x::AuthorDraft` | +| A segment that is exactly `**` matches **one or more** whole segments | `acme::**` matches `acme::Author` and `acme::a::b::Author`, but **not** the bare `acme` | +| `**` mid-pattern still requires at least one segment | `acme::**::Author` does **not** match `acme::Author` | +| Every other character is **literal**, regex metacharacters included | `acme::v1.0::*` matches a segment literally named `v1.0` | +| Absent or empty `include` means **everything** | `{ "exclude": ["acme::blog::internal::**"] }` narrows the default | +| Multiple `include` patterns are a **union** | a name matches if any one matches | +| `exclude` applies **after** `include` | an excluded name stays excluded no matter which `include` admitted it | +| Matching is **case-sensitive** | `acme::Author` does not match `acme::author` | + +An unparseable pattern is an error (`ERR_SCOPE_PATTERN_INVALID`), never a silent +non-match. Empty patterns, empty segments, and a malformed separator (an odd run of +`:`) all fail loudly at load. + +A common first mistake: `acme::*` matches only objects **one** segment below `acme`. +For a package tree, you want `acme::**`. + +### Where `scope` applies + +| Command | Scoped by top-level `scope`? | +|---|---| +| `meta gen` | **Yes** — an object is generated only when its FQN is in scope | +| `meta verify --codegen` | **Yes** — it regenerates under the same scope, so a scoped `gen` cannot be reported as drift | +| `meta docs` | No | +| `meta export` | No | +| `meta migrate`, `meta verify --db` | No — those take [`migrate.scope`](#migratescope--who-owns-which-tables) instead | + +`docs` and `export` are **inspection surfaces over the loaded collection**, not code +emitters. Scoping them would make the tools you reach for to answer "what is +actually in this model?" answer a narrower question than the one you asked. + +`meta gen ` arguments **intersect** with scope: both must pass. If a scope +leaves nothing to generate, `gen` says so and names the scope as the reason rather +than blaming the entity filter. + +### The one sharp edge + +An in-scope object may reference an out-of-scope one — an FK target, a +relationship `@objectRef`, a projection's base. The reference resolves perfectly at +load time (everything loaded), but the code emitted for the in-scope object names a +symbol that was never generated here. + +This is left to fail loudly rather than silently auto-widening the scope: you +declared the scope precisely because something else owns those objects. The failure +is an unresolved import — a plain compiler error at build time, not a surprise at +runtime. + +### Per-generator scope is not phase 1 + +The TypeScript-only per-generator **`filter` function** in +`metaobjects.config.ts` is unchanged and remains supported as an escape hatch: + +```ts +entityFile({ filter: (e) => e.name !== "Legacy" }) +``` + +It is deliberately not the thing a cross-port feature depends on — a JavaScript +predicate cannot be written in a `pom.xml`, a Python config, or a C# CLI flag, and +no conformance corpus can gate it. Package patterns are strings and port unchanged +to all five config surfaces. A declarative per-generator `scope` key is deferred. + +--- + +## `migrate.scope` — who owns which tables + +A database is often shared: this consumer owns one package tree's tables, another +tool owns the rest. Without a declaration, `meta migrate` treats every table it does +not model as a table to **drop**. + +```jsonc +"migrate": { + "outDir": "./.metaobjects/migrations", + "databaseUrl": "postgres://localhost:5432/acme", + "dialect": "postgres", + "scope": ["acme::billing::**"] +} +``` + +`migrate.scope` is a **plain array of include patterns** — the same grammar as +top-level `scope`, with no `exclude` arm. A migration run is scoped to what it +governs, not filtered down from "everything". + +Tables and views whose declaring object falls outside the scope are **neither created +nor dropped**. That takes two suppressions, and the toolchain does both: the objects +leave the *expected* schema, and their physical names are suppressed on the *actual* +side too. Doing only the first would be strictly worse than doing nothing — every +out-of-scope table that already exists would become a proposed `DROP TABLE`. + +- **`meta migrate`** prints what it left alone: `N object(s) out-of-scope (outside + migrate.scope, governed elsewhere)`, naming the tables. Without that line, "no + changes" and "no changes to the half of the model this run governs" read + identically. +- **`meta verify --db`** reports out-of-scope objects as out-of-scope rather than as + drift, and applies the same narrowing to the committed schema snapshot. +- **`meta migrate baseline` is deliberately unscoped.** A `--from-db` baseline + records a starting point read out of the database; it has no metadata provenance to + scope by. An out-of-scope table sitting in that snapshot is harmless — the diff is + scoped on every subsequent run. +- A table or view with **no recorded provenance is kept**. Scope decides on the + declaring object's FQN, and an object whose FQN is unknown was never proven to be + anyone else's. + +### Put the `migrate` block where the ledger lives + +**Whoever holds `.metaobjects/migrations/` and the schema snapshot owns the schema.** +A repository with six codegen consumers over one database has at most one schema +owner; if each declared a `migrate` block you would get six partial migrations, which +is worse than having none. + +This is also mechanically required today: `migrate.scope` is read from the +**discovered** config, but the rest of the `migrate` block (`outDir`, +`databaseUrl`, `dialect`, `allow`, `d1`) is read from `.metaobjects/config.json` in +the directory you run the command in. Run `meta migrate` from the directory that +holds both the config and the ledger, or pass `--cwd` to point at it. + +`meta verify --db` may run from any consumer — it reports rather than writes. + +--- + +## Vendoring — airgapped and hermetic builds + +There is no separate vendoring mechanism, and none is needed. Because a `path` +source is **read in place and never installed**, vendoring is: + +1. Copy the dependency's metadata into a directory in your repository. +2. Point a `path` at it. +3. Commit it. + +```jsonc +{ + "schema_version": 1, + "sources": [ + { "path": "vendor/acme-common-model" }, + { "path": "metaobjects" } + ] +} +``` + +The build now resolves entirely from committed files, with no network access and no +resolution step that could produce a different answer tomorrow than it did today — +the `go mod vendor` property, obtained by declaring a directory. + +Because sources are a set, the vendored entry needs no particular position. If the +vendored tree and your own both declare the same node, ordinary overlay merge rules +apply — see [`loaders.md`](loaders.md). + +--- + +## A worked polyglot example + +A repository where a Maven module owns the model, two Node consumers generate from +it, and exactly one of them owns the database. + +``` +acme-platform/ +├── .git/ +├── model/ # Maven module — the model, no CLI config +│ └── src/main/resources/metadata/ +│ ├── meta.common.json # package acme::common +│ ├── meta.billing.json # package acme::billing +│ └── meta.blog.json # package acme::blog +├── services/billing/ # Node consumer — SCHEMA OWNER +│ └── .metaobjects/ +│ ├── config.json +│ └── migrations/ # the ledger lives here +└── apps/web/ # Node consumer — codegen only + └── .metaobjects/ + └── config.json +``` + +`services/billing/.metaobjects/config.json` — reaches the Maven module's resource +directory as a plain path, generates only the billing tree, and owns the billing +tables: + +```json +{ + "schema_version": 1, + "sources": [ + { "path": "../../model/src/main/resources/metadata" } + ], + "scope": { + "include": ["acme::billing::**", "acme::common::**"] + }, + "migrate": { + "outDir": "./.metaobjects/migrations", + "databaseUrl": "postgres://localhost:5432/acme", + "dialect": "postgres", + "scope": ["acme::billing::**"] + } +} +``` + +`apps/web/.metaobjects/config.json` — same model, different slice, **no `migrate` +block** because it does not own the schema: + +```json +{ + "schema_version": 1, + "sources": [ + { "path": "../../model/src/main/resources/metadata" } + ], + "scope": { + "include": ["acme::blog::**", "acme::common::**"], + "exclude": ["acme::blog::internal::**"] + } +} +``` + +What this buys: + +- **No symlinks and no copied files.** The Maven module stays the single home of the + metadata; both Node consumers read it in place. The Java build is untouched — it + keeps using its own Maven configuration. +- **Running `meta gen` in `apps/web/` finds `apps/web`'s config**, because discovery + walks up from the working directory and takes the nearest one. It never reaches + `services/billing`, and the `.git` at `acme-platform/` stops it from escaping the + checkout. +- **`meta migrate` from `services/billing/`** proposes changes to `acme::billing` + tables only. The `acme::blog` tables — owned by a different tool sharing the same + database — are neither created nor dropped, and are reported as out-of-scope. +- **`meta verify --db` from either consumer** reports honestly: the web app sees the + billing tables as out-of-scope, not as drift. + +- **Running a command at `acme-platform/` itself fails**, rather than guessing. There + is no config there, `.git` stops the walk, and no default `metaobjects/` directory + exists — so `ERR_COLLECTION_NOT_FOUND` names both halves. Run from a consumer, or + point `--cwd` at one. + +To make this repository build with no network access, copy +`model/src/main/resources/metadata` to `vendor/model/` in each consumer and change +one line per config. + +--- + +## Order independence is three layers + +Worth knowing precisely, because the layers are easy to conflate and they are not +redundant. + +1. **`resolveSources` canonicalizes.** It sorts resolved absolute paths, so in + production the loader never sees a permuted file list at all. +2. **The loader resolves content order-independently.** Overlay-only sources are + stable-partitioned to merge last, so an overlay reaching a base declared in + another file resolves the same regardless of which arrived first. +3. **Sibling order of unrelated top-level nodes still follows load order**, and that + is *not* a contract — the canonical serializer only ever promised attribute-key + alphabetization. Do not expect byte-identical whole-tree serialization across + permuted loader inputs. + +Layers 1 and 2 are pinned by +[`server/typescript/packages/sdk/test/order-independence.test.ts`](../../server/typescript/packages/sdk/test/order-independence.test.ts). + +--- + +## Errors + +| Code | Raised when | +|---|---| +| `ERR_SOURCE_UNRESOLVED` | A declared `path` source does not exist on disk | +| `ERR_SOURCE_KIND_UNSUPPORTED` | A `resource` or `package` source was declared; this toolchain resolves `path` only | +| `ERR_SCOPE_PATTERN_INVALID` | A scope pattern is empty, has an empty segment, or has a malformed `::` separator | +| `ERR_COLLECTION_NOT_FOUND` | No `sources` were declared **and** no default `metaobjects/` directory exists | + +A schema violation in `.metaobjects/config.json` itself (an unknown key, a wrong +type) surfaces as the config load error and stops the command. + +--- + +## What is deferred + +Phase 1 ships the spine. Explicitly **not** built yet, so you do not go looking: + +- **`resource` sources** (JVM classpath roots) and **`package` sources** (a published + metadata package) — declared in the config shape, rejected at resolution. +- **`url` sources** and **named `collection` references**. +- **Per-generator declarative `scope`.** The TypeScript `filter` function is the + escape hatch and is unchanged. +- **The other four ports' CLIs reading `.metaobjects/config.json`.** The pattern + grammar is corpus-gated so they cannot diverge when they land. +- **Database and other runtime metadata sources.** Ruled a runtime-metadata concern + rather than a build-time one. + +Design rationale and the full phase plan: +[`docs/superpowers/specs/2026-08-17-metadata-source-resolution-design.md`](../superpowers/specs/2026-08-17-metadata-source-resolution-design.md). + +--- + +## Verified by + +**Cross-port pattern semantics** + +- [`fixtures/scope-conformance/`](../../fixtures/scope-conformance/) — 10 cases + pinning `*` / `**`, include-union, exclude-after-include, literal metacharacters, + and case sensitivity. TypeScript runs it today; the other four ports are deferred + to the phase-1 ports plan. See [`CONFORMANCE.md`](../CONFORMANCE.md). + +**TypeScript gates** + +- `server/typescript/packages/sdk/test/scope.test.ts` — the pattern engine +- `server/typescript/packages/sdk/test/scope-conformance.test.ts` — the corpus runner +- `server/typescript/packages/sdk/test/sources.test.ts` — `path` resolution, `_pending` + exclusion, unsupported kinds +- `server/typescript/packages/sdk/test/discovery.test.ts` — nearest-ancestor walk and + the `.git` boundary +- `server/typescript/packages/sdk/test/collection.test.ts` — `resolveCollection` + precedence, the `metaobjects/` default, and error propagation +- `server/typescript/packages/sdk/test/order-independence.test.ts` — layers 1 and 2 above +- `server/typescript/packages/sdk/test/dogfood-examples.test.ts` — a consumer reaching + a real committed metadata tree, with scope evaluated over the FQNs the loader + actually produced +- `server/typescript/packages/cli/test/collection-routing.test.ts` — every command + routing through `resolveCollection` +- `server/typescript/packages/cli/test/migrate-scope.test.ts` and + `server/typescript/packages/migrate-ts/test/expected-schema-scope.test.ts` — + both-sided `migrate.scope` suppression +- `server/typescript/packages/codegen-ts/test/run-gen.test.ts` — scope intersecting + the entity filter at the `gen` choke point + +## See also + +- [`loaders.md`](loaders.md) — how the resolved file set is merged +- [`cli.md`](cli.md) — the locked CLI architecture (ADR-0015) and which port owns what +- [`migrations-and-drift.md`](migrations-and-drift.md) — `meta migrate` and + `meta verify --db` +- [`own-your-codegen.md`](own-your-codegen.md) — generator ownership and the `filter` + escape hatch diff --git a/docs/superpowers/specs/2026-08-17-metadata-source-resolution-design.md b/docs/superpowers/specs/2026-08-17-metadata-source-resolution-design.md index b0f629ccb..1faa9f285 100644 --- a/docs/superpowers/specs/2026-08-17-metadata-source-resolution-design.md +++ b/docs/superpowers/specs/2026-08-17-metadata-source-resolution-design.md @@ -65,6 +65,25 @@ Order still matters in exactly three places, and only one concerns the source li **Therefore the declared source order carries no information the loader needs. Sources are a set.** +**Order-independence is three layers, not one, and they are not redundant.** Implementation +confirmed this empirically, and the sharper statement belongs here because two readers in a row +conflated the layers and wrote a vacuous test as a result: + +1. **`resolveSources` canonicalizes.** It sorts resolved absolute paths, so in production the + loader never sees a permuted file list at all. +2. **The loader resolves CONTENT order-independently.** `_partitionOverlayLast` stable-partitions + overlay-only sources to merge last, so an overlay reaching a base declared in another file + resolves the same regardless of which arrived first. Disabling it throws `ERR_OVERLAY_NO_TARGET` + on half of the permutations of a two-file overlay set and silently drops the overlay's fields on + the rest. +3. **SIBLING ORDER of unrelated top-level nodes still follows load order, and that is *not* a + contract** — `canonicalSerialize` only ever promised attr-key alphabetization. A test that + demands byte-identical whole-tree serialization across permuted loader inputs is asserting a bar + the design never set, and it will fail on correct code. + +Layers 1 and 2 are pinned by `server/typescript/packages/sdk/test/order-independence.test.ts` — the +executable form of this statement. + What this deletes, rather than adds: - No ordered-list semantics to specify, document, or port. @@ -328,6 +347,11 @@ Two new corpora, plus one gate that is the linchpin of the whole design. the canonical serialization must be **byte-identical** across all of them, in all five ports. Without this, set semantics is an aspiration that decays the first time someone adds an order-sensitive code path. With it, the property is enforced rather than believed. + **Corrected during implementation — see §3's three-layer statement:** whole-tree byte-identity + is too strong a bar, because sibling order of unrelated top-level nodes legitimately follows + load order and was never a contract. The gate asserts layers 1 and 2 (identical `resolveSources` + output across permutations; identical resolved CONTENT across permuted loader inputs), which is + the property this item was reaching for. 2. **Scope-pattern corpus.** A matrix of patterns × fully-qualified names → expected match/no-match, byte-matched across all five ports. This is what stops `*` and `**` from meaning five different things — the failure mode that produced the `like`/`ILIKE` divergence. diff --git a/server/typescript/packages/cli/README.md b/server/typescript/packages/cli/README.md index 7f4f33d78..7a93fd44a 100644 --- a/server/typescript/packages/cli/README.md +++ b/server/typescript/packages/cli/README.md @@ -278,6 +278,84 @@ For D1 projects, the `migrate` block instead looks like: Precedence for `meta migrate`: CLI flag > env var (`DATABASE_URL` only) > `.metaobjects/config.json` > built-in default. +### Metadata sources (`sources`) + +`sources` is the single authority on **where metadata lives**. Every read command +(`gen`, `migrate`, `verify`, `docs`, `export`) resolves it. When the key is absent or +empty — which is what `meta init` scaffolds — it takes its default value, the +`metaobjects/` directory beside the config, so existing projects are unaffected. + +```jsonc +"sources": [ + { "path": "metaobjects" }, // a directory, relative to this config + { "path": "../model/src/main/resources/metadata" }, // a sibling module — read IN PLACE + { "path": "vendor/model/meta.catalog.json" } // a single file +] +``` + +- A relative `path` resolves against the directory holding this `.metaobjects/` + folder, never against the ambient cwd. A directory is walked recursively for + `.json` / `.yaml` / `.yml`, skipping `_pending/`. +- **`sources` is a set** — reordering it cannot change what resolves, and two entries + may overlap. +- A `path` that does not exist is an error (`ERR_SOURCE_UNRESOLVED`), never a silent + skip. +- `{ "resource": "…" }` and `{ "package": "…" }` parse but do not resolve yet + (`ERR_SOURCE_KIND_UNSUPPORTED`). The file is validated **strictly**, so a + misspelled key is a load error rather than a silently ignored extra. + +**Discovery.** The CLI walks **up** from the working directory (`--cwd` moves the +start) for the nearest `.metaobjects/config.json` — nearest wins — and stops after a +directory containing `.git`, so a checkout never adopts a parent checkout's config. + +### Output scope (`scope`) + +```jsonc +"scope": { + "include": ["acme::billing::**", "acme::common::*"], + "exclude": ["acme::billing::internal::**"] +} +``` + +Patterns match an object's fully-qualified name (`::`). `*` matches +within **one** segment and never crosses `::`; a segment that is exactly `**` matches +**one or more** whole segments; everything else is literal; absent/empty `include` +means everything; `exclude` applies after `include`; matching is case-sensitive. + +**The collection always loads in full — `scope` filters output, never input.** It +applies to `meta gen` and to `meta verify --codegen` (which regenerates under the +same scope, so a scoped `gen` is not reported as drift). `meta docs` and `meta export` +are deliberately **not** scoped — they inspect the loaded collection. `meta gen +` arguments intersect with `scope`; both must pass. + +The TypeScript-only per-generator `filter` function is unchanged and remains the +escape hatch for predicate-shaped filtering. + +### Per-command scope (`migrate.scope`) + +```jsonc +"migrate": { + "outDir": "./.metaobjects/migrations", + "scope": ["acme::billing::**"] +} +``` + +An include-only array of the same patterns, for a database this project **shares with +another owner**. Tables and views whose declaring object falls outside it are neither +created nor dropped — they leave the expected schema *and* are suppressed on the +actual side — and `meta migrate` prints how many it left alone. `meta verify --db` +reports them as out-of-scope instead of as drift. `meta migrate baseline` is +deliberately unscoped: a `--from-db` baseline has no metadata provenance to scope by. + +Declare a `migrate` block only in the project that holds `.metaobjects/migrations/` +and the schema snapshot — that project owns the schema. Run `meta migrate` from that +directory (or point `--cwd` at it): `migrate.scope` comes from the discovered config, +but the rest of the block is read from `.metaobjects/config.json` in the directory the +command runs in. + +Full adopter guide, including vendoring for airgapped builds and a worked polyglot +example: [docs/features/metadata-sources.md](../../../../docs/features/metadata-sources.md). + ## Metadata format See `.metaobjects/AGENTS.md` (scaffolded by `meta init`) for the metaobjects metamodel rules, attribute conventions, and worked examples. Deeper references: From fece4e5f80c474b2a4081ce23d2f47a740b06a7e Mon Sep 17 00:00:00 2001 From: Doug Mealing Date: Tue, 18 Aug 2026 07:50:39 -0400 Subject: [PATCH 26/44] fix(sdk): resolved sources keep the loader's per-level walk order MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `resolveSources` returned a flat lexicographic sort of absolute paths, while every read path before this branch walked metadata through `listMetadataFiles` — the files at a level, then that level's subdirectories, depth-first. The two disagree the moment a subdirectory name sorts before a sibling file (`metaobjects/common/…` ahead of `metaobjects/meta.users.json`), and that order is observable in output: the barrel generator emits exports straight from `root.objects()` order, and the same order flows into the shared `enums.ts`, `meta docs` page ordering and `meta export`'s sibling order. A project with a nested metadata directory therefore got REORDERED generated code — against the absolute promise that a project declaring no `sources` and no `scope` is byte-identical to before. There is now exactly ONE walker: `listMetadataFiles` is exported and `resolveSources` calls it rather than keeping a second recursive walk of its own (the two had already drifted once, on case-sensitivity). Across specs the order is decided by spec CONTENT — the specs are walked in `JSON.stringify` order, not declared order — so the full result stays a pure function of the declared SET, and `order-independence.test.ts` passes unchanged. Every `metaobjects/` tree committed in this repository is flat, so nothing here could observe the property. `sdk/test/source-order.test.ts` builds the shape that discriminates and pins it four ways: the explicit expected order, a check that the fixture would sort the other way (so the gate cannot go vacuous), equality with `listMetadataFiles` on the same tree, and the loaded tree's sibling order. `dogfood-examples.test.ts` now asserts the committed examples tree resolves in that same walk order instead of merely "sorted", and takes its directory name from `DEFAULT_METADATA_DIR` rather than the literal. Co-Authored-By: Claude Opus 5 (1M context) --- docs/features/metadata-sources.md | 22 ++- server/typescript/packages/sdk/src/memory.ts | 49 +++++-- server/typescript/packages/sdk/src/sources.ts | 110 +++++++-------- .../sdk/test/dogfood-examples.test.ts | 22 +-- .../sdk/test/order-independence.test.ts | 12 +- .../packages/sdk/test/source-order.test.ts | 129 ++++++++++++++++++ 6 files changed, 250 insertions(+), 94 deletions(-) create mode 100644 server/typescript/packages/sdk/test/source-order.test.ts diff --git a/docs/features/metadata-sources.md b/docs/features/metadata-sources.md index 3ee6bdec6..a415da1f4 100644 --- a/docs/features/metadata-sources.md +++ b/docs/features/metadata-sources.md @@ -37,10 +37,10 @@ four CLIs to the same config file is the phase-1 ports plan. ## `sources` — a set, not an ordered list `sources` is an array for authoring convenience, but it is **specified as a set**. -Reordering it cannot change what resolves: `resolveSources` sorts the resolved -absolute paths, and when two entries overlap on the same file the one recorded as -its provenance is chosen by comparing the entry's content, never by which was -declared first. +Reordering it cannot change what resolves: `resolveSources` walks the entries in +**content order** rather than declared order, so both the resolved file order and — +when two entries overlap on the same file — the entry recorded as its provenance are +decided by content, never by which was declared first. That is a real guarantee rather than a stylistic claim, because the loader does not need an order either — it derives overlay precedence from the files themselves @@ -380,8 +380,14 @@ one line per config. Worth knowing precisely, because the layers are easy to conflate and they are not redundant. -1. **`resolveSources` canonicalizes.** It sorts resolved absolute paths, so in - production the loader never sees a permuted file list at all. +1. **`resolveSources` canonicalizes.** It processes the entries in content order, so + in production the loader never sees a permuted file list at all. Canonical is not + the same as "sorted": within one directory entry the files keep the walk order the + toolchain has always used — the files at a level, then that level's + subdirectories, depth-first — because declaration order survives into generated + output (a barrel's export list, the shared `enums.ts`, `meta docs` page order, + `meta export`'s sibling order). Flat-sorting the paths would silently reorder any + project holding a subdirectory whose name sorts before a sibling file. 2. **The loader resolves content order-independently.** Overlay-only sources are stable-partitioned to merge last, so an overlay reaching a base declared in another file resolves the same regardless of which arrived first. @@ -391,7 +397,9 @@ redundant. permuted loader inputs. Layers 1 and 2 are pinned by -[`server/typescript/packages/sdk/test/order-independence.test.ts`](../../server/typescript/packages/sdk/test/order-independence.test.ts). +[`server/typescript/packages/sdk/test/order-independence.test.ts`](../../server/typescript/packages/sdk/test/order-independence.test.ts); +the per-level walk order layer 1 preserves is pinned by +[`server/typescript/packages/sdk/test/source-order.test.ts`](../../server/typescript/packages/sdk/test/source-order.test.ts). --- diff --git a/server/typescript/packages/sdk/src/memory.ts b/server/typescript/packages/sdk/src/memory.ts index 2257fe419..d3c3caa0e 100644 --- a/server/typescript/packages/sdk/src/memory.ts +++ b/server/typescript/packages/sdk/src/memory.ts @@ -27,11 +27,9 @@ export const DEFAULT_METAOBJECTS_DIR = ".metaobjects"; /** Recognized metadata file extensions, matched case-insensitively — mirrors * `DirectorySource` in `@metaobjectsdev/metadata`, which checks * `extname().toLowerCase()`. The single definition every metadata-file - * walker in this package uses: `sources.ts`'s `resolveSources` imports - * `isMetadataFile` from here rather than keeping its own copy, so the - * package's two walkers cannot silently disagree about whether e.g. - * `meta.JSON` counts (a real drift this fixes — this walk used to be - * case-sensitive while `sources.ts`'s was already case-insensitive). */ + * walker in this package uses — and since `resolveSources` (`sources.ts`) + * now calls {@link listMetadataFiles} outright rather than keeping a second + * recursive walk of its own, there is exactly one walker to keep honest. */ export const METADATA_EXTENSIONS = new Set([".json", ".yaml", ".yml"]); export function isMetadataFile(name: string): boolean { @@ -171,23 +169,46 @@ async function collectMetadataPaths(repoRoot: string): Promise { return listMetadataFiles(join(repoRoot, DEFAULT_METADATA_DIR)); } +/** Directory excluded at every level of {@link listMetadataFiles} — drafts + * that are deliberately not part of the loaded model. */ +const PENDING_DIR = "_pending"; + /** * Recursively list metadata files (*.json, *.yaml, *.yml, matched * case-insensitively — see `isMetadataFile` above) under a directory, * excluding _pending/ at any level. Subdirectories (e.g. projections/) are * walked depth-first. Files within a directory are sorted alphabetically for - * deterministic load order; subdirectories are visited after files at the + * deterministic load order; subdirectories are visited AFTER the files at the * same level. * + * That per-level rule is a contract, not an implementation detail. This is the + * order production has always handed the loader, and declaration order survives + * into generated output: `codegen-ts`'s barrel emits from `root.objects()` + * order, and so do the shared `enums.ts`, `meta docs` page ordering and `meta + * export`'s `canonicalSerialize` sibling order. A flat lexicographic sort of + * absolute paths is NOT the same list — it disagrees whenever a subdirectory + * name sorts before a sibling file (`common/` before `meta.users.json`) — so + * `resolveSources` calls this function rather than re-walking and re-sorting. + * Pinned by `test/source-order.test.ts`. + * + * Exported for that gate and for `sources.ts`; not re-exported from the package + * index — `resolveCollection` is the public door. + * + * An entry whose `stat` fails (a dangling symlink, a TOCTOU removal between + * `readdir` and `stat`, an EACCES entry) is SKIPPED, matching `DirectorySource` + * in `@metaobjectsdev/metadata`, which this walk otherwise mirrors. A failure to + * read the directory itself still throws — that is the "you have no metadata + * here" case callers report. + * * Format selection (parsing) happens downstream in `FileSource` from * `@metaobjectsdev/metadata`, which infers the parser from file extension. */ -async function listMetadataFiles(dir: string): Promise { +export async function listMetadataFiles(dir: string): Promise { let entries: string[]; try { entries = await readdir(dir); } catch (err) { - throw new Error(`loadMemory: cannot read ${dir}: ${(err as Error).message}`); + throw new Error(`cannot read metadata directory ${dir}: ${(err as Error).message}`); } const paths: string[] = []; const subdirs: string[] = []; @@ -198,17 +219,21 @@ async function listMetadataFiles(dir: string): Promise { // deterministic-enumeration FLOOR, not the fix; it keeps every derived artifact // that preserves declaration order, e.g. serialization, stable across runtimes.) for (const entry of [...entries].sort()) { - if (entry === "_pending") continue; + if (entry === PENDING_DIR) continue; const full = join(dir, entry); - const s = await stat(full); + // `stat` (not `lstat`/`Dirent.isDirectory()`) so a symlinked subdirectory is + // traversed — `DirectorySource` has always followed symlinks this way. + const s = await stat(full).catch(() => undefined); + if (s === undefined) continue; if (s.isDirectory()) { subdirs.push(full); } else if (s.isFile() && isMetadataFile(entry)) { paths.push(full); } } - // Recurse into subdirectories after collecting files at this level - for (const sub of subdirs.sort()) { + // Recurse into subdirectories after collecting files at this level. + // `subdirs` is already in sorted order (built from the sorted `entries` above). + for (const sub of subdirs) { paths.push(...(await listMetadataFiles(sub))); } return paths; diff --git a/server/typescript/packages/sdk/src/sources.ts b/server/typescript/packages/sdk/src/sources.ts index c94c47340..7b48f7866 100644 --- a/server/typescript/packages/sdk/src/sources.ts +++ b/server/typescript/packages/sdk/src/sources.ts @@ -3,17 +3,27 @@ // Phase-1 metadata-source-resolution — source spec resolution. // // Turns a declared source SET (`.metaobjects/config.json`'s `sources`) into a -// canonically-sorted, de-duplicated list of metadata file paths. The FULL +// canonically-ordered, de-duplicated list of metadata file paths. The FULL // result — including which spec each entry attributes to — is a pure // function of the source SET, never of declaration order: permuting `specs` -// cannot change the output, even when two specs overlap on the same file. A -// later phase-1 task pins this with a permutation test (the design's -// linchpin), so both the sort-by-absolute-path step and the content-based -// overlap tie-break below are load-bearing. -import { readdir, stat } from "node:fs/promises"; -import { isAbsolute, join, resolve } from "node:path"; +// cannot change the output, even when two specs overlap on the same file. +// `test/order-independence.test.ts` pins that (the design's linchpin), so the +// canonical spec ordering below is load-bearing. +// +// Canonical is NOT the same as "flat-sorted". Within one directory spec the +// order is `listMetadataFiles`'s (memory.ts) — files at a level, then that +// level's subdirectories, depth-first — because that is the order production +// has always handed the loader, and declaration order survives into generated +// output (the barrel's export list, the shared `enums.ts`, `meta docs` page +// order, `meta export`'s sibling order). A flat sort of absolute paths +// silently reorders any project with a subdirectory whose name sorts before a +// sibling file. Across specs, order is decided by spec CONTENT, which is what +// keeps the whole result permutation-invariant. `test/source-order.test.ts` +// pins both halves. +import { stat } from "node:fs/promises"; +import { isAbsolute, resolve } from "node:path"; import { ParseError, codeSource } from "@metaobjectsdev/metadata"; -import { DEFAULT_METADATA_DIR, isMetadataFile } from "./memory.js"; +import { DEFAULT_METADATA_DIR, listMetadataFiles } from "./memory.js"; /** Tagged union of source kinds. `resource` and `package` are declared now so * the config shape is stable across phases; only `path` resolves in phase 1 — @@ -42,36 +52,6 @@ export interface ResolvedSource { * whole mechanism exists to eliminate. */ export const DEFAULT_SOURCES: readonly SourceSpec[] = [{ path: DEFAULT_METADATA_DIR }]; -const PENDING_DIR = "_pending"; - -/** Recursively collect metadata files under `dir`, excluding `_pending/` at any - * depth. Uses `stat` (follows symlinks) rather than `lstat` or - * `Dirent.isDirectory()` — `DirectorySource` in `@metaobjectsdev/metadata` has - * always followed symlinks this way, so a symlinked subdirectory must be - * traversed here too, or this walk and the loader's would silently disagree - * about the same tree. */ -async function collectDir(dir: string, out: string[]): Promise { - const entries = await readdir(dir); - for (const entry of entries) { - if (entry === PENDING_DIR) continue; - const full = join(dir, entry); - let s; - try { - s = await stat(full); - } catch { - // A dangling symlink, a TOCTOU removal between readdir and stat, or an - // inaccessible (EACCES) entry — skip it, matching DirectorySource in - // @metaobjectsdev/metadata (directory-source.ts), which this walk - // otherwise mirrors. An uncaught stat() here would crash - // resolveSources with a raw Node ENOENT on a tree the loader reads - // fine. - continue; - } - if (s.isDirectory()) await collectDir(full, out); - else if (s.isFile() && isMetadataFile(entry)) out.push(full); - } -} - /** Narrows `spec` to its `path` arm, throwing `ERR_SOURCE_KIND_UNSUPPORTED` * for `resource`/`package` — phase 1 resolves `path` only. Called in two * separate passes by {@link resolveSources} (see the comment there): an @@ -87,17 +67,20 @@ function assertPathSpec(spec: SourceSpec): asserts spec is { readonly path: stri } /** - * Resolve a declared source SET to a canonically-sorted list of metadata files. + * Resolve a declared source SET to a canonically-ordered list of metadata files. * * The full result — each entry's `.file` AND its `.spec` — is a pure function - * of the SET of `specs`: permuting `specs` cannot change the output. Two parts - * make that hold: entries are sorted by absolute path (so file ORDER carries no - * declaration-order information), and when two specs overlap on the same file, - * the one attributed is chosen by comparing `JSON.stringify(spec)` — a - * content-only tie-break, so which spec "wins" never depends on which was - * processed first. Declared order carries no information anywhere in this - * function (the loader derives whatever order it needs from the files - * themselves). + * of the SET of `specs`: permuting `specs` cannot change the output. One thing + * makes that hold: the specs are processed in CONTENT order + * (`JSON.stringify(spec)`, ascending) rather than declared order, so both the + * emitted file order and the spec attributed to a file overlapping two specs + * are decided by content alone. Declared order carries no information anywhere + * in this function. + * + * Within one directory spec the file order is `listMetadataFiles`'s — files at + * a level, then that level's subdirectories, depth-first. That is deliberately + * NOT a flat sort of absolute paths: see the file header, and + * `test/source-order.test.ts`. * * Only `path` specs resolve in phase 1: a directory is walked recursively, a * file is taken as-is. An unresolvable `path` throws `ERR_SOURCE_UNRESOLVED` @@ -121,34 +104,37 @@ export async function resolveSources( // "pure function of the SET" invariant (see the file header). for (const spec of specs) assertPathSpec(spec); + // Content order, computed once. This is the ONLY place declaration order is + // discarded, and everything below depends on it: the output file order, the + // spec attributed to an overlapping file, and which of several unresolvable + // paths reports its ERR_SOURCE_UNRESOLVED first. + const ordered = [...specs].sort((a, b) => { + const [ja, jb] = [JSON.stringify(a), JSON.stringify(b)]; + return ja < jb ? -1 : ja > jb ? 1 : 0; + }); + + // Insertion order IS output order — a Map preserves it, so the per-spec walk + // order above survives to the caller. First contributor wins a shared file, + // which is content-determined because `ordered` is. const byFile = new Map(); - for (const spec of specs) { + for (const spec of ordered) { assertPathSpec(spec); // already validated above; narrows `spec.path` for TS below. const target = isAbsolute(spec.path) ? spec.path : resolve(configDir, spec.path); - let stats; - try { - stats = await stat(target); - } catch { + const stats = await stat(target).catch(() => undefined); + if (stats === undefined) { throw new ParseError( `source path "${spec.path}" does not exist (resolved to ${target}, relative to ${configDir})`, { code: "ERR_SOURCE_UNRESOLVED", source: codeSource("resolveSources") }, ); } - const found: string[] = []; - if (stats.isDirectory()) await collectDir(target, found); - else found.push(target); + const found = stats.isDirectory() ? await listMetadataFiles(target) : [target]; for (const file of found) { - const existing = byFile.get(file); - // Content-only tie-break: never "first spec processed wins", or the - // attributed `.spec` would depend on declaration order. - if (existing === undefined || JSON.stringify(spec) < JSON.stringify(existing)) { - byFile.set(file, spec); - } + if (!byFile.has(file)) byFile.set(file, spec); } } - return [...byFile.keys()].sort().map((file) => ({ file, spec: byFile.get(file)! })); + return [...byFile].map(([file, spec]) => ({ file, spec })); } diff --git a/server/typescript/packages/sdk/test/dogfood-examples.test.ts b/server/typescript/packages/sdk/test/dogfood-examples.test.ts index 54428374f..cd5a119e8 100644 --- a/server/typescript/packages/sdk/test/dogfood-examples.test.ts +++ b/server/typescript/packages/sdk/test/dogfood-examples.test.ts @@ -16,13 +16,15 @@ import { mkdtempSync, rmSync, mkdirSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { basename, join, resolve } from "node:path"; import { resolveCollection } from "../src/collection.js"; -import { loadMemory } from "../src/memory.js"; +import { DEFAULT_METADATA_DIR, listMetadataFiles, loadMemory } from "../src/memory.js"; import { compileScope, matchesScope } from "../src/scope.js"; // Located relative to this test file (never a hardcoded absolute home path — // this repo is public) so the test runs unchanged on any checkout. const EXAMPLES_PROJECT = resolve(import.meta.dir, "../../../../../examples/advanced-modeling"); -const EXAMPLES = join(EXAMPLES_PROJECT, "metaobjects"); +// `DEFAULT_METADATA_DIR`, not the literal: this file dogfoods the rule that no +// call site may assume that directory name, so it must not assume it either. +const EXAMPLES = join(EXAMPLES_PROJECT, DEFAULT_METADATA_DIR); // The tree holds exactly these three files today (verified at HEAD by the // controller before dispatching this task) — asserted as a floor ("contains @@ -31,14 +33,16 @@ const EXAMPLES = join(EXAMPLES_PROJECT, "metaobjects"); const KNOWN_BASENAMES = ["meta.catalog.yaml", "meta.content.yaml", "meta.prompts.yaml"]; /** Shared shape for both dogfood file-set assertions (F22): every resolved - * path sits under `under`, the known files are all present, and the set is - * sorted — `resolveSources` canonicalizes by sorting absolute paths, so - * pinning that here is what makes load order irrelevant to a consumer. */ -function assertKnownFileSet(files: readonly string[], under: string): void { + * path sits under `under`, the known files are all present, and the order is + * exactly the walk order the toolchain has always used — asserted by calling + * `listMetadataFiles` on the same tree rather than restating a rule, so a + * consumer reaching this collection from elsewhere gets byte-identical + * generated output to a consumer sitting on top of it. */ +async function assertKnownFileSet(files: readonly string[], under: string): Promise { expect(files.every((f) => f.startsWith(under))).toBe(true); const names = files.map((f) => basename(f)); for (const known of KNOWN_BASENAMES) expect(names).toContain(known); - expect([...files]).toEqual([...files].sort()); + expect([...files]).toEqual(await listMetadataFiles(under)); } describe("dogfood: a consumer reaches the in-repo examples tree", () => { @@ -59,7 +63,7 @@ describe("dogfood: a consumer reaches the in-repo examples tree", () => { test("resolves every metadata file in it", async () => { const c = await resolveCollection(join(consumer, "apps/ui")); - assertKnownFileSet(c.files, EXAMPLES); + await assertKnownFileSet(c.files, EXAMPLES); }); test("the resolved set loads without errors", async () => { @@ -73,7 +77,7 @@ describe("dogfood: the examples project's own committed config (sources: [])", ( test("falls back to metaobjects/ under the project root, exactly like every adopter's default config", async () => { const c = await resolveCollection(EXAMPLES_PROJECT); expect(c.configDir).toBe(EXAMPLES_PROJECT); - assertKnownFileSet(c.files, EXAMPLES); + await assertKnownFileSet(c.files, EXAMPLES); }); }); diff --git a/server/typescript/packages/sdk/test/order-independence.test.ts b/server/typescript/packages/sdk/test/order-independence.test.ts index c6e9ce45f..9d4b149e0 100644 --- a/server/typescript/packages/sdk/test/order-independence.test.ts +++ b/server/typescript/packages/sdk/test/order-independence.test.ts @@ -10,10 +10,14 @@ // documentation of record on how each one is satisfied — deliberately not // collapsed into one over-broad assertion, because two earlier drafts of // this gate got that collapse wrong in opposite directions: -// 1. `resolveSources` CANONICALIZES file order — it sorts its output by -// absolute path (sources.ts:127), so every permutation of a declared -// source SET collapses to the same file list before the loader ever -// runs. Test 1 pins this directly. +// 1. `resolveSources` CANONICALIZES file order — it walks the specs in +// CONTENT order rather than declared order, so every permutation of a +// declared source SET collapses to the same file list before the loader +// ever runs. Test 1 pins this directly. (What that canonical order IS — +// per-directory-level, files before subdirectories, never a flat sort of +// absolute paths — is a separate contract, pinned by +// `source-order.test.ts`. This file only asserts it does not depend on +// declaration order.) // 2. The LOADER resolves CONTENT order-independently, given whatever file // list it's handed — including an overlay arriving before its base. // `_partitionOverlayLast` is the mechanism (stable-partitions diff --git a/server/typescript/packages/sdk/test/source-order.test.ts b/server/typescript/packages/sdk/test/source-order.test.ts new file mode 100644 index 000000000..7bce9ace0 --- /dev/null +++ b/server/typescript/packages/sdk/test/source-order.test.ts @@ -0,0 +1,129 @@ +// server/typescript/packages/sdk/test/source-order.test.ts +// +// The load-ORDER gate for resolved sources. +// +// Order independence (`order-independence.test.ts`) proves the resolved SET +// is a pure function of the declared spec set. It says nothing about the +// order that set is handed to the loader in — and that order IS observable in +// generated output: `codegen-ts`'s barrel emits exports straight from +// `root.objects()` order, and the same order flows into the shared `enums.ts`, +// `meta docs` page ordering, and `meta export`'s `canonicalSerialize` sibling +// order. +// +// The pre-source-resolution toolchain read every file through +// `listMetadataFiles` (memory.ts), which visits FILES at a level before +// descending into that level's subdirectories. A flat lexicographic sort of +// absolute paths disagrees with it the moment a subdirectory name sorts before +// a sibling file — `metaobjects/common/…` before `metaobjects/meta.users.json` +// is exactly that shape, and it is the shape this fixture builds. Every +// `metaobjects/` tree committed in this repository is FLAT, so nothing else +// here can observe the property. +// +// The structural half of the fix is that there is now ONE walker: +// `resolveSources` calls `listMetadataFiles` rather than keeping a second +// recursive walk of its own. These tests pin the resulting order directly, so +// re-splitting the walkers (or "simplifying" either one back to a flat sort) +// goes red rather than silently reordering everyone's generated code. +import { describe, test, expect, beforeEach, afterEach } from "bun:test"; +import { mkdtempSync, rmSync, mkdirSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import { DEFAULT_METADATA_DIR, listMetadataFiles, loadMemory } from "../src/memory.js"; +import { resolveSources, type SourceSpec } from "../src/sources.js"; +import { resolveCollection } from "../src/collection.js"; + +let root: string; + +const write = (rel: string, body: object): void => { + const full = join(root, rel); + mkdirSync(dirname(full), { recursive: true }); + writeFileSync(full, JSON.stringify(body), "utf8"); +}; + +const entity = (pkg: string, name: string): object => ({ + "metadata.root": { + package: pkg, + children: [ + { "object.entity": { name, children: [{ "field.string": { name: "id" } }] } }, + ], + }, +}); + +const relative = (files: readonly string[]): string[] => + files.map((f) => f.slice(root.length + 1)); + +beforeEach(() => { + root = mkdtempSync(join(tmpdir(), "metaobjects-source-order-")); + // `common` sorts BEFORE `meta.users.json` as a plain string, so a flat sort + // of absolute paths puts the SUBDIRECTORY first. The walker production used + // before this branch puts the file first. + write(join(DEFAULT_METADATA_DIR, "common", "meta.base.json"), entity("acme", "BaseThing")); + write(join(DEFAULT_METADATA_DIR, "meta.users.json"), entity("acme", "User")); +}); +afterEach(() => { + rmSync(root, { recursive: true, force: true }); +}); + +describe("resolved source order — nested directories", () => { + test("files at a level come before that level's subdirectories", async () => { + const out = await resolveSources(root, [{ path: DEFAULT_METADATA_DIR }]); + expect(relative(out.map((r) => r.file))).toEqual([ + "metaobjects/meta.users.json", + "metaobjects/common/meta.base.json", + ]); + }); + + test("the fixture actually discriminates — a flat sort would order it the other way", async () => { + // Without this the assertion above could pass on a flat-sorting resolver + // and nobody would notice the gate had stopped testing anything. + const out = await resolveSources(root, [{ path: DEFAULT_METADATA_DIR }]); + const files = out.map((r) => r.file); + expect([...files].sort()).not.toEqual(files); + }); + + test("resolveSources agrees with listMetadataFiles, the walker production used before this branch", async () => { + const out = await resolveSources(root, [{ path: DEFAULT_METADATA_DIR }]); + const legacy = await listMetadataFiles(join(root, DEFAULT_METADATA_DIR)); + expect(out.map((r) => r.file)).toEqual(legacy); + }); + + test("resolveCollection's default path resolves that same order", async () => { + const collection = await resolveCollection(root); + const legacy = await listMetadataFiles(join(root, DEFAULT_METADATA_DIR)); + expect([...collection.files]).toEqual(legacy); + }); + + test("the loaded tree's sibling order follows it — the observable half", async () => { + // The reason any of this matters: declaration order survives into + // `root.children()`, which is what the barrel generator emits from. + const collection = await resolveCollection(root); + const loaded = await loadMemory(collection.configDir, { files: collection.files }); + expect(loaded.children().map((c) => c.name)).toEqual(["User", "BaseThing"]); + }); +}); + +describe("resolved source order — across several specs", () => { + beforeEach(() => { + write(join("extra", "nested", "meta.deep.json"), entity("acme", "Deep")); + write(join("extra", "meta.top.json"), entity("acme", "Top")); + }); + + test("each spec contributes its own per-level order, and specs are ordered by content", async () => { + // "extra" sorts before "metaobjects", so its files lead; within each spec + // the per-level rule applies. + const out = await resolveSources(root, [{ path: DEFAULT_METADATA_DIR }, { path: "extra" }]); + expect(relative(out.map((r) => r.file))).toEqual([ + "extra/meta.top.json", + "extra/nested/meta.deep.json", + "metaobjects/meta.users.json", + "metaobjects/common/meta.base.json", + ]); + }); + + test("permuting the specs cannot change the order (set purity survives the per-level walk)", async () => { + const specs: SourceSpec[] = [{ path: DEFAULT_METADATA_DIR }, { path: "extra" }]; + const forward = await resolveSources(root, specs); + const reverse = await resolveSources(root, [...specs].reverse()); + expect(forward).toEqual(reverse); + }); +}); From 202ad65ca7693281cbb8bb9250762607ac966701 Mon Sep 17 00:00:00 2001 From: Doug Mealing Date: Tue, 18 Aug 2026 07:50:54 -0400 Subject: [PATCH 27/44] fix(migrate): a migrate.scope matching nothing no longer proposes DROP TABLE MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Narrowing the expected side could WIDEN what the diff governs. `diff` derives its SCHEMA scope from the schemas the expected side mentions and falls back to "no schema scoping at all" when expected is empty (the legacy whole-database path for a project with no model). A `migrate.scope` matching nothing empties `expected`, reaches that fallback, and every actual table in every schema becomes a drop candidate — another owner's included, which was never in `expected` so it carries no provenance and never lands in `outOfScope` either. So the declaration whose entire purpose is to stop migrate touching another owner's tables CAUSED migrate to propose dropping one, silently, whenever the scope was wrong. The same mechanism reached `verify --db` as phantom drift. Structural fix: `scopeExpectedSchema` reports the UNSCOPED model's schemas as `declaredSchemas`, and every narrowing caller threads it into `diff`'s existing `scopeSchemas` — `planOffline`, `computeDriftFromActual`, both CLI migrate paths, and verify's committed-snapshot gate (which narrows the snapshot by the same `outOfScope` set and had the identical hole). The schema scope is now a property of the whole model, which `migrate.scope` cannot move in either direction. An UNSCOPED project passes no `scopeSchemas` at all, exactly as before — `declaredSchemas` is undefined without a predicate, so its arguments to `diff` are unchanged. Defensive fix: a `migrate.scope` matching zero loaded objects is refused, on all four command paths, naming the patterns that missed and the FQNs that were loaded. It can never be what someone meant, and left alone it is silent — migrate reports "no changes" having compared nothing. `Collection` carries `migrateScopePatterns` so the message can show what the author wrote rather than a compiled regex source. Refused inside verify's schema gate rather than beside its collection load, so a stale pattern cannot fail a `--templates` run that never consults it. Gated by `migrate-ts/test/scope-empty-match.test.ts` (the reviewer's probe: control drops nothing, empty-match scope must too) and `cli/test/integration/migrate-db-scope.test.ts` on the online `--db` path, which had no scope test at all. Co-Authored-By: Claude Opus 5 (1M context) --- .../packages/cli/src/commands/migrate.ts | 31 +++- .../packages/cli/src/commands/verify.ts | 23 ++- .../packages/cli/src/lib/migrate-scope.ts | 55 ++++++- .../test/integration/migrate-db-scope.test.ts | 135 ++++++++++++++++++ .../packages/migrate-ts/src/drift/drift.ts | 6 + .../packages/migrate-ts/src/index.ts | 2 +- .../packages/migrate-ts/src/scope.ts | 51 +++++++ .../packages/migrate-ts/src/snapshot/plan.ts | 6 + .../migrate-ts/test/scope-empty-match.test.ts | 135 ++++++++++++++++++ .../typescript/packages/sdk/src/collection.ts | 12 +- 10 files changed, 451 insertions(+), 5 deletions(-) create mode 100644 server/typescript/packages/cli/test/integration/migrate-db-scope.test.ts create mode 100644 server/typescript/packages/migrate-ts/test/scope-empty-match.test.ts diff --git a/server/typescript/packages/cli/src/commands/migrate.ts b/server/typescript/packages/cli/src/commands/migrate.ts index de7665906..9d691f308 100644 --- a/server/typescript/packages/cli/src/commands/migrate.ts +++ b/server/typescript/packages/cli/src/commands/migrate.ts @@ -12,7 +12,7 @@ import { buildKyselyFromUrl, redactUrl } from "../lib/kysely.js"; import { log } from "../lib/log.js"; import { loadMemory, resolveCollection } from "@metaobjectsdev/sdk"; import { loadMetaobjectsConfig } from "../lib/load-metaobjects-config.js"; -import { toObjectScope } from "../lib/migrate-scope.js"; +import { migrateScopeMismatch, toObjectScope } from "../lib/migrate-scope.js"; import { buildExpectedSchemaWithProvenance, scopeExpectedSchema, @@ -428,6 +428,13 @@ export async function migrateCommand( return 2; } + const scopeMismatch = migrateScopeMismatch(collection, metadata); + if (scopeMismatch !== undefined) { + log.error(`migrate: ${scopeMismatch}`); + emitStructuredError(`migrate: ${scopeMismatch}`, "fix or remove migrate.scope in .metaobjects/config.json", fmt); + return 2; + } + let kysely; try { kysely = await buildKyselyFromUrl(config.databaseUrl, config.dialect); @@ -500,6 +507,11 @@ export async function migrateCommand( // actual side so migrate proposes neither create nor drop for them. Objects // outside `migrate.scope` ride the same seam, for the same reason. unmanagedNames: [...collectUnmanagedNames(metadata), ...scoped.outOfScope], + // Pin the schema scope to the UNSCOPED model's schemas (see migrate-ts's + // scope.ts header): a `migrate.scope` matching nothing would otherwise empty + // `expected`, which `diff` reads as "no model, govern the whole database". + // Absent when no scope was given, so an unscoped run is unchanged. + ...(scoped.declaredSchemas !== undefined ? { scopeSchemas: scoped.declaredSchemas } : {}), onAmbiguous: async (a) => { collectedAmbiguous.push(a); return onAmbiguousResolution; @@ -933,6 +945,13 @@ export async function runOfflineGenerate( return 2; } + const offlineScopeMismatch = migrateScopeMismatch(collection, metadata); + if (offlineScopeMismatch !== undefined) { + log.error(`migrate: ${offlineScopeMismatch}`); + emitStructuredError(`migrate: ${offlineScopeMismatch}`, "fix or remove migrate.scope in .metaobjects/config.json", fmt); + return 2; + } + const outDir = resolvePath(metaRoot, config.outDir); const path = snapshotPath(outDir, config.dialect); let snapshot; @@ -1171,6 +1190,13 @@ async function runD1Migrate( return 2; } + const d1ScopeMismatch = migrateScopeMismatch(collection, metadata); + if (d1ScopeMismatch !== undefined) { + log.error(`migrate: ${d1ScopeMismatch}`); + emitStructuredError(`migrate: ${d1ScopeMismatch}`, "fix or remove migrate.scope in .metaobjects/config.json", fmt); + return 2; + } + // 4. Build expected schema + introspect actual. let columnNamingStrategy: "snake_case" | "literal" | "kebab-case" = "snake_case"; try { @@ -1221,6 +1247,9 @@ async function runD1Migrate( // #208 §7 — declared-@unmanaged objects are external (see the online path above), // and so are objects outside `migrate.scope`. unmanagedNames: [...collectUnmanagedNames(metadata), ...scoped.outOfScope], + // Schema scope pinned to the UNSCOPED model's schemas — same reasoning as the + // online path above (migrate-ts's scope.ts header has the mechanism). + ...(scoped.declaredSchemas !== undefined ? { scopeSchemas: scoped.declaredSchemas } : {}), onAmbiguous: async (a) => { collectedAmbiguous.push(a); return onAmbiguousResolution; diff --git a/server/typescript/packages/cli/src/commands/verify.ts b/server/typescript/packages/cli/src/commands/verify.ts index 14f745214..e2e82ea47 100644 --- a/server/typescript/packages/cli/src/commands/verify.ts +++ b/server/typescript/packages/cli/src/commands/verify.ts @@ -33,6 +33,7 @@ import { computeDrift, computeDriftFromActual, collectUnmanagedNames, + declaredSchemasOf, qualifiedDbName, introspect, diff, @@ -49,7 +50,7 @@ import { type DriftResult, } from "@metaobjectsdev/migrate-ts"; import { loadMemory, resolveCollection, matchesScope } from "@metaobjectsdev/sdk"; -import { toObjectScope } from "../lib/migrate-scope.js"; +import { migrateScopeMismatch, toObjectScope } from "../lib/migrate-scope.js"; import { TYPE_TEMPLATE, TEMPLATE_SUBTYPE_PROMPT, @@ -400,6 +401,17 @@ export async function verifyCommand( const usingD1 = flags.dialect === "d1"; if ((flags.db === undefined && !usingD1) || flags.skipSchema) return 0; + // A `migrate.scope` matching nothing is refused, not tolerated — it would make + // this gate compare zero objects and report "in sync" (see `migrateScopeMismatch`). + // Checked HERE rather than beside the other collection work at the top of + // `verifyCommand`, because `migrate.scope` governs only the schema gate: a stale + // pattern must not fail a `--templates` run that never consults it. + const scopeMismatch = migrateScopeMismatch(collection, root); + if (scopeMismatch !== undefined) { + log.error(`verify: ${scopeMismatch}`); + return 2; + } + if (usingD1 && flags.db !== undefined) { log.error(`verify: --db is not used for dialect 'd1' — wrangler.toml owns the connection; pass --d1 instead`); return 2; @@ -619,6 +631,15 @@ export async function verifyCommand( actual, allow: {}, unmanagedNames: [...collectUnmanagedNames(root), ...outOfScope], + // Pin the schema scope to the UNFILTERED snapshot's schemas whenever the + // filter above removed anything (migrate-ts's scope.ts header has the + // mechanism): filtering `expected` down to empty would otherwise reach + // `diff`'s "no model, govern the whole database" fallback and report every + // table another owner has as a snapshot disagreement. Nothing filtered ⇒ + // nothing passed ⇒ an unscoped project's arguments are unchanged. + ...(excluded.size > 0 && declaredSchemasOf(snapshot).length > 0 + ? { scopeSchemas: declaredSchemasOf(snapshot) } + : {}), }); if (result.changes.length === 0) return []; diff --git a/server/typescript/packages/cli/src/lib/migrate-scope.ts b/server/typescript/packages/cli/src/lib/migrate-scope.ts index ab84f2741..fe0d717ed 100644 --- a/server/typescript/packages/cli/src/lib/migrate-scope.ts +++ b/server/typescript/packages/cli/src/lib/migrate-scope.ts @@ -11,7 +11,8 @@ // the identical object set, so they share the one declaration rather than each // growing a key of its own. -import { matchesScope, type CompiledScope } from "@metaobjectsdev/sdk"; +import { matchesScope, type Collection, type CompiledScope } from "@metaobjectsdev/sdk"; +import type { MetaRoot } from "@metaobjectsdev/metadata"; import type { ObjectScopePredicate } from "@metaobjectsdev/migrate-ts"; /** @@ -22,3 +23,55 @@ import type { ObjectScopePredicate } from "@metaobjectsdev/migrate-ts"; export function toObjectScope(scope: CompiledScope | undefined): ObjectScopePredicate | undefined { return scope === undefined ? undefined : (fqn: string): boolean => matchesScope(fqn, scope); } + +/** How many loaded FQNs to name in the refusal below — enough to show the shape + * an author's patterns have to match, short enough to stay readable. */ +const EXAMPLE_FQN_CAP = 3; + +/** + * The refusal for a `migrate.scope` that matches NOTHING. + * + * A scope matching zero loaded objects can never be what someone meant — it says + * "every table in this model belongs to somebody else", which is a project with no + * schema to migrate at all, expressed the hard way. In practice it is a typo'd or + * stale package pattern, and it is silent: migrate reports "no changes" while + * having compared nothing. + * + * It is also actively dangerous, which is why this is a refusal and not a warning. + * An empty expected side is what `diff` reads as "no model, govern the whole + * database" — the inversion `scopeExpectedSchema`'s `declaredSchemas` closes + * structurally (migrate-ts `scope.ts`). This is the second lock on the same door: + * the structural fix stops a wrong scope proposing a destructive change, and this + * stops the wrong scope going unnoticed in the first place. + * + * Returns the message to report, or `undefined` when there is nothing to refuse — + * no scope declared, or at least one loaded object inside it. Callers report it and + * exit 2 (a configuration error), rather than this throwing, so it reads like every + * other config failure in these commands. + */ +export function migrateScopeMismatch( + collection: Collection, + root: MetaRoot, +): string | undefined { + const { migrateScope, migrateScopePatterns } = collection; + if (migrateScope === undefined) return undefined; + + // ADR-0039: `objects()` is the resolving accessor — the loaded object set, which + // is exactly what `migrate.scope` claims to be a subset of. + const fqns = root.objects().map((o) => o.resolutionKey()); + // No objects at all is not a scope error: there is nothing for a pattern to miss, + // and an empty model has its own (much louder) failure modes downstream. + if (fqns.length === 0) return undefined; + if (fqns.some((fqn) => matchesScope(fqn, migrateScope))) return undefined; + + const patterns = JSON.stringify(migrateScopePatterns ?? []); + const examples = fqns.slice(0, EXAMPLE_FQN_CAP).join(", "); + const more = fqns.length > EXAMPLE_FQN_CAP ? `, …and ${fqns.length - EXAMPLE_FQN_CAP} more` : ""; + return ( + `migrate.scope matched none of the ${fqns.length} object(s) loaded, so this run would ` + + `treat every one of them as another owner's and compare nothing. ` + + `Patterns: ${patterns}. Loaded: ${examples}${more}. ` + + `Fix the patterns in .metaobjects/config.json (migrate.scope), or remove the key to ` + + `govern everything loaded.` + ); +} diff --git a/server/typescript/packages/cli/test/integration/migrate-db-scope.test.ts b/server/typescript/packages/cli/test/integration/migrate-db-scope.test.ts new file mode 100644 index 000000000..118c4a687 --- /dev/null +++ b/server/typescript/packages/cli/test/integration/migrate-db-scope.test.ts @@ -0,0 +1,135 @@ +/** + * `meta migrate --db` (the ONLINE, live-introspection path) and `migrate.scope`. + * + * The scope feature shipped with no test on this path at all — every existing scope + * test drives either the offline diff or `verify --db`. It is also the path where a + * wrong scope is most expensive: it introspects a real database and writes DDL. + * + * The case under test is the one that inverts: a scope matching NOTHING. It is + * always an authoring error (a typo'd or stale package pattern), it can never be + * what someone meant, and left alone it is silent — migrate reports "no changes" + * having compared nothing, while an empty expected side is exactly what the diff + * reads as "no model, govern the whole database". Refused, with the patterns and + * the loaded FQNs named, so the author can see what missed. + */ +import { describe, test, expect, beforeEach, afterEach } from "bun:test"; +import { mkdtempSync, mkdirSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { run } from "../../src/index.js"; + +const PLATFORM = JSON.stringify({ + "metadata.root": { + package: "acme::platform", + children: [{ + "object.entity": { + name: "Job", + children: [ + { "source.rdb": { name: "src", "@table": "jobs" } }, + { "field.long": { name: "id" } }, + { "field.string": { name: "title" } }, + { "identity.primary": { name: "pk", "@fields": ["id"] } }, + ], + }, + }], + }, +}); + +/** Another owner's package, sharing the database. */ +const ARENA = JSON.stringify({ + "metadata.root": { + package: "arena", + children: [{ + "object.entity": { + name: "Match", + children: [ + { "source.rdb": { name: "src", "@table": "matches" } }, + { "field.long": { name: "id" } }, + { "identity.primary": { name: "pk", "@fields": ["id"] } }, + ], + }, + }], + }, +}); + +function scaffold(): { repo: string; dbUrl: string } { + const repo = mkdtempSync(join(tmpdir(), "metaobjects-migrate-scope-")); + mkdirSync(join(repo, "metaobjects"), { recursive: true }); + writeFileSync(join(repo, "metaobjects", "meta.platform.json"), PLATFORM, "utf8"); + writeFileSync(join(repo, "metaobjects", "meta.arena.json"), ARENA, "utf8"); + return { repo, dbUrl: `file:${join(repo, "local.db")}` }; +} + +function declareScope(repo: string, scope: string[]): void { + mkdirSync(join(repo, ".metaobjects"), { recursive: true }); + writeFileSync( + join(repo, ".metaobjects", "config.json"), + JSON.stringify({ schema_version: 1, migrate: { scope } }), + "utf8", + ); +} + +const migrateFromDb = (repo: string, dbUrl: string): Promise => + run(["migrate", "--from-db", "--cwd", repo, "--db", dbUrl, "--dialect", "sqlite", "--slug", "initial"]); + +let out: string[]; +let err: string[]; +let origLog: typeof console.log; +let origErr: typeof console.error; + +beforeEach(() => { + out = []; + err = []; + origLog = console.log; + origErr = console.error; + console.log = (...a: unknown[]) => { out.push(a.map(String).join(" ")); }; + console.error = (...a: unknown[]) => { err.push(a.map(String).join(" ")); }; +}); +afterEach(() => { + console.log = origLog; + console.error = origErr; +}); + +describe("meta migrate --db — migrate.scope", () => { + test("a scope matching NO loaded object is refused, naming the patterns and what was loaded", async () => { + const { repo, dbUrl } = scaffold(); + try { + declareScope(repo, ["typo::**"]); + expect(await migrateFromDb(repo, dbUrl)).toBe(2); + const all = [...out, ...err].join("\n"); + expect(all).toContain("matched none"); + // The patterns that missed, and the shape they had to match — an author + // cannot fix a typo from "your scope matched nothing" alone. + expect(all).toContain("typo::**"); + expect(all).toContain("acme::platform::Job"); + } finally { + rmSync(repo, { recursive: true, force: true }); + } + }); + + test("a scope that matches something still runs (the refusal is not a blanket break)", async () => { + const { repo, dbUrl } = scaffold(); + try { + declareScope(repo, ["acme::platform::**"]); + expect(await migrateFromDb(repo, dbUrl)).toBe(0); + const all = [...out, ...err].join("\n"); + expect(all).not.toContain("matched none"); + // `matches` belongs to the other owner: reported as out-of-scope, never created. + expect(all).toContain("out-of-scope"); + } finally { + rmSync(repo, { recursive: true, force: true }); + } + }); + + test("no migrate.scope declared — unchanged, both tables governed", async () => { + const { repo, dbUrl } = scaffold(); + try { + expect(await migrateFromDb(repo, dbUrl)).toBe(0); + const all = [...out, ...err].join("\n"); + expect(all).not.toContain("matched none"); + expect(all).not.toContain("out-of-scope"); + } finally { + rmSync(repo, { recursive: true, force: true }); + } + }); +}); diff --git a/server/typescript/packages/migrate-ts/src/drift/drift.ts b/server/typescript/packages/migrate-ts/src/drift/drift.ts index bf8dda28c..28abd9285 100644 --- a/server/typescript/packages/migrate-ts/src/drift/drift.ts +++ b/server/typescript/packages/migrate-ts/src/drift/drift.ts @@ -107,6 +107,12 @@ export async function computeDriftFromActual( // Out-of-scope objects join it: dropping them from `expected` alone would turn each // one that EXISTS in the database into a spurious drop-* drift. unmanagedNames: [...collectUnmanagedNames(metadata), ...scoped.outOfScope], + // Pin the schema scope to the UNSCOPED model's schemas (see scope.ts's header): + // a scope matching nothing would otherwise empty `expected`, which `diff` reads + // as "no model, govern the whole database" — reporting phantom drift for every + // table another owner has in a schema this model never mentions. Absent when no + // scope was given, so an unscoped run is unchanged. + ...(scoped.declaredSchemas !== undefined ? { scopeSchemas: scoped.declaredSchemas } : {}), ...(opts?.ignoreTables !== undefined ? { ignoreTables: opts.ignoreTables } : {}), }); return { ...result, outOfScope: scoped.outOfScope }; diff --git a/server/typescript/packages/migrate-ts/src/index.ts b/server/typescript/packages/migrate-ts/src/index.ts index c1770220b..12d189b01 100644 --- a/server/typescript/packages/migrate-ts/src/index.ts +++ b/server/typescript/packages/migrate-ts/src/index.ts @@ -15,7 +15,7 @@ export { diff } from "./diff/index.js"; export { collectUnmanagedNames } from "./unmanaged.js"; // Per-command scope (`migrate.scope`) — see scope.ts for why the suppression is // two-sided and why the pattern engine stays in @metaobjectsdev/sdk. -export { scopeExpectedSchema } from "./scope.js"; +export { scopeExpectedSchema, declaredSchemasOf } from "./scope.js"; export type { ObjectScopePredicate, ScopedExpectedSchema } from "./scope.js"; export { qualifiedDbName } from "./qualified-name.js"; export { computeDrift, computeDriftFromActual, type ComputeDriftOptions, type DriftResult } from "./drift/drift.js"; diff --git a/server/typescript/packages/migrate-ts/src/scope.ts b/server/typescript/packages/migrate-ts/src/scope.ts index 797911e30..ece625509 100644 --- a/server/typescript/packages/migrate-ts/src/scope.ts +++ b/server/typescript/packages/migrate-ts/src/scope.ts @@ -12,7 +12,19 @@ // EXISTS in the database becomes a proposed `DROP TABLE` — the precise hazard this // feature exists to remove. `scopeExpectedSchema` therefore returns both halves and // callers must thread `outOfScope` into the diff. +// +// There is a THIRD half, and it is the one that bites hardest when the scope is +// wrong. `diff` derives its SCHEMA scope from the schemas the expected side +// mentions, falling back to "no schema scoping at all" when expected is empty (the +// legacy whole-DB path for a project with no model). A scope matching NOTHING +// empties `expected`, reaches that fallback, and every actual table in every schema +// becomes a drop candidate — another owner's included, which was never in `expected` +// so it has no provenance and never lands in `outOfScope`. Narrowing must never +// WIDEN. `declaredSchemas` below reports the UNSCOPED model's schemas so callers can +// pin `diff`'s `scopeSchemas` to a property of the whole model, which `migrate.scope` +// then cannot move in either direction. +import { DEFAULT_DB_SCHEMA_POSTGRES } from "@metaobjectsdev/metadata"; import type { ExpectedSchemaWithProvenance } from "./expected-schema.js"; import { qualifiedDbName } from "./qualified-name.js"; import type { SchemaSnapshot } from "./types.js"; @@ -36,6 +48,39 @@ export interface ScopedExpectedSchema { * too — see the module header. */ outOfScope: string[]; + /** + * The database schemas the UNSCOPED model declares, for `diff`'s `scopeSchemas`. + * MUST be threaded there by every caller that narrows — see the module header: + * without it a scope matching nothing hands `diff` an empty expected side, which + * it reads as "no model, govern the whole database". + * + * `undefined` when no predicate was supplied (so `diff` derives its own set from + * an untouched `expected`, exactly as before — an unscoped project's arguments are + * unchanged) and also when the unscoped model declares no tables or views at all + * (nothing to derive from; `diff`'s legacy whole-DB fallback is preserved). + */ + declaredSchemas?: string[]; +} + +/** + * The distinct database schemas a snapshot's tables and views sit in, absent + * normalized to the Postgres default — the value `diff` derives for itself when no + * `scopeSchemas` is supplied. The ONE definition: any caller narrowing an expected + * side must pin `diff`'s schema scope to the UNNARROWED snapshot's schemas, and a + * second encoding of "absent means public" here would silently disagree with the + * one inside `diff`. + * + * Empty in ⇒ empty out, which callers translate to "pass nothing", preserving + * `diff`'s legacy whole-database fallback for a genuinely empty model. + */ +export function declaredSchemasOf(snapshot: SchemaSnapshot): string[] { + return [ + ...new Set( + [...snapshot.tables, ...snapshot.views].map( + (o) => o.schema ?? DEFAULT_DB_SCHEMA_POSTGRES, + ), + ), + ].sort(); } /** @@ -56,6 +101,11 @@ export function scopeExpectedSchema( ): ScopedExpectedSchema { if (inScope === undefined) return { snapshot: built.snapshot, outOfScope: [] }; + // Computed from `built.snapshot` — the UNSCOPED side — deliberately, and before + // the filter below runs. Deriving it from the survivors would reproduce exactly + // the defect this exists to close. + const declared = declaredSchemasOf(built.snapshot); + const outOfScope: string[] = []; const governed = (obj: T): boolean => { const qualified = qualifiedDbName(obj); @@ -72,5 +122,6 @@ export function scopeExpectedSchema( views: built.snapshot.views.filter(governed), }, outOfScope, + ...(declared.length > 0 ? { declaredSchemas: declared } : {}), }; } diff --git a/server/typescript/packages/migrate-ts/src/snapshot/plan.ts b/server/typescript/packages/migrate-ts/src/snapshot/plan.ts index f05a9f67d..0ee766a17 100644 --- a/server/typescript/packages/migrate-ts/src/snapshot/plan.ts +++ b/server/typescript/packages/migrate-ts/src/snapshot/plan.ts @@ -62,6 +62,12 @@ export async function planOffline(args: PlanOfflineArgs): Promise { + const loaded = await new MetaDataLoader().load([new InMemoryStringSource(PLATFORM)]); + expect(loaded.errors).toHaveLength(0); + return loaded.root; +} + +/** Another owner's table, in the default schema this model never mentions. */ +const OTHER_APP_TABLE: TableDescriptor = { + name: "other_app_table", + schema: "public", + columns: [{ name: "id", sqlType: { kind: "integer", bits: 64 }, nullable: false }], + indexes: [], + foreignKeys: [], + checks: [], + primaryKey: ["id"], +}; + +/** The live database: this consumer's table exactly as declared, plus a table + * belonging to someone else entirely. */ +async function actualSchema(root: MetaRoot): Promise { + const mine = buildExpectedSchema(root, { dialect: "postgres" }); + return { ...mine, tables: [...mine.tables, OTHER_APP_TABLE] }; +} + +/** A `migrate.scope` that matches nothing — a typo'd or stale package pattern. */ +const matchesNothing = (): boolean => false; + +/** The proposed `DROP TABLE` names, by table name — the whole point of the probe. + * Narrowed on the discriminant so `table` is the string arm, never a descriptor. */ +const dropped = (changes: readonly Change[]): string[] => + changes.filter((c) => c.kind === "drop-table").map((c) => c.table); + +describe("migrate.scope matching nothing", () => { + test("CONTROL — with no scope at all, another owner's table in an undeclared schema is left alone", async () => { + const root = await loadPlatform(); + const plan = await planOffline({ + metadata: root, + dialect: "postgres", + snapshot: await actualSchema(root), + allow: { dropTable: true }, + }); + expect(dropped(plan.diff.changes)).toEqual([]); + }); + + test("a scope that matches nothing must NOT propose dropping it either", async () => { + const root = await loadPlatform(); + const plan = await planOffline({ + metadata: root, + dialect: "postgres", + snapshot: await actualSchema(root), + inScope: matchesNothing, + allow: { dropTable: true }, + }); + // The consumer's own table left the expected side, as declared... + expect(plan.outOfScope).toEqual(["acme.jobs"]); + // ...and nothing at all is proposed for the schema the model never mentions. + expect(dropped(plan.diff.changes)).toEqual([]); + }); + + test("verify --db sees the same thing — no phantom drift for another owner's table", async () => { + const root = await loadPlatform(); + const drift = await computeDriftFromActual( + await actualSchema(root), + "postgres", + root, + { inScope: matchesNothing }, + ); + expect(drift.outOfScope).toEqual(["acme.jobs"]); + expect(drift.changes).toEqual([]); + }); + + test("the schema scope reported is the UNSCOPED model's, so narrowing can never widen it", async () => { + const root = await loadPlatform(); + const built = buildExpectedSchemaWithProvenance(root, { dialect: "postgres" }); + + // Unscoped: nothing reported, so `diff` derives its own exactly as before — + // an unscoped project reaches the diff through byte-identical arguments. + expect(scopeExpectedSchema(built, undefined).declaredSchemas).toBeUndefined(); + + // Scoped: the full model's schemas, not the survivors'. + expect(scopeExpectedSchema(built, matchesNothing).declaredSchemas).toEqual(["acme"]); + expect(scopeExpectedSchema(built, () => true).declaredSchemas).toEqual(["acme"]); + }); +}); diff --git a/server/typescript/packages/sdk/src/collection.ts b/server/typescript/packages/sdk/src/collection.ts index b928e5c0e..2e0e1bdaa 100644 --- a/server/typescript/packages/sdk/src/collection.ts +++ b/server/typescript/packages/sdk/src/collection.ts @@ -23,7 +23,10 @@ export interface Collection { /** Directory whose config declared this collection (or the resolved start * directory, when nothing was discovered and the default applies). */ readonly configDir: string; - /** Canonically-sorted absolute metadata file paths — see `resolveSources`. */ + /** Canonically-ordered absolute metadata file paths — see `resolveSources`. + * Canonical, not sorted: within a directory source the walk order the + * toolchain has always used is preserved, because it survives into + * generated output. */ readonly files: readonly string[]; /** Same set, carrying the contributing spec for provenance. */ readonly sources: readonly ResolvedSource[]; @@ -32,6 +35,12 @@ export interface Collection { /** Output filter for migrate/verify --db. Undefined => the command governs * everything in scope. */ readonly migrateScope: CompiledScope | undefined; + /** The patterns `migrateScope` was compiled FROM, for diagnostics only — + * `compileScope` produces RegExps, and a regex source is not something to + * show an author who wrote `acme::platform::**`. Carried so the "your scope + * matched nothing" refusal can name the patterns that missed. Always in + * lockstep with `migrateScope`: both undefined, or both present. */ + readonly migrateScopePatterns: readonly string[] | undefined; } // Deliberately NOT deduped with `exists` (imported from `./discovery.js`) @@ -142,5 +151,6 @@ export async function resolveCollection( sources, scope: compileScope(toScope(scopeSpec)), migrateScope: migrateSpec === undefined ? undefined : compileScope({ include: migrateSpec }), + migrateScopePatterns: migrateSpec, }; } From 181d6bb5ed03cd63ebad2f3c1d2cf8edde140888 Mon Sep 17 00:00:00 2001 From: Doug Mealing Date: Tue, 18 Aug 2026 07:57:04 -0400 Subject: [PATCH 28/44] fix(cli): route prompt-snapshot through resolveCollection; close two fail-open config blocks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit I3 — `meta prompt-snapshot` was the one live read path still deciding for itself where metadata lives: `loadMemory(cwd)` with no `files`, plus the ENOENT sniff every other site removed. `--check` is a drift GATE, so a project declaring `sources` elsewhere either got "no metaobjects/ found" or, with a stale `metaobjects/` still on disk, gated silently against the wrong model. Routed exactly as `gen.ts` does, and everything project-relative (the `.metaobjects/ snapshots/` goldens, the `prompts/` a `@textRef` resolves in) now hangs off the resolved config dir rather than ambient cwd, so the metadata and the text it names can never come from two different projects. This was a spec gap, not a missed task: design §4.6.0's site table enumerated nine reads and omitted this one, which is why nothing scheduled it. The row is added, with a note saying why — the ports plan is written from that table and would otherwise inherit the omission in four more languages. I4 — `MigrateBlock` and `D1Block` were `.partial()` under zod's default STRIP policy, so `{ migrate: { scopee: [...], dialect: "postgres" } }` parsed to `{ dialect: "postgres" }`: a typo'd `scope` key silently meant UNSCOPED, which is the "migrate.scope matched nothing" hazard through a second door. Both are `.strict()` now, matching the top level, the `sources` arms and `ScopeSchema`, each of which already carried a comment about this exact fail-open. Co-Authored-By: Claude Opus 5 (1M context) --- ...08-17-metadata-source-resolution-design.md | 9 +++- .../cli/src/commands/prompt-snapshot.ts | 41 ++++++++++++++----- server/typescript/packages/sdk/src/config.ts | 12 +++++- .../packages/sdk/test/config.test.ts | 18 ++++++++ 4 files changed, 66 insertions(+), 14 deletions(-) diff --git a/docs/superpowers/specs/2026-08-17-metadata-source-resolution-design.md b/docs/superpowers/specs/2026-08-17-metadata-source-resolution-design.md index 1faa9f285..2b263dd1c 100644 --- a/docs/superpowers/specs/2026-08-17-metadata-source-resolution-design.md +++ b/docs/superpowers/specs/2026-08-17-metadata-source-resolution-design.md @@ -270,17 +270,24 @@ able to create and read it**, or the neutral file is neutral in name only. when the key is absent or empty — never a requirement, and never assumed by any code path. The rule: **`sources` is the single authority on where metadata lives, and everything that needs to -find metadata reads it.** Today that is false in TypeScript in nine places, which is the concrete +find metadata reads it.** Today that is false in TypeScript in ten places, which is the concrete phase-1 work item: | Site | Kind | Phase 1 | |---|---|---| | `cli/commands/docs.ts` (×3), `export.ts`, `gen.ts` | read | route through resolved `sources` | +| `cli/commands/prompt-snapshot.ts` | read | route | | `cli/index.ts` — the "is this a MetaObjects project?" probe | read | route | | `cli/lib/detect-stack.ts` — concern detection | read | route | | `sdk/memory.ts` (×2) — the loader entry itself | read | route | | `cli/commands/init.ts` (×2) | **write** | **keep the literal** — scaffolding the default is the one place it belongs | +`prompt-snapshot.ts` was missing from the first draft of this table, which is why nothing scheduled +it; it is listed now because the ports plan is written from this table and would otherwise inherit +the omission in four more languages. It matters more than its size suggests: `--check` is a drift +GATE, so a project declaring `sources` elsewhere would gate against a stale `metaobjects/` rather +than fail. + **Python is already the reference implementation of this shape**, not a laggard: its project config reads `metadata` from the config file with the directory name as a *fallback* (`raw.get("metadata", DEFAULT_METADATA_DIR)`). It needs widening from one string to a set, not diff --git a/server/typescript/packages/cli/src/commands/prompt-snapshot.ts b/server/typescript/packages/cli/src/commands/prompt-snapshot.ts index d5978f69f..be80b7b22 100644 --- a/server/typescript/packages/cli/src/commands/prompt-snapshot.ts +++ b/server/typescript/packages/cli/src/commands/prompt-snapshot.ts @@ -15,7 +15,7 @@ import { log } from "../lib/log.js"; import { FileProvider } from "../lib/file-provider.js"; import { snapshotPaths, unifiedDiff } from "../lib/snapshot.js"; import { loadMetaobjectsConfig } from "../lib/load-metaobjects-config.js"; -import { loadMemory } from "@metaobjectsdev/sdk"; +import { loadMemory, resolveCollection } from "@metaobjectsdev/sdk"; import { TYPE_TEMPLATE, TEMPLATE_ATTR_TEXT_REF, TEMPLATE_ATTR_FORMAT } from "@metaobjectsdev/metadata"; import { render, ESCAPERS, type RenderFormat } from "@metaobjectsdev/render"; @@ -30,13 +30,36 @@ export async function promptSnapshotCommand(args: string[], cwd: string): Promis return 2; } + // Where the metadata lives is `resolveCollection`'s decision, not this + // command's — `--check` is a drift GATE, so a project declaring `sources` + // elsewhere would otherwise gate against a stale `metaobjects/` (or report + // "no metaobjects/ found" for metadata it can see perfectly well). Discovery + // and load stay separate failure modes, the `meta gen` pattern: a broad catch + // around both reports a genuine ParseError as "no metadata found". + // `resolveCollection` raises ERR_COLLECTION_NOT_FOUND with its own message, + // replacing the hand-rolled ENOENT sniff that used to live here. + let collection; + try { + collection = await resolveCollection(cwd); + } catch (err) { + log.error((err as Error).message); + return 2; + } + + // Everything project-relative below hangs off the DECLARING directory, not + // ambient cwd: `.metaobjects/snapshots/` is that config's own state, and the + // prompt text belongs to the same project as the metadata that references it. + // Identical to cwd for a run from the project root, which is the only + // invocation that worked before metadata sources were resolvable at all. + const projectRoot = collection.configDir; + // Best-effort load of metaobjects.config.ts to pick up consumer-supplied // providers. prompt-snapshot doesn't require codegen config; if it's absent // or invalid, fall back to defaults — the loader still works for any // metadata that only uses core+forge subtypes. let configProviders: NonNullable>["providers"]> | undefined; try { - const forgeConfig = await loadMetaobjectsConfig(cwd); + const forgeConfig = await loadMetaobjectsConfig(projectRoot); configProviders = forgeConfig.providers; } catch { configProviders = undefined; @@ -44,20 +67,16 @@ export async function promptSnapshotCommand(args: string[], cwd: string): Promis let root; try { - root = await loadMemory(cwd, { + root = await loadMemory(collection.configDir, { + files: collection.files, ...(configProviders !== undefined ? { providers: configProviders } : {}), }); } catch (err) { - const msg = (err as Error).message; - if (msg.includes("ENOENT") || msg.includes("no such") || msg.includes("cannot read")) { - log.error(`no metaobjects/ found in ${cwd}; run 'meta init' to scaffold`); - return 2; - } - log.error(`failed to load metadata: ${msg}`); + log.error(`failed to load metadata: ${(err as Error).message}`); return 1; } - const promptsDir = join(cwd, flags.prompts ?? DEFAULT_PROMPTS_DIR); + const promptsDir = join(projectRoot, flags.prompts ?? DEFAULT_PROMPTS_DIR); const provider = new FileProvider(promptsDir); // ADR-0039: effective children — resolve rather than rely on root being unextended. @@ -79,7 +98,7 @@ export async function promptSnapshotCommand(args: string[], cwd: string): Promis // Absent/typeless required attrs are a loader-schema concern, not ours. if (typeof textRef !== "string") continue; - const { dir, payloadPath, snapPath } = snapshotPaths(cwd, tmpl.name); + const { dir, payloadPath, snapPath } = snapshotPaths(projectRoot, tmpl.name); if (!existsSync(payloadPath)) { log.info(`[${tmpl.name}] skipped — no payload at ${payloadPath}`); skipped++; diff --git a/server/typescript/packages/sdk/src/config.ts b/server/typescript/packages/sdk/src/config.ts index 8dae20f27..5f16a466e 100644 --- a/server/typescript/packages/sdk/src/config.ts +++ b/server/typescript/packages/sdk/src/config.ts @@ -33,12 +33,15 @@ export const AllowTokenEnum = z.enum([ "drop-identity-default", ]); +// .strict(), like every other object in this schema: `.partial()` alone leaves +// zod's default STRIP policy in place, so a misspelled key is silently deleted +// and the block reads as if the author never wrote it. const D1Block = z.object({ binding: z.string(), remote: z.boolean(), autoApply: z.boolean(), wranglerConfigPath: z.string(), -}).partial(); +}).partial().strict(); /** #192 — migration output-format adapters; orthogonal to dialect. */ const MigrateFormatEnum = z.enum(["default", "flyway"]); @@ -56,7 +59,12 @@ const MigrateBlock = z.object({ * Include-only — there's no `migrate.scope.exclude`, since a migration run * is scoped to what it's touching, not filtered down from "everything". */ scope: z.array(z.string().min(1)), -}).partial(); +// .strict() for the same reason as the top level and the source arms below, and +// with sharper teeth here: `{ migrate: { scopee: [...], dialect: "postgres" } }` +// used to parse to `{ dialect: "postgres" }`, so a typo'd `scope` key meant +// "unscoped" — silently governing every table in the database, which is the +// hazard `migrate.scope` exists to remove. +}).partial().strict(); /** * Mirrors the hand-written `SourceSpec` union in `./sources.ts` — a diff --git a/server/typescript/packages/sdk/test/config.test.ts b/server/typescript/packages/sdk/test/config.test.ts index f3b89d56b..1a17d27d5 100644 --- a/server/typescript/packages/sdk/test/config.test.ts +++ b/server/typescript/packages/sdk/test/config.test.ts @@ -142,6 +142,24 @@ describe("ConfigSchema — phase-1 source resolution", () => { }); expect(p.migrate?.scope).toEqual(["acme::platform::**"]); }); + test("rejects a typo'd key inside `migrate` — a stripped `scopee` silently means UNSCOPED", () => { + // The nested blocks were `.partial()` under zod's default strip policy, so + // `{ migrate: { scopee: [...], dialect: "postgres" } }` parsed to + // `{ dialect: "postgres" }` — the typo vanished and the run governed the whole + // database. That is the `migrate.scope`-matched-nothing hazard through a + // second door, and the exact fail-open `.strict()` exists to prevent. + expect(() => + ConfigSchema.parse({ + schema_version: 1, + migrate: { scopee: ["acme::platform::**"], dialect: "postgres" }, + }), + ).toThrow(); + }); + test("rejects a typo'd key inside `migrate.d1`", () => { + expect(() => + ConfigSchema.parse({ schema_version: 1, migrate: { d1: { bindingg: "DB" } } }), + ).toThrow(); + }); test("an existing config with no new keys still parses (back-compat)", () => { const p = ConfigSchema.parse({ schema_version: 1, pending_in_git: true, From 0c8fd136eea4eae3977bf1f8e771c7410679db5e Mon Sep 17 00:00:00 2001 From: Doug Mealing Date: Tue, 18 Aug 2026 07:57:04 -0400 Subject: [PATCH 29/44] fix(cli): read per-project config from the resolved config dir, not ambient cwd MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit I5 — metadata resolved through `resolveCollection` (nearest ancestor holding `.metaobjects/config.json`) while `metaobjects.config.ts` and the `.metaobjects/config.json` operational block were still read from cwd, at six sites. Run `meta migrate` from a subdirectory of a project whose root declares `columnNamingStrategy: "literal"`: the metadata comes from the ancestor, the strategy silently defaults to `snake_case`, and the emitted migration RENAMES EVERY COLUMN. Newly reachable — pre-branch that invocation failed outright with "no metaobjects/ found" — and it aligns the code with design §4.6.1, which already said per-port generator config is read from that same directory. The line drawn, and it is the same in all three commands: anything named BY the metadata or its config resolves against the config dir — `metaobjects.config.ts`, `.metaobjects/config.json`, the `outDir` and `wranglerConfigPath` they carry, the `prompts/` a `@textRef` resolves in, the test files a `@verifiedBy` names. Anything that is merely "the tree the user is standing in" stays on cwd: the agent-context staleness nudge and the advisory anti-pattern scan, both warnings-only. For a run from the project root every path is identical, which is the only invocation that worked before. The relative-path interaction the fix had to decide: a relative `outDir` resolved against the resolved cwd (a deliberate fix in 0.19.2). It now resolves against the config dir, because the config it comes from does — otherwise a subdirectory run writes its migration where the next run cannot find it. `migrate` computes its root with `findConfigDir` rather than `resolveCollection`, deliberately: `apply-pending` and `--rollback` replay committed SQL and load no metadata at all, so requiring metadata to exist would be a regression. It falls back to cwd exactly as `resolveCollection` does, so the two agree by construction. Gated by `cli/test/integration/migrate-config-dir.test.ts`: a subdirectory run must emit BYTE-IDENTICAL SQL to a project-root run, and must write under the project root. Both fail on the pre-fix `metaRoot = cwd`. Co-Authored-By: Claude Opus 5 (1M context) --- .../packages/cli/src/commands/gen.ts | 30 ++-- .../packages/cli/src/commands/migrate.ts | 25 +++- .../packages/cli/src/commands/verify.ts | 60 ++++---- .../integration/migrate-config-dir.test.ts | 128 ++++++++++++++++++ 4 files changed, 207 insertions(+), 36 deletions(-) create mode 100644 server/typescript/packages/cli/test/integration/migrate-config-dir.test.ts diff --git a/server/typescript/packages/cli/src/commands/gen.ts b/server/typescript/packages/cli/src/commands/gen.ts index 592a19556..f658db1ca 100644 --- a/server/typescript/packages/cli/src/commands/gen.ts +++ b/server/typescript/packages/cli/src/commands/gen.ts @@ -38,26 +38,36 @@ export async function genCommand(args: string[], cwd: string, fmt: OutputFormat // Advisory: nudge to refresh the .claude/skills docs if they predate this CLI. warnIfAgentContextStale(cwd); - const projectRoot = cwd; const cliConfig = resolveGenConfig(flags); - let forgeConfig; - try { - forgeConfig = await loadMetaobjectsConfig(projectRoot); - } catch (err) { - log.error((err as Error).message); - return 2; - } - // Discovery and load are two separate failure modes, kept in separate try // blocks deliberately: a broad catch around both previously swallowed // genuine ParseErrors (e.g. `origin.@via "X.y" ...: no such relationship // "y" on X`) as "no metaobjects/ found", masking the real failure. // `resolveCollection` raises `ERR_COLLECTION_NOT_FOUND` with its own // message when nothing is discovered and no default directory exists. + // + // Discovery runs BEFORE the config read, deliberately: the project root is + // whichever directory `resolveCollection` decided the metadata belongs to, + // and everything project-relative — `metaobjects.config.ts`, the `outDir` + // its generators name, `.metaobjects/.gen-state/` — has to come from that + // same directory. Reading the config from ambient cwd while the metadata + // came from an ancestor is the config-half of the very divergence this + // design exists to remove (design §4.6.1: "Per-port generator config is then + // read from that same directory"). For a run from the project root the two + // are the same path, which is the only invocation that worked before. let collection; try { - collection = await resolveCollection(projectRoot); + collection = await resolveCollection(cwd); + } catch (err) { + log.error((err as Error).message); + return 2; + } + const projectRoot = collection.configDir; + + let forgeConfig; + try { + forgeConfig = await loadMetaobjectsConfig(projectRoot); } catch (err) { log.error((err as Error).message); return 2; diff --git a/server/typescript/packages/cli/src/commands/migrate.ts b/server/typescript/packages/cli/src/commands/migrate.ts index 9d691f308..93184c2f8 100644 --- a/server/typescript/packages/cli/src/commands/migrate.ts +++ b/server/typescript/packages/cli/src/commands/migrate.ts @@ -10,7 +10,7 @@ import type { OutputFormat } from "../lib/format.js"; import { toonEncode } from "../lib/format.js"; import { buildKyselyFromUrl, redactUrl } from "../lib/kysely.js"; import { log } from "../lib/log.js"; -import { loadMemory, resolveCollection } from "@metaobjectsdev/sdk"; +import { findConfigDir, loadMemory, resolveCollection } from "@metaobjectsdev/sdk"; import { loadMetaobjectsConfig } from "../lib/load-metaobjects-config.js"; import { migrateScopeMismatch, toObjectScope } from "../lib/migrate-scope.js"; import { @@ -272,7 +272,28 @@ export async function migrateCommand( return 2; } - const metaRoot = cwd; + // The project root is the directory whose `.metaobjects/config.json` governs + // this run — the same directory `resolveCollection` resolves the metadata + // from, found the same way (design §4.6.1: "Per-port generator config is then + // read from that same directory"). Everything below is relative to it: the + // `.metaobjects/config.json` operational block, `metaobjects.config.ts`'s + // `columnNamingStrategy`, the migrations `outDir`, `wrangler.toml` discovery. + // + // Read from ambient cwd instead, as this did, they DIVERGE the moment the two + // differ: run `meta migrate` from a subdirectory of a project whose root + // declares `columnNamingStrategy: "literal"` and the metadata resolves from + // the ancestor while the strategy silently defaults to snake_case — emitting a + // migration that renames every column. Newly reachable, too: before metadata + // sources were resolvable that invocation just failed with "no metaobjects/ + // found". + // + // `findConfigDir` rather than `resolveCollection` deliberately: this must not + // require metadata to EXIST. `migrate apply-pending` and `--rollback` replay + // committed SQL and load no metadata at all, and making them fail on a project + // with no model would be a regression. Falls back to cwd when no config is + // found anywhere, which is exactly what `resolveCollection` does, so the two + // agree by construction. + const metaRoot = (await findConfigDir(cwd)) ?? resolvePath(cwd); const config = await resolveMigrateConfig(flags, metaRoot); try { diff --git a/server/typescript/packages/cli/src/commands/verify.ts b/server/typescript/packages/cli/src/commands/verify.ts index e2e82ea47..17cab54aa 100644 --- a/server/typescript/packages/cli/src/commands/verify.ts +++ b/server/typescript/packages/cli/src/commands/verify.ts @@ -129,20 +129,6 @@ export async function verifyCommand( ); } - // Best-effort load of metaobjects.config.ts. Two consumers: - // 1) consumer-supplied providers (e.g. a `template.toolcall` subtype) threaded - // into loadMemory — verify doesn't REQUIRE codegen config for templates/db; - // 2) the full config object, which `--codegen` needs to locate outDir/targets. - // If absent/invalid we fall back to defaults; `--codegen` then reports a clear - // error (it can't diff without knowing where the committed output lives). - let forgeConfig: MetaobjectsGenConfig | undefined; - try { - forgeConfig = await loadMetaobjectsConfig(cwd); - } catch { - forgeConfig = undefined; - } - const configProviders = forgeConfig?.providers; - // Where the metadata lives is `resolveCollection`'s decision, not a hardcoded // directory. It also carries the per-command `migrate.scope` the schema gate below // honours — `verify --db` and `migrate` govern the identical object set — and the @@ -160,6 +146,32 @@ export async function verifyCommand( return 2; } + // The project root is whichever directory `resolveCollection` decided the + // metadata belongs to (design §4.6.1: "Per-port generator config is then read + // from that same directory"). The line this draws, applied throughout this + // command: anything named BY the metadata or its config resolves against + // `projectRoot` — `metaobjects.config.ts`, `.metaobjects/config.json`, the + // `outDir` and `wranglerConfigPath` they carry, the `prompts/` a `@textRef` + // resolves in, the test files a `@verifiedBy` names. Anything that is merely + // "the tree the user is standing in" stays on `cwd` — the agent-context + // staleness nudge and the advisory anti-pattern scan, both warnings-only. + // Identical paths for a run from the project root. + const projectRoot = collection.configDir; + + // Best-effort load of metaobjects.config.ts. Two consumers: + // 1) consumer-supplied providers (e.g. a `template.toolcall` subtype) threaded + // into loadMemory — verify doesn't REQUIRE codegen config for templates/db; + // 2) the full config object, which `--codegen` needs to locate outDir/targets. + // If absent/invalid we fall back to defaults; `--codegen` then reports a clear + // error (it can't diff without knowing where the committed output lives). + let forgeConfig: MetaobjectsGenConfig | undefined; + try { + forgeConfig = await loadMetaobjectsConfig(projectRoot); + } catch { + forgeConfig = undefined; + } + const configProviders = forgeConfig?.providers; + // ADR-0023 strict-by-default (#96): verify loads strict unless --lax is passed, // so an undeclared/typo'd own @attr fails verify (matching Java's Maven goal). let root: Awaited>; @@ -192,7 +204,7 @@ export async function verifyCommand( // loaded, which is every project that declares no scope. const schemaScope = toObjectScope(collection.migrateScope); - const promptsDir = join(cwd, flags.prompts ?? DEFAULT_PROMPTS_DIR); + const promptsDir = join(projectRoot, flags.prompts ?? DEFAULT_PROMPTS_DIR); const provider = new FileProvider(promptsDir); // Exit-code composition: the overall result is the MAX across every selected @@ -223,7 +235,7 @@ export async function verifyCommand( // authority — see the verified-by-scan header. const diags = [ ...checkRequirements(root), - ...checkVerifiedBy(root, cwd, forgeConfig?.verify?.testFiles), + ...checkVerifiedBy(root, projectRoot, forgeConfig?.verify?.testFiles), ]; // Printed on EVERY run, clean or not — a gate that says nothing when it @@ -498,14 +510,14 @@ export async function verifyCommand( // computeDriftFromActual and the SAME reportSchemaDrift the sqlite/postgres // path uses — no forked reporting/exit-code logic. async function runD1SchemaVerify(ledgerDrift: string[]): Promise { - const d1Config = await resolveD1Config({ d1Binding: flags.d1, remote: flags.remote }, cwd); + const d1Config = await resolveD1Config({ d1Binding: flags.d1, remote: flags.remote }, projectRoot); const wranglerConfigPath = d1Config.wranglerConfigPath - ? resolvePath(cwd, d1Config.wranglerConfigPath) - : findWranglerConfig(cwd); + ? resolvePath(projectRoot, d1Config.wranglerConfigPath) + : findWranglerConfig(projectRoot); if (wranglerConfigPath === undefined && d1Config.binding === undefined) { - log.error(`verify: no wrangler.toml found in ${cwd} or parents; pass --d1 to bypass`); + log.error(`verify: no wrangler.toml found in ${projectRoot} or parents; pass --d1 to bypass`); return 2; } @@ -531,7 +543,7 @@ export async function verifyCommand( command: sql, configPath: wranglerConfigPath, }); - const { stdout } = await activeWranglerRunner(wranglerArgs, cwd); + const { stdout } = await activeWranglerRunner(wranglerArgs, projectRoot); return stdout; }; @@ -603,8 +615,8 @@ export async function verifyCommand( // default) rather than re-deriving it, so verify can never look somewhere migrate // does not write. Only `outDir` is consumed; the rest of the resolved config is // migrate's business. - const migrateConfig = await resolveMigrateConfig(EMPTY_MIGRATE_FLAGS, cwd); - const dir = resolvePath(cwd, migrateConfig.outDir); + const migrateConfig = await resolveMigrateConfig(EMPTY_MIGRATE_FLAGS, projectRoot); + const dir = resolvePath(projectRoot, migrateConfig.outDir); let snapshot: SchemaSnapshot | null; try { snapshot = await readSnapshot(snapshotPath(dir, dialect)); @@ -709,7 +721,7 @@ export async function verifyCommand( // files should exist, reporting every out-of-scope entity as drift. let result; try { - result = await computeCodegenDrift(forgeConfig, root, cwd, (fqn) => matchesScope(fqn, collection.scope)); + result = await computeCodegenDrift(forgeConfig, root, projectRoot, (fqn) => matchesScope(fqn, collection.scope)); } catch (err) { log.error(`verify --codegen: regeneration failed: ${(err as Error).message}`); return 1; diff --git a/server/typescript/packages/cli/test/integration/migrate-config-dir.test.ts b/server/typescript/packages/cli/test/integration/migrate-config-dir.test.ts new file mode 100644 index 000000000..3c03df7c9 --- /dev/null +++ b/server/typescript/packages/cli/test/integration/migrate-config-dir.test.ts @@ -0,0 +1,128 @@ +/** + * The config a command reads comes from the directory the METADATA was resolved + * from, never from ambient cwd. + * + * `metaobjects.config.ts` carries `columnNamingStrategy`. Once metadata resolves + * from the nearest ancestor holding `.metaobjects/config.json`, reading that file + * from cwd instead silently splits the two: run `meta migrate` from a subdirectory + * of a project whose root declares `literal` and the metadata comes from the + * ancestor while the strategy defaults to `snake_case` — emitting a migration that + * RENAMES EVERY COLUMN. Newly reachable, too: before metadata sources were + * resolvable, that invocation just failed with "no metaobjects/ found". + * + * The gate is byte-level and comparative, not a spot-check on one identifier: the + * SQL a subdirectory run emits must be byte-identical to the SQL the project-root + * run emits. A drifting default shows up as a diff whether or not anyone thought + * to assert on the setting that drifted. + */ +import { describe, test, expect } from "bun:test"; +import { mkdtempSync, mkdirSync, rmSync, readdirSync, readFileSync, writeFileSync } from "node:fs"; +import { join, resolve } from "node:path"; +import { run } from "../../src/index.js"; + +// Temp dirs live inside the monorepo so jiti can resolve @metaobjectsdev/* when +// it loads metaobjects.config.ts (same rationale as gen-sqlite.test.ts). +const WORKSPACE_TMP = resolve(import.meta.dirname, "../fixtures/__tmp__"); + +const USERS = JSON.stringify({ + "metadata.root": { + package: "acme", + children: [{ + "object.entity": { + name: "User", + children: [ + { "source.rdb": { name: "src", "@table": "users" } }, + { "field.long": { name: "id" } }, + // Two words, so `literal` and `snake_case` produce DIFFERENT column names. + { "field.string": { name: "firstName" } }, + { "identity.primary": { name: "pk", "@fields": ["id"] } }, + ], + }, + }], + }, +}); + +/** A project whose ROOT declares `columnNamingStrategy: "literal"`, with an + * otherwise-empty subdirectory to run from. */ +function scaffold(): string { + mkdirSync(WORKSPACE_TMP, { recursive: true }); + const repo = mkdtempSync(join(WORKSPACE_TMP, "migrate-config-dir-")); + mkdirSync(join(repo, "metaobjects"), { recursive: true }); + writeFileSync(join(repo, "metaobjects", "meta.users.json"), USERS, "utf8"); + mkdirSync(join(repo, ".metaobjects"), { recursive: true }); + writeFileSync( + join(repo, ".metaobjects", "config.json"), + JSON.stringify({ schema_version: 1 }), + "utf8", + ); + writeFileSync( + join(repo, "metaobjects.config.ts"), + ` +import { defineConfig } from "@metaobjectsdev/codegen-ts"; +export default defineConfig({ + outDir: ${JSON.stringify(join(repo, "generated"))}, + dialect: "sqlite", + columnNamingStrategy: "literal", + generators: [], +}); +`, + "utf8", + ); + mkdirSync(join(repo, "apps", "api"), { recursive: true }); + return repo; +} + +/** The `up.sql` the run wrote, read out of the project root's migrations dir. */ +function emittedUpSql(repo: string): string { + const migrations = join(repo, ".metaobjects", "migrations"); + const dirs = readdirSync(migrations).filter((d) => d.endsWith("-init")); + expect(dirs).toHaveLength(1); + return readFileSync(join(migrations, dirs[0]!, "up.sql"), "utf8"); +} + +async function migrateFrom(repo: string, runDir: string): Promise { + const exit = await run([ + "migrate", "--from-db", "--cwd", runDir, + "--db", `file:${join(repo, "local.db")}`, + "--dialect", "sqlite", "--slug", "init", + ]); + expect(exit).toBe(0); + return emittedUpSql(repo); +} + +describe("meta migrate — config comes from the resolved config dir", () => { + test("a subdirectory run emits byte-identical SQL to a project-root run", async () => { + const fromRoot = scaffold(); + const fromSubdir = scaffold(); + try { + const rootSql = await migrateFrom(fromRoot, fromRoot); + const subdirSql = await migrateFrom(fromSubdir, join(fromSubdir, "apps", "api")); + + // Both runs honour the root's `literal` strategy. Asserted explicitly as + // well as comparatively, so a failure says WHICH way it went rather than + // only that the two disagree. + expect(rootSql).toContain("firstName"); + expect(rootSql).not.toContain("first_name"); + + // The paths differ per temp dir, so compare the SQL bodies only — nothing + // in generated DDL should mention an absolute path anyway. + expect(subdirSql).toBe(rootSql); + } finally { + rmSync(fromRoot, { recursive: true, force: true }); + rmSync(fromSubdir, { recursive: true, force: true }); + } + }); + + test("a subdirectory run writes its migration under the project root, not the subdirectory", async () => { + const repo = scaffold(); + try { + await migrateFrom(repo, join(repo, "apps", "api")); + // `outDir` is relative (`./.metaobjects/migrations`) and must resolve against + // the config dir, or the migration lands somewhere the next run cannot find. + expect(readdirSync(join(repo, ".metaobjects", "migrations")).length).toBeGreaterThan(0); + expect(readdirSync(join(repo, "apps", "api"))).toEqual([]); + } finally { + rmSync(repo, { recursive: true, force: true }); + } + }); +}); From 5cbdeac3796ee6a827a580ac6775d9231866ec73 Mon Sep 17 00:00:00 2001 From: Doug Mealing Date: Tue, 18 Aug 2026 07:57:04 -0400 Subject: [PATCH 30/44] docs: an Upgrading section for the three adopter-visible changes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit I6 — `loadMemory(configDir, { files })` bypasses `collectMetadataPaths`'s `package.meta.json` peer-package walk entirely. Design §11 rules that intentional and it fails loudly (`ERR_UNRESOLVED_SUPER`) rather than silently, but the adopter guide did not mention `package.meta.json` or workspaces anywhere — so the one mechanism a project could be relying on was documented nowhere, including its replacement. Documented in an Upgrading section together with the other two changes an existing project can notice: `ConfigSchema` is now `.strict()` (a previously stripped key is a load error), and `ExpectedView.fqn` became required on a public `codegen-ts` export. Mirrored under the existing `## [Unreleased]` heading in CHANGELOG.md. Also corrects the two places the guide described `resolveSources` as sorting absolute paths, which stopped being true when the resolver was fixed to keep the loader's per-level walk order. Co-Authored-By: Claude Opus 5 (1M context) --- CHANGELOG.md | 28 ++++++++++++++ docs/features/metadata-sources.md | 64 +++++++++++++++++++++++++++++++ 2 files changed, 92 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 481468c2d..414881453 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -91,6 +91,34 @@ the local release path published `docs-site` ahead of `metadata` and `render`, t packages it depends on. The tier is declared now, and an undeclared one is an error instead of an accidental position. +### Metadata source resolution — adopter-visible changes + +`.metaobjects/config.json` gains `sources`, `scope` and `migrate.scope`, and every +command resolves where metadata lives through one authority instead of reading a +hardcoded `metaobjects/` directory. A project with one config at its root, no +`sources` and no `scope` resolves the same files, generates the same code and emits +the same migrations. Three changes are visible even to that project. Adopter guide: +[`docs/features/metadata-sources.md`](docs/features/metadata-sources.md#upgrading). + +- **The workspace `extends:` walk is retired.** `loadMemory` used to have a second, + undocumented way of finding metadata: a `package.meta.json` declaring `extends:` + dependencies, inside a discoverable workspace (`pnpm-workspace.yaml` or + `package.json` `workspaces`), pulled in each peer package's `metaobjects/` + directory first, in topological order. Every CLI read path now resolves through + `sources`, which does no such walk. It fails LOUDLY — `ERR_UNRESOLVED_SUPER` + naming the target it cannot find, never a half-resolved model — and the + replacement is an explicit `{ "path": "../shared-model/metaobjects" }` source, + which works in any layout and needs no topological ordering. +- **`.metaobjects/config.json` rejects unknown keys.** `ConfigSchema` is `.strict()` + at every level, so a key that was previously stripped in silence is now a load + error naming the key. Silently dropping a key means the setting you wrote does not + exist: `{ "migrate": { "scopee": [...] } }` used to mean *unscoped*, governing + every table in a database you were trying to share. +- **`ExpectedView.fqn` is required.** On the public `@metaobjectsdev/codegen-ts` + export, the declaring object's fully-qualified name is no longer optional — + `migrate.scope` decides ownership on that name, and a view arriving without one + cannot be scoped at all. `buildProjectionViews` already supplies it; only + hand-built `ExpectedView` values need the field added. ## [0.23.2] — npm `0.23.2` · PyPI `0.23.2` · NuGet `0.23.2` · Maven `7.23.2` diff --git a/docs/features/metadata-sources.md b/docs/features/metadata-sources.md index a415da1f4..595776a1c 100644 --- a/docs/features/metadata-sources.md +++ b/docs/features/metadata-sources.md @@ -417,6 +417,70 @@ type) surfaces as the config load error and stops the command. --- +## Upgrading + +A project with one config at its root, no `sources` and no `scope` resolves the same +files it always did and generates the same code. Three changes are still worth +knowing about before you upgrade. + +### The workspace `extends:` walk is retired + +`loadMemory` used to have a second, hidden way of finding metadata: if the project +carried a `package.meta.json` declaring `extends:` dependencies **and** a workspace +could be discovered (`pnpm-workspace.yaml`, or `package.json` `workspaces`), it +walked that dependency graph and loaded each peer package's `metaobjects/` directory +first, in topological order. + +Every CLI read path now resolves its files through `sources`, which does no such +walk. Two ways to find metadata, one of them implicit and reachable only from a +particular repository layout, is precisely the divergence this feature exists to +remove — and one of them was undocumented. + +**It fails loudly, not silently.** A model that depended on a peer package's +declarations now fails to load with `ERR_UNRESOLVED_SUPER` naming the `extends:` +target it cannot find; nothing generates from a half-resolved model. + +**The replacement is a declared source**, which is explicit and works in any layout, +workspace or not: + +```json +{ + "schema_version": 1, + "sources": [ + { "path": "../shared-model/metaobjects" }, + { "path": "metaobjects" } + ] +} +``` + +Order does not matter (see [`sources` — a set, not an ordered +list](#sources--a-set-not-an-ordered-list)), so there is no topological ordering to +reproduce. + +### `.metaobjects/config.json` rejects unknown keys + +`ConfigSchema` is `.strict()` at every level — the top level, `sources` entries, +`scope`, `migrate`, and `migrate.d1`. A key that used to be silently dropped is now +a load error that stops the command. + +This is deliberate and it is the whole point: a stripped key means the setting you +wrote does not exist, and the command runs as if you had never written it. +`{ "migrate": { "scopee": [...] } }` used to mean *unscoped* — governing every table +in a database you were trying to share. + +If a command starts failing on a config that used to load, the message names the +unrecognized key; fix the spelling or delete the key. + +### `ExpectedView.fqn` is required + +`ExpectedView` is a public `@metaobjectsdev/codegen-ts` export. Its `fqn` — the +declaring object's fully-qualified name — is now **required** rather than optional, +because `migrate.scope` decides on that name and a view arriving without one cannot +be scoped at all. Code that builds `ExpectedView` values by hand (the normal path, +`buildProjectionViews`, already supplies it) has to add the field. + +--- + ## What is deferred Phase 1 ships the spine. Explicitly **not** built yet, so you do not go looking: From e929bd87cf482ee6b210ef13f2a5e92954fc93cd Mon Sep 17 00:00:00 2001 From: Doug Mealing Date: Tue, 18 Aug 2026 08:01:59 -0400 Subject: [PATCH 31/44] fix: the seven fix-before-merge minors from the whole-branch review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 1. `fixtures/scope-conformance/README.md` named `isInScope(...)`; the symbol is `matchesScope`. Four ports implement from that file. 2. `logOutOfScope` used `log.info` unconditionally, so a scoped `meta migrate --format json` put a prose line on stdout ahead of the JSON and broke `| jq`. Text format keeps stdout; every other format routes it to stderr rather than dropping it — the non-TTY default format is toon, so suppressing outright would silence the note for every piped and CI run. `verify` is NOT changed: it never receives `fmt` (index.ts passes it only to `gen` and `migrate`) and emits prose throughout, so there is no structured document to interleave with. 3. `gen-scope.test.ts`'s "byte-identical to today" case asserted three FILENAMES and read no bytes — a title claiming a byte guarantee over a test that cannot see one is how the real guarantee went unexamined. It now compares an unscoped run against an all-matching scope file-for-file, byte-for-byte, and is titled for what it checks. 4. (Shipped with the C1 commit) `dogfood-examples.test.ts` builds its metadata dir from `DEFAULT_METADATA_DIR`, not the literal. 5. CLAUDE.md claimed `meta init` was the sanctioned exception to hardcoding the directory name; `init.ts` has always IMPORTED the constant. The only literal is `sdk/src/memory.ts:18`, which is now what the sentence says. 6. "the phase-1 ports plan" was cited three times as if it named a findable artifact; no such document exists. Reworded to describe the future work. 7. An accepted scoped run persisted the NARROWED schema as the committed snapshot, deleting every out-of-scope entry — so later widening or removing `migrate.scope` proposed CREATE TABLE for a table that exists and failed at apply. `carryForwardOutOfScope` keeps those entries, on both the offline and the `--from-db` paths. `PlanOfflineResult` now separates `nextSnapshot` (what to COMMIT) from `expected` (the governed side the diff compared and the emitter renders against), so the emitter's input is unchanged. Unscoped runs commit the same object, so the snapshot stays byte-identical. The `migrate baseline` comment that claimed an out-of-scope entry in the snapshot was harmless is now true, and says why. Co-Authored-By: Claude Opus 5 (1M context) --- AGENTS.md | 2 +- docs/CONFORMANCE.md | 5 +- docs/features/metadata-sources.md | 6 +-- fixtures/scope-conformance/README.md | 2 +- .../packages/cli/src/commands/migrate.ts | 47 ++++++++++++++----- .../cli/test/integration/gen-scope.test.ts | 41 +++++++++++----- .../test/integration/migrate-db-scope.test.ts | 22 +++++++++ .../packages/migrate-ts/src/index.ts | 2 +- .../packages/migrate-ts/src/scope.ts | 30 ++++++++++++ .../packages/migrate-ts/src/snapshot/plan.ts | 27 +++++++++-- .../test/expected-schema-scope.test.ts | 44 +++++++++++++++++ 11 files changed, 192 insertions(+), 36 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 3edc033f3..4e1500dc1 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -200,7 +200,7 @@ import { EntityFetcherProvider, EntityGrid } from "@metaobjectsdev/tanstack"; - **Codegen substrate**: ts-poet for greenfield emit, ts-morph for in-place edits, Biome for format pass, `git merge-file --diff3` for hand-edit-preserving regen. - **Runtime substrate**: Kysely for TS (user-provided connection, async-only). - **Migration substrate**: Postgres + SQLite for TS v0.3. -- **Metadata location**: resolved via `resolveCollection()` (`@metaobjectsdev/sdk`) — the single authority. No code path may hardcode the `metaobjects/` directory name except `meta init`, which scaffolds it. See [docs/features/metadata-sources.md](docs/features/metadata-sources.md). +- **Metadata location**: resolved via `resolveCollection()` (`@metaobjectsdev/sdk`) — the single authority. The directory name lives in ONE literal, `DEFAULT_METADATA_DIR` in `sdk/src/memory.ts`; every other site (`meta init`'s scaffold included) imports it. No code path may write the literal again. See [docs/features/metadata-sources.md](docs/features/metadata-sources.md). ## Explicitly out of scope diff --git a/docs/CONFORMANCE.md b/docs/CONFORMANCE.md index 42c8e2c4e..dafb26fd6 100644 --- a/docs/CONFORMANCE.md +++ b/docs/CONFORMANCE.md @@ -158,8 +158,9 @@ produced the cross-port `LIKE`/`ILIKE` divergence fixed in 0.21.6. **TypeScript is the only port with a runner today.** The reference implementation is [`server/typescript/packages/sdk/src/scope.ts`](../server/typescript/packages/sdk/src/scope.ts) (`compilePattern` / `compileScope` / `matchesScope`), and the corpus was authored -against it. Java, Kotlin, C# and Python are deferred to the phase-1 ports plan; the -corpus exists now precisely so those four land on one grammar rather than four. +against it. Java, Kotlin, C# and Python have no runner yet; when each gains one, this corpus is +what it implements against — it exists now precisely so those four land on one +grammar rather than four. ## Orphaned fixtures (tested but not yet documented) diff --git a/docs/features/metadata-sources.md b/docs/features/metadata-sources.md index 595776a1c..93c3f9f80 100644 --- a/docs/features/metadata-sources.md +++ b/docs/features/metadata-sources.md @@ -30,7 +30,7 @@ files it always did. location their own way (a Maven `` element, a positional directory, a `metadata` config key). The cross-port pattern grammar is already pinned by [`fixtures/scope-conformance/`](../../fixtures/scope-conformance/); wiring the other -four CLIs to the same config file is the phase-1 ports plan. +four CLIs to the same config file is future work, not yet planned or scheduled. --- @@ -506,8 +506,8 @@ Design rationale and the full phase plan: - [`fixtures/scope-conformance/`](../../fixtures/scope-conformance/) — 10 cases pinning `*` / `**`, include-union, exclude-after-include, literal metacharacters, - and case sensitivity. TypeScript runs it today; the other four ports are deferred - to the phase-1 ports plan. See [`CONFORMANCE.md`](../CONFORMANCE.md). + and case sensitivity. TypeScript runs it today; the other four ports have no runner + yet. See [`CONFORMANCE.md`](../CONFORMANCE.md). **TypeScript gates** diff --git a/fixtures/scope-conformance/README.md b/fixtures/scope-conformance/README.md index fff022c86..ec1d70f3f 100644 --- a/fixtures/scope-conformance/README.md +++ b/fixtures/scope-conformance/README.md @@ -42,7 +42,7 @@ README.md Each port's runner reads `cases.json`, and for every case: compiles `scope` with its native pattern compiler, then for every `expect` entry asserts -`isInScope(fqn, compiledScope) === matches`. All ports assert the same +`matchesScope(fqn, compiledScope) === matches`. All ports assert the same booleans — single-source, byte-identical expectations. ## Reference implementation diff --git a/server/typescript/packages/cli/src/commands/migrate.ts b/server/typescript/packages/cli/src/commands/migrate.ts index 93184c2f8..d75f01079 100644 --- a/server/typescript/packages/cli/src/commands/migrate.ts +++ b/server/typescript/packages/cli/src/commands/migrate.ts @@ -19,6 +19,7 @@ import { introspect, diff, collectUnmanagedNames, + carryForwardOutOfScope, emit, writeMigration, baselineFromMetadata, @@ -128,13 +129,21 @@ function resolveFormatOutDir(config: ResolvedMigrateConfig, metaRoot: string): s * Say what a declared `migrate.scope` left out. An excluded object produces neither * a create nor a drop, so without this line "no changes" and "no changes to the half * of the model this run governs" read identically. + * + * STDOUT in text format, STDERR otherwise. `--format json` / `--format toon` put a + * single machine-readable document on stdout, and a prose line ahead of it breaks + * `| jq` outright — the same split `emitStructuredError` makes two functions down. + * Routed to stderr rather than dropped, because the non-TTY default format is toon + * (`resolveFormat`): suppressing it outright would silence the note for every + * piped and CI run, which is most of them. */ -function logOutOfScope(names: readonly string[]): void { +function logOutOfScope(names: readonly string[], fmt: OutputFormat): void { if (names.length === 0) return; - log.info( + const msg = `meta migrate — ${names.length} object(s) out-of-scope (outside migrate.scope, ` + - `governed elsewhere): ${names.join(", ")}`, - ); + `governed elsewhere): ${names.join(", ")}`; + if (fmt === "text") log.info(msg); + else log.warn(msg); } function emitStructuredError(error: string, hint: string, fmt: OutputFormat): void { @@ -500,7 +509,7 @@ export async function migrateCommand( toObjectScope(collection.migrateScope), ); const expected = scoped.snapshot; - logOutOfScope(scoped.outOfScope); + logOutOfScope(scoped.outOfScope, fmt); let actual; try { actual = await introspect(kysely.db, kysely.dialect); @@ -677,9 +686,16 @@ export async function migrateCommand( // native ALTER vs recreate-and-copy on older SQLite. if (!config.dryRun && exitCode === 0 && !applyFailed && writtenPaths.length > 0) { try { + // The COMMITTED snapshot keeps what `migrate.scope` excluded: writing the + // narrowed schema would delete every out-of-scope entry, so a later widening + // would propose CREATE TABLE for a table that exists and fail at apply. The + // out-of-scope entries come from `actual` — they are in the database, which + // is the same thing `baseline --from-db` records. Identical object, and so a + // byte-identical snapshot, for an unscoped run. + const committed = carryForwardOutOfScope(expected, actual, scoped.outOfScope); await writeSnapshot( snapshotPath(resolvePath(metaRoot, config.outDir), kysely.dialect), - actual.meta !== undefined ? { ...expected, meta: actual.meta } : expected, + actual.meta !== undefined ? { ...committed, meta: actual.meta } : committed, ); } catch (err) { // The migration itself is written (and possibly applied) — report the @@ -815,8 +831,12 @@ export async function runBaseline( // `baseline` records a STARTING POINT, so it is deliberately NOT scoped: the // `--from-db` arm captures whatever the database holds (there is no provenance // for an introspected table), and an offline baseline that recorded less would - // disagree with it. An out-of-scope table sitting in the snapshot is harmless — - // every later run suppresses it on both sides. + // disagree with it. An out-of-scope table sitting in the snapshot is harmless: + // every later run suppresses it on both sides of the diff, and an accepted + // scoped run carries it FORWARD (`carryForwardOutOfScope`) rather than dropping + // it — which is what makes that true. Committing the narrowed schema instead + // would delete the entry, and removing the scope later would then propose + // CREATE TABLE for a table that exists. try { const collection = await resolveCollection(metaRoot); metadata = await loadMemory(collection.configDir, { @@ -1033,8 +1053,11 @@ export async function runOfflineGenerate( throw err; } - const { diff: diffResult, nextSnapshot } = plan; - logOutOfScope(plan.outOfScope); + // `nextSnapshot` is what gets COMMITTED (it retains this run's out-of-scope + // entries); `expected` is the governed side the emitter renders against. Equal + // for an unscoped run. + const { diff: diffResult, nextSnapshot, expected: governedExpected } = plan; + logOutOfScope(plan.outOfScope, fmt); if (diffResult.blocked.length > 0) { log.error(`migrate: ${diffResult.blocked.length} destructive change(s) blocked; re-run with --allow `); @@ -1051,7 +1074,7 @@ export async function runOfflineGenerate( const emitResult = emit(diffResult.changes, { dialect: config.dialect, - expectedSchema: nextSnapshot, + expectedSchema: governedExpected, actualSchema: snapshot, ...(snapshot.meta ? { actualMeta: snapshot.meta } : {}), }); @@ -1233,7 +1256,7 @@ async function runD1Migrate( toObjectScope(collection.migrateScope), ); const expected = scoped.snapshot; - logOutOfScope(scoped.outOfScope); + logOutOfScope(scoped.outOfScope, fmt); let actual; try { actual = await introspectD1({ diff --git a/server/typescript/packages/cli/test/integration/gen-scope.test.ts b/server/typescript/packages/cli/test/integration/gen-scope.test.ts index bdf10406e..b2fc1f15c 100644 --- a/server/typescript/packages/cli/test/integration/gen-scope.test.ts +++ b/server/typescript/packages/cli/test/integration/gen-scope.test.ts @@ -4,7 +4,7 @@ * collection still loads the whole model; only the emitted file set narrows. */ import { describe, test, expect } from "bun:test"; -import { cpSync, mkdtempSync, mkdirSync, rmSync, readdirSync, writeFileSync } from "node:fs"; +import { cpSync, mkdtempSync, mkdirSync, rmSync, readdirSync, readFileSync, writeFileSync } from "node:fs"; import { join, resolve } from "node:path"; import { run } from "../../src/index.js"; @@ -67,20 +67,39 @@ describe("meta gen — collection scope", () => { } }); - test("no declared scope emits every entity (byte-identical to today)", async () => { - const root = setupRepo(); + test("an unscoped run emits every entity, byte-for-byte the same as an all-matching scope", async () => { + // Previously titled "byte-identical to today" while asserting only that three + // FILENAMES were present. A title claiming a byte guarantee over a test that + // reads no bytes is how the real guarantee went unexamined — the resolver was + // reordering the loaded file list, and therefore the emitted content, with + // nothing here able to see it. + const unscoped = setupRepo(); + const allMatching = setupRepo(); try { // No .metaobjects/config.json at all — the default, unscoped path. - const exit = await run(["gen", "--cwd", root]); - expect(exit).toBe(0); + expect(await run(["gen", "--cwd", unscoped])).toBe(0); + // A scope that admits everything must be indistinguishable from no scope. + declareScope(allMatching, ["trainerWebsite::**"]); + expect(await run(["gen", "--cwd", allMatching])).toBe(0); - const outDir = genOutDir(root); - const files = readdirSync(outDir); - expect(files).toContain("Post.ts"); - expect(files).toContain("User.ts"); - expect(files).toContain("Tag.ts"); + const names = readdirSync(genOutDir(unscoped)).sort(); + expect(names).toContain("Post.ts"); + expect(names).toContain("User.ts"); + expect(names).toContain("Tag.ts"); + expect(readdirSync(genOutDir(allMatching)).sort()).toEqual(names); + + for (const name of names) { + const a = readFileSync(join(genOutDir(unscoped), name), "utf8"); + const b = readFileSync(join(genOutDir(allMatching), name), "utf8"); + // Real bytes, not a filename listing. The outDir is baked into each + // repo's config, so nothing generated should mention it — if that ever + // changes, this is the assertion that says so. + expect(a).not.toContain(unscoped); + expect(b).toBe(a); + } } finally { - rmSync(root, { recursive: true, force: true }); + rmSync(unscoped, { recursive: true, force: true }); + rmSync(allMatching, { recursive: true, force: true }); } }); }); diff --git a/server/typescript/packages/cli/test/integration/migrate-db-scope.test.ts b/server/typescript/packages/cli/test/integration/migrate-db-scope.test.ts index 118c4a687..b5f699dc4 100644 --- a/server/typescript/packages/cli/test/integration/migrate-db-scope.test.ts +++ b/server/typescript/packages/cli/test/integration/migrate-db-scope.test.ts @@ -121,6 +121,28 @@ describe("meta migrate --db — migrate.scope", () => { } }); + test("--format json stays parseable under a scope — the out-of-scope note is text-format only", async () => { + const { repo, dbUrl } = scaffold(); + try { + declareScope(repo, ["acme::platform::**"]); + expect(await run([ + "--format", "json", "migrate", "--from-db", "--cwd", repo, + "--db", dbUrl, "--dialect", "sqlite", "--slug", "initial", + ])).toBe(0); + // The whole point: stdout is ONE machine-readable document. A prose line + // ahead of it breaks `| jq` outright, which is how the out-of-scope note + // shipped — unconditional `log.info`. + const stdout = out.join("\n"); + expect(stdout).not.toContain("out-of-scope"); + expect(() => JSON.parse(stdout)).not.toThrow(); + // Moved to stderr, not dropped — an object that was neither created nor + // dropped has to be reported somewhere, in every format. + expect(err.join("\n")).toContain("out-of-scope"); + } finally { + rmSync(repo, { recursive: true, force: true }); + } + }); + test("no migrate.scope declared — unchanged, both tables governed", async () => { const { repo, dbUrl } = scaffold(); try { diff --git a/server/typescript/packages/migrate-ts/src/index.ts b/server/typescript/packages/migrate-ts/src/index.ts index 12d189b01..1392e3097 100644 --- a/server/typescript/packages/migrate-ts/src/index.ts +++ b/server/typescript/packages/migrate-ts/src/index.ts @@ -15,7 +15,7 @@ export { diff } from "./diff/index.js"; export { collectUnmanagedNames } from "./unmanaged.js"; // Per-command scope (`migrate.scope`) — see scope.ts for why the suppression is // two-sided and why the pattern engine stays in @metaobjectsdev/sdk. -export { scopeExpectedSchema, declaredSchemasOf } from "./scope.js"; +export { scopeExpectedSchema, declaredSchemasOf, carryForwardOutOfScope } from "./scope.js"; export type { ObjectScopePredicate, ScopedExpectedSchema } from "./scope.js"; export { qualifiedDbName } from "./qualified-name.js"; export { computeDrift, computeDriftFromActual, type ComputeDriftOptions, type DriftResult } from "./drift/drift.js"; diff --git a/server/typescript/packages/migrate-ts/src/scope.ts b/server/typescript/packages/migrate-ts/src/scope.ts index ece625509..f6c48afab 100644 --- a/server/typescript/packages/migrate-ts/src/scope.ts +++ b/server/typescript/packages/migrate-ts/src/scope.ts @@ -62,6 +62,36 @@ export interface ScopedExpectedSchema { declaredSchemas?: string[]; } +/** + * Carry an out-of-scope object forward into the snapshot a run is about to commit. + * + * The committed snapshot is built from the metadata-expected schema, which a scoped + * run has already narrowed — so accepting a scoped run DELETES every out-of-scope + * entry the previous snapshot held. Widening or removing `migrate.scope` later then + * proposes `CREATE TABLE` for a table that exists, and the migration fails at apply. + * + * `prior` is the snapshot (or introspected schema) the run diffed against, and the + * entries taken from it are exactly the ones this run excluded — nothing else is + * carried, so a table the model never declared is unaffected either way. An empty + * `outOfScope` returns the SAME object, so an unscoped run commits a byte-identical + * snapshot. + */ +export function carryForwardOutOfScope( + next: SchemaSnapshot, + prior: SchemaSnapshot, + outOfScope: readonly string[], +): SchemaSnapshot { + if (outOfScope.length === 0) return next; + const excluded = new Set(outOfScope); + const keep = (objs: readonly T[]): T[] => + objs.filter((o) => excluded.has(qualifiedDbName(o))); + return { + ...next, + tables: [...next.tables, ...keep(prior.tables)], + views: [...next.views, ...keep(prior.views)], + }; +} + /** * The distinct database schemas a snapshot's tables and views sit in, absent * normalized to the Postgres default — the value `diff` derives for itself when no diff --git a/server/typescript/packages/migrate-ts/src/snapshot/plan.ts b/server/typescript/packages/migrate-ts/src/snapshot/plan.ts index 0ee766a17..c56b47c6e 100644 --- a/server/typescript/packages/migrate-ts/src/snapshot/plan.ts +++ b/server/typescript/packages/migrate-ts/src/snapshot/plan.ts @@ -3,7 +3,7 @@ import type { ColumnNamingStrategy, MetaData } from "@metaobjectsdev/metadata"; import { buildExpectedSchema, buildExpectedSchemaWithProvenance } from "../expected-schema.js"; import { diff, type DiffArgs } from "../diff/index.js"; import { collectUnmanagedNames } from "../unmanaged.js"; -import { scopeExpectedSchema, type ObjectScopePredicate } from "../scope.js"; +import { carryForwardOutOfScope, scopeExpectedSchema, type ObjectScopePredicate } from "../scope.js"; import type { Dialect, DiffResult, SchemaSnapshot } from "../types.js"; import type { ExpectedViewInput } from "../expected-schema.js"; @@ -27,8 +27,20 @@ export interface PlanOfflineArgs extends Pick { }); }); +describe("an accepted scoped run keeps out-of-scope entries in the committed snapshot", () => { + test("the snapshot planOffline hands back retains the excluded table", async () => { + const root = await loadBoth(); + const prior = buildExpectedSchema(root, { dialect: "sqlite" }); + expect(prior.tables.map((t) => t.name).sort()).toEqual(["jobs", "matches"]); + + const plan = await planOffline({ + metadata: root, + dialect: "sqlite", + snapshot: prior, + inScope: platformOnly, + }); + + // The DIFF side is narrowed — the run governs `jobs` only... + expect(plan.expected.tables.map((t) => t.name)).toEqual(["jobs"]); + // ...but the snapshot it commits still holds `matches`. Committing the narrowed + // schema would delete it, and removing `migrate.scope` later would then propose + // CREATE TABLE for a table that exists — a migration that fails at apply. + expect(plan.nextSnapshot.tables.map((t) => t.name).sort()).toEqual(["jobs", "matches"]); + }); + + test("an unscoped run commits the SAME object — byte-identical snapshot", async () => { + const root = await loadBoth(); + const prior = buildExpectedSchema(root, { dialect: "sqlite" }); + const plan = await planOffline({ metadata: root, dialect: "sqlite", snapshot: prior }); + expect(plan.nextSnapshot).toBe(plan.expected); + }); + + test("removing the scope after an accepted run proposes nothing (the round trip)", async () => { + const root = await loadBoth(); + const prior = buildExpectedSchema(root, { dialect: "sqlite" }); + + const scopedRun = await planOffline({ + metadata: root, dialect: "sqlite", snapshot: prior, inScope: platformOnly, + }); + // Second run, scope removed, diffing against what the first run committed. + const unscopedRun = await planOffline({ + metadata: root, dialect: "sqlite", snapshot: scopedRun.nextSnapshot, + }); + expect(unscopedRun.diff.changes).toEqual([]); + }); +}); + describe("provenance never reaches the committed snapshot", () => { test("a view's declaring FQN is recorded but never serialized; formatVersion stays 3", async () => { const root = await loadBoth(); From 5ee93a81c9a32b8569dda82edc2a7aed0a65b061 Mon Sep 17 00:00:00 2001 From: Doug Mealing Date: Tue, 18 Aug 2026 08:03:37 -0400 Subject: [PATCH 32/44] refactor(cli): compute the snapshot gate's schema scope once MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Self-review follow-up to the C2 fix: `declaredSchemasOf(snapshot)` was called twice in the same expression — once to decide whether to pass `scopeSchemas`, once to build it. One binding, named for what it is, with the reasoning hoisted above the `diff` call where a reader meets it. Co-Authored-By: Claude Opus 5 (1M context) --- .../packages/cli/src/commands/verify.ts | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/server/typescript/packages/cli/src/commands/verify.ts b/server/typescript/packages/cli/src/commands/verify.ts index 17cab54aa..32ab71309 100644 --- a/server/typescript/packages/cli/src/commands/verify.ts +++ b/server/typescript/packages/cli/src/commands/verify.ts @@ -638,20 +638,20 @@ export async function verifyCommand( views: snapshot.views.filter((v) => !excluded.has(qualifiedDbName(v))), }; + // Pin the schema scope to the UNFILTERED snapshot's schemas whenever the filter + // above removed anything (migrate-ts's scope.ts header has the mechanism): + // filtering `expected` down to empty would otherwise reach `diff`'s "no model, + // govern the whole database" fallback and report every table another owner has + // as a snapshot disagreement. Nothing filtered ⇒ nothing passed ⇒ an unscoped + // project's arguments are unchanged. + const snapshotSchemas = excluded.size === 0 ? [] : declaredSchemasOf(snapshot); + const result = await diff({ expected: scopedSnapshot, actual, allow: {}, unmanagedNames: [...collectUnmanagedNames(root), ...outOfScope], - // Pin the schema scope to the UNFILTERED snapshot's schemas whenever the - // filter above removed anything (migrate-ts's scope.ts header has the - // mechanism): filtering `expected` down to empty would otherwise reach - // `diff`'s "no model, govern the whole database" fallback and report every - // table another owner has as a snapshot disagreement. Nothing filtered ⇒ - // nothing passed ⇒ an unscoped project's arguments are unchanged. - ...(excluded.size > 0 && declaredSchemasOf(snapshot).length > 0 - ? { scopeSchemas: declaredSchemasOf(snapshot) } - : {}), + ...(snapshotSchemas.length > 0 ? { scopeSchemas: snapshotSchemas } : {}), }); if (result.changes.length === 0) return []; From bdef4fc19b88b8f0dfa39f4470298d5829f2da4e Mon Sep 17 00:00:00 2001 From: Doug Mealing Date: Tue, 18 Aug 2026 08:56:02 -0400 Subject: [PATCH 33/44] fix: nine correctness findings from the whole-branch quality pass MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Part A of the consolidated quality pass over the metadata-source-resolution branch. Every item is a guarantee the toolchain already made and did not keep. A1 — `meta docs --site` hard-failed on the flagship multi-source config. The CLI's dedup set was SEEDED from the source dirs, so it only ever guarded the `templates/` additions, and two declared sources sharing a basename (`metaobjects` plus `../shared-model/metaobjects`) reached the site loader as a "duplicate source dir basename" error. The site keys its source groups by that basename, so the fix is in the loader: collisions are qualified by their parent directory, then by a counter, and the returned group names are the staging names `treeOf` actually matches. The CLI dedups by resolved PATH — two different dirs sharing a name are legitimate; the same dir twice is the real hazard. A2 — discovery walked past a LOCAL `metaobjects/`. A nested project holding its own metadata and no config of its own read that metadata before this branch; the config-only stop condition made it silently load the ANCESTOR's model and write generated output to the ancestor's outDir. The walk now stops at the first ancestor carrying EITHER `.metaobjects/config.json` OR a `metaobjects/` directory, config checked first. Stopping on the latter means the default sources apply, which is exactly the pre-branch behaviour for that directory. Design §4.6.1 amended. `findConfigDir` is replaced by `discoverCollectionRoot` (dir + hasConfig) and `resolveConfigDir`, which `migrate` now calls — two walks with different stop conditions is the drift this branch exists to remove. A3 — `meta docs` never adopted `projectRoot`. It read `metaobjects.config.ts`, the docs `outDir`, `templates/` and the owned `codegen/docs-site/` theme from the ambient `` argument, resolving the collection only afterwards, so a run from a subdirectory rendered the ancestor's metadata with the subdirectory's absent providers. Discovery now runs first and `projectRoot = collection.configDir`, matching `gen`, `verify` and `prompt-snapshot`. `--scaffold-site` resolves the same root, so it writes where `--site` reads. A4 — `meta export`'s two adopter-visible changes (files-before-subdirectories sibling order, `_pending/` excluded) are documented in the Upgrading section and the changelog. Both are correct; neither was written down. A5 — the last whole-database door. The committed-snapshot gate guarded on `snapshotSchemas.length > 0`, so a snapshot that EXISTS but is empty combined with a non-empty out-of-scope set still reached `diff`'s "no model, govern the whole database" fallback: a third party's table, in a schema this model never declares, became a drop candidate reported as a snapshot disagreement. Closed through the shared helper rather than beside it — the gate now takes the scope decision the drift comparison already made (`DriftResult` reports its `declaredSchemas`) instead of re-deriving a pin from the snapshot. A6 — the previous brief's premise was wrong in the other direction, so the real semantics is pinned by test and stated in `scope.ts`'s header: a scope narrows which OBJECTS the tool governs, never which SCHEMAS it may see. A schema whose every declared object is excluded stays in scope, so another owner's undeclared table in it is still a drop candidate — the same verdict an unscoped run gives. Deriving the schema set from the survivors instead reintroduces the inversion. A7 — the migrations directory follows the project root, which moves the ledger for the two subcommands that load no metadata (`apply-pending`, `--rollback`). Kept, because the ledger belongs with the config that declares it, but made loud: `migrate` names the directory it is using whenever it differs from `/.metaobjects/migrations` and that local directory exists. A8 — `gen` and `verify` describe the anti-pattern scan and the agent-context nudge as the same advisory pass and scanned two different trees for it. Both now root at `projectRoot`; a `verify` run from a subdirectory previously found no agent-context manifest at all, so the nudge silently never fired. A9 — one comment naming the carry-forward trade-off: the entries carried into a scoped run's committed snapshot are INTROSPECTED descriptors, so removing the scope later can produce one round of cosmetic alter churn. Strictly better than the `CREATE TABLE`-on-an-existing-table it replaced, and identical in kind to `baseline --from-db`. The shared `scopedDiffInputs` / `excludeFromSnapshot` helper (Part B's highest- value finding) lands here because A5 cannot be fixed correctly without it: the three-part scoped-diff contract was enforced by prose at five call sites, and site five had already drifted into its own guard. All five now compose through one door. Co-Authored-By: Claude Opus 5 (1M context) --- CHANGELOG.md | 20 +++ docs/features/metadata-sources.md | 50 ++++++- ...08-17-metadata-source-resolution-design.md | 12 +- .../packages/cli/src/commands/docs.ts | 114 +++++++++------- .../packages/cli/src/commands/gen.ts | 11 +- .../packages/cli/src/commands/migrate.ts | 83 ++++++++---- .../packages/cli/src/commands/verify.ts | 63 ++++----- .../packages/cli/test/docs-command.test.ts | 50 +++++++ .../typescript/packages/docs-site/src/load.ts | 45 +++++-- .../packages/migrate-ts/src/drift/drift.ts | 35 +++-- .../packages/migrate-ts/src/index.ts | 10 +- .../packages/migrate-ts/src/scope.ts | 124 ++++++++++++++++-- .../packages/migrate-ts/src/snapshot/plan.ts | 21 +-- .../test/expected-schema-scope.test.ts | 99 +++++++++++++- .../test/scope-snapshot-gate.test.ts | 94 +++++++++++++ .../typescript/packages/sdk/src/collection.ts | 53 ++++---- .../typescript/packages/sdk/src/discovery.ts | 107 +++++++++++---- server/typescript/packages/sdk/src/index.ts | 6 +- .../packages/sdk/test/collection.test.ts | 27 ++++ .../packages/sdk/test/discovery.test.ts | 54 ++++++-- 20 files changed, 848 insertions(+), 230 deletions(-) create mode 100644 server/typescript/packages/migrate-ts/test/scope-snapshot-gate.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 414881453..4e09c812b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -119,6 +119,26 @@ the same migrations. Three changes are visible even to that project. Adopter gui `migrate.scope` decides ownership on that name, and a view arriving without one cannot be scoped at all. `buildProjectionViews` already supplies it; only hand-built `ExpectedView` values need the field added. +- **`meta export` output order changed, and `_pending/` is excluded.** `export` now + serializes the file set `resolveCollection` resolved rather than scanning a + directory through `DirectorySource`, so siblings emit files-before-subdirectories + (the overlay-safe order the loader has always been given) instead of a flat + basename sort, and staged `_pending/` files — skipped by every other read path — + are no longer exported. The canonical JSON content is unchanged; a committed + export diffed against a fresh one shows a reordering. +- **The migrations directory follows the project root.** `.metaobjects/migrations` + and the schema snapshot resolve from the directory whose `.metaobjects/config.json` + governs the run, found by walking up from the working directory. `meta migrate + apply-pending` and `--rollback` load no metadata and previously used the working + directory unconditionally, so a subdirectory holding a ledger but no config of its + own now replays the project root's history. `migrate` says so out loud when the + resolved directory differs from `/.metaobjects/migrations` and that local + directory exists; `--out-dir` overrides, and giving the subdirectory its own + `.metaobjects/config.json` makes it a project root. +- **Discovery stops at a `metaobjects/` directory, not only at a config.** A nested + project holding its own `metaobjects/` and no `.metaobjects/config.json` keeps + reading its own metadata, as it always did, rather than adopting an ancestor's + model and `outDir`. ## [0.23.2] — npm `0.23.2` · PyPI `0.23.2` · NuGet `0.23.2` · Maven `7.23.2` diff --git a/docs/features/metadata-sources.md b/docs/features/metadata-sources.md index 93c3f9f80..16c40f974 100644 --- a/docs/features/metadata-sources.md +++ b/docs/features/metadata-sources.md @@ -420,8 +420,8 @@ type) surfaces as the config load error and stops the command. ## Upgrading A project with one config at its root, no `sources` and no `scope` resolves the same -files it always did and generates the same code. Three changes are still worth -knowing about before you upgrade. +files it always did and generates the same code. Six changes are still worth knowing +about before you upgrade. ### The workspace `extends:` walk is retired @@ -479,6 +479,52 @@ because `migrate.scope` decides on that name and a view arriving without one can be scoped at all. Code that builds `ExpectedView` values by hand (the normal path, `buildProjectionViews`, already supplies it) has to add the field. +### `meta export` walks one tree, and skips `_pending/` + +`meta export` used to scan a directory through `DirectorySource`; it now serializes +the file set `resolveCollection` resolved, which is the same walk every other command +uses. Two things change in its output, for every project — declared `sources` or not: + +- **Sibling order is files-before-subdirectories**, not a flat basename sort. A + directory whose subdirectory name sorts before a sibling file (`admin/` before + `user.json`) therefore emits in a different order than it used to. This is the + order the loader has always been given, and it is the overlay-safe one — a base + file must load before an overlay nested under it — so `export` and the loader now + agree instead of disagreeing. +- **`_pending/` is excluded.** Those files are staged, not active metadata; every + other read path already skipped them, and `export` was the one that did not. + +The canonical JSON *content* is unchanged. If you diff a committed export against a +freshly generated one, expect a reordering and the loss of any `_pending/` entries. + +### The migrations directory follows the project root, not the shell + +`.metaobjects/migrations` (and the schema snapshot beside it) is resolved from the +directory whose `.metaobjects/config.json` governs the run, discovered by walking up +from the working directory. It used to come from the working directory +unconditionally for `meta migrate apply-pending` and `meta migrate --rollback`, which +load no metadata at all. + +The ledger belongs with the config that declares it, so this is the intended +behaviour — but it moves the ledger for one layout: a subdirectory holding +`.metaobjects/migrations` with **no** `.metaobjects/config.json` of its own, sitting +under a project root that has both. Running `apply-pending` there now replays the +root's history. + +It is not silent. When the resolved migrations directory differs from +`/.metaobjects/migrations` **and** that local directory exists, the command +prints which one it is using and how to override: + +``` +migrate: using the migrations directory /repo/.metaobjects/migrations, not the +/repo/apps/api/.metaobjects/migrations in this working directory — the ledger belongs +to the project root that declares it. Pass --out-dir … to use the local one. +``` + +`--out-dir` (or `migrate.outDir` in the config) selects the directory explicitly, and +adding a `.metaobjects/config.json` to that subdirectory makes it a project root in +its own right — which is what a directory holding its own ledger almost always wants. + --- ## What is deferred diff --git a/docs/superpowers/specs/2026-08-17-metadata-source-resolution-design.md b/docs/superpowers/specs/2026-08-17-metadata-source-resolution-design.md index 2b263dd1c..f79990d52 100644 --- a/docs/superpowers/specs/2026-08-17-metadata-source-resolution-design.md +++ b/docs/superpowers/specs/2026-08-17-metadata-source-resolution-design.md @@ -304,8 +304,16 @@ from the same root cause — so routing every read through one authority closes Running a CLI inside an app must find that app's configuration. -- **Walk up from cwd** for the nearest `.metaobjects/config.json` declaring a non-empty `sources`. - Nearest wins. Per-port generator config is then read from that same directory. +- **Walk up from cwd** for the nearest project root. Nearest wins. Per-port generator config is + then read from that same directory. +- **A directory is a project root when it carries EITHER `.metaobjects/config.json` OR a + `metaobjects/` directory.** The config is checked first, so a directory carrying both resolves + as declared. The second marker is a back-compat obligation, not a convenience: before source + resolution existed, a nested project holding its own `metaobjects/` and no config of its own + read its own metadata. A config-only stop condition walks straight past it and silently loads + the ANCESTOR's model, then writes generated output to the ancestor's `outDir` — a silent + regression on a layout that worked. Stopping there with no config found means the default + sources apply, which is exactly the pre-branch behaviour for that directory. - **Stop at a repository boundary** (`.git`) or the filesystem root, so a monorepo can never silently adopt a parent checkout's configuration. - **Explicit override wins** — the existing `--cwd` / `-C` flag and project-root positional are diff --git a/server/typescript/packages/cli/src/commands/docs.ts b/server/typescript/packages/cli/src/commands/docs.ts index cde8bbded..8410bc766 100644 --- a/server/typescript/packages/cli/src/commands/docs.ts +++ b/server/typescript/packages/cli/src/commands/docs.ts @@ -15,7 +15,7 @@ import { resolve as resolvePath, basename, isAbsolute } from "node:path"; import { mkdir, writeFile } from "node:fs/promises"; import { log } from "../lib/log.js"; import { loadMetaobjectsConfig } from "../lib/load-metaobjects-config.js"; -import { loadMemory, resolveCollection, type Collection } from "@metaobjectsdev/sdk"; +import { loadMemory, resolveCollection, resolveConfigDir, type Collection } from "@metaobjectsdev/sdk"; import { existsSync } from "node:fs"; import { join } from "node:path"; import { @@ -206,16 +206,41 @@ export async function docsCommand(args: string[], cwd: string): Promise // `--scaffold-site`: copy the docs-site templates + assets into codegen/docs-site/ // so the consumer owns them (ADR-0034 scaffold-and-own). Scaffold and return — - // it does not also generate. + // it does not also generate. `resolveConfigDir` rather than `resolveCollection`: + // scaffolding needs no metadata, but it must write where `emitSite` will READ + // (below, under the resolved project root), and that is the same walk. if (flags.scaffoldSite) { - return scaffoldSiteCommand(metaRoot); + return scaffoldSiteCommand(await resolveConfigDir(metaRoot)); + } + + // Discovery and load are two separate failure modes, kept in separate try + // blocks deliberately — same reasoning as `meta gen` (gen.ts): a broad + // catch around both would swallow a genuine ParseError as "no metaobjects/ + // found", masking the real failure. + // + // Discovery runs BEFORE the config read, deliberately, and `meta gen` calls + // out the same ordering as the thing it fixed: the project root is whichever + // directory `resolveCollection` decided the metadata belongs to, so + // everything project-relative — `metaobjects.config.ts` and its providers, + // the `docs.outDir` it names, the adopter `templates/` overrides, the owned + // `codegen/docs-site/` theme — has to come from that same directory. Reading + // the config from the ambient `` argument while the metadata came + // from an ancestor renders the ancestor's model with the subdirectory's + // (absent) providers. For a run at the project root the two are the same path. + let collection: Awaited>; + try { + collection = await resolveCollection(metaRoot); + } catch (err) { + log.error(`docs: ${(err as Error).message}`); + return 2; } // The project root used to resolve adopter `templates/` overrides; the - // framework defaults sit underneath via projectProvider's chain. + // framework defaults sit underneath via projectProvider's chain. `--templates` + // is the one explicit override. const projectRoot = flags.templates !== undefined ? resolvePath(cwd, flags.templates) - : metaRoot; + : collection.configDir; // Best-effort load of metaobjects.config.ts to pick up consumer-supplied // providers (e.g. a project's custom field/object subtypes). Unlike `gen`, @@ -228,9 +253,10 @@ export async function docsCommand(args: string[], cwd: string): Promise // hasConfig gates the api surface: api docs describe the GENERATED REST // surface, which only exists when there is a (loadable) gen config. A config // that EXISTS but fails to load degrades to model-only with a warning. - const hasConfig = existsSync(join(metaRoot, "metaobjects.config.ts")); - // The config lives alongside metaobjects/ at the metadata root (metaRoot); - // projectRoot only diverges when --templates overrides the template lookup. + const hasConfig = existsSync(join(collection.configDir, "metaobjects.config.ts")); + // The config lives beside the `.metaobjects/` that declared this collection — + // `collection.configDir`, never ambient cwd. `projectRoot` only diverges from + // it when --templates overrides the template lookup. // Only attempt the load when the file is actually present: absence is the // expected config-less case (stay silent), but a config that EXISTS yet fails // to load is surfaced as a warning rather than silently degrading to @@ -238,7 +264,7 @@ export async function docsCommand(args: string[], cwd: string): Promise // cryptic unknown-subtype error instead of the real config error. if (hasConfig) { try { - loadedConfig = await loadMetaobjectsConfig(metaRoot); + loadedConfig = await loadMetaobjectsConfig(collection.configDir); configProviders = loadedConfig.providers; } catch (err) { log.warn( @@ -269,7 +295,7 @@ export async function docsCommand(args: string[], cwd: string): Promise cliOverrides, loadedConfig?.outputLayout ?? "flat", ); - const outDir = resolvePath(metaRoot, docsCfg.outDir); + const outDir = resolvePath(collection.configDir, docsCfg.outDir); // SITE surface has its OWN model loader (docs-site's loadModel — NOT the sdk // loadMemory below) and needs no gen config. When the site is the ONLY @@ -277,19 +303,7 @@ export async function docsCommand(args: string[], cwd: string): Promise // WITHOUT building the markdown GenContext — decoupled and one fewer failure // surface. Combined with --model/--api it is emitted after them (below). if (flags.site && docsCfg.surfaces.length === 0) { - return emitSite(metaRoot, outDir, configProviders, promptsDir); - } - - // Discovery and load are two separate failure modes, kept in separate try - // blocks deliberately — same reasoning as `meta gen` (gen.ts): a broad - // catch around both would swallow a genuine ParseError as "no metaobjects/ - // found", masking the real failure. - let collection; - try { - collection = await resolveCollection(metaRoot); - } catch (err) { - log.error(`docs: ${(err as Error).message}`); - return 2; + return emitSite(collection, projectRoot, outDir, configProviders, promptsDir); } // Load metadata standalone — same loader path as migrate/gen. Threads any @@ -452,7 +466,7 @@ export async function docsCommand(args: string[], cwd: string): Promise // SITE surface (additive) — emit after the markdown surfaces so both coexist. if (flags.site) { - const siteRc = await emitSite(metaRoot, outDir, configProviders, promptsDir); + const siteRc = await emitSite(collection, projectRoot, outDir, configProviders, promptsDir); if (siteRc !== 0) return siteRc; } @@ -479,9 +493,9 @@ export async function docsCommand(args: string[], cwd: string): Promise * into `/codegen/docs-site/{templates,assets}`, writing each file ONLY if * absent so a re-run never clobbers a hand-edited file. */ -async function scaffoldSiteCommand(metaRoot: string): Promise { - const tplDir = join(metaRoot, "codegen/docs-site/templates"); - const astDir = join(metaRoot, "codegen/docs-site/assets"); +async function scaffoldSiteCommand(projectRoot: string): Promise { + const tplDir = join(projectRoot, "codegen/docs-site/templates"); + const astDir = join(projectRoot, "codegen/docs-site/assets"); const created: string[] = []; const preserved: string[] = []; try { @@ -507,7 +521,7 @@ async function scaffoldSiteCommand(metaRoot: string): Promise { } log.info( `meta docs --scaffold-site — ${created.length} created, ${preserved.length} preserved ` + - `→ ${join(metaRoot, "codegen/docs-site")} (edit these to own your theme)`, + `→ ${join(projectRoot, "codegen/docs-site")} (edit these to own your theme)`, ); return 0; } @@ -540,11 +554,16 @@ function collectionSourceDirs(collection: Collection): string[] { * this is independent of the sdk loadMemory path used for the markdown * surfaces. Writes under `/site` so it can coexist with the markdown * output. Scaffold-and-own: when the consumer has copied templates/assets - * into `/codegen/docs-site/` (via `--scaffold-site`), those win + * into `/codegen/docs-site/` (via `--scaffold-site`), those win * over the bundled defaults. + * + * Takes the ALREADY-RESOLVED collection: `docsCommand` resolved it to read the + * config from the right directory, and resolving a second time here made the + * combined `--model --site` path do the whole discovery-and-config walk twice. */ async function emitSite( - metaRoot: string, + collection: Collection, + projectRoot: string, outDir: string, configProviders?: readonly MetaDataTypeProvider[], promptsDir?: string, @@ -555,35 +574,36 @@ async function emitSite( // additionally searched in the conventional /templates/ and any // explicit --prompts dir (for a project whose templates live elsewhere, // e.g. data/templates/) — else the site can't show the prompt TEXT and - // prints a "source missing" note. Only existing dirs are added, and dirs are - // deduped by BASENAME (the site keys source groups by basename, and rejects a dup). - let collection; - try { - collection = await resolveCollection(metaRoot); - } catch (err) { - log.error(`docs: ${(err as Error).message}`); - return 2; - } + // prints a "source missing" note. Only existing dirs are added. + // + // Deduped by resolved PATH, not by basename. Two DIFFERENT directories that + // happen to share a basename are a legitimate multi-source project (`metaobjects` + // plus `../shared-model/metaobjects`); `loadModel` disambiguates their site + // group names, so refusing the pair here — which a basename key did, by + // dropping the second — would break the feature this branch exists to ship. + // The same directory named twice is the real hazard: it would be symlinked + // and loaded twice. const sourceDirs = collectionSourceDirs(collection); - const seenBasenames = new Set(sourceDirs.map((d) => basename(d))); + const seenDirs = new Set(sourceDirs); if (promptsDir !== undefined && !existsSync(promptsDir)) { log.warn(`docs: --prompts dir does not exist: ${promptsDir}`); } - for (const d of [join(metaRoot, "templates"), ...(promptsDir !== undefined ? [promptsDir] : [])]) { - if (existsSync(d) && !seenBasenames.has(basename(d))) { - sourceDirs.push(d); - seenBasenames.add(basename(d)); + for (const d of [join(projectRoot, "templates"), ...(promptsDir !== undefined ? [promptsDir] : [])]) { + const abs = resolvePath(d); + if (existsSync(abs) && !seenDirs.has(abs)) { + sourceDirs.push(abs); + seenDirs.add(abs); } } // Scaffold-and-own: when the consumer has copied templates/assets into // codegen/docs-site/ (via --scaffold-site), use those; else the bundled defaults. - const ownedTemplates = join(metaRoot, "codegen/docs-site/templates"); - const ownedAssets = join(metaRoot, "codegen/docs-site/assets"); + const ownedTemplates = join(projectRoot, "codegen/docs-site/templates"); + const ownedAssets = join(projectRoot, "codegen/docs-site/assets"); try { const r = await generateSite({ sourceDirs, outDir: siteOutDir, - title: basename(metaRoot) || "Metadata", + title: basename(collection.configDir) || "Metadata", stamp: new Date().toISOString().slice(0, 10), commit: "", core: { n: 15 }, diff --git a/server/typescript/packages/cli/src/commands/gen.ts b/server/typescript/packages/cli/src/commands/gen.ts index f658db1ca..3122a568a 100644 --- a/server/typescript/packages/cli/src/commands/gen.ts +++ b/server/typescript/packages/cli/src/commands/gen.ts @@ -35,9 +35,6 @@ export async function genCommand(args: string[], cwd: string, fmt: OutputFormat return listGeneratorsCommand(); } - // Advisory: nudge to refresh the .claude/skills docs if they predate this CLI. - warnIfAgentContextStale(cwd); - const cliConfig = resolveGenConfig(flags); // Discovery and load are two separate failure modes, kept in separate try @@ -65,6 +62,14 @@ export async function genCommand(args: string[], cwd: string, fmt: OutputFormat } const projectRoot = collection.configDir; + // Advisory: nudge to refresh the .claude/skills docs if they predate this CLI. + // Rooted at `projectRoot`, not ambient cwd — the scaffolded agent context sits + // with the project that declares the metadata, so a run from a subdirectory + // would find no manifest there and silently skip the nudge. `meta verify` makes + // the same call for the same reason; the two commands describe this and the + // anti-pattern scan below as one advisory pass, so they must scan one tree. + warnIfAgentContextStale(projectRoot); + let forgeConfig; try { forgeConfig = await loadMetaobjectsConfig(projectRoot); diff --git a/server/typescript/packages/cli/src/commands/migrate.ts b/server/typescript/packages/cli/src/commands/migrate.ts index d75f01079..52a125b3b 100644 --- a/server/typescript/packages/cli/src/commands/migrate.ts +++ b/server/typescript/packages/cli/src/commands/migrate.ts @@ -1,5 +1,6 @@ import { resolve as resolvePath } from "node:path"; import { mkdir } from "node:fs/promises"; +import { existsSync } from "node:fs"; import { spawn } from "node:child_process"; import { parseMigrateArgs } from "../lib/args.js"; import { resolveMigrateConfig, MIGRATE_DEFAULT_OUT_DIR } from "../lib/config.js"; @@ -10,12 +11,13 @@ import type { OutputFormat } from "../lib/format.js"; import { toonEncode } from "../lib/format.js"; import { buildKyselyFromUrl, redactUrl } from "../lib/kysely.js"; import { log } from "../lib/log.js"; -import { findConfigDir, loadMemory, resolveCollection } from "@metaobjectsdev/sdk"; +import { loadMemory, resolveCollection, resolveConfigDir } from "@metaobjectsdev/sdk"; import { loadMetaobjectsConfig } from "../lib/load-metaobjects-config.js"; import { migrateScopeMismatch, toObjectScope } from "../lib/migrate-scope.js"; import { buildExpectedSchemaWithProvenance, scopeExpectedSchema, + scopedDiffInputs, introspect, diff, collectUnmanagedNames, @@ -125,6 +127,35 @@ function resolveFormatOutDir(config: ResolvedMigrateConfig, metaRoot: string): s return resolvePath(metaRoot, config.outDir); } +/** + * Say so when the migrations directory this run will use is NOT the one sitting + * in the working directory. + * + * `metaRoot` is now the discovered project root rather than ambient cwd, and the + * migrations directory follows it. That is the right call — the ledger belongs + * with the config that declares it — but it is a behaviour change for the two + * subcommands that load no metadata at all: `apply-pending` and `--rollback` + * used cwd unconditionally, so a subdirectory holding `.metaobjects/migrations` + * under a project root that also has one now replays the ROOT's ledger. + * Replaying somebody else's migration history silently is the worst outcome + * available here, so it is announced. + * + * Conditioned on the local directory EXISTING, so the ordinary case — a run from + * anywhere inside a project with one ledger at its root — says nothing. + * `--out-dir` (and a `migrate.outDir` in the config) is honoured: the caller + * passes the RESOLVED directory, so a deliberate redirection is compared, not + * the default that was overridden. + */ +function warnIfLedgerRelocated(cwd: string, resolvedOutDir: string): void { + const local = resolvePath(cwd, MIGRATE_DEFAULT_OUT_DIR); + if (resolvedOutDir === local || !existsSync(local)) return; + log.warn( + `migrate: using the migrations directory ${resolvedOutDir}, not the ${local} ` + + `in this working directory — the ledger belongs to the project root that declares it. ` + + `Pass --out-dir ${local} to use the local one.`, + ); +} + /** * Say what a declared `migrate.scope` left out. An excluded object produces neither * a create nor a drop, so without this line "no changes" and "no changes to the half @@ -296,14 +327,16 @@ export async function migrateCommand( // sources were resolvable that invocation just failed with "no metaobjects/ // found". // - // `findConfigDir` rather than `resolveCollection` deliberately: this must not - // require metadata to EXIST. `migrate apply-pending` and `--rollback` replay - // committed SQL and load no metadata at all, and making them fail on a project - // with no model would be a regression. Falls back to cwd when no config is - // found anywhere, which is exactly what `resolveCollection` does, so the two - // agree by construction. - const metaRoot = (await findConfigDir(cwd)) ?? resolvePath(cwd); + // `resolveConfigDir` rather than `resolveCollection` deliberately: this must + // not require metadata to EXIST. `migrate apply-pending` and `--rollback` + // replay committed SQL and load no metadata at all, and making them fail on a + // project with no model would be a regression. It is the SAME walk + // `resolveCollection` runs (one exported definition in the sdk's + // `discovery.ts`, not two that agree by construction), so the directory this + // resolves and the directory the metadata comes from cannot diverge. + const metaRoot = await resolveConfigDir(cwd); const config = await resolveMigrateConfig(flags, metaRoot); + warnIfLedgerRelocated(cwd, resolvePath(metaRoot, config.outDir)); try { // #192 — Flyway owns apply + history (flyway_schema_history). We generate the @@ -525,7 +558,12 @@ export async function migrateCommand( let diffResult; try { diffResult = await diff({ - expected, + // The three scoped-diff obligations as one value (migrate-ts's scope.ts + // header has the mechanism): the narrowed expected side, `unmanagedNames` + // merging #208 §7's declared-@unmanaged set with the out-of-scope names so + // neither is created or dropped, and the schema scope pinned to the + // UNSCOPED model so narrowing can never widen the run. + ...scopedDiffInputs(scoped, collectUnmanagedNames(metadata)), actual, dialect: kysely.dialect, allow: tokensToAllowOptions(config.allow), @@ -533,15 +571,6 @@ export async function migrateCommand( // has no expressible migration; refuse loudly instead of emitting SQL that drops // the constraint and breaks referencing FKs at apply. refusePrimaryKeyChange: true, - // #208 §7 — declared-@unmanaged objects are external: exclude them from the - // actual side so migrate proposes neither create nor drop for them. Objects - // outside `migrate.scope` ride the same seam, for the same reason. - unmanagedNames: [...collectUnmanagedNames(metadata), ...scoped.outOfScope], - // Pin the schema scope to the UNSCOPED model's schemas (see migrate-ts's - // scope.ts header): a `migrate.scope` matching nothing would otherwise empty - // `expected`, which `diff` reads as "no model, govern the whole database". - // Absent when no scope was given, so an unscoped run is unchanged. - ...(scoped.declaredSchemas !== undefined ? { scopeSchemas: scoped.declaredSchemas } : {}), onAmbiguous: async (a) => { collectedAmbiguous.push(a); return onAmbiguousResolution; @@ -692,6 +721,15 @@ export async function migrateCommand( // out-of-scope entries come from `actual` — they are in the database, which // is the same thing `baseline --from-db` records. Identical object, and so a // byte-identical snapshot, for an unscoped run. + // + // The trade-off, stated so it is not rediscovered: those carried entries are + // INTROSPECTED descriptors, not metadata-built ones, so they can differ + // cosmetically from what this model would have emitted for the same table + // (column order, a default's rendered form). Removing the scope later can + // therefore produce one round of alter churn. That is strictly better than + // the alternative it replaced — a `CREATE TABLE` for a table that exists, + // which fails at apply — and it is the same mixed-provenance snapshot + // `baseline --from-db` writes for every table it adopts. const committed = carryForwardOutOfScope(expected, actual, scoped.outOfScope); await writeSnapshot( snapshotPath(resolvePath(metaRoot, config.outDir), kysely.dialect), @@ -1276,7 +1314,8 @@ async function runD1Migrate( let diffResult; try { diffResult = await diff({ - expected, + // The three scoped-diff obligations, exactly as on the online path above. + ...scopedDiffInputs(scoped, collectUnmanagedNames(metadata)), actual, // D1 is SQLite at the SQL level — the dialect activates the sqlite diff // semantics (structural FK matching: SQLite stores no FK names; CHECK @@ -1288,12 +1327,6 @@ async function runD1Migrate( // has no expressible migration; refuse loudly instead of emitting SQL that drops // the constraint and breaks referencing FKs at apply (same failure as the online path). refusePrimaryKeyChange: true, - // #208 §7 — declared-@unmanaged objects are external (see the online path above), - // and so are objects outside `migrate.scope`. - unmanagedNames: [...collectUnmanagedNames(metadata), ...scoped.outOfScope], - // Schema scope pinned to the UNSCOPED model's schemas — same reasoning as the - // online path above (migrate-ts's scope.ts header has the mechanism). - ...(scoped.declaredSchemas !== undefined ? { scopeSchemas: scoped.declaredSchemas } : {}), onAmbiguous: async (a) => { collectedAmbiguous.push(a); return onAmbiguousResolution; diff --git a/server/typescript/packages/cli/src/commands/verify.ts b/server/typescript/packages/cli/src/commands/verify.ts index 32ab71309..f81a3248c 100644 --- a/server/typescript/packages/cli/src/commands/verify.ts +++ b/server/typescript/packages/cli/src/commands/verify.ts @@ -33,8 +33,9 @@ import { computeDrift, computeDriftFromActual, collectUnmanagedNames, - declaredSchemasOf, - qualifiedDbName, + excludeFromSnapshot, + scopedDiffInputs, + type GovernedScope, introspect, diff, readSnapshot, @@ -110,9 +111,6 @@ export async function verifyCommand( return 2; } - // Advisory: nudge to refresh the .claude/skills docs if they predate this CLI. - warnIfAgentContextStale(cwd); - // ADR-0021 D2 — explicit verify subverbs. Each flag selects one drift mode; // any combination runs each and the overall exit code is the MAX (non-zero on // any drift). A bare `verify` (no explicit subverb) keeps its documented @@ -152,12 +150,19 @@ export async function verifyCommand( // command: anything named BY the metadata or its config resolves against // `projectRoot` — `metaobjects.config.ts`, `.metaobjects/config.json`, the // `outDir` and `wranglerConfigPath` they carry, the `prompts/` a `@textRef` - // resolves in, the test files a `@verifiedBy` names. Anything that is merely - // "the tree the user is standing in" stays on `cwd` — the agent-context - // staleness nudge and the advisory anti-pattern scan, both warnings-only. - // Identical paths for a run from the project root. + // resolves in, the test files a `@verifiedBy` names. Identical paths for a run + // from the project root. + // + // The two advisory passes — the agent-context staleness nudge and the + // anti-pattern scan — are rooted here too, matching `meta gen`. Both commands + // describe them as the same pass, and scanning two different trees for it made + // that false: a `verify` run from a subdirectory scanned only that subtree and + // found no agent-context manifest at all, so the nudge silently never fired. const projectRoot = collection.configDir; + // Advisory: nudge to refresh the .claude/skills docs if they predate this CLI. + warnIfAgentContextStale(projectRoot); + // Best-effort load of metaobjects.config.ts. Two consumers: // 1) consumer-supplied providers (e.g. a `template.toolcall` subtype) threaded // into loadMemory — verify doesn't REQUIRE codegen config for templates/db; @@ -280,7 +285,7 @@ export async function verifyCommand( function runAntiPatternAdvisory(): void { let findings; try { - findings = scanSourceForAntiPatterns(cwd); + findings = scanSourceForAntiPatterns(projectRoot); } catch { return; // never let an advisory scan break verify } @@ -486,7 +491,7 @@ export async function verifyCommand( const snapshotDrift = driftResult.changes.length === 0 - ? await checkCommittedSnapshot(actual, kysely.dialect, kysely.displayUrl, driftResult.outOfScope) + ? await checkCommittedSnapshot(actual, kysely.dialect, kysely.displayUrl, driftResult) : []; return reportSchemaDrift(driftResult, [...ledgerDrift, ...snapshotDrift], kysely.displayUrl); @@ -608,7 +613,7 @@ export async function verifyCommand( actual: SchemaSnapshot, dialect: Dialect, displayUrl: string, - outOfScope: readonly string[], + governed: GovernedScope, ): Promise { if (dialect === "d1") return []; // d1 keeps migrations Wrangler-native; no offline snapshot // Resolve the migrations dir through migrate's OWN precedence (flag > config > @@ -625,33 +630,19 @@ export async function verifyCommand( } if (snapshot === null) return []; - // Out-of-scope objects leave BOTH sides of this comparison. `unmanagedNames` - // suppresses the actual side only, which is right for the metadata↔DB diff (the - // expected side is already scoped) but not here: the committed snapshot is the - // expected side, and a snapshot written before the scope was declared still - // carries the other owner's tables — leaving them in would report a phantom - // "snapshot disagrees" for an object this consumer does not manage. - const excluded = new Set(outOfScope); - const scopedSnapshot: SchemaSnapshot = excluded.size === 0 ? snapshot : { - ...snapshot, - tables: snapshot.tables.filter((t) => !excluded.has(qualifiedDbName(t))), - views: snapshot.views.filter((v) => !excluded.has(qualifiedDbName(v))), - }; - - // Pin the schema scope to the UNFILTERED snapshot's schemas whenever the filter - // above removed anything (migrate-ts's scope.ts header has the mechanism): - // filtering `expected` down to empty would otherwise reach `diff`'s "no model, - // govern the whole database" fallback and report every table another owner has - // as a snapshot disagreement. Nothing filtered ⇒ nothing passed ⇒ an unscoped - // project's arguments are unchanged. - const snapshotSchemas = excluded.size === 0 ? [] : declaredSchemasOf(snapshot); - + // Out-of-scope objects leave BOTH sides of this comparison, and the schema pin + // comes from the scope decision the DRIFT comparison already made — one door + // (migrate-ts's `excludeFromSnapshot` + `scopedDiffInputs`), not a fifth + // hand-rolled copy of the three-part contract. `unmanagedNames` suppresses the + // actual side only, which is right for the metadata↔DB diff (its expected side + // is already scoped) but not here: the committed snapshot IS the expected side, + // and a snapshot written before the scope was declared still carries the other + // owner's tables. Re-deriving the pin from the snapshot is what left an empty + // (never-migrated) snapshot reaching `diff`'s whole-database fallback. const result = await diff({ - expected: scopedSnapshot, + ...scopedDiffInputs(excludeFromSnapshot(snapshot, governed), collectUnmanagedNames(root)), actual, allow: {}, - unmanagedNames: [...collectUnmanagedNames(root), ...outOfScope], - ...(snapshotSchemas.length > 0 ? { scopeSchemas: snapshotSchemas } : {}), }); if (result.changes.length === 0) return []; diff --git a/server/typescript/packages/cli/test/docs-command.test.ts b/server/typescript/packages/cli/test/docs-command.test.ts index fcd1ea756..026e7a004 100644 --- a/server/typescript/packages/cli/test/docs-command.test.ts +++ b/server/typescript/packages/cli/test/docs-command.test.ts @@ -37,6 +37,21 @@ const META = { }, }; +/** A second model in its own package, for the shared-source `--site` case. */ +const SHARED_META = { + "metadata.root": { + package: "acme::shared", + children: [ + { + "object.value": { + name: "SharedThing", + children: [{ "field.string": { name: "label" } }], + }, + }, + ], + }, +}; + const dirs: string[] = []; /** Build a standalone project root holding metaobjects/ — NO gen config. Also @@ -450,6 +465,41 @@ describe("meta docs --site — HTML documentation site", () => { expect(code).toBe(0); expect(existsSync(join(out, "site", "index.html"))).toBe(true); }); + + test("two declared sources sharing a basename are disambiguated, not refused", async () => { + // The flagship shape of the multi-source feature: a project's own + // `metaobjects/` plus a shared model's `metaobjects/` next door. Both have + // the basename the site keys its source groups by, and the site used to + // refuse the pair outright ("duplicate source dir basename"). + const workspace = await mkdtemp(join(tmpdir(), "meta-docs-multi-")); + dirs.push(workspace); + const root = join(workspace, "app"); + await mkdir(join(root, ".metaobjects"), { recursive: true }); + await mkdir(join(root, "metaobjects"), { recursive: true }); + await writeFile(join(root, "metaobjects", "meta.json"), JSON.stringify(META), "utf8"); + await mkdir(join(workspace, "shared-model", "metaobjects"), { recursive: true }); + await writeFile( + join(workspace, "shared-model", "metaobjects", "meta.json"), + JSON.stringify(SHARED_META), + "utf8", + ); + await writeFile( + join(root, ".metaobjects", "config.json"), + JSON.stringify({ + schema_version: 1, + sources: [{ path: "metaobjects" }, { path: "../shared-model/metaobjects" }], + }), + "utf8", + ); + + const out = join(root, "out-site-multi"); + expect(await docsCommand([root, "--site", "--out", out], root)).toBe(0); + expect(existsSync(join(out, "site", "index.html"))).toBe(true); + // Both models are in the site, so neither source was dropped to dodge the collision. + const index = await readFile(join(out, "site", "index.html"), "utf8"); + expect(index).toContain("Welcome"); + expect(index).toContain("SharedThing"); + }); }); describe("meta docs --scaffold-site — own your theme", () => { diff --git a/server/typescript/packages/docs-site/src/load.ts b/server/typescript/packages/docs-site/src/load.ts index a3d40d81e..5ee9b171d 100644 --- a/server/typescript/packages/docs-site/src/load.ts +++ b/server/typescript/packages/docs-site/src/load.ts @@ -1,6 +1,6 @@ import { mkdtempSync, readdirSync, rmSync, statSync, symlinkSync } from "node:fs"; import { tmpdir } from "node:os"; -import { basename, join, resolve } from "node:path"; +import { basename, dirname, join, resolve } from "node:path"; import { MetaDataLoader, composeRegistry, coreTypesProvider, dbProvider, docProvider, promptProvider, uiProvider } from "@metaobjectsdev/metadata"; import type { MetaData, MetaRoot, MetaDataTypeProvider } from "@metaobjectsdev/metadata"; import { FileSource } from "@metaobjectsdev/metadata/core"; @@ -26,14 +26,19 @@ export async function loadModel( ): Promise { const staging = mkdtempSync(join(tmpdir(), "metadocs-")); try { - const usedBasenames = new Set(); + // Source groups are keyed by the staging entry's name, so two source dirs + // sharing a basename need distinct names — `metaobjects/` plus a shared + // model's `../shared-model/metaobjects` is the ordinary shape of a + // multi-source project, and refusing it would make the feature unusable + // with `--site`. Collisions are qualified by their parent directory (the + // name a reader recognises) before falling back to a counter. + const used = new Set(); + const groupNames: string[] = []; for (const dir of sourceDirs) { - const baseName = basename(dir); - if (usedBasenames.has(baseName)) { - throw new Error(`duplicate source dir basename: ${baseName}`); - } - usedBasenames.add(baseName); - symlinkSync(resolve(dir), join(staging, baseName)); + const name = uniqueGroupName(dir, used); + used.add(name); + groupNames.push(name); + symlinkSync(resolve(dir), join(staging, name)); } const registry = composeRegistry([coreTypesProvider, dbProvider, docProvider, promptProvider, uiProvider, ...extraProviders]); // Feed files in files-before-subdirs order (the same order the sdk's loadMemory @@ -53,13 +58,35 @@ export async function loadModel( return { root: result.root, warnings: result.warnings.map((w) => w.message), - sourceDirs: sourceDirs.map((d) => basename(resolve(d))), + // The staging entry names, NOT raw basenames: `treeOf` matches a node's + // source path segment against this list, and a collision-qualified name + // is what that segment actually is. + sourceDirs: groupNames, }; } finally { rmSync(staging, { recursive: true, force: true }); } } +/** + * A staging-directory entry name for `dir` that no earlier source dir already + * took. The basename when it is free (so a single-source project's group name + * is unchanged); otherwise `-`, then a counter — deterministic + * for a given source list, which keeps the emitted site byte-stable. + */ +function uniqueGroupName(dir: string, used: ReadonlySet): string { + const abs = resolve(dir); + const base = basename(abs); + if (!used.has(base)) return base; + const parent = basename(dirname(abs)); + const qualified = parent === "" || parent === base ? base : `${parent}-${base}`; + if (!used.has(qualified)) return qualified; + for (let n = 2; ; n++) { + const candidate = `${qualified}-${n}`; + if (!used.has(candidate)) return candidate; + } +} + /** Metadata files under `dir`, files-before-subdirs with each level sorted — the * overlay-safe order the sdk's loadMemory uses, so a base loads before an overlay * nested under it. Symlinks (the staging dir uses them) are followed. */ diff --git a/server/typescript/packages/migrate-ts/src/drift/drift.ts b/server/typescript/packages/migrate-ts/src/drift/drift.ts index 28abd9285..eb068c02a 100644 --- a/server/typescript/packages/migrate-ts/src/drift/drift.ts +++ b/server/typescript/packages/migrate-ts/src/drift/drift.ts @@ -20,7 +20,7 @@ import { buildExpectedSchemaWithProvenance } from "../expected-schema.js"; import { introspect } from "../introspect/index.js"; import { diff } from "../diff/index.js"; import { collectUnmanagedNames } from "../unmanaged.js"; -import { scopeExpectedSchema, type ObjectScopePredicate } from "../scope.js"; +import { scopeExpectedSchema, scopedDiffInputs, type ObjectScopePredicate } from "../scope.js"; import type { AllowOptions, Dialect, DiffResult, SchemaSnapshot } from "../types.js"; export interface ComputeDriftOptions { @@ -67,6 +67,16 @@ export interface DriftResult extends DiffResult { * comparison is indistinguishable from one that was checked and found clean. */ outOfScope: readonly string[]; + /** + * The schemas this comparison governed (`ScopedExpectedSchema.declaredSchemas`), + * `undefined` when no scope was given and `diff` derived its own. + * + * Reported so a SECOND comparison over the same run — `verify`'s committed-snapshot + * gate — can govern exactly the same schemas instead of re-deriving them from a + * different expected side. Together with `outOfScope` this pair is a + * `GovernedScope`, which is what `excludeFromSnapshot` takes. + */ + declaredSchemas: readonly string[] | undefined; } /** @@ -98,24 +108,21 @@ export async function computeDriftFromActual( opts?.inScope, ); const result = await diff({ - expected: scoped.snapshot, + // The three scoped-diff obligations as one value (see scope.ts's header): + // the narrowed expected side, `unmanagedNames` merging @unmanaged with the + // out-of-scope names so neither is proposed for drop, and the schema scope + // pinned to the UNSCOPED model so a narrow scope can never widen the run. + ...scopedDiffInputs(scoped, collectUnmanagedNames(metadata)), actual, dialect, allow: opts?.allow ?? {}, - // #208 §7 — a declared-@unmanaged object is external, so it is not drift: exclude it - // from the actual side (same as `meta migrate`) rather than surface a false drop-*. - // Out-of-scope objects join it: dropping them from `expected` alone would turn each - // one that EXISTS in the database into a spurious drop-* drift. - unmanagedNames: [...collectUnmanagedNames(metadata), ...scoped.outOfScope], - // Pin the schema scope to the UNSCOPED model's schemas (see scope.ts's header): - // a scope matching nothing would otherwise empty `expected`, which `diff` reads - // as "no model, govern the whole database" — reporting phantom drift for every - // table another owner has in a schema this model never mentions. Absent when no - // scope was given, so an unscoped run is unchanged. - ...(scoped.declaredSchemas !== undefined ? { scopeSchemas: scoped.declaredSchemas } : {}), ...(opts?.ignoreTables !== undefined ? { ignoreTables: opts.ignoreTables } : {}), }); - return { ...result, outOfScope: scoped.outOfScope }; + return { + ...result, + outOfScope: scoped.outOfScope, + declaredSchemas: scoped.declaredSchemas, + }; } /** diff --git a/server/typescript/packages/migrate-ts/src/index.ts b/server/typescript/packages/migrate-ts/src/index.ts index 1392e3097..b4c8c9208 100644 --- a/server/typescript/packages/migrate-ts/src/index.ts +++ b/server/typescript/packages/migrate-ts/src/index.ts @@ -15,8 +15,14 @@ export { diff } from "./diff/index.js"; export { collectUnmanagedNames } from "./unmanaged.js"; // Per-command scope (`migrate.scope`) — see scope.ts for why the suppression is // two-sided and why the pattern engine stays in @metaobjectsdev/sdk. -export { scopeExpectedSchema, declaredSchemasOf, carryForwardOutOfScope } from "./scope.js"; -export type { ObjectScopePredicate, ScopedExpectedSchema } from "./scope.js"; +export { + scopeExpectedSchema, + declaredSchemasOf, + carryForwardOutOfScope, + excludeFromSnapshot, + scopedDiffInputs, +} from "./scope.js"; +export type { ObjectScopePredicate, ScopedExpectedSchema, GovernedScope } from "./scope.js"; export { qualifiedDbName } from "./qualified-name.js"; export { computeDrift, computeDriftFromActual, type ComputeDriftOptions, type DriftResult } from "./drift/drift.js"; export { classifyDrift, driftAgainstSnapshot } from "./drift/classify.js"; diff --git a/server/typescript/packages/migrate-ts/src/scope.ts b/server/typescript/packages/migrate-ts/src/scope.ts index f6c48afab..0fa36ed27 100644 --- a/server/typescript/packages/migrate-ts/src/scope.ts +++ b/server/typescript/packages/migrate-ts/src/scope.ts @@ -23,8 +23,22 @@ // WIDEN. `declaredSchemas` below reports the UNSCOPED model's schemas so callers can // pin `diff`'s `scopeSchemas` to a property of the whole model, which `migrate.scope` // then cannot move in either direction. +// +// THE RULE THAT FOLLOWS FROM THAT, stated once because it is easy to read the other +// way: **a scope narrows which OBJECTS the tool governs, never which SCHEMAS it is +// allowed to see.** Pinning `scopeSchemas` to the unscoped model means a scope that +// excludes every declared object in schema `X` leaves `X` in scope, so another +// owner's UNDECLARED table in `X` stays a drop candidate — exactly as it would be on +// an unscoped run of the same model. That is deliberate: a schema this model +// declares into is a schema this model manages, and deriving the schema set from the +// survivors instead is precisely the inversion above. Declaring a scope is not a way +// to hand a schema over; removing the objects from the model is. +// +// `scopedDiffInputs` exists so no caller has to remember any of this: it returns all +// three obligations as one object, and every scoped `diff` call goes through it. import { DEFAULT_DB_SCHEMA_POSTGRES } from "@metaobjectsdev/metadata"; +import type { DiffArgs } from "./diff/index.js"; import type { ExpectedSchemaWithProvenance } from "./expected-schema.js"; import { qualifiedDbName } from "./qualified-name.js"; import type { SchemaSnapshot } from "./types.js"; @@ -43,16 +57,16 @@ export interface ScopedExpectedSchema { snapshot: SchemaSnapshot; /** * Qualified physical names (`.`) of the tables and views removed - * above. MUST be threaded into `diff`'s `unmanagedNames` (merged with - * `collectUnmanagedNames`, never replacing it) so the actual side is suppressed - * too — see the module header. + * above. Reaches `diff`'s `unmanagedNames` (MERGED with `collectUnmanagedNames`, + * never replacing it) so the actual side is suppressed too — `scopedDiffInputs` + * does that merge; see the module header for why omitting it inverts the feature. */ outOfScope: string[]; /** * The database schemas the UNSCOPED model declares, for `diff`'s `scopeSchemas`. - * MUST be threaded there by every caller that narrows — see the module header: - * without it a scope matching nothing hands `diff` an empty expected side, which - * it reads as "no model, govern the whole database". + * `scopedDiffInputs` threads it — see the module header: without it a scope + * matching nothing hands `diff` an empty expected side, which it reads as "no + * model, govern the whole database". * * `undefined` when no predicate was supplied (so `diff` derives its own set from * an untouched `expected`, exactly as before — an unscoped project's arguments are @@ -83,12 +97,102 @@ export function carryForwardOutOfScope( ): SchemaSnapshot { if (outOfScope.length === 0) return next; const excluded = new Set(outOfScope); - const keep = (objs: readonly T[]): T[] => - objs.filter((o) => excluded.has(qualifiedDbName(o))); return { ...next, - tables: [...next.tables, ...keep(prior.tables)], - views: [...next.views, ...keep(prior.views)], + tables: [...next.tables, ...splitOnName(prior.tables, excluded).named], + views: [...next.views, ...splitOnName(prior.views, excluded).named], + }; +} + +/** + * Drop the out-of-scope entries from a COMMITTED SNAPSHOT, producing the same + * three-part shape `scopeExpectedSchema` produces so the result can go straight + * through {@link scopedDiffInputs}. + * + * `verify`'s committed-snapshot gate (#292) needs this: `unmanagedNames` suppresses + * only the ACTUAL side, which is right when the expected side is the metadata (it is + * already scoped) and wrong here, where the expected side IS the snapshot — a + * snapshot written before the scope was declared still carries the other owner's + * tables, and leaving them in reports a phantom disagreement about an object this + * consumer does not manage. + * + * `governed` is the scope decision the caller's drift comparison already made — pass + * the `DriftResult` itself, which satisfies this shape. Taking `declaredSchemas` + * from there rather than re-deriving it from the snapshot is what closes the last + * whole-database door: a snapshot that is present but EMPTY (a never-migrated + * project) declares no schemas at all, so deriving from it hands `diff` nothing and + * reaches its "no model, govern the whole database" fallback — the very inversion + * this module exists to prevent, at the one call site that was still re-deriving. + * + * An empty `outOfScope` returns the SAME snapshot object with no schema pin, so an + * unscoped project's `diff` arguments are byte-for-byte what they always were. + */ +export function excludeFromSnapshot( + snapshot: SchemaSnapshot, + governed: GovernedScope, +): ScopedExpectedSchema { + if (governed.outOfScope.length === 0) return { snapshot, outOfScope: [] }; + const excluded = new Set(governed.outOfScope); + const declared = governed.declaredSchemas ?? declaredSchemasOf(snapshot); + return { + snapshot: { + ...snapshot, + tables: splitOnName(snapshot.tables, excluded).rest, + views: splitOnName(snapshot.views, excluded).rest, + }, + outOfScope: [...governed.outOfScope], + declaredSchemas: [...declared], + }; +} + +/** The scope decision a run made, as `DriftResult` reports it. */ +export interface GovernedScope { + /** Qualified physical names (`.`) the run does not govern. */ + readonly outOfScope: readonly string[]; + /** The schemas the run governs — `ScopedExpectedSchema.declaredSchemas`. */ + readonly declaredSchemas?: readonly string[] | undefined; +} + +/** + * Partition `objs` on whether `qualifiedDbName(o)` is in `names`. + * + * `carryForwardOutOfScope` wants the `named` half (carry the excluded entries + * forward) and `excludeFromSnapshot` wants the `rest` half (drop them). They are + * exact complements over the same key function, so they share one traversal rather + * than two filters that could come to key differently. + */ +function splitOnName( + objs: readonly T[], + names: ReadonlySet, +): { named: T[]; rest: T[] } { + const named: T[] = []; + const rest: T[] = []; + for (const o of objs) (names.has(qualifiedDbName(o)) ? named : rest).push(o); + return { named, rest }; +} + +/** + * The three `diff` arguments a scoped run owes, as ONE value. + * + * The module header lists them as three separate obligations, and five call sites + * re-derived them by hand — one of which had already drifted into its own guard. + * Every scoped `diff` call is now + * `diff({ ...scopedDiffInputs(scoped, collectUnmanagedNames(metadata)), actual, ... })`, + * so the rule is enforced by the type rather than by the comment. + * + * `unmanaged` is the `@unmanaged`-declared set (`collectUnmanagedNames`); it is + * MERGED with `outOfScope`, never replaced by it — both must reach `diff`. + * `scopeSchemas` is omitted entirely when the run narrowed nothing, so an unscoped + * project's arguments are unchanged. + */ +export function scopedDiffInputs( + scoped: ScopedExpectedSchema, + unmanaged: readonly string[], +): Pick { + return { + expected: scoped.snapshot, + unmanagedNames: [...unmanaged, ...scoped.outOfScope], + ...(scoped.declaredSchemas !== undefined ? { scopeSchemas: scoped.declaredSchemas } : {}), }; } diff --git a/server/typescript/packages/migrate-ts/src/snapshot/plan.ts b/server/typescript/packages/migrate-ts/src/snapshot/plan.ts index c56b47c6e..01ad1787d 100644 --- a/server/typescript/packages/migrate-ts/src/snapshot/plan.ts +++ b/server/typescript/packages/migrate-ts/src/snapshot/plan.ts @@ -3,7 +3,7 @@ import type { ColumnNamingStrategy, MetaData } from "@metaobjectsdev/metadata"; import { buildExpectedSchema, buildExpectedSchemaWithProvenance } from "../expected-schema.js"; import { diff, type DiffArgs } from "../diff/index.js"; import { collectUnmanagedNames } from "../unmanaged.js"; -import { carryForwardOutOfScope, scopeExpectedSchema, type ObjectScopePredicate } from "../scope.js"; +import { carryForwardOutOfScope, scopeExpectedSchema, scopedDiffInputs, type ObjectScopePredicate } from "../scope.js"; import type { Dialect, DiffResult, SchemaSnapshot } from "../types.js"; import type { ExpectedViewInput } from "../expected-schema.js"; @@ -66,25 +66,18 @@ export async function planOffline(args: PlanOfflineArgs): Promise { }); }); +/** + * A scope narrows which OBJECTS the run governs — never which SCHEMAS it may see. + * + * This is the consequence of pinning `scopeSchemas` to the UNSCOPED model, and it is + * easy to read the other way round, so it is pinned here rather than left in a review + * transcript. Excluding every declared object in a schema does NOT hand that schema + * over: it stays in scope, so an UNDECLARED table sitting in it is still a drop + * candidate — exactly as it would be on an unscoped run of the same model. + * + * The alternative (deriving the schema set from the survivors) reintroduces the + * inversion the pin exists to close: a scope matching nothing empties `expected`, + * `diff` reads that as "no model, govern the whole database", and every table in + * every schema becomes a drop candidate. Narrowing must never widen. + * + * The way to stop managing a schema is to remove its objects from the MODEL, or to + * declare them `@unmanaged` — both of which change what the model claims. A + * `migrate.scope` says who runs the migration, not what the model describes. + */ +const REPORTING = JSON.stringify({ + "metadata.root": { + package: "arena", + children: [ + { + "object.entity": { + name: "Standing", + children: [ + { "source.rdb": { name: "src", "@table": "standings", "@schema": "reporting" } }, + { "field.long": { name: "id" } }, + { "identity.primary": { name: "pk", "@fields": ["id"], "@generation": "increment" } }, + ], + }, + }, + ], + }, +}); + +describe("scope narrows objects, never schemas", () => { + test("a schema whose every declared object is excluded STAYS in scope", async () => { + const loaded = await new MetaDataLoader().load([ + new InMemoryStringSource(PLATFORM), + new InMemoryStringSource(REPORTING), + ]); + const built = buildExpectedSchemaWithProvenance(loaded.root, { dialect: "postgres" }); + const scoped = scopeExpectedSchema(built, platformOnly); + + // `reporting` lost its only declared object, and is still pinned. + expect(scoped.snapshot.tables.map((t) => t.name)).toEqual(["jobs"]); + expect(scoped.declaredSchemas).toEqual(["public", "reporting"]); + + // A table nobody declared, living in that schema. It has no provenance, so it + // never reaches `outOfScope` and nothing suppresses it on the actual side. + const actual: SchemaSnapshot = { + tables: [ + ...built.snapshot.tables, + { name: "legacy_stats", schema: "reporting", columns: [], indexes: [], foreignKeys: [], checks: [], primaryKey: [] }, + ], + views: [], + }; + const result = await diff({ + ...scopedDiffInputs(scoped, []), + actual, + dialect: "postgres", + allow: { dropTable: true }, + }); + + // It IS a drop candidate — the same verdict an unscoped run of this model gives. + const drops = result.changes.filter((c) => c.kind === "drop-table"); + expect(drops).toHaveLength(1); + if (drops[0]?.kind !== "drop-table") throw new Error("expected a drop-table"); + expect(drops[0].table).toBe("legacy_stats"); + }); + + test("a schema the model never declares at all is untouched (the pin is not a widening)", async () => { + const loaded = await new MetaDataLoader().load([ + new InMemoryStringSource(PLATFORM), + new InMemoryStringSource(REPORTING), + ]); + const built = buildExpectedSchemaWithProvenance(loaded.root, { dialect: "postgres" }); + const scoped = scopeExpectedSchema(built, platformOnly); + + const actual: SchemaSnapshot = { + tables: [ + ...built.snapshot.tables, + { name: "events", schema: "analytics", columns: [], indexes: [], foreignKeys: [], checks: [], primaryKey: [] }, + ], + views: [], + }; + const result = await diff({ + ...scopedDiffInputs(scoped, []), + actual, + dialect: "postgres", + allow: { dropTable: true }, + }); + expect(result.changes).toEqual([]); + }); +}); + describe("an accepted scoped run keeps out-of-scope entries in the committed snapshot", () => { test("the snapshot planOffline hands back retains the excluded table", async () => { const root = await loadBoth(); diff --git a/server/typescript/packages/migrate-ts/test/scope-snapshot-gate.test.ts b/server/typescript/packages/migrate-ts/test/scope-snapshot-gate.test.ts new file mode 100644 index 000000000..8f3e38c67 --- /dev/null +++ b/server/typescript/packages/migrate-ts/test/scope-snapshot-gate.test.ts @@ -0,0 +1,94 @@ +/** + * `excludeFromSnapshot` — the scoped-diff door for a COMMITTED SNAPSHOT. + * + * `verify`'s committed-snapshot gate (#292) runs a SECOND comparison over the same + * run: the committed `.schema..json` against the live database. It owes the + * same three obligations as every other scoped diff (migrate-ts `scope.ts` header), + * and it was the one call site re-deriving them by hand — including the schema pin, + * which it derived from the SNAPSHOT rather than from the model. + * + * That re-derivation left one door open: a snapshot that is present but EMPTY (a + * never-migrated project) declares no schemas at all, so the pin came out empty, the + * caller's `length > 0` guard dropped it, and `diff` fell back to "no model, govern + * the whole database" — reporting every table another owner has, in schemas this + * model never mentions, as a snapshot disagreement. + */ +import { describe, test, expect } from "bun:test"; +import { diff } from "../src/diff/index.js"; +import { excludeFromSnapshot, scopedDiffInputs } from "../src/scope.js"; +import type { SchemaSnapshot, TableDescriptor } from "../src/types.js"; + +function table(name: string, schema: string): TableDescriptor { + return { + name, + schema, + columns: [ + { name: "id", sqlType: { kind: "integer", bits: 64 }, nullable: false, identity: "increment" }, + ], + indexes: [], + foreignKeys: [], + primaryKey: ["id"], + checks: [], + }; +} + +/** A never-migrated committed snapshot: the file exists, it records nothing. */ +const EMPTY_SNAPSHOT: SchemaSnapshot = { tables: [], views: [] }; + +/** The live database: this consumer's table, the co-owner's table in the SAME + * schema (declared by the model, excluded by `migrate.scope`), and a third + * party's table in a schema this model never mentions at all. */ +const ACTUAL: SchemaSnapshot = { + tables: [table("jobs", "public"), table("matches", "public"), table("events", "analytics")], + views: [], +}; + +/** The scope decision the drift comparison already made for this run: `matches` is + * another owner's, and the model declares into `public` only. */ +const GOVERNED = { outOfScope: ["public.matches"], declaredSchemas: ["public"] }; + +describe("excludeFromSnapshot", () => { + test("an EMPTY committed snapshot under a scope does not govern the whole database", async () => { + const scoped = excludeFromSnapshot(EMPTY_SNAPSHOT, GOVERNED); + const result = await diff({ + ...scopedDiffInputs(scoped, []), + actual: ACTUAL, + allow: {}, + }); + + // analytics.events is a third party's, in a schema this model never declares. + // It has no provenance, so it can never reach `outOfScope` — the SCHEMA pin is + // the only thing that can protect it, and re-deriving that pin from the empty + // snapshot is what handed `diff` the whole database instead. + expect(result.changes.map((c) => JSON.stringify(c)).join("\n")).not.toContain("events"); + // The governed table is still compared: an empty snapshot really does disagree + // with a database that holds `public.jobs`, and that finding must survive. + expect(result.changes.filter((c) => c.kind === "drop-table")).toHaveLength(1); + // …and the co-owner's in-schema table is suppressed through `unmanagedNames`. + expect(result.changes.map((c) => JSON.stringify(c)).join("\n")).not.toContain("matches"); + }); + + test("nothing out of scope returns the SAME snapshot object and no schema pin", () => { + const scoped = excludeFromSnapshot(EMPTY_SNAPSHOT, { outOfScope: [], declaredSchemas: ["public"] }); + // Identity, not equality: an unscoped run's `diff` arguments must be exactly + // what they were before scope existed. + expect(scoped.snapshot).toBe(EMPTY_SNAPSHOT); + expect(scoped.declaredSchemas).toBeUndefined(); + expect(scopedDiffInputs(scoped, ["public.legacy"])).toEqual({ + expected: EMPTY_SNAPSHOT, + unmanagedNames: ["public.legacy"], + }); + }); + + test("out-of-scope entries leave the snapshot's own tables and views", () => { + const snapshot: SchemaSnapshot = { + tables: [table("jobs", "public"), table("matches", "public")], + views: [], + }; + const scoped = excludeFromSnapshot(snapshot, GOVERNED); + expect(scoped.snapshot.tables.map((t) => t.name)).toEqual(["jobs"]); + // The pin is the schemas the RUN governs, which is a property of the model — + // not of whatever survived the filter. + expect(scoped.declaredSchemas).toEqual(["public"]); + }); +}); diff --git a/server/typescript/packages/sdk/src/collection.ts b/server/typescript/packages/sdk/src/collection.ts index 2e0e1bdaa..a8d0bfd10 100644 --- a/server/typescript/packages/sdk/src/collection.ts +++ b/server/typescript/packages/sdk/src/collection.ts @@ -10,11 +10,10 @@ // today (`DEFAULT_SOURCES` in `sources.ts`); a project that declares // `sources` can point anywhere. No other call site may assume the directory // name — this is where that assumption is allowed to live, exactly once. -import { stat } from "node:fs/promises"; import { join, resolve } from "node:path"; import { ParseError, codeSource } from "@metaobjectsdev/metadata"; import { CONFIG_FILE, loadConfig, type Config } from "./config.js"; -import { exists, findConfigDir } from "./discovery.js"; +import { discoverCollectionRoot, exists, isDir } from "./discovery.js"; import { compileScope, type CompiledScope, type Scope } from "./scope.js"; import { DEFAULT_METADATA_DIR, DEFAULT_METAOBJECTS_DIR } from "./memory.js"; import { DEFAULT_SOURCES, resolveSources, type ResolvedSource, type SourceSpec } from "./sources.js"; @@ -43,20 +42,6 @@ export interface Collection { readonly migrateScopePatterns: readonly string[] | undefined; } -// Deliberately NOT deduped with `exists` (imported from `./discovery.js`) -// even though both wrap a bare stat/catch: this predicate exists to produce -// the friendlier `ERR_COLLECTION_NOT_FOUND` diagnostic below rather than the -// raw `ERR_SOURCE_UNRESOLVED` `resolveSources` would throw on a genuinely -// missing default directory — trading that clearer error for one syscall is -// a bad trade, so the redundant `stat` here is intentional, not an oversight. -async function isDir(p: string): Promise { - try { - return (await stat(p)).isDirectory(); - } catch { - return false; - } -} - /** Narrow the zod-inferred `Config["scope"]` (whose `.optional()` fields are * typed `T | undefined` even when present) down to `Scope`'s * exactOptionalPropertyTypes-safe shape — a key is omitted entirely rather @@ -74,9 +59,11 @@ function toScope(spec: Config["scope"]): Scope { * assumption baked into a call site. * * Resolution order: an explicit `opts.explicitDir` wins outright; otherwise - * `findConfigDir` walks up from `startDir` for the nearest - * `.metaobjects/config.json`, falling back to `startDir` itself when none is - * found. When the resolved directory carries a config, its declared + * `discoverCollectionRoot` walks up from `startDir` for the nearest directory + * carrying `.metaobjects/config.json` OR a `metaobjects/` directory (see + * `discovery.ts` — the second marker is what keeps a nested project reading + * its OWN metadata), falling back to `startDir` itself when neither is found. + * When the resolved directory carries a config, its declared * `sources`/`scope`/`migrate.scope` govern. Only a genuinely ABSENT * `config.json` falls through to `DEFAULT_SOURCES` — the same `metaobjects/` * directory the pre-source-resolution toolchain always read; a config.json @@ -98,23 +85,20 @@ export async function resolveCollection( const explicit = opts?.explicitDir; // Whether `configDir` carries a `config.json` — threaded through rather - // than re-`stat`'d below. On the non-explicit path, `findConfigDir` - // already proved this: it returns a directory ONLY after confirming - // `.metaobjects/config.json` exists there (discovery.ts's own `exists` - // check), and returns undefined only after confirming the same file is - // absent at every directory it examined, `resolve(startDir)` included. A - // second `stat` of the identical file would just re-prove what discovery - // already established. The check is only load-bearing on the - // `explicitDir` path, where `findConfigDir` never runs at all. + // than re-`stat`'d below. On the non-explicit path, `discoverCollectionRoot` + // already proved it either way: it reports `hasConfig` from the same + // `.metaobjects/config.json` probe that decided where to stop, and reports + // false only after confirming that file is absent at every directory it + // examined, `resolve(startDir)` included. A second `stat` of the identical + // file would just re-prove what discovery established. The check is only + // load-bearing on the `explicitDir` path, where discovery never runs at all. let configDir: string; let hasConfig: boolean; if (explicit !== undefined) { configDir = resolve(explicit); hasConfig = await exists(join(configDir, DEFAULT_METAOBJECTS_DIR, CONFIG_FILE)); } else { - const found = await findConfigDir(startDir); - configDir = found ?? resolve(startDir); - hasConfig = found !== undefined; + ({ dir: configDir, hasConfig } = await discoverCollectionRoot(startDir)); } let specs: readonly SourceSpec[] = DEFAULT_SOURCES; @@ -136,6 +120,15 @@ export async function resolveCollection( // Only the DEFAULT is allowed to be absent — an explicitly declared source // that does not resolve is `resolveSources`'s ERR_SOURCE_UNRESOLVED, not this. + // + // This re-`stat`s a directory the non-explicit discovery walk may already + // have probed, and that redundancy is intentional: it exists to produce the + // friendlier `ERR_COLLECTION_NOT_FOUND` diagnostic below rather than the raw + // `ERR_SOURCE_UNRESOLVED` `resolveSources` would throw on a genuinely missing + // default directory. Trading that clearer error for one syscall is a bad + // trade. It is also load-bearing outright on the `explicitDir` path and + // whenever a discovered config declares no `sources`, where nothing has + // probed it at all. if (specs === DEFAULT_SOURCES && !(await isDir(join(configDir, DEFAULT_METADATA_DIR)))) { throw new ParseError( `no metadata sources declared in ${configDir} and no default "${DEFAULT_METADATA_DIR}" directory found. ` + diff --git a/server/typescript/packages/sdk/src/discovery.ts b/server/typescript/packages/sdk/src/discovery.ts index 09c5f0909..9f1db2884 100644 --- a/server/typescript/packages/sdk/src/discovery.ts +++ b/server/typescript/packages/sdk/src/discovery.ts @@ -1,22 +1,33 @@ // server/typescript/packages/sdk/src/discovery.ts // -// Phase-1 metadata-source-resolution — nearest-ancestor config discovery. +// Phase-1 metadata-source-resolution — nearest-ancestor collection discovery. // -// Walks up from a starting directory to find the nearest `.metaobjects/` -// carrying `config.json`. This is what makes a CLI *contextual*: run it -// inside an app in a monorepo and it finds that app's config rather than the -// repo root's. Two properties are load-bearing: nearest wins (a config in a -// subdirectory beats one in an ancestor — the walk returns on the FIRST -// directory found), and the walk stops at a repository boundary (`.git`), so -// a monorepo checkout can never silently adopt a *parent checkout's* -// configuration. The config check runs BEFORE the `.git` check within each -// directory — reversed, a config at the repo root (where `.git` also lives) -// would be unreachable from any subdirectory, since the boundary would stop -// the walk one directory too early. +// Walks up from a starting directory to find the nearest directory that IS a +// project root. This is what makes a CLI *contextual*: run it inside an app in +// a monorepo and it finds that app's configuration rather than the repo root's. +// +// Three properties are load-bearing. +// +// 1. **Two markers, not one.** A directory is a project root when it carries +// `.metaobjects/config.json` OR a `metaobjects/` directory. The config half +// is the declared form; the `metaobjects/` half is the pre-source-resolution +// convention, and skipping past it would silently load an ANCESTOR's model +// (and write generated output to the ancestor's `outDir`) for a nested +// project that has always read its own — a back-compat regression on a +// layout that worked. The config is checked first so a directory carrying +// both resolves as declared. See design §4.6.1. +// 2. **Nearest wins** — the walk returns on the FIRST directory carrying either +// marker, so a config in a subdirectory beats one in an ancestor. +// 3. **The walk stops at a repository boundary** (`.git`), so a monorepo +// checkout can never silently adopt a *parent checkout's* configuration. The +// marker checks run BEFORE the `.git` check within each directory — +// reversed, a root-level project (where `.git` also lives) would be +// unreachable from any subdirectory, since the boundary would stop the walk +// one directory too early. import { stat } from "node:fs/promises"; import { dirname, join, resolve } from "node:path"; import { CONFIG_FILE } from "./config.js"; -import { DEFAULT_METAOBJECTS_DIR } from "./memory.js"; +import { DEFAULT_METADATA_DIR, DEFAULT_METAOBJECTS_DIR } from "./memory.js"; const GIT_DIR = ".git"; @@ -31,22 +42,70 @@ export async function exists(path: string): Promise { } } +/** `exists`, narrowed to directories. A plain FILE named `metaobjects` is not a + * metadata home, so it must not stop the walk below — and `collection.ts`'s + * default-directory probe needs the same distinction to raise its friendlier + * `ERR_COLLECTION_NOT_FOUND`. */ +export async function isDir(path: string): Promise { + try { + return (await stat(path)).isDirectory(); + } catch { + return false; + } +} + +/** Where the discovery walk stopped, and what it found there. */ +export interface DiscoveredRoot { + /** The project root the walk settled on. Always absolute; falls back to the + * resolved start directory when the walk found no marker at all. */ + readonly dir: string; + /** Whether `dir` carries `.metaobjects/config.json`. False both for a + * `metaobjects/`-only directory and for the no-marker fallback — in either + * case the DEFAULT sources apply, which is the pre-branch behaviour. */ + readonly hasConfig: boolean; +} + /** - * Walk up from `startDir` for the nearest directory holding - * `.metaobjects/config.json`. The walk stops after examining a directory - * that contains `.git`, so a monorepo can never silently adopt a parent - * checkout's configuration. Returns the containing directory (not the - * `.metaobjects` directory itself), or undefined when nothing is found. + * Walk up from `startDir` for the nearest project root — a directory holding + * `.metaobjects/config.json` or a `metaobjects/` directory (see the file + * header for why both count). The walk stops after examining a directory that + * contains `.git`, so a monorepo can never silently adopt a parent checkout's + * configuration. + * + * Never fails: with no marker anywhere below the boundary it reports the + * resolved `startDir` with `hasConfig: false`, which is what + * `resolveCollection` turns into either the default `metaobjects/` source or + * `ERR_COLLECTION_NOT_FOUND`. */ -export async function findConfigDir(startDir: string): Promise { - let dir = resolve(startDir); +export async function discoverCollectionRoot(startDir: string): Promise { + const start = resolve(startDir); + let dir = start; for (;;) { - if (await exists(join(dir, DEFAULT_METAOBJECTS_DIR, CONFIG_FILE))) return dir; - // Boundary check AFTER the config check: a repo-root config (sharing its + if (await exists(join(dir, DEFAULT_METAOBJECTS_DIR, CONFIG_FILE))) { + return { dir, hasConfig: true }; + } + if (await isDir(join(dir, DEFAULT_METADATA_DIR))) return { dir, hasConfig: false }; + // Boundary check AFTER the marker checks: a repo-root project (sharing its // directory with `.git`) is still findable from any subdirectory. - if (await exists(join(dir, GIT_DIR))) return undefined; + if (await exists(join(dir, GIT_DIR))) break; const parent = dirname(dir); - if (parent === dir) return undefined; + if (parent === dir) break; dir = parent; } + return { dir: start, hasConfig: false }; +} + +/** + * The directory whose configuration governs a run started in `startDir` — the + * `dir` half of {@link discoverCollectionRoot}. + * + * Exists as its own export for the callers that must NOT require metadata to + * exist: `meta migrate apply-pending` and `--rollback` replay committed SQL and + * load no model at all, so they resolve their `.metaobjects/` directory through + * this rather than through `resolveCollection`. Sharing the walk is the point — + * a second "find the project root" implementation is how the migrations + * directory and the metadata directory come to disagree. + */ +export async function resolveConfigDir(startDir: string): Promise { + return (await discoverCollectionRoot(startDir)).dir; } diff --git a/server/typescript/packages/sdk/src/index.ts b/server/typescript/packages/sdk/src/index.ts index d4051db1e..f0be02ee3 100644 --- a/server/typescript/packages/sdk/src/index.ts +++ b/server/typescript/packages/sdk/src/index.ts @@ -95,8 +95,10 @@ export type { Scope, CompiledScope } from "./scope.js"; export { resolveSources, DEFAULT_SOURCES } from "./sources.js"; export type { SourceSpec, ResolvedSource } from "./sources.js"; -// Discovery — nearest-ancestor `.metaobjects/config.json`, bounded by the repo root -export { findConfigDir } from "./discovery.js"; +// Discovery — nearest-ancestor project root (a `.metaobjects/config.json` OR a +// `metaobjects/` directory), bounded by the repo root +export { discoverCollectionRoot, resolveConfigDir } from "./discovery.js"; +export type { DiscoveredRoot } from "./discovery.js"; // Collection — the single authority on where a project's metadata lives export { resolveCollection } from "./collection.js"; diff --git a/server/typescript/packages/sdk/test/collection.test.ts b/server/typescript/packages/sdk/test/collection.test.ts index a88eec76e..6b6bafaf9 100644 --- a/server/typescript/packages/sdk/test/collection.test.ts +++ b/server/typescript/packages/sdk/test/collection.test.ts @@ -93,4 +93,31 @@ describe("resolveCollection", () => { const c = await resolveCollection(root); expect(c.files.map((f) => f.replace(root + "/", ""))).toEqual(["metaobjects/meta.a.json"]); }); + + test("BACK-COMPAT: a LOCAL metaobjects/ stops the walk, even under an ancestor config", async () => { + // The pre-branch layout: a nested project holding its own `metaobjects/` and + // no config of its own read ITS OWN metadata. Walking past it to an ancestor + // config silently loads the ancestor's model AND writes generated output to + // the ancestor's outDir — a silent regression on a layout that worked. + config(".", {}); + write("metaobjects/meta.root.json", "{}"); + write("apps/ui/metaobjects/meta.ui.json", "{}"); + const c = await resolveCollection(join(root, "apps/ui")); + expect(c.configDir).toBe(join(root, "apps/ui")); + expect(c.files.map((f) => f.replace(root + "/", ""))).toEqual([ + "apps/ui/metaobjects/meta.ui.json", + ]); + }); + + test("a nearer config still wins over a further-down metaobjects/ in an ancestor", async () => { + // The stop condition is per-DIRECTORY, first-match-wins: the nearest ancestor + // holding EITHER marker stops the walk, so a config beside the start dir is + // not skipped just because an ancestor also has a `metaobjects/`. + write("metaobjects/meta.root.json", "{}"); + write("model/meta.a.json", "{}"); + config("apps/ui", { sources: [{ path: "../../model" }] }); + const c = await resolveCollection(join(root, "apps/ui/src")); + expect(c.configDir).toBe(join(root, "apps/ui")); + expect(c.files.map((f) => f.replace(root + "/", ""))).toEqual(["model/meta.a.json"]); + }); }); diff --git a/server/typescript/packages/sdk/test/discovery.test.ts b/server/typescript/packages/sdk/test/discovery.test.ts index 64689e3e0..c66c54d0a 100644 --- a/server/typescript/packages/sdk/test/discovery.test.ts +++ b/server/typescript/packages/sdk/test/discovery.test.ts @@ -2,7 +2,7 @@ import { describe, test, expect, beforeEach, afterEach } from "bun:test"; import { mkdtempSync, rmSync, mkdirSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; -import { findConfigDir } from "../src/discovery.js"; +import { discoverCollectionRoot, resolveConfigDir } from "../src/discovery.js"; let root: string; const mk = (rel: string) => mkdirSync(join(root, rel), { recursive: true }); @@ -10,22 +10,26 @@ const cfg = (rel: string) => { mk(join(rel, ".metaobjects")); writeFileSync(join(root, rel, ".metaobjects/config.json"), '{"schema_version":1}', "utf8"); }; +/** The second stop marker: a `metaobjects/` directory, no config. */ +const meta = (rel: string) => mk(join(rel, "metaobjects")); beforeEach(() => { root = mkdtempSync(join(tmpdir(), "metaobjects-discovery-")); mk(".git"); }); afterEach(() => { rmSync(root, { recursive: true, force: true }); }); -describe("findConfigDir", () => { +describe("discoverCollectionRoot — config marker", () => { test("finds a config in the start directory", async () => { cfg("apps/ui"); mk("apps/ui/src"); - expect(await findConfigDir(join(root, "apps/ui"))).toBe(join(root, "apps/ui")); + expect(await discoverCollectionRoot(join(root, "apps/ui"))).toEqual({ + dir: join(root, "apps/ui"), hasConfig: true, + }); }); test("walks up to the nearest ancestor config", async () => { cfg("apps/ui"); mk("apps/ui/src/deep"); - expect(await findConfigDir(join(root, "apps/ui/src/deep"))).toBe(join(root, "apps/ui")); + expect(await resolveConfigDir(join(root, "apps/ui/src/deep"))).toBe(join(root, "apps/ui")); }); test("nearest wins over a further ancestor", async () => { cfg("."); cfg("apps/ui"); mk("apps/ui/src"); - expect(await findConfigDir(join(root, "apps/ui/src"))).toBe(join(root, "apps/ui")); + expect(await resolveConfigDir(join(root, "apps/ui/src"))).toBe(join(root, "apps/ui")); }); test("stops at the repository boundary — never adopts a parent checkout's config", async () => { // A config ABOVE the .git boundary must not be found. @@ -35,17 +39,49 @@ describe("findConfigDir", () => { mkdirSync(join(outer, ".metaobjects"), { recursive: true }); writeFileSync(join(outer, ".metaobjects/config.json"), '{"schema_version":1}', "utf8"); mkdirSync(join(outer, "inner/src"), { recursive: true }); - expect(await findConfigDir(join(outer, "inner/src"))).toBeUndefined(); + const start = join(outer, "inner/src"); + expect(await discoverCollectionRoot(start)).toEqual({ dir: start, hasConfig: false }); } finally { rmSync(outer, { recursive: true, force: true }); } }); test("a repo-root config IS found from a subdirectory", async () => { cfg("."); mk("apps/ui"); - expect(await findConfigDir(join(root, "apps/ui"))).toBe(root); + expect(await resolveConfigDir(join(root, "apps/ui"))).toBe(root); }); - test("returns undefined when nothing is found", async () => { + test("falls back to the start directory when nothing is found", async () => { mk("apps/ui"); - expect(await findConfigDir(join(root, "apps/ui"))).toBeUndefined(); + const start = join(root, "apps/ui"); + expect(await discoverCollectionRoot(start)).toEqual({ dir: start, hasConfig: false }); + }); +}); + +describe("discoverCollectionRoot — metaobjects/ marker", () => { + test("a LOCAL metaobjects/ stops the walk, reported as config-less", async () => { + cfg("."); meta("."); meta("apps/ui"); mk("apps/ui/src"); + expect(await discoverCollectionRoot(join(root, "apps/ui"))).toEqual({ + dir: join(root, "apps/ui"), hasConfig: false, + }); + }); + test("the walk reaches a metaobjects/ marker from a subdirectory", async () => { + meta("apps/ui"); mk("apps/ui/src/deep"); + expect(await resolveConfigDir(join(root, "apps/ui/src/deep"))).toBe(join(root, "apps/ui")); + }); + test("a config in the SAME directory wins — hasConfig is true", async () => { + cfg("apps/ui"); meta("apps/ui"); + expect(await discoverCollectionRoot(join(root, "apps/ui"))).toEqual({ + dir: join(root, "apps/ui"), hasConfig: true, + }); + }); + test("a nearer config beats a further ancestor's metaobjects/", async () => { + meta("."); cfg("apps/ui"); mk("apps/ui/src"); + expect(await discoverCollectionRoot(join(root, "apps/ui/src"))).toEqual({ + dir: join(root, "apps/ui"), hasConfig: true, + }); + }); + test("a FILE named metaobjects is not a metadata home", async () => { + cfg("."); mk("apps/ui"); + writeFileSync(join(root, "apps/ui/metaobjects"), "not a directory", "utf8"); + expect(await resolveConfigDir(join(root, "apps/ui"))).toBe(root); }); }); From aab1bfae7fa28a0b28b3ebebd1c00b8e664dc915 Mon Sep 17 00:00:00 2001 From: Doug Mealing Date: Tue, 18 Aug 2026 09:02:27 -0400 Subject: [PATCH 34/44] refactor: collapse the rules this branch enforced by repetition MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Part B of the quality pass. Every item here is a rule that was stated in prose and re-derived at N call sites — the shape whose failure at site N+1 is this branch's entire premise. (B1 and B3 shipped with Part A: A5 could not be fixed without B1's shared helper, and A2's stop-condition change forced B3's single discovery definition.) B2 — `Collection` exposed a `CompiledScope` and every consumer immediately wrapped it in the same lambda; nothing ever consumed one as a compiled scope, which is why `migrateScopePatterns` had to exist alongside it (a compiled scope cannot be shown to a human). It now carries predicates: `inScope`, always defined so callers pass it through without branching, and `inMigrateScope`, whose undefined-ness is load-bearing — that is what leaves an unscoped run's expected schema untouched. `toObjectScope` is deleted; `compileScope`/`matchesScope` stay exported for the conformance corpus. The window was now: `Collection` is new on this branch. B4 — the scope-mismatch refusal was three byte-identical copies differing only in a local variable name. One `refuseScopeMismatch`, so the hint string and the exit code are one decision. B5 — `docs.ts` resolved the collection twice on the `--model --site` path, and re-derived spec→path resolution byte-for-byte from `sources.ts`. `emitSite` now takes the resolved collection (and loses its own try/catch), and the source roots come from the collection, which derives them from the DECLARED specs in `resolveSources`'s own canonical order. That last part is a fix, not a tidy: dirs were derived from resolved FILES, so a declared source directory holding no metadata vanished from the site's group list entirely and `sourceDirs` could come back empty where the pre-branch code always passed `/metaobjects`. B6 — `assertPathSpec` was called twice per spec, the second call carrying a comment admitting it existed to re-narrow for TypeScript. A returning `toPathSpec` does both jobs once, and the validate-then-order pass it feeds is now a named `orderedPathSpecs` — exported, because `sourceRoots` must use the identical canonical order and a second sort would be a second definition of "canonical". B7 — the three-level nested ternary in `runner.ts` is an if/else chain. Behaviour-preserving, quirk included: the unscoped arm still blames `entityFilter` for an empty root, and the warning strings are untouched. B8 — `loadMemory`'s `repoRoot` is inert whenever `files` is supplied, which is all eight routed call sites. Documented rather than re-signed: a ninth call site copying the shape but forgetting `files` silently loads `/metaobjects/`, which is the divergence this design closes. C2 lands here too — `outOfScopeNote` is the same sentence `migrate` and `verify` built separately, and both were already being rewritten for B2/B4. Byte-identical output for both commands. Co-Authored-By: Claude Opus 5 (1M context) --- .../packages/cli/src/commands/docs.ts | 37 +++------- .../packages/cli/src/commands/gen.ts | 9 ++- .../packages/cli/src/commands/migrate.ts | 66 +++++++++-------- .../packages/cli/src/commands/verify.ts | 16 ++--- .../packages/cli/src/lib/migrate-scope.ts | 48 +++++++------ .../packages/cli/test/migrate-scope.test.ts | 15 ---- .../packages/codegen-ts/src/runner.ts | 23 ++++-- .../typescript/packages/sdk/src/collection.ts | 57 ++++++++++++--- server/typescript/packages/sdk/src/index.ts | 2 +- server/typescript/packages/sdk/src/memory.ts | 6 ++ server/typescript/packages/sdk/src/sources.ts | 72 ++++++++++++------- .../packages/sdk/test/collection.test.ts | 22 ++++-- 12 files changed, 215 insertions(+), 158 deletions(-) diff --git a/server/typescript/packages/cli/src/commands/docs.ts b/server/typescript/packages/cli/src/commands/docs.ts index 8410bc766..3aedbb18c 100644 --- a/server/typescript/packages/cli/src/commands/docs.ts +++ b/server/typescript/packages/cli/src/commands/docs.ts @@ -11,7 +11,7 @@ // output is therefore guaranteed — it is byte-for-byte the same generator the // `meta gen` pipeline runs (gated by the docs conformance fixture). -import { resolve as resolvePath, basename, isAbsolute } from "node:path"; +import { resolve as resolvePath, basename } from "node:path"; import { mkdir, writeFile } from "node:fs/promises"; import { log } from "../lib/log.js"; import { loadMetaobjectsConfig } from "../lib/load-metaobjects-config.js"; @@ -526,33 +526,12 @@ async function scaffoldSiteCommand(projectRoot: string): Promise { return 0; } -/** - * Distinct directories declared by the collection's source specs, resolved - * absolute against `collection.configDir`. The site's own loader - * (`@metaobjectsdev/docs-site`) wants whole directories — each becomes a - * symlinked, basename-keyed source group — unlike the per-file list - * `resolveCollection` produces for the sdk `loadMemory` path. - */ -function collectionSourceDirs(collection: Collection): string[] { - const dirs = new Map(); - for (const { spec } of collection.sources) { - if (!("path" in spec)) continue; // phase 1 resolves "path" specs only - const key = JSON.stringify(spec); - if (dirs.has(key)) continue; - dirs.set( - key, - isAbsolute(spec.path) ? spec.path : resolvePath(collection.configDir, spec.path), - ); - } - return [...dirs.values()]; -} - /** * Emit the browsable HTML documentation site via `@metaobjectsdev/docs-site`. - * The site loads the model with its OWN loader from the resolved metadata - * source directories (see `resolveCollection`/`collectionSourceDirs`), so - * this is independent of the sdk loadMemory path used for the markdown - * surfaces. Writes under `/site` so it can coexist with the markdown + * The site loads the model with its OWN loader from the collection's declared + * source ROOTS (whole directories, one page group each) rather than from the + * per-file list the sdk `loadMemory` path takes, so this is independent of the + * markdown surfaces. Writes under `/site` so it can coexist with the markdown * output. Scaffold-and-own: when the consumer has copied templates/assets * into `/codegen/docs-site/` (via `--scaffold-site`), those win * over the bundled defaults. @@ -583,7 +562,11 @@ async function emitSite( // dropping the second — would break the feature this branch exists to ship. // The same directory named twice is the real hazard: it would be symlinked // and loaded twice. - const sourceDirs = collectionSourceDirs(collection); + // The DECLARED source roots, not directories re-derived from the resolved + // files: a declared source directory holding no metadata yet would otherwise + // vanish from the site's group list entirely, and `sourceDirs` could come back + // empty where the pre-branch code always passed `/metaobjects`. + const sourceDirs = [...collection.sourceRoots]; const seenDirs = new Set(sourceDirs); if (promptsDir !== undefined && !existsSync(promptsDir)) { log.warn(`docs: --prompts dir does not exist: ${promptsDir}`); diff --git a/server/typescript/packages/cli/src/commands/gen.ts b/server/typescript/packages/cli/src/commands/gen.ts index 3122a568a..6f7ae4238 100644 --- a/server/typescript/packages/cli/src/commands/gen.ts +++ b/server/typescript/packages/cli/src/commands/gen.ts @@ -8,7 +8,7 @@ import type { OutputFormat } from "../lib/format.js"; import { log } from "../lib/log.js"; import { warnIfAgentContextStale } from "../lib/agent-context-staleness.js"; import { scanSourceForAntiPatterns } from "../lib/anti-patterns.js"; -import { loadMemory, resolveCollection, matchesScope } from "@metaobjectsdev/sdk"; +import { loadMemory, resolveCollection } from "@metaobjectsdev/sdk"; import { runGen, listGenerators } from "@metaobjectsdev/codegen-ts"; import type { WriteStatus } from "@metaobjectsdev/codegen-ts"; @@ -101,10 +101,9 @@ export async function genCommand(args: string[], cwd: string, fmt: OutputFormat dryRun: cliConfig.dryRun, // Collection-level `scope` (Task 12b) — the output filter over // GENERATED entities, never over what the collection loads. Always - // passed: an unconfigured project's `collection.scope` compiles to an - // empty include/exclude, and `matchesScope` treats that as "everything" - // — so this is a no-op for the common case, not a behavior change. - scope: (fqn) => matchesScope(fqn, collection.scope), + // passed: an unconfigured project's predicate admits everything, so this + // is a no-op for the common case, not a behavior change. + scope: collection.inScope, ...(cliConfig.entities.length > 0 ? { entityFilter: cliConfig.entities } : {}), }); } catch (err) { diff --git a/server/typescript/packages/cli/src/commands/migrate.ts b/server/typescript/packages/cli/src/commands/migrate.ts index 52a125b3b..b78795dbd 100644 --- a/server/typescript/packages/cli/src/commands/migrate.ts +++ b/server/typescript/packages/cli/src/commands/migrate.ts @@ -11,9 +11,10 @@ import type { OutputFormat } from "../lib/format.js"; import { toonEncode } from "../lib/format.js"; import { buildKyselyFromUrl, redactUrl } from "../lib/kysely.js"; import { log } from "../lib/log.js"; -import { loadMemory, resolveCollection, resolveConfigDir } from "@metaobjectsdev/sdk"; +import { loadMemory, resolveCollection, resolveConfigDir, type Collection } from "@metaobjectsdev/sdk"; +import type { MetaRoot } from "@metaobjectsdev/metadata"; import { loadMetaobjectsConfig } from "../lib/load-metaobjects-config.js"; -import { migrateScopeMismatch, toObjectScope } from "../lib/migrate-scope.js"; +import { migrateScopeMismatch, outOfScopeNote } from "../lib/migrate-scope.js"; import { buildExpectedSchemaWithProvenance, scopeExpectedSchema, @@ -157,22 +158,18 @@ function warnIfLedgerRelocated(cwd: string, resolvedOutDir: string): void { } /** - * Say what a declared `migrate.scope` left out. An excluded object produces neither - * a create nor a drop, so without this line "no changes" and "no changes to the half - * of the model this run governs" read identically. + * Report what a declared `migrate.scope` left out (wording: `outOfScopeNote`). * * STDOUT in text format, STDERR otherwise. `--format json` / `--format toon` put a * single machine-readable document on stdout, and a prose line ahead of it breaks - * `| jq` outright — the same split `emitStructuredError` makes two functions down. + * `| jq` outright — the same split `emitStructuredError` makes just below. * Routed to stderr rather than dropped, because the non-TTY default format is toon * (`resolveFormat`): suppressing it outright would silence the note for every * piped and CI run, which is most of them. */ function logOutOfScope(names: readonly string[], fmt: OutputFormat): void { if (names.length === 0) return; - const msg = - `meta migrate — ${names.length} object(s) out-of-scope (outside migrate.scope, ` + - `governed elsewhere): ${names.join(", ")}`; + const msg = outOfScopeNote("migrate", names); if (fmt === "text") log.info(msg); else log.warn(msg); } @@ -187,6 +184,27 @@ function emitStructuredError(error: string, hint: string, fmt: OutputFormat): vo // text format: errors go to stderr via log.error() — the caller handles that path } +/** + * The refusal for a `migrate.scope` that matches nothing, as all three of migrate's + * pipelines (online, offline, D1) issue it. + * + * Returns the exit code to return, or `undefined` when there is nothing to refuse. + * Three byte-identical copies of the report-and-exit differed only in a local + * variable name; the hint string and the exit code are one decision, recorded once + * — a configuration error, so exit 2. + */ +function refuseScopeMismatch( + collection: Collection, + root: MetaRoot, + fmt: OutputFormat, +): number | undefined { + const mismatch = migrateScopeMismatch(collection, root); + if (mismatch === undefined) return undefined; + log.error(`migrate: ${mismatch}`); + emitStructuredError(`migrate: ${mismatch}`, "fix or remove migrate.scope in .metaobjects/config.json", fmt); + return 2; +} + /** * Sentinel thrown by sub-functions that have already emitted a structured error * via emitStructuredError(). The top-level catch in migrateCommand re-throws @@ -491,12 +509,8 @@ export async function migrateCommand( return 2; } - const scopeMismatch = migrateScopeMismatch(collection, metadata); - if (scopeMismatch !== undefined) { - log.error(`migrate: ${scopeMismatch}`); - emitStructuredError(`migrate: ${scopeMismatch}`, "fix or remove migrate.scope in .metaobjects/config.json", fmt); - return 2; - } + const scopeRc = refuseScopeMismatch(collection, metadata, fmt); + if (scopeRc !== undefined) return scopeRc; let kysely; try { @@ -539,7 +553,7 @@ export async function migrateCommand( columnNamingStrategy, views: expectedViews, }), - toObjectScope(collection.migrateScope), + collection.inMigrateScope, ); const expected = scoped.snapshot; logOutOfScope(scoped.outOfScope, fmt); @@ -1024,12 +1038,8 @@ export async function runOfflineGenerate( return 2; } - const offlineScopeMismatch = migrateScopeMismatch(collection, metadata); - if (offlineScopeMismatch !== undefined) { - log.error(`migrate: ${offlineScopeMismatch}`); - emitStructuredError(`migrate: ${offlineScopeMismatch}`, "fix or remove migrate.scope in .metaobjects/config.json", fmt); - return 2; - } + const scopeRc = refuseScopeMismatch(collection, metadata, fmt); + if (scopeRc !== undefined) return scopeRc; const outDir = resolvePath(metaRoot, config.outDir); const path = snapshotPath(outDir, config.dialect); @@ -1059,7 +1069,7 @@ export async function runOfflineGenerate( const onAmbiguousResolution = mapOnAmbiguous(config.onAmbiguous); const offlineViews = buildProjectionViews(metadata, { dialect: config.dialect, columnNamingStrategy: offlineStrategy }); - const offlineScope = toObjectScope(collection.migrateScope); + const offlineScope = collection.inMigrateScope; let plan; try { @@ -1272,12 +1282,8 @@ async function runD1Migrate( return 2; } - const d1ScopeMismatch = migrateScopeMismatch(collection, metadata); - if (d1ScopeMismatch !== undefined) { - log.error(`migrate: ${d1ScopeMismatch}`); - emitStructuredError(`migrate: ${d1ScopeMismatch}`, "fix or remove migrate.scope in .metaobjects/config.json", fmt); - return 2; - } + const scopeRc = refuseScopeMismatch(collection, metadata, fmt); + if (scopeRc !== undefined) return scopeRc; // 4. Build expected schema + introspect actual. let columnNamingStrategy: "snake_case" | "literal" | "kebab-case" = "snake_case"; @@ -1291,7 +1297,7 @@ async function runD1Migrate( // Per-command scope — both-sided, exactly as on the Kysely path above. const scoped = scopeExpectedSchema( buildExpectedSchemaWithProvenance(metadata, { dialect: "d1", columnNamingStrategy, views: expectedViews }), - toObjectScope(collection.migrateScope), + collection.inMigrateScope, ); const expected = scoped.snapshot; logOutOfScope(scoped.outOfScope, fmt); diff --git a/server/typescript/packages/cli/src/commands/verify.ts b/server/typescript/packages/cli/src/commands/verify.ts index f81a3248c..fbdbfa763 100644 --- a/server/typescript/packages/cli/src/commands/verify.ts +++ b/server/typescript/packages/cli/src/commands/verify.ts @@ -50,8 +50,8 @@ import { type D1Runner, type DriftResult, } from "@metaobjectsdev/migrate-ts"; -import { loadMemory, resolveCollection, matchesScope } from "@metaobjectsdev/sdk"; -import { migrateScopeMismatch, toObjectScope } from "../lib/migrate-scope.js"; +import { loadMemory, resolveCollection } from "@metaobjectsdev/sdk"; +import { migrateScopeMismatch, outOfScopeNote } from "../lib/migrate-scope.js"; import { TYPE_TEMPLATE, TEMPLATE_SUBTYPE_PROMPT, @@ -207,7 +207,7 @@ export async function verifyCommand( // declaration (`migrate.scope`), not a second key: a drift gate that fails on // tables migrate deliberately does not own is incoherent. Undefined ⇒ everything // loaded, which is every project that declares no scope. - const schemaScope = toObjectScope(collection.migrateScope); + const schemaScope = collection.inMigrateScope; const promptsDir = join(projectRoot, flags.prompts ?? DEFAULT_PROMPTS_DIR); const provider = new FileProvider(promptsDir); @@ -666,12 +666,10 @@ export async function verifyCommand( } // Same reasoning for the per-command scope: an object `migrate.scope` excluded - // was NOT checked, and silence would misreport it as checked-and-clean. + // was NOT checked, and silence would misreport it as checked-and-clean. Shared + // wording with `meta migrate` — one declaration, one sentence about it. if (driftResult.outOfScope.length > 0) { - log.info( - `meta verify — ${driftResult.outOfScope.length} object(s) out-of-scope ` + - `(outside migrate.scope, governed elsewhere): ${driftResult.outOfScope.join(", ")}`, - ); + log.info(outOfScopeNote("verify", driftResult.outOfScope)); } const changes = driftResult.changes; @@ -712,7 +710,7 @@ export async function verifyCommand( // files should exist, reporting every out-of-scope entity as drift. let result; try { - result = await computeCodegenDrift(forgeConfig, root, projectRoot, (fqn) => matchesScope(fqn, collection.scope)); + result = await computeCodegenDrift(forgeConfig, root, projectRoot, collection.inScope); } catch (err) { log.error(`verify --codegen: regeneration failed: ${(err as Error).message}`); return 1; diff --git a/server/typescript/packages/cli/src/lib/migrate-scope.ts b/server/typescript/packages/cli/src/lib/migrate-scope.ts index fe0d717ed..568afd9b1 100644 --- a/server/typescript/packages/cli/src/lib/migrate-scope.ts +++ b/server/typescript/packages/cli/src/lib/migrate-scope.ts @@ -1,27 +1,35 @@ -// The one adapter between a declared `migrate.scope` and migrate-ts's scope seam. +// What `meta migrate` and `meta verify --db` SAY about a declared `migrate.scope`. // -// `resolveCollection` compiles `.metaobjects/config.json`'s `migrate.scope` into a -// `CompiledScope`; migrate-ts takes a plain predicate over an object's -// fully-qualified name so it never carries a second implementation of the pattern -// grammar. `matchesScope` (@metaobjectsdev/sdk) is THE pattern engine — there is no -// other, and adding one would let `migrate` and `gen` disagree about what -// `acme::platform::**` means. +// The scope itself needs no adapter: `resolveCollection` hands back +// `inMigrateScope` already in migrate-ts's predicate shape, so the pattern grammar +// lives in exactly one place (`matchesScope`, @metaobjectsdev/sdk) and `migrate` and +// `gen` cannot come to disagree about what `acme::platform::**` means. // -// Both `meta migrate` and `meta verify --db` import this: the two commands govern -// the identical object set, so they share the one declaration rather than each -// growing a key of its own. +// What DOES need one home is the user-facing language, and both commands import it +// from here: they govern the identical object set from one declaration, so a note +// or a refusal that drifted between them would be drift the user reads. -import { matchesScope, type Collection, type CompiledScope } from "@metaobjectsdev/sdk"; +import type { Collection } from "@metaobjectsdev/sdk"; import type { MetaRoot } from "@metaobjectsdev/metadata"; -import type { ObjectScopePredicate } from "@metaobjectsdev/migrate-ts"; /** - * Adapt a compiled `migrate.scope` to migrate-ts's predicate seam. Undefined in, - * undefined out — a project that declared no scope governs everything it loaded, - * and the undefined predicate is what keeps its expected schema untouched. + * Say what a declared `migrate.scope` left out, for `migrate` and `verify --db` + * alike. + * + * An excluded object produces neither a create nor a drop and is neither checked + * nor reported as drift, so without this line "no changes" and "no changes to the + * half of the model this run governs" read identically — and an unchecked table is + * indistinguishable from a checked-and-clean one. + * + * One sentence, one definition: the two commands say the same thing about the same + * declaration, and this is a string a user reads, so drift between two copies of it + * is drift the user sees. */ -export function toObjectScope(scope: CompiledScope | undefined): ObjectScopePredicate | undefined { - return scope === undefined ? undefined : (fqn: string): boolean => matchesScope(fqn, scope); +export function outOfScopeNote(command: string, names: readonly string[]): string { + return ( + `meta ${command} — ${names.length} object(s) out-of-scope ` + + `(outside migrate.scope, governed elsewhere): ${names.join(", ")}` + ); } /** How many loaded FQNs to name in the refusal below — enough to show the shape @@ -53,8 +61,8 @@ export function migrateScopeMismatch( collection: Collection, root: MetaRoot, ): string | undefined { - const { migrateScope, migrateScopePatterns } = collection; - if (migrateScope === undefined) return undefined; + const { inMigrateScope, migrateScopePatterns } = collection; + if (inMigrateScope === undefined) return undefined; // ADR-0039: `objects()` is the resolving accessor — the loaded object set, which // is exactly what `migrate.scope` claims to be a subset of. @@ -62,7 +70,7 @@ export function migrateScopeMismatch( // No objects at all is not a scope error: there is nothing for a pattern to miss, // and an empty model has its own (much louder) failure modes downstream. if (fqns.length === 0) return undefined; - if (fqns.some((fqn) => matchesScope(fqn, migrateScope))) return undefined; + if (fqns.some(inMigrateScope)) return undefined; const patterns = JSON.stringify(migrateScopePatterns ?? []); const examples = fqns.slice(0, EXAMPLE_FQN_CAP).join(", "); diff --git a/server/typescript/packages/cli/test/migrate-scope.test.ts b/server/typescript/packages/cli/test/migrate-scope.test.ts index 5c977e3a1..d91f9fcf1 100644 --- a/server/typescript/packages/cli/test/migrate-scope.test.ts +++ b/server/typescript/packages/cli/test/migrate-scope.test.ts @@ -15,9 +15,7 @@ import { describe, test, expect, afterAll } from "bun:test"; import { mkdtemp, rm, mkdir, writeFile, readdir, readFile, unlink } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; -import { compileScope } from "@metaobjectsdev/sdk"; import { runBaseline, runOfflineGenerate } from "../src/commands/migrate.js"; -import { toObjectScope } from "../src/lib/migrate-scope.js"; const dirs: string[] = []; afterAll(async () => { for (const d of dirs) await rm(d, { recursive: true, force: true }); }); @@ -83,19 +81,6 @@ const cfg = () => const migrationDirs = async (root: string): Promise => (await readdir(join(root, ".metaobjects/migrations"))).filter((e) => !e.startsWith(".")); -describe("toObjectScope", () => { - test("matchesScope drives the decision — no second pattern implementation", () => { - const inScope = toObjectScope(compileScope({ include: ["acme::platform::**"] }))!; - expect(inScope("acme::platform::Job")).toBe(true); - expect(inScope("acme::platform::billing::Invoice")).toBe(true); - expect(inScope("arena::Match")).toBe(false); - }); - - test("no declared scope → no predicate (the command governs everything loaded)", () => { - expect(toObjectScope(undefined)).toBeUndefined(); - }); -}); - describe("meta migrate — migrate.scope", () => { test("an out-of-scope table is neither altered nor dropped", async () => { const root = await project(); diff --git a/server/typescript/packages/codegen-ts/src/runner.ts b/server/typescript/packages/codegen-ts/src/runner.ts index bc7ad62f5..c85a7a12e 100644 --- a/server/typescript/packages/codegen-ts/src/runner.ts +++ b/server/typescript/packages/codegen-ts/src/runner.ts @@ -195,13 +195,22 @@ export async function runGen(opts: RunGenOpts): Promise { // children" for a scoped-out model, or "...entityFilter" for a scope // that admitted everything entityFilter then excluded, are both false // statements that send the reader to the wrong file. - const reason = scope === undefined - ? (entityFilter ? "no object children match the provided entityFilter" : "root has no object children") - : (allObjects.length === 0 - ? "root has no object children" - : afterEntityFilter.length === 0 - ? "no object children match the provided entityFilter" - : "no object children match the configured scope"); + let reason: string; + if (scope === undefined) { + // Byte-identical to the pre-scope branch, quirk included: an EMPTY root + // with an entityFilter set still blames the filter. Wrong, and untouched + // — changing what an unscoped project reads is a behaviour change, and + // this is a shape change. + reason = entityFilter + ? "no object children match the provided entityFilter" + : "root has no object children"; + } else if (allObjects.length === 0) { + reason = "root has no object children"; + } else if (afterEntityFilter.length === 0) { + reason = "no object children match the provided entityFilter"; + } else { + reason = "no object children match the configured scope"; + } warnings.push(`No entities to generate — ${reason}.`); return { files: [], warnings, conflicts: [] }; } diff --git a/server/typescript/packages/sdk/src/collection.ts b/server/typescript/packages/sdk/src/collection.ts index a8d0bfd10..adcd30975 100644 --- a/server/typescript/packages/sdk/src/collection.ts +++ b/server/typescript/packages/sdk/src/collection.ts @@ -14,9 +14,16 @@ import { join, resolve } from "node:path"; import { ParseError, codeSource } from "@metaobjectsdev/metadata"; import { CONFIG_FILE, loadConfig, type Config } from "./config.js"; import { discoverCollectionRoot, exists, isDir } from "./discovery.js"; -import { compileScope, type CompiledScope, type Scope } from "./scope.js"; +import { compileScope, matchesScope, type Scope } from "./scope.js"; import { DEFAULT_METADATA_DIR, DEFAULT_METAOBJECTS_DIR } from "./memory.js"; -import { DEFAULT_SOURCES, resolveSources, type ResolvedSource, type SourceSpec } from "./sources.js"; +import { + DEFAULT_SOURCES, + orderedPathSpecs, + resolveSpecPath, + resolveSources, + type ResolvedSource, + type SourceSpec, +} from "./sources.js"; export interface Collection { /** Directory whose config declared this collection (or the resolved start @@ -29,16 +36,35 @@ export interface Collection { readonly files: readonly string[]; /** Same set, carrying the contributing spec for provenance. */ readonly sources: readonly ResolvedSource[]; - /** Output filter for codegen. Empty include => everything. */ - readonly scope: CompiledScope; - /** Output filter for migrate/verify --db. Undefined => the command governs - * everything in scope. */ - readonly migrateScope: CompiledScope | undefined; - /** The patterns `migrateScope` was compiled FROM, for diagnostics only — + /** The distinct roots the declared source specs resolve to, absolute, in the + * same canonical (content) order `files` uses. Derived from the DECLARED + * specs, not from the resolved files, so a source directory that legitimately + * holds no metadata still appears — a consumer listing "where this model + * comes from" (`meta docs --site` groups its pages by source root) must not + * silently lose a declared source because it happens to be empty today. */ + readonly sourceRoots: readonly string[]; + /** + * Output filter for codegen: does this fully-qualified name survive the + * collection's `scope`? Always defined — an unconfigured project compiles to + * an empty include/exclude, which admits everything, so callers pass this + * through unconditionally rather than branching. + * + * A PREDICATE rather than the `CompiledScope` it closes over, because nothing + * consumes a compiled scope as a compiled scope: every consumer immediately + * wrapped it in exactly this lambda, and `migrateScopePatterns` exists + * precisely because the compiled form cannot be shown to a human. + * `compileScope`/`matchesScope` stay exported for the conformance corpus. + */ + readonly inScope: (fqn: string) => boolean; + /** Output filter for migrate/verify --db (`migrate.scope`). Undefined => the + * command governs everything loaded, and that undefined is load-bearing: it + * is what leaves the expected schema untouched (migrate-ts `scope.ts`). */ + readonly inMigrateScope: ((fqn: string) => boolean) | undefined; + /** The patterns `inMigrateScope` was compiled FROM, for diagnostics only — * `compileScope` produces RegExps, and a regex source is not something to * show an author who wrote `acme::platform::**`. Carried so the "your scope * matched nothing" refusal can name the patterns that missed. Always in - * lockstep with `migrateScope`: both undefined, or both present. */ + * lockstep with `inMigrateScope`: both undefined, or both present. */ readonly migrateScopePatterns: readonly string[] | undefined; } @@ -138,12 +164,21 @@ export async function resolveCollection( } const sources = await resolveSources(configDir, specs); + const scope = compileScope(toScope(scopeSpec)); + const migrateScope = + migrateSpec === undefined ? undefined : compileScope({ include: migrateSpec }); return { configDir, files: sources.map((s) => s.file), sources, - scope: compileScope(toScope(scopeSpec)), - migrateScope: migrateSpec === undefined ? undefined : compileScope({ include: migrateSpec }), + // Canonical (content) order, from `resolveSources`'s own ordering — so this + // list is a pure function of the source SET, exactly like `files`. + sourceRoots: [ + ...new Set(orderedPathSpecs(specs).map((spec) => resolveSpecPath(configDir, spec))), + ], + inScope: (fqn: string): boolean => matchesScope(fqn, scope), + inMigrateScope: + migrateScope === undefined ? undefined : (fqn: string): boolean => matchesScope(fqn, migrateScope), migrateScopePatterns: migrateSpec, }; } diff --git a/server/typescript/packages/sdk/src/index.ts b/server/typescript/packages/sdk/src/index.ts index f0be02ee3..e5d53506f 100644 --- a/server/typescript/packages/sdk/src/index.ts +++ b/server/typescript/packages/sdk/src/index.ts @@ -92,7 +92,7 @@ export { compileScope, matchesScope } from "./scope.js"; export type { Scope, CompiledScope } from "./scope.js"; // Source resolution — a declared source SET to a canonically-sorted file list -export { resolveSources, DEFAULT_SOURCES } from "./sources.js"; +export { resolveSources, resolveSpecPath, orderedPathSpecs, DEFAULT_SOURCES } from "./sources.js"; export type { SourceSpec, ResolvedSource } from "./sources.js"; // Discovery — nearest-ancestor project root (a `.metaobjects/config.json` OR a diff --git a/server/typescript/packages/sdk/src/memory.ts b/server/typescript/packages/sdk/src/memory.ts index d3c3caa0e..6ddf7177b 100644 --- a/server/typescript/packages/sdk/src/memory.ts +++ b/server/typescript/packages/sdk/src/memory.ts @@ -101,6 +101,12 @@ export const defaultLoadMemoryProviders: readonly MetaDataTypeProvider[] = [ * @param repoRoot The project's working-directory root (e.g. process.cwd()). * `loadMemory` resolves `metaobjects/` and (if workspace-aware) the * transitive `extends:` graph automatically. + * **Ignored entirely when `options.files` is supplied** — that list has + * already been resolved (by `resolveCollection`, which owns the decision), so + * no discovery runs and nothing reads this path. Every routed CLI command + * passes both, and the argument is inert at all of them; a caller that copies + * that shape but omits `files` silently loads `/metaobjects/` + * instead, which is the divergence this design exists to close. * @param options Optional {@link LoadMemoryOptions} — supply additional * providers or replace the default bundle entirely. */ diff --git a/server/typescript/packages/sdk/src/sources.ts b/server/typescript/packages/sdk/src/sources.ts index 7b48f7866..f33a68833 100644 --- a/server/typescript/packages/sdk/src/sources.ts +++ b/server/typescript/packages/sdk/src/sources.ts @@ -53,12 +53,12 @@ export interface ResolvedSource { export const DEFAULT_SOURCES: readonly SourceSpec[] = [{ path: DEFAULT_METADATA_DIR }]; /** Narrows `spec` to its `path` arm, throwing `ERR_SOURCE_KIND_UNSUPPORTED` - * for `resource`/`package` — phase 1 resolves `path` only. Called in two - * separate passes by {@link resolveSources} (see the comment there): an - * unsupported kind must be reported regardless of where it sits in the - * declared list. */ -function assertPathSpec(spec: SourceSpec): asserts spec is { readonly path: string } { - if ("path" in spec) return; + * for `resource`/`package` — phase 1 resolves `path` only. Returns rather than + * asserting so one call both validates and narrows: an `asserts` signature has + * to be re-invoked wherever TypeScript's control-flow analysis cannot carry the + * narrowing, which is a language workaround masquerading as a second check. */ +function toPathSpec(spec: SourceSpec): { readonly path: string } { + if ("path" in spec) return spec; const kind = "resource" in spec ? "resource" : "package"; throw new ParseError( `source kind "${kind}" is not supported by this toolchain yet; use a "path" source`, @@ -66,6 +66,43 @@ function assertPathSpec(spec: SourceSpec): asserts spec is { readonly path: stri ); } +/** + * The declared source SET in CANONICAL order — kind-validated, then sorted by + * spec CONTENT rather than by declaration order. + * + * This is the ONE place declaration order is discarded, and the module's "pure + * function of the SET" invariant rests on it: the emitted file order, the spec + * attributed to a file two specs both reach, and which of several unresolvable + * paths reports its `ERR_SOURCE_UNRESOLVED` first are all decided here. + * Validation runs across the WHOLE list before any sorting or filesystem I/O — + * interleaved with resolution, which error code came back would depend on + * declaration order, contradicting that same invariant. + * + * Exported because `resolveCollection` derives `sourceRoots` from the declared + * specs and must use this identical ordering; a second sort would be a second + * definition of "canonical". + */ +export function orderedPathSpecs(specs: readonly SourceSpec[]): { readonly path: string }[] { + return specs.map(toPathSpec).sort((a, b) => { + const [ja, jb] = [JSON.stringify(a), JSON.stringify(b)]; + return ja < jb ? -1 : ja > jb ? 1 : 0; + }); +} + +/** + * Where a declared `path` source lives on disk: absolute as written, otherwise + * relative to the DECLARING config's directory — never to ambient + * `process.cwd()`. + * + * One definition, because this expression *is* the rule for where a declared + * source lives, which is the single piece of knowledge this module exists to + * own. A caller that needs a source's root directory (rather than its files) + * calls this rather than restating it. + */ +export function resolveSpecPath(configDir: string, spec: { readonly path: string }): string { + return isAbsolute(spec.path) ? spec.path : resolve(configDir, spec.path); +} + /** * Resolve a declared source SET to a canonically-ordered list of metadata files. * @@ -95,23 +132,8 @@ export async function resolveSources( configDir: string, specs: readonly SourceSpec[], ): Promise { - // Validate every spec's KIND up front, before any filesystem I/O. Without - // this separate pass, kind-validation and path resolution were - // interleaved in one loop, so which error code came back depended on - // DECLARATION ORDER: an unsupported-kind spec placed after an - // unresolvable path spec never got reached (the path spec's - // ERR_SOURCE_UNRESOLVED fired first) — contradicting this module's own - // "pure function of the SET" invariant (see the file header). - for (const spec of specs) assertPathSpec(spec); - - // Content order, computed once. This is the ONLY place declaration order is - // discarded, and everything below depends on it: the output file order, the - // spec attributed to an overlapping file, and which of several unresolvable - // paths reports its ERR_SOURCE_UNRESOLVED first. - const ordered = [...specs].sort((a, b) => { - const [ja, jb] = [JSON.stringify(a), JSON.stringify(b)]; - return ja < jb ? -1 : ja > jb ? 1 : 0; - }); + // Kind-validated and content-ordered in one pass — see `orderedPathSpecs`. + const ordered = orderedPathSpecs(specs); // Insertion order IS output order — a Map preserves it, so the per-spec walk // order above survives to the caller. First contributor wins a shared file, @@ -119,9 +141,7 @@ export async function resolveSources( const byFile = new Map(); for (const spec of ordered) { - assertPathSpec(spec); // already validated above; narrows `spec.path` for TS below. - - const target = isAbsolute(spec.path) ? spec.path : resolve(configDir, spec.path); + const target = resolveSpecPath(configDir, spec); const stats = await stat(target).catch(() => undefined); if (stats === undefined) { throw new ParseError( diff --git a/server/typescript/packages/sdk/test/collection.test.ts b/server/typescript/packages/sdk/test/collection.test.ts index 6b6bafaf9..df8a3172b 100644 --- a/server/typescript/packages/sdk/test/collection.test.ts +++ b/server/typescript/packages/sdk/test/collection.test.ts @@ -3,7 +3,6 @@ import { mkdtempSync, rmSync, mkdirSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { resolveCollection } from "../src/collection.js"; -import { matchesScope } from "../src/scope.js"; import { rejectedCode } from "./support/error-code.js"; let root: string; @@ -39,26 +38,35 @@ describe("resolveCollection", () => { expect(c.files.map((f) => f.replace(root + "/", ""))).toEqual(["model/meta.a.json"]); }); - test("scope compiles and is applied by matchesScope", async () => { + test("scope compiles into a predicate the caller passes straight through", async () => { write("model/meta.a.json", "{}"); config("apps/ui", { sources: [{ path: "../../model" }], scope: { include: ["acme::**"] } }); const c = await resolveCollection(join(root, "apps/ui")); - expect(matchesScope("acme::Order", c.scope)).toBe(true); - expect(matchesScope("other::Order", c.scope)).toBe(false); + expect(c.inScope("acme::Order")).toBe(true); + expect(c.inScope("other::Order")).toBe(false); + }); + + test("an undeclared scope admits everything — the predicate is always defined", async () => { + write("metaobjects/meta.a.json", "{}"); + config(".", {}); + expect((await resolveCollection(root)).inScope("anything::at::All")).toBe(true); }); test("migrateScope is undefined when not declared", async () => { write("metaobjects/meta.a.json", "{}"); config(".", {}); - expect((await resolveCollection(root)).migrateScope).toBeUndefined(); + expect((await resolveCollection(root)).inMigrateScope).toBeUndefined(); }); test("migrateScope compiles when declared", async () => { write("metaobjects/meta.a.json", "{}"); config(".", { migrate: { scope: ["acme::platform::**"] } }); const c = await resolveCollection(root); - expect(matchesScope("acme::platform::Job", c.migrateScope!)).toBe(true); - expect(matchesScope("arena::Match", c.migrateScope!)).toBe(false); + expect(c.inMigrateScope!("acme::platform::Job")).toBe(true); + // `**` spans any number of segments — `matchesScope` decides, here as + // everywhere: there is exactly one implementation of the pattern grammar. + expect(c.inMigrateScope!("acme::platform::billing::Invoice")).toBe(true); + expect(c.inMigrateScope!("arena::Match")).toBe(false); }); test("an explicit dir overrides discovery", async () => { From ab6fc04902a9bfcd95cde2a71cdb7bd224975b7f Mon Sep 17 00:00:00 2001 From: Doug Mealing Date: Tue, 18 Aug 2026 09:02:35 -0400 Subject: [PATCH 35/44] test(cli): one fixture for the two migrate.scope integration suites MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Part C. `migrate-db-scope` and `verify-db-scope` scaffold the identical two-owner project — the same `PLATFORM` metadata, the same `declareScope`, a near-identical `scaffold` — because `migrate` and `verify --db` govern the identical object set from ONE declaration. Two copies of that project had already drifted: `ARENA` was a constant in one file and a `(venue: boolean)` factory in the other, which is how two suites meant to prove the same contract quietly stop testing the same thing. `test/integration/support/scope-fixture.ts` exports `PLATFORM`, `arena(opts)`, `arenaFile`, `scaffold(prefix)` and `declareScope`. The `prefix` argument keeps each suite's temp directories self-identifying. The `console.log` capture boilerplate stays where it is — it is a pre-existing convention across fifteen CLI test files, and consolidating it is a different change. Co-Authored-By: Claude Opus 5 (1M context) --- .../test/integration/migrate-db-scope.test.ts | 64 ++------------- .../test/integration/support/scope-fixture.ts | 82 +++++++++++++++++++ .../test/integration/verify-db-scope.test.ts | 67 ++------------- 3 files changed, 95 insertions(+), 118 deletions(-) create mode 100644 server/typescript/packages/cli/test/integration/support/scope-fixture.ts diff --git a/server/typescript/packages/cli/test/integration/migrate-db-scope.test.ts b/server/typescript/packages/cli/test/integration/migrate-db-scope.test.ts index b5f699dc4..612e44b2c 100644 --- a/server/typescript/packages/cli/test/integration/migrate-db-scope.test.ts +++ b/server/typescript/packages/cli/test/integration/migrate-db-scope.test.ts @@ -13,61 +13,9 @@ * the loaded FQNs named, so the author can see what missed. */ import { describe, test, expect, beforeEach, afterEach } from "bun:test"; -import { mkdtempSync, mkdirSync, rmSync, writeFileSync } from "node:fs"; -import { tmpdir } from "node:os"; -import { join } from "node:path"; +import { rmSync } from "node:fs"; import { run } from "../../src/index.js"; - -const PLATFORM = JSON.stringify({ - "metadata.root": { - package: "acme::platform", - children: [{ - "object.entity": { - name: "Job", - children: [ - { "source.rdb": { name: "src", "@table": "jobs" } }, - { "field.long": { name: "id" } }, - { "field.string": { name: "title" } }, - { "identity.primary": { name: "pk", "@fields": ["id"] } }, - ], - }, - }], - }, -}); - -/** Another owner's package, sharing the database. */ -const ARENA = JSON.stringify({ - "metadata.root": { - package: "arena", - children: [{ - "object.entity": { - name: "Match", - children: [ - { "source.rdb": { name: "src", "@table": "matches" } }, - { "field.long": { name: "id" } }, - { "identity.primary": { name: "pk", "@fields": ["id"] } }, - ], - }, - }], - }, -}); - -function scaffold(): { repo: string; dbUrl: string } { - const repo = mkdtempSync(join(tmpdir(), "metaobjects-migrate-scope-")); - mkdirSync(join(repo, "metaobjects"), { recursive: true }); - writeFileSync(join(repo, "metaobjects", "meta.platform.json"), PLATFORM, "utf8"); - writeFileSync(join(repo, "metaobjects", "meta.arena.json"), ARENA, "utf8"); - return { repo, dbUrl: `file:${join(repo, "local.db")}` }; -} - -function declareScope(repo: string, scope: string[]): void { - mkdirSync(join(repo, ".metaobjects"), { recursive: true }); - writeFileSync( - join(repo, ".metaobjects", "config.json"), - JSON.stringify({ schema_version: 1, migrate: { scope } }), - "utf8", - ); -} +import { declareScope, scaffold } from "./support/scope-fixture.js"; const migrateFromDb = (repo: string, dbUrl: string): Promise => run(["migrate", "--from-db", "--cwd", repo, "--db", dbUrl, "--dialect", "sqlite", "--slug", "initial"]); @@ -92,7 +40,7 @@ afterEach(() => { describe("meta migrate --db — migrate.scope", () => { test("a scope matching NO loaded object is refused, naming the patterns and what was loaded", async () => { - const { repo, dbUrl } = scaffold(); + const { repo, dbUrl } = scaffold("metaobjects-migrate-scope-"); try { declareScope(repo, ["typo::**"]); expect(await migrateFromDb(repo, dbUrl)).toBe(2); @@ -108,7 +56,7 @@ describe("meta migrate --db — migrate.scope", () => { }); test("a scope that matches something still runs (the refusal is not a blanket break)", async () => { - const { repo, dbUrl } = scaffold(); + const { repo, dbUrl } = scaffold("metaobjects-migrate-scope-"); try { declareScope(repo, ["acme::platform::**"]); expect(await migrateFromDb(repo, dbUrl)).toBe(0); @@ -122,7 +70,7 @@ describe("meta migrate --db — migrate.scope", () => { }); test("--format json stays parseable under a scope — the out-of-scope note is text-format only", async () => { - const { repo, dbUrl } = scaffold(); + const { repo, dbUrl } = scaffold("metaobjects-migrate-scope-"); try { declareScope(repo, ["acme::platform::**"]); expect(await run([ @@ -144,7 +92,7 @@ describe("meta migrate --db — migrate.scope", () => { }); test("no migrate.scope declared — unchanged, both tables governed", async () => { - const { repo, dbUrl } = scaffold(); + const { repo, dbUrl } = scaffold("metaobjects-migrate-scope-"); try { expect(await migrateFromDb(repo, dbUrl)).toBe(0); const all = [...out, ...err].join("\n"); diff --git a/server/typescript/packages/cli/test/integration/support/scope-fixture.ts b/server/typescript/packages/cli/test/integration/support/scope-fixture.ts new file mode 100644 index 000000000..2d917fa3d --- /dev/null +++ b/server/typescript/packages/cli/test/integration/support/scope-fixture.ts @@ -0,0 +1,82 @@ +/** + * The two-owner project every `migrate.scope` integration test drives. + * + * `meta migrate` and `meta verify --db` govern the identical object set from ONE + * declaration, so their integration tests scaffold the identical project — and + * two copies of it had already drifted (`ARENA` was a constant in one file and a + * `(venue: boolean)` factory in the other), which is how two tests that are + * supposed to prove the same contract quietly stop testing the same thing. + * + * The shape: this consumer's `acme::platform` package owns `jobs`; a second + * owner's `arena` package owns `matches` in the same database. A scope of + * `["acme::platform::**"]` therefore governs exactly one of the two. + */ +import { mkdtempSync, mkdirSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +/** This consumer's package. */ +export const PLATFORM = JSON.stringify({ + "metadata.root": { + package: "acme::platform", + children: [{ + "object.entity": { + name: "Job", + children: [ + { "source.rdb": { name: "src", "@table": "jobs" } }, + { "field.long": { name: "id" } }, + { "field.string": { name: "title" } }, + { "identity.primary": { name: "pk", "@fields": ["id"] } }, + ], + }, + }], + }, +}); + +/** + * Another owner's package, sharing the database. + * + * `venue` models a column the other owner has declared but not migrated yet — + * drift for THEM, never for this consumer. Pass `false` for the base shape. + */ +export const arena = (opts: { venue: boolean } = { venue: false }): string => JSON.stringify({ + "metadata.root": { + package: "arena", + children: [{ + "object.entity": { + name: "Match", + children: [ + { "source.rdb": { name: "src", "@table": "matches" } }, + { "field.long": { name: "id" } }, + ...(opts.venue ? [{ "field.string": { name: "venue" } }] : []), + { "identity.primary": { name: "pk", "@fields": ["id"] } }, + ], + }, + }], + }, +}); + +/** Absolute path of the arena metadata file, for a test that rewrites it. */ +export const arenaFile = (repo: string): string => join(repo, "metaobjects", "meta.arena.json"); + +/** + * A throwaway project holding both packages, plus the sqlite URL beside it. + * `prefix` names the temp directory so a failing run says which suite made it. + */ +export function scaffold(prefix: string): { repo: string; dbUrl: string } { + const repo = mkdtempSync(join(tmpdir(), prefix)); + mkdirSync(join(repo, "metaobjects"), { recursive: true }); + writeFileSync(join(repo, "metaobjects", "meta.platform.json"), PLATFORM, "utf8"); + writeFileSync(arenaFile(repo), arena(), "utf8"); + return { repo, dbUrl: `file:${join(repo, "local.db")}` }; +} + +/** Declare `migrate.scope` on an existing scaffold — the ONE key both commands read. */ +export function declareScope(repo: string, scope: string[]): void { + mkdirSync(join(repo, ".metaobjects"), { recursive: true }); + writeFileSync( + join(repo, ".metaobjects", "config.json"), + JSON.stringify({ schema_version: 1, migrate: { scope } }), + "utf8", + ); +} diff --git a/server/typescript/packages/cli/test/integration/verify-db-scope.test.ts b/server/typescript/packages/cli/test/integration/verify-db-scope.test.ts index 7cdbf2135..215169e97 100644 --- a/server/typescript/packages/cli/test/integration/verify-db-scope.test.ts +++ b/server/typescript/packages/cli/test/integration/verify-db-scope.test.ts @@ -8,64 +8,11 @@ * alone would misreport an unchecked table as a checked one. */ import { describe, test, expect, beforeEach, afterEach } from "bun:test"; -import { mkdtempSync, mkdirSync, rmSync, writeFileSync, readFileSync, readdirSync } from "node:fs"; -import { tmpdir } from "node:os"; +import { rmSync, writeFileSync, readFileSync, readdirSync } from "node:fs"; import { join } from "node:path"; import { createClient } from "@libsql/client"; import { run } from "../../src/index.js"; - -const PLATFORM = JSON.stringify({ - "metadata.root": { - package: "acme::platform", - children: [{ - "object.entity": { - name: "Job", - children: [ - { "source.rdb": { name: "src", "@table": "jobs" } }, - { "field.long": { name: "id" } }, - { "field.string": { name: "title" } }, - { "identity.primary": { name: "pk", "@fields": ["id"] } }, - ], - }, - }], - }, -}); - -/** Another owner's package, sharing the database. `venue` models a column the - * other owner has not migrated yet — drift for THEM, never for this consumer. */ -const ARENA = (venue: boolean): string => JSON.stringify({ - "metadata.root": { - package: "arena", - children: [{ - "object.entity": { - name: "Match", - children: [ - { "source.rdb": { name: "src", "@table": "matches" } }, - { "field.long": { name: "id" } }, - ...(venue ? [{ "field.string": { name: "venue" } }] : []), - { "identity.primary": { name: "pk", "@fields": ["id"] } }, - ], - }, - }], - }, -}); - -function scaffold(): { repo: string; dbUrl: string } { - const repo = mkdtempSync(join(tmpdir(), "metaobjects-verify-scope-")); - mkdirSync(join(repo, "metaobjects"), { recursive: true }); - writeFileSync(join(repo, "metaobjects", "meta.platform.json"), PLATFORM, "utf8"); - writeFileSync(join(repo, "metaobjects", "meta.arena.json"), ARENA(false), "utf8"); - return { repo, dbUrl: `file:${join(repo, "local.db")}` }; -} - -function declareScope(repo: string, scope: string[]): void { - mkdirSync(join(repo, ".metaobjects"), { recursive: true }); - writeFileSync( - join(repo, ".metaobjects", "config.json"), - JSON.stringify({ schema_version: 1, migrate: { scope } }), - "utf8", - ); -} +import { arena, arenaFile, declareScope, PLATFORM, scaffold } from "./support/scope-fixture.js"; /** Materialize the current metadata schema into the DB via the real migrate path. */ async function materialize(repo: string, dbUrl: string): Promise { @@ -101,14 +48,14 @@ afterEach(() => { describe("meta verify --db — migrate.scope", () => { test("an out-of-scope object's divergence is reported as out-of-scope, not as drift", async () => { - const { repo, dbUrl } = scaffold(); + const { repo, dbUrl } = scaffold("metaobjects-verify-scope-"); try { await materialize(repo, dbUrl); expect(await run(["verify", "--cwd", repo, "--db", dbUrl, "--dialect", "sqlite"])).toBe(0); declareScope(repo, ["acme::platform::**"]); // The other owner's model gains a column its own migration has not applied. - writeFileSync(join(repo, "metaobjects", "meta.arena.json"), ARENA(true), "utf8"); + writeFileSync(arenaFile(repo), arena({ venue: true }), "utf8"); out = []; err = []; @@ -123,7 +70,7 @@ describe("meta verify --db — migrate.scope", () => { }); test("in-scope drift still fails the gate under a scope", async () => { - const { repo, dbUrl } = scaffold(); + const { repo, dbUrl } = scaffold("metaobjects-verify-scope-"); try { await materialize(repo, dbUrl); declareScope(repo, ["acme::platform::**"]); @@ -147,10 +94,10 @@ describe("meta verify --db — migrate.scope", () => { }); test("with no migrate.scope declared, the same divergence IS drift (unchanged)", async () => { - const { repo, dbUrl } = scaffold(); + const { repo, dbUrl } = scaffold("metaobjects-verify-scope-"); try { await materialize(repo, dbUrl); - writeFileSync(join(repo, "metaobjects", "meta.arena.json"), ARENA(true), "utf8"); + writeFileSync(arenaFile(repo), arena({ venue: true }), "utf8"); out = []; err = []; From ce5724d3de2697bcf092dfcf983c7ca67b11d20a Mon Sep 17 00:00:00 2001 From: Doug Mealing Date: Tue, 18 Aug 2026 09:04:08 -0400 Subject: [PATCH 36/44] fix(cli): the docs-site theme is read where --scaffold-site writes it Self-review of the A3 change. `--templates` redirects the adopter RENDER template chain, and routing the owned `codegen/docs-site/` theme through the same `projectRoot` would have made `meta docs --site --templates ` look for a theme in a directory `--scaffold-site` never writes to. Both now key on `collection.configDir`, so the two halves of scaffold-and-own cannot separate. Co-Authored-By: Claude Opus 5 (1M context) --- server/typescript/packages/cli/src/commands/docs.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/server/typescript/packages/cli/src/commands/docs.ts b/server/typescript/packages/cli/src/commands/docs.ts index 3aedbb18c..84ebf67b6 100644 --- a/server/typescript/packages/cli/src/commands/docs.ts +++ b/server/typescript/packages/cli/src/commands/docs.ts @@ -580,8 +580,11 @@ async function emitSite( } // Scaffold-and-own: when the consumer has copied templates/assets into // codegen/docs-site/ (via --scaffold-site), use those; else the bundled defaults. - const ownedTemplates = join(projectRoot, "codegen/docs-site/templates"); - const ownedAssets = join(projectRoot, "codegen/docs-site/assets"); + // Keyed on `configDir`, NOT `projectRoot`: `--templates` redirects the adopter + // RENDER template chain (the `templates/` above), and letting it also move the + // docs-site theme would read it from somewhere `--scaffold-site` never writes. + const ownedTemplates = join(collection.configDir, "codegen/docs-site/templates"); + const ownedAssets = join(collection.configDir, "codegen/docs-site/assets"); try { const r = await generateSite({ sourceDirs, From 39678a7eef5ffdfdb29b613c394cc1ba4efd2929 Mon Sep 17 00:00:00 2001 From: Doug Mealing Date: Tue, 18 Aug 2026 09:25:31 -0400 Subject: [PATCH 37/44] =?UTF-8?q?fix(sdk):=20the=20config=20is=20the=20onl?= =?UTF-8?q?y=20authority=20=E2=80=94=20at=20the=20walk=20and=20at=20the=20?= =?UTF-8?q?loader=20entry?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two sites still answered "where does metadata live?" without reading the config. Both are read paths, so both could silently load from somewhere the project never declared. 1 — discovery stops at `.metaobjects/config.json` and nothing else. A quality-pass ruling had added a second stop marker: a bare `metaobjects/` directory also ended the walk, to keep a nested config-less project reading its own metadata. That is withdrawn. `metaobjects/` is the DEFAULT VALUE of `sources` and carries no other meaning, so a directory of that name says nothing about whether a project lives there — and a second marker is a second definition of where metadata lives, which is precisely the duplication `resolveCollection` exists to be the only instance of. It was also wrong on its own terms: a project pointing `sources` at a sibling module has no such directory, and one with both would be governed by whichever the walk noticed first. The consequence is real and is documented as the rule rather than as a caveat: a subdirectory holding metadata but no config now resolves to its nearest ancestor config. A subdirectory that should own its metadata declares one — `meta init` writes it, and `"sources": []` is enough to claim the directory. 2 — `loadMemory` resolves through `resolveCollection` too. Its no-`files` arm scanned `/` directly and walked `package.meta.json` workspace peers, so a caller that copied the routed shape but forgot `files` loaded from a directory the config may never have mentioned — the ninth call site, arriving by omission. Both arms are now the same answer: one already computed by the caller, one computed here. The peer walk goes with it (design §11 pre-ruled the retirement; the Upgrading section documents it). The constants and the metadata-file walk move to a new leaf module, `metadata-files.ts`, which imports nothing from its siblings. Homing them in `memory.ts` closes an ESM cycle whose failure mode is a crash, not a warning: `DEFAULT_SOURCES` reads `DEFAULT_METADATA_DIR` at module top level, so the cycle surfaces as `ReferenceError: Cannot access 'DEFAULT_METADATA_DIR' before initialization`. A lazy `await import()` would hide that rather than remove it. Tests: two discovery cases retargeted to assert the withdrawn behaviour's absence, one dropped (it only guarded that marker); `resolveCollection`'s back-compat case inverted to assert the ancestor config governs. The two retired-peer-walk cases now pin what the Upgrading section PROMISES about the removal — that it fails loudly with `ERR_UNRESOLVED_SUPER`, and that a declared source replaces it — rather than being deleted along with the feature. `loadMemory` with nothing to resolve now reports `ERR_COLLECTION_NOT_FOUND`, the same structured code every other command gives. sdk 243 -> 242 (one test dropped, six retargeted); cli, migrate-ts and codegen-ts unchanged. Co-Authored-By: Claude Opus 5 (1M context) --- ...08-17-metadata-source-resolution-design.md | 20 +- .../typescript/packages/sdk/src/collection.ts | 17 +- .../typescript/packages/sdk/src/discovery.ts | 51 +++-- server/typescript/packages/sdk/src/index.ts | 22 +-- server/typescript/packages/sdk/src/memory.ts | 185 ++++-------------- .../packages/sdk/src/metadata-files.ts | 126 ++++++++++++ server/typescript/packages/sdk/src/sources.ts | 12 +- .../packages/sdk/test/collection.test.ts | 25 +-- .../packages/sdk/test/discovery.test.ts | 20 +- .../sdk/test/dogfood-examples.test.ts | 3 +- .../packages/sdk/test/memory.test.ts | 59 +++--- .../packages/sdk/test/source-order.test.ts | 5 +- 12 files changed, 287 insertions(+), 258 deletions(-) create mode 100644 server/typescript/packages/sdk/src/metadata-files.ts diff --git a/docs/superpowers/specs/2026-08-17-metadata-source-resolution-design.md b/docs/superpowers/specs/2026-08-17-metadata-source-resolution-design.md index f79990d52..4a8725883 100644 --- a/docs/superpowers/specs/2026-08-17-metadata-source-resolution-design.md +++ b/docs/superpowers/specs/2026-08-17-metadata-source-resolution-design.md @@ -304,16 +304,16 @@ from the same root cause — so routing every read through one authority closes Running a CLI inside an app must find that app's configuration. -- **Walk up from cwd** for the nearest project root. Nearest wins. Per-port generator config is - then read from that same directory. -- **A directory is a project root when it carries EITHER `.metaobjects/config.json` OR a - `metaobjects/` directory.** The config is checked first, so a directory carrying both resolves - as declared. The second marker is a back-compat obligation, not a convenience: before source - resolution existed, a nested project holding its own `metaobjects/` and no config of its own - read its own metadata. A config-only stop condition walks straight past it and silently loads - the ANCESTOR's model, then writes generated output to the ancestor's `outDir` — a silent - regression on a layout that worked. Stopping there with no config found means the default - sources apply, which is exactly the pre-branch behaviour for that directory. +- **Walk up from cwd** for the nearest `.metaobjects/config.json`. Nearest wins. Per-port + generator config is then read from that same directory. +- **That file is the ONLY project marker.** A directory that merely *holds* metadata is not a + project boundary. Where metadata lives is the `sources` key's answer, and the default + directory name is only that key's default *value* — so stopping the walk on a directory of + that name would put a second definition of "where metadata lives" back into the toolchain, + which is the exact duplication §4.6 exists to remove. It would also be wrong on its own + terms: a project whose config points `sources` at a sibling module has no such directory at + all, and one that has both would be governed by whichever the walk noticed first. A + subdirectory that should own its metadata declares a config — `meta init` writes one. - **Stop at a repository boundary** (`.git`) or the filesystem root, so a monorepo can never silently adopt a parent checkout's configuration. - **Explicit override wins** — the existing `--cwd` / `-C` flag and project-root positional are diff --git a/server/typescript/packages/sdk/src/collection.ts b/server/typescript/packages/sdk/src/collection.ts index adcd30975..5b915cd6e 100644 --- a/server/typescript/packages/sdk/src/collection.ts +++ b/server/typescript/packages/sdk/src/collection.ts @@ -15,7 +15,7 @@ import { ParseError, codeSource } from "@metaobjectsdev/metadata"; import { CONFIG_FILE, loadConfig, type Config } from "./config.js"; import { discoverCollectionRoot, exists, isDir } from "./discovery.js"; import { compileScope, matchesScope, type Scope } from "./scope.js"; -import { DEFAULT_METADATA_DIR, DEFAULT_METAOBJECTS_DIR } from "./memory.js"; +import { DEFAULT_METADATA_DIR, DEFAULT_METAOBJECTS_DIR } from "./metadata-files.js"; import { DEFAULT_SOURCES, orderedPathSpecs, @@ -86,19 +86,18 @@ function toScope(spec: Config["scope"]): Scope { * * Resolution order: an explicit `opts.explicitDir` wins outright; otherwise * `discoverCollectionRoot` walks up from `startDir` for the nearest directory - * carrying `.metaobjects/config.json` OR a `metaobjects/` directory (see - * `discovery.ts` — the second marker is what keeps a nested project reading - * its OWN metadata), falling back to `startDir` itself when neither is found. - * When the resolved directory carries a config, its declared - * `sources`/`scope`/`migrate.scope` govern. Only a genuinely ABSENT - * `config.json` falls through to `DEFAULT_SOURCES` — the same `metaobjects/` + * carrying `.metaobjects/config.json` — the ONLY project marker (`discovery.ts` + * says why a directory that merely holds metadata is not one) — falling back to + * `startDir` itself when none is found. When the resolved directory carries a + * config, its declared `sources`/`scope`/`migrate.scope` govern. Only a + * genuinely ABSENT `config.json` falls through to `DEFAULT_SOURCES` — the same * directory the pre-source-resolution toolchain always read; a config.json * that EXISTS but fails to load (malformed JSON, schema violation) is the * author's error and propagates rather than silently degrading — a source * that fails to resolve must never look like one that was never declared. * Throws `ERR_COLLECTION_NOT_FOUND` only when BOTH have failed: no - * `sources` were declared AND the default `metaobjects/` directory does not - * exist either. + * `sources` were declared AND the default source directory does not exist + * either. * * A declared source that fails to resolve is a different, louder failure — * `resolveSources` throws `ERR_SOURCE_UNRESOLVED` for that case; only the diff --git a/server/typescript/packages/sdk/src/discovery.ts b/server/typescript/packages/sdk/src/discovery.ts index 9f1db2884..b66922945 100644 --- a/server/typescript/packages/sdk/src/discovery.ts +++ b/server/typescript/packages/sdk/src/discovery.ts @@ -8,26 +8,27 @@ // // Three properties are load-bearing. // -// 1. **Two markers, not one.** A directory is a project root when it carries -// `.metaobjects/config.json` OR a `metaobjects/` directory. The config half -// is the declared form; the `metaobjects/` half is the pre-source-resolution -// convention, and skipping past it would silently load an ANCESTOR's model -// (and write generated output to the ancestor's `outDir`) for a nested -// project that has always read its own — a back-compat regression on a -// layout that worked. The config is checked first so a directory carrying -// both resolves as declared. See design §4.6.1. -// 2. **Nearest wins** — the walk returns on the FIRST directory carrying either +// 1. **One marker.** A directory is a project root when it carries +// `.metaobjects/config.json`, and on no other evidence. A directory that +// merely *holds* metadata is not a project boundary: where metadata lives is +// the `sources` key's answer, and `metaobjects/` is only that key's default +// value. Stopping on a bare `metaobjects/` directory would put a second +// definition of "where metadata lives" back into the walk — the exact +// duplication `resolveCollection` exists to be the only instance of — and it +// would ignore a project whose config points its `sources` somewhere else +// entirely. See design §4.6.1. +// 2. **Nearest wins** — the walk returns on the FIRST directory carrying the // marker, so a config in a subdirectory beats one in an ancestor. // 3. **The walk stops at a repository boundary** (`.git`), so a monorepo // checkout can never silently adopt a *parent checkout's* configuration. The -// marker checks run BEFORE the `.git` check within each directory — +// marker check runs BEFORE the `.git` check within each directory — // reversed, a root-level project (where `.git` also lives) would be // unreachable from any subdirectory, since the boundary would stop the walk // one directory too early. import { stat } from "node:fs/promises"; import { dirname, join, resolve } from "node:path"; import { CONFIG_FILE } from "./config.js"; -import { DEFAULT_METADATA_DIR, DEFAULT_METAOBJECTS_DIR } from "./memory.js"; +import { DEFAULT_METAOBJECTS_DIR } from "./metadata-files.js"; const GIT_DIR = ".git"; @@ -42,10 +43,11 @@ export async function exists(path: string): Promise { } } -/** `exists`, narrowed to directories. A plain FILE named `metaobjects` is not a - * metadata home, so it must not stop the walk below — and `collection.ts`'s - * default-directory probe needs the same distinction to raise its friendlier - * `ERR_COLLECTION_NOT_FOUND`. */ +/** `exists`, narrowed to directories — `collection.ts`'s default-source probe + * needs that distinction to raise its friendlier `ERR_COLLECTION_NOT_FOUND` + * (a plain FILE where the default source directory should be is not a + * metadata home). Lives here beside `exists` so there is one filesystem + * predicate pair in the package rather than a copy per caller. */ export async function isDir(path: string): Promise { try { return (await stat(path)).isDirectory(); @@ -59,22 +61,20 @@ export interface DiscoveredRoot { /** The project root the walk settled on. Always absolute; falls back to the * resolved start directory when the walk found no marker at all. */ readonly dir: string; - /** Whether `dir` carries `.metaobjects/config.json`. False both for a - * `metaobjects/`-only directory and for the no-marker fallback — in either - * case the DEFAULT sources apply, which is the pre-branch behaviour. */ + /** Whether `dir` carries `.metaobjects/config.json`. False only on the + * no-marker fallback, where the DEFAULT sources apply. */ readonly hasConfig: boolean; } /** * Walk up from `startDir` for the nearest project root — a directory holding - * `.metaobjects/config.json` or a `metaobjects/` directory (see the file - * header for why both count). The walk stops after examining a directory that - * contains `.git`, so a monorepo can never silently adopt a parent checkout's - * configuration. + * `.metaobjects/config.json`, and nothing else (see the file header). The walk + * stops after examining a directory that contains `.git`, so a monorepo can + * never silently adopt a parent checkout's configuration. * - * Never fails: with no marker anywhere below the boundary it reports the + * Never fails: with no config anywhere below the boundary it reports the * resolved `startDir` with `hasConfig: false`, which is what - * `resolveCollection` turns into either the default `metaobjects/` source or + * `resolveCollection` turns into either the default source or * `ERR_COLLECTION_NOT_FOUND`. */ export async function discoverCollectionRoot(startDir: string): Promise { @@ -84,8 +84,7 @@ export async function discoverCollectionRoot(startDir: string): Promise/metaobjects/` into a single - * MetaData. If `/.meta/package.meta.json` declares `extends:` deps - * and a workspace can be discovered (pnpm-workspace.yaml or package.json - * workspaces), peer packages are loaded too in topological dep-first order. - * Pass {@link LoadMemoryOptions.files} to bypass this discovery entirely and - * load an already-resolved file list instead (e.g. from `resolveCollection`). + * Load a project's metadata into a single MetaData tree. + * + * Which files those are is `resolveCollection`'s decision, never this + * function's: with no {@link LoadMemoryOptions.files} it calls + * `resolveCollection(repoRoot)` — nearest-ancestor `.metaobjects/config.json`, + * then that config's declared `sources`, falling back to the default source + * directory only when a project declares none. `loadMemory` names no directory + * of its own, so a caller cannot end up loading from somewhere the rest of the + * toolchain does not. * * Excludes `_pending/`. Registers metaobjects core types plus Meta Forge's * descriptive top-level types (decision, principle, etc.) so mixed content @@ -95,18 +76,15 @@ export const defaultLoadMemoryProviders: readonly MetaDataTypeProvider[] = [ * {@link LoadMemoryOptions.providers}) are composed AFTER the defaults so * they may depend on core/forge ids. * - * Throws if `metaobjects/` doesn't exist (callers should run `meta init`), - * unless `options.files` is supplied. + * Throws `ERR_COLLECTION_NOT_FOUND` when nothing resolves (callers should run + * `meta init`), unless `options.files` is supplied. * - * @param repoRoot The project's working-directory root (e.g. process.cwd()). - * `loadMemory` resolves `metaobjects/` and (if workspace-aware) the - * transitive `extends:` graph automatically. - * **Ignored entirely when `options.files` is supplied** — that list has - * already been resolved (by `resolveCollection`, which owns the decision), so - * no discovery runs and nothing reads this path. Every routed CLI command - * passes both, and the argument is inert at all of them; a caller that copies - * that shape but omits `files` silently loads `/metaobjects/` - * instead, which is the divergence this design exists to close. + * @param repoRoot Where resolution STARTS — the working directory, typically + * `process.cwd()`. The walk goes up from here for the governing config, so + * this need not be the project root itself. + * **Ignored entirely when `options.files` is supplied**: that list is already + * resolved, so nothing reads this path. Every routed CLI command passes both, + * and the argument is inert at all of them. * @param options Optional {@link LoadMemoryOptions} — supply additional * providers or replace the default bundle entirely. */ @@ -128,14 +106,15 @@ export async function loadMemory( } const registry = composeRegistry(providers); - // Collect all metadata file paths to load. Order matters for the parser's - // deferred-resolution pass (it parses in array order, then resolves supers - // against the merged tree afterwards) — dep packages first, current last. - // An explicit `files` list (already resolved, e.g. by `resolveCollection`) - // wins outright and skips discovery entirely. + // Both arms are `resolveCollection`'s answer — one already computed by the + // caller, one computed here. There is no third way to find metadata, and + // that is the whole of this line's design: the previous no-`files` arm + // scanned `/` directly, so a caller that copied the + // routed shape but forgot `files` silently loaded from a directory the + // project's config may never have mentioned. const paths = options?.files !== undefined ? [...options.files] - : await collectMetadataPaths(repoRoot); + : [...(await resolveCollection(repoRoot)).files]; const loader = new MetaDataLoader({ registry, @@ -150,97 +129,3 @@ export async function loadMemory( return result.root; } - -// Dep packages' metaobjects/ files first (topological order), then current. -async function collectMetadataPaths(repoRoot: string): Promise { - const currentMetaDir = join(repoRoot, ".meta"); - const ws = await discoverWorkspace(repoRoot); - - // Workspace path: walk extends, load dep metaobjects/ dirs first - if (ws !== undefined) { - const currentPkg = ws.packages.find((p) => p.metaDir === currentMetaDir); - if (currentPkg !== undefined && currentPkg.manifest.extends.length > 0) { - const ordered = resolveExtendsOrder(ws, currentMetaDir); - const paths: string[] = []; - for (const pkg of ordered) { - // Each workspace package's metadata lives alongside its .meta/ dir - const pkgRoot = join(pkg.metaDir, ".."); - paths.push(...(await listMetadataFiles(join(pkgRoot, DEFAULT_METADATA_DIR)))); - } - return paths; - } - } - - // Single-package path: scan metaobjects/ at the project root - return listMetadataFiles(join(repoRoot, DEFAULT_METADATA_DIR)); -} - -/** Directory excluded at every level of {@link listMetadataFiles} — drafts - * that are deliberately not part of the loaded model. */ -const PENDING_DIR = "_pending"; - -/** - * Recursively list metadata files (*.json, *.yaml, *.yml, matched - * case-insensitively — see `isMetadataFile` above) under a directory, - * excluding _pending/ at any level. Subdirectories (e.g. projections/) are - * walked depth-first. Files within a directory are sorted alphabetically for - * deterministic load order; subdirectories are visited AFTER the files at the - * same level. - * - * That per-level rule is a contract, not an implementation detail. This is the - * order production has always handed the loader, and declaration order survives - * into generated output: `codegen-ts`'s barrel emits from `root.objects()` - * order, and so do the shared `enums.ts`, `meta docs` page ordering and `meta - * export`'s `canonicalSerialize` sibling order. A flat lexicographic sort of - * absolute paths is NOT the same list — it disagrees whenever a subdirectory - * name sorts before a sibling file (`common/` before `meta.users.json`) — so - * `resolveSources` calls this function rather than re-walking and re-sorting. - * Pinned by `test/source-order.test.ts`. - * - * Exported for that gate and for `sources.ts`; not re-exported from the package - * index — `resolveCollection` is the public door. - * - * An entry whose `stat` fails (a dangling symlink, a TOCTOU removal between - * `readdir` and `stat`, an EACCES entry) is SKIPPED, matching `DirectorySource` - * in `@metaobjectsdev/metadata`, which this walk otherwise mirrors. A failure to - * read the directory itself still throws — that is the "you have no metadata - * here" case callers report. - * - * Format selection (parsing) happens downstream in `FileSource` from - * `@metaobjectsdev/metadata`, which infers the parser from file extension. - */ -export async function listMetadataFiles(dir: string): Promise { - let entries: string[]; - try { - entries = await readdir(dir); - } catch (err) { - throw new Error(`cannot read metadata directory ${dir}: ${(err as Error).message}`); - } - const paths: string[] = []; - const subdirs: string[] = []; - // #188: sort the raw `readdir` entries so file order is deterministic across - // runtimes/filesystems (Node vs Bun return different `readdir` orders), matching - // this function's docstring and the metadata package's own `DirectorySource`. - // (Resolution is now order-INDEPENDENT — super-resolve.ts #188 — so this is the - // deterministic-enumeration FLOOR, not the fix; it keeps every derived artifact - // that preserves declaration order, e.g. serialization, stable across runtimes.) - for (const entry of [...entries].sort()) { - if (entry === PENDING_DIR) continue; - const full = join(dir, entry); - // `stat` (not `lstat`/`Dirent.isDirectory()`) so a symlinked subdirectory is - // traversed — `DirectorySource` has always followed symlinks this way. - const s = await stat(full).catch(() => undefined); - if (s === undefined) continue; - if (s.isDirectory()) { - subdirs.push(full); - } else if (s.isFile() && isMetadataFile(entry)) { - paths.push(full); - } - } - // Recurse into subdirectories after collecting files at this level. - // `subdirs` is already in sorted order (built from the sorted `entries` above). - for (const sub of subdirs) { - paths.push(...(await listMetadataFiles(sub))); - } - return paths; -} diff --git a/server/typescript/packages/sdk/src/metadata-files.ts b/server/typescript/packages/sdk/src/metadata-files.ts new file mode 100644 index 000000000..bcddc4679 --- /dev/null +++ b/server/typescript/packages/sdk/src/metadata-files.ts @@ -0,0 +1,126 @@ +// server/typescript/packages/sdk/src/metadata-files.ts +// +// The project's default directory names, what counts as a metadata file, and +// the one walk that turns a directory into an ordered file list. +// +// **This module imports nothing from its siblings, and that is the point.** +// `resolveCollection` (`collection.ts`) is the single authority on where +// metadata lives, so `memory.ts`'s `loadMemory` must call it — while +// `collection.ts` and `sources.ts` need the constants and the walk below. +// Homing those in `memory.ts` closes an ESM cycle whose failure mode is not a +// warning but a crash: `DEFAULT_SOURCES` (`sources.ts`) reads +// `DEFAULT_METADATA_DIR` at module top level, so the cycle surfaces as +// `ReferenceError: Cannot access 'DEFAULT_METADATA_DIR' before initialization` +// on whichever module the entry point happens to reach first. A leaf both +// sides import is the fix; a lazy `await import()` inside `loadMemory` is not +// — that hides the cycle rather than removing it. +import { extname, join } from "node:path"; +import { readdir, stat } from "node:fs/promises"; + +/** + * The DEFAULT value of `sources` — the directory scanned when + * `.metaobjects/config.json` declares no sources. Scaffold via `meta init`; + * the directory is committed to git. + * + * **A default, and nothing else.** No read path may assume a directory of this + * name exists or is where metadata lives: that question is answered by + * `resolveCollection`, which applies this constant exactly once (via + * `DEFAULT_SOURCES` in `sources.ts`) when a project declares nothing. A + * project that declares `sources` may put its metadata anywhere, and every + * command follows the config. `test/no-hardcoded-metadata-dir.test.ts` is the + * enforcer. + */ +export const DEFAULT_METADATA_DIR = "metaobjects"; + +/** + * Default directory name (relative to project root) for MetaObjects' own + * runtime state: config.json, .gen-state/, package.meta.json, agent docs. + * Scaffold via `meta init`; most contents are committed to git. + * + * Unlike {@link DEFAULT_METADATA_DIR} this one IS a fixed convention — it is + * where the config that answers "where is the metadata?" lives, so it cannot + * itself be configured. + */ +export const DEFAULT_METAOBJECTS_DIR = ".metaobjects"; + +/** Recognized metadata file extensions, matched case-insensitively — mirrors + * `DirectorySource` in `@metaobjectsdev/metadata`, which checks + * `extname().toLowerCase()`. The single definition every metadata-file + * walker in this package uses — and since `resolveSources` (`sources.ts`) + * calls {@link listMetadataFiles} outright rather than keeping a second + * recursive walk of its own, there is exactly one walker to keep honest. */ +export const METADATA_EXTENSIONS = new Set([".json", ".yaml", ".yml"]); + +export function isMetadataFile(name: string): boolean { + return METADATA_EXTENSIONS.has(extname(name).toLowerCase()); +} + +/** Directory excluded at every level of {@link listMetadataFiles} — drafts + * that are deliberately not part of the loaded model. */ +const PENDING_DIR = "_pending"; + +/** + * Recursively list metadata files (*.json, *.yaml, *.yml, matched + * case-insensitively — see `isMetadataFile` above) under a directory, + * excluding _pending/ at any level. Subdirectories (e.g. projections/) are + * walked depth-first. Files within a directory are sorted alphabetically for + * deterministic load order; subdirectories are visited AFTER the files at the + * same level. + * + * That per-level rule is a contract, not an implementation detail. This is the + * order production has always handed the loader, and declaration order survives + * into generated output: `codegen-ts`'s barrel emits from `root.objects()` + * order, and so do the shared `enums.ts`, `meta docs` page ordering and `meta + * export`'s `canonicalSerialize` sibling order. A flat lexicographic sort of + * absolute paths is NOT the same list — it disagrees whenever a subdirectory + * name sorts before a sibling file (`common/` before `meta.users.json`) — so + * `resolveSources` calls this function rather than re-walking and re-sorting. + * Pinned by `test/source-order.test.ts`. + * + * Exported for that gate and for `sources.ts`; not re-exported from the package + * index — `resolveCollection` is the public door. + * + * An entry whose `stat` fails (a dangling symlink, a TOCTOU removal between + * `readdir` and `stat`, an EACCES entry) is SKIPPED, matching `DirectorySource` + * in `@metaobjectsdev/metadata`, which this walk otherwise mirrors. A failure to + * read the directory itself still throws — that is the "you have no metadata + * here" case callers report. + * + * Format selection (parsing) happens downstream in `FileSource` from + * `@metaobjectsdev/metadata`, which infers the parser from file extension. + */ +export async function listMetadataFiles(dir: string): Promise { + let entries: string[]; + try { + entries = await readdir(dir); + } catch (err) { + throw new Error(`cannot read metadata directory ${dir}: ${(err as Error).message}`); + } + const paths: string[] = []; + const subdirs: string[] = []; + // #188: sort the raw `readdir` entries so file order is deterministic across + // runtimes/filesystems (Node vs Bun return different `readdir` orders), matching + // this function's docstring and the metadata package's own `DirectorySource`. + // (Resolution is now order-INDEPENDENT — super-resolve.ts #188 — so this is the + // deterministic-enumeration FLOOR, not the fix; it keeps every derived artifact + // that preserves declaration order, e.g. serialization, stable across runtimes.) + for (const entry of [...entries].sort()) { + if (entry === PENDING_DIR) continue; + const full = join(dir, entry); + // `stat` (not `lstat`/`Dirent.isDirectory()`) so a symlinked subdirectory is + // traversed — `DirectorySource` has always followed symlinks this way. + const s = await stat(full).catch(() => undefined); + if (s === undefined) continue; + if (s.isDirectory()) { + subdirs.push(full); + } else if (s.isFile() && isMetadataFile(entry)) { + paths.push(full); + } + } + // Recurse into subdirectories after collecting files at this level. + // `subdirs` is already in sorted order (built from the sorted `entries` above). + for (const sub of subdirs) { + paths.push(...(await listMetadataFiles(sub))); + } + return paths; +} diff --git a/server/typescript/packages/sdk/src/sources.ts b/server/typescript/packages/sdk/src/sources.ts index f33a68833..5f301d85a 100644 --- a/server/typescript/packages/sdk/src/sources.ts +++ b/server/typescript/packages/sdk/src/sources.ts @@ -11,8 +11,8 @@ // canonical spec ordering below is load-bearing. // // Canonical is NOT the same as "flat-sorted". Within one directory spec the -// order is `listMetadataFiles`'s (memory.ts) — files at a level, then that -// level's subdirectories, depth-first — because that is the order production +// order is `listMetadataFiles`'s (metadata-files.ts) — files at a level, then +// that level's subdirectories, depth-first — because that is the order production // has always handed the loader, and declaration order survives into generated // output (the barrel's export list, the shared `enums.ts`, `meta docs` page // order, `meta export`'s sibling order). A flat sort of absolute paths @@ -23,7 +23,7 @@ import { stat } from "node:fs/promises"; import { isAbsolute, resolve } from "node:path"; import { ParseError, codeSource } from "@metaobjectsdev/metadata"; -import { DEFAULT_METADATA_DIR, listMetadataFiles } from "./memory.js"; +import { DEFAULT_METADATA_DIR, listMetadataFiles } from "./metadata-files.js"; /** Tagged union of source kinds. `resource` and `package` are declared now so * the config shape is stable across phases; only `path` resolves in phase 1 — @@ -42,9 +42,9 @@ export interface ResolvedSource { /** Used when `sources` is absent or empty in `.metaobjects/config.json`. A * DEFAULT, never a requirement — a project that declares `sources` explicitly - * need not include `metaobjects/` at all. Built from `DEFAULT_METADATA_DIR` - * (`memory.ts`'s own default-directory constant) rather than restating the - * literal "metaobjects" here: a second independent encoding of the same + * need not include the default directory at all. Built from + * `DEFAULT_METADATA_DIR` (`metadata-files.ts`'s single definition) rather than + * restating that name here: a second independent encoding of the same * default would let `resolveCollection`'s "does the default dir exist" * check (`collection.ts`) desync from what `resolveSources` actually * resolves the moment the default ever changed — silently reproducing the diff --git a/server/typescript/packages/sdk/test/collection.test.ts b/server/typescript/packages/sdk/test/collection.test.ts index df8a3172b..b60f71ebf 100644 --- a/server/typescript/packages/sdk/test/collection.test.ts +++ b/server/typescript/packages/sdk/test/collection.test.ts @@ -102,25 +102,28 @@ describe("resolveCollection", () => { expect(c.files.map((f) => f.replace(root + "/", ""))).toEqual(["metaobjects/meta.a.json"]); }); - test("BACK-COMPAT: a LOCAL metaobjects/ stops the walk, even under an ancestor config", async () => { - // The pre-branch layout: a nested project holding its own `metaobjects/` and - // no config of its own read ITS OWN metadata. Walking past it to an ancestor - // config silently loads the ancestor's model AND writes generated output to - // the ancestor's outDir — a silent regression on a layout that worked. + test("a bare metaobjects/ is NOT a project boundary — the ancestor config governs", async () => { + // A project boundary is a `.metaobjects/config.json`, nothing else. A + // subdirectory holding only a `metaobjects/` directory declares no project, + // so the nearest ancestor config governs it — including its `sources`, which + // may point nowhere near either directory. The alternative (treating a bare + // directory as a second stop marker) puts a second definition of "where + // metadata lives" back into the walk, which is exactly what + // `resolveCollection` exists to be the only one of. A subdirectory that + // should own its metadata declares a config; `meta init` writes one. config(".", {}); write("metaobjects/meta.root.json", "{}"); write("apps/ui/metaobjects/meta.ui.json", "{}"); const c = await resolveCollection(join(root, "apps/ui")); - expect(c.configDir).toBe(join(root, "apps/ui")); + expect(c.configDir).toBe(root); expect(c.files.map((f) => f.replace(root + "/", ""))).toEqual([ - "apps/ui/metaobjects/meta.ui.json", + "metaobjects/meta.root.json", ]); }); - test("a nearer config still wins over a further-down metaobjects/ in an ancestor", async () => { - // The stop condition is per-DIRECTORY, first-match-wins: the nearest ancestor - // holding EITHER marker stops the walk, so a config beside the start dir is - // not skipped just because an ancestor also has a `metaobjects/`. + test("a nearer config wins over an ancestor's", async () => { + // Nearest-ancestor, first-match-wins: a config beside the start dir governs, + // and a `metaobjects/` sitting in an ancestor changes nothing. write("metaobjects/meta.root.json", "{}"); write("model/meta.a.json", "{}"); config("apps/ui", { sources: [{ path: "../../model" }] }); diff --git a/server/typescript/packages/sdk/test/discovery.test.ts b/server/typescript/packages/sdk/test/discovery.test.ts index c66c54d0a..c4ca08778 100644 --- a/server/typescript/packages/sdk/test/discovery.test.ts +++ b/server/typescript/packages/sdk/test/discovery.test.ts @@ -56,16 +56,21 @@ describe("discoverCollectionRoot — config marker", () => { }); }); -describe("discoverCollectionRoot — metaobjects/ marker", () => { - test("a LOCAL metaobjects/ stops the walk, reported as config-less", async () => { +describe("discoverCollectionRoot — a metadata directory is not a marker", () => { + // `.metaobjects/config.json` is the ONLY stop condition (plus the `.git` + // boundary). A directory that merely holds metadata declares no project, so + // the walk goes straight past it. Anything else would be a second definition + // of "where metadata lives" living outside `resolveCollection`. + test("a LOCAL metaobjects/ does not stop the walk — the ancestor config governs", async () => { cfg("."); meta("."); meta("apps/ui"); mk("apps/ui/src"); expect(await discoverCollectionRoot(join(root, "apps/ui"))).toEqual({ - dir: join(root, "apps/ui"), hasConfig: false, + dir: root, hasConfig: true, }); }); - test("the walk reaches a metaobjects/ marker from a subdirectory", async () => { + test("with no config anywhere, a metaobjects/ up the tree is passed over", async () => { meta("apps/ui"); mk("apps/ui/src/deep"); - expect(await resolveConfigDir(join(root, "apps/ui/src/deep"))).toBe(join(root, "apps/ui")); + const start = join(root, "apps/ui/src/deep"); + expect(await resolveConfigDir(start)).toBe(start); }); test("a config in the SAME directory wins — hasConfig is true", async () => { cfg("apps/ui"); meta("apps/ui"); @@ -79,9 +84,4 @@ describe("discoverCollectionRoot — metaobjects/ marker", () => { dir: join(root, "apps/ui"), hasConfig: true, }); }); - test("a FILE named metaobjects is not a metadata home", async () => { - cfg("."); mk("apps/ui"); - writeFileSync(join(root, "apps/ui/metaobjects"), "not a directory", "utf8"); - expect(await resolveConfigDir(join(root, "apps/ui"))).toBe(root); - }); }); diff --git a/server/typescript/packages/sdk/test/dogfood-examples.test.ts b/server/typescript/packages/sdk/test/dogfood-examples.test.ts index cd5a119e8..160bc0f0e 100644 --- a/server/typescript/packages/sdk/test/dogfood-examples.test.ts +++ b/server/typescript/packages/sdk/test/dogfood-examples.test.ts @@ -16,7 +16,8 @@ import { mkdtempSync, rmSync, mkdirSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { basename, join, resolve } from "node:path"; import { resolveCollection } from "../src/collection.js"; -import { DEFAULT_METADATA_DIR, listMetadataFiles, loadMemory } from "../src/memory.js"; +import { loadMemory } from "../src/memory.js"; +import { DEFAULT_METADATA_DIR, listMetadataFiles } from "../src/metadata-files.js"; import { compileScope, matchesScope } from "../src/scope.js"; // Located relative to this test file (never a hardcoded absolute home path — diff --git a/server/typescript/packages/sdk/test/memory.test.ts b/server/typescript/packages/sdk/test/memory.test.ts index bc5f9da17..2de96af8c 100644 --- a/server/typescript/packages/sdk/test/memory.test.ts +++ b/server/typescript/packages/sdk/test/memory.test.ts @@ -3,6 +3,7 @@ import { mkdtempSync, mkdirSync, writeFileSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { loadMemory } from "../src/memory.js"; +import { rejectedCode } from "./support/error-code.js"; function makeMetaRoot(): string { const root = mkdtempSync(join(tmpdir(), "memory-load-")); @@ -131,10 +132,14 @@ describe("loadMemory", () => { } }); - test("throws if metaobjects/ doesn't exist", async () => { + test("nothing to resolve is ERR_COLLECTION_NOT_FOUND, the same code every command reports", async () => { + // `loadMemory` resolves through `resolveCollection` like every other read + // path, so the "you have no metadata here" failure is that function's + // structured code rather than a bare `readdir` ENOENT from a directory + // `loadMemory` picked on its own. const root = mkdtempSync(join(tmpdir(), "memory-load-nodir-")); try { - await expect(loadMemory(root)).rejects.toThrow(/cannot read|ENOENT|no such/i); + expect(await rejectedCode(loadMemory(root))).toBe("ERR_COLLECTION_NOT_FOUND"); } finally { rmSync(root, { recursive: true, force: true }); } @@ -200,8 +205,17 @@ describe("loadMemory", () => { }); }); -describe("loadMemory — cross-package loading via workspace", () => { - test("loads transitive extends: deps from workspace peers", async () => { +// The `package.meta.json` + workspace `extends:` peer walk is RETIRED (design +// §11; `docs/features/metadata-sources.md` → Upgrading). `loadMemory` had two +// ways of finding metadata, one of them implicit and reachable only from a +// particular repository layout. It now has none of its own: it asks +// `resolveCollection`, exactly like every other read path. +// +// These tests pin what the Upgrading section PROMISES about that removal — +// that it fails loudly, and that a declared source replaces it — rather than +// merely deleting the coverage along with the feature. +describe("loadMemory — the retired workspace peer walk", () => { + test("a declared source is the replacement, and it needs no topological order", async () => { const wsRoot = mkdtempSync(join(tmpdir(), "ws-loadmem-")); try { // Workspace setup: shared package + billing package that extends shared @@ -231,16 +245,16 @@ describe("loadMemory — cross-package loading via workspace", () => { }), ); - // billing package: extends shared; defines an Invoice entity - mkdirSync(join(wsRoot, "packages", "billing", ".meta"), { recursive: true }); + // billing package: reaches shared by DECLARING it as a source. The + // shared entry is written SECOND on purpose — `sources` is a set, so + // there is no topological order to reproduce. + mkdirSync(join(wsRoot, "packages", "billing", ".metaobjects"), { recursive: true }); mkdirSync(join(wsRoot, "packages", "billing", "metaobjects"), { recursive: true }); writeFileSync( - join(wsRoot, "packages", "billing", ".meta", "package.meta.json"), + join(wsRoot, "packages", "billing", ".metaobjects", "config.json"), JSON.stringify({ - name: "@acme/billing", - version: "1.0.0", - metaobjectsPackage: "acme::billing", - extends: ["@acme/shared"], + schema_version: 1, + sources: [{ path: "metaobjects" }, { path: "../shared/metaobjects" }], }), ); writeFileSync( @@ -266,8 +280,9 @@ describe("loadMemory — cross-package loading via workspace", () => { } }); - test("single-package mode works unchanged when no workspace present", async () => { - // No workspace config — loadMemory falls back to current package only + test("a project declaring nothing resolves its own default source, and only that", async () => { + // The other side of the removal: with no config and no peer walk, the + // default source is the whole of what loads. const root = makeMetaRoot(); try { writeFileSync( @@ -288,7 +303,12 @@ describe("loadMemory — cross-package loading via workspace", () => { } }); - test("cross-package super: resolves via extends graph", async () => { + test("a model that leaned on the peer walk fails LOUDLY, with ERR_UNRESOLVED_SUPER", async () => { + // The Upgrading section's promise, gated: nothing generates from a + // half-resolved model. `domain` declares no `sources`, so it resolves its + // own default directory and nothing else — and the `extends:` into + // `acme::common` that the workspace walk used to satisfy now names a target + // no loaded file declares. const wsRoot = mkdtempSync(join(tmpdir(), "ws-crossref-")); try { writeFileSync(join(wsRoot, "pnpm-workspace.yaml"), "packages:\n - 'packages/*'\n"); @@ -348,14 +368,9 @@ describe("loadMemory — cross-package loading via workspace", () => { }), ); - const meta = await loadMemory(join(wsRoot, "packages", "domain")); - const widget = meta.ownChildren().find((c) => c.name === "Widget"); - expect(widget).toBeDefined(); - const idField = widget!.ownChildren().find((c) => c.name === "id"); - expect(idField).toBeDefined(); - // super resolved across package boundary - expect(idField!.superResolved).toBeDefined(); - expect(idField!.superResolved!.typeId.subType).toBe("long"); + expect(await rejectedCode(loadMemory(join(wsRoot, "packages", "domain")))).toBe( + "ERR_UNRESOLVED_SUPER", + ); } finally { rmSync(wsRoot, { recursive: true, force: true }); } diff --git a/server/typescript/packages/sdk/test/source-order.test.ts b/server/typescript/packages/sdk/test/source-order.test.ts index 7bce9ace0..de8c9bc38 100644 --- a/server/typescript/packages/sdk/test/source-order.test.ts +++ b/server/typescript/packages/sdk/test/source-order.test.ts @@ -11,7 +11,7 @@ // order. // // The pre-source-resolution toolchain read every file through -// `listMetadataFiles` (memory.ts), which visits FILES at a level before +// `listMetadataFiles` (metadata-files.ts), which visits FILES at a level before // descending into that level's subdirectories. A flat lexicographic sort of // absolute paths disagrees with it the moment a subdirectory name sorts before // a sibling file — `metaobjects/common/…` before `metaobjects/meta.users.json` @@ -28,7 +28,8 @@ import { describe, test, expect, beforeEach, afterEach } from "bun:test"; import { mkdtempSync, rmSync, mkdirSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { dirname, join } from "node:path"; -import { DEFAULT_METADATA_DIR, listMetadataFiles, loadMemory } from "../src/memory.js"; +import { loadMemory } from "../src/memory.js"; +import { DEFAULT_METADATA_DIR, listMetadataFiles } from "../src/metadata-files.js"; import { resolveSources, type SourceSpec } from "../src/sources.js"; import { resolveCollection } from "../src/collection.js"; From e550fd66cde488433ea3e6c792575cc6efc6cc3d Mon Sep 17 00:00:00 2001 From: Doug Mealing Date: Tue, 18 Aug 2026 09:28:16 -0400 Subject: [PATCH 38/44] fix(cli): what the CLI prints must not assert a directory the project may not have MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The no-args probe already routed through `resolveCollection`, and the comment above it said so — then the next three lines printed `metaobjects/ found`, `no metaobjects/ here`, and `Scaffold metaobjects/ in this directory`. For a project whose config points `sources` at a sibling module the first is simply false, and the second blames a missing default directory when the real problem may be a declared source that failed to resolve. The status line now reports what the probe actually established — metadata found, or no MetaObjects project here — and the next step offers to scaffold a project rather than a named directory. Swept the rest of `cli/src` for user-facing strings making the same claim: `gen`'s two help summaries, the `` positional in both docs help slices, the `--prompts` note, and `gen`'s empty-result hint. `meta init`'s own output keeps naming the directory — it CREATES it, which is the one place the name belongs. Comments are untouched. Two tests pin the rule at the seam it broke: a project whose sources point elsewhere, and a directory with no project at all, each asserting the printed status contains no directory name. Both were confirmed red against the previous strings. The committed help snapshot caught the help-text edits, as intended, and is regenerated. Co-Authored-By: Claude Opus 5 (1M context) --- server/typescript/packages/cli/src/index.ts | 29 ++++---- .../typescript/packages/cli/src/lib/output.ts | 2 +- .../cli/test/__snapshots__/cli.test.ts.snap | 4 +- .../cli/test/collection-routing.test.ts | 67 ++++++++++++++++++- 4 files changed, 86 insertions(+), 16 deletions(-) diff --git a/server/typescript/packages/cli/src/index.ts b/server/typescript/packages/cli/src/index.ts index b6180fff4..8f6bff229 100644 --- a/server/typescript/packages/cli/src/index.ts +++ b/server/typescript/packages/cli/src/index.ts @@ -17,7 +17,7 @@ COMMANDS: init Scaffold metaobjects/ + .metaobjects/ in the current repo init --refresh-docs Refresh .metaobjects/AGENTS.md + CLAUDE.md after CLI upgrades agent-docs Scaffold only the agent-context (.metaobjects/ + .claude/skills/) — canonical redirect target for all language ports - gen [...] Codegen TS targets from metaobjects/ entities + gen [...] Codegen TS targets from your declared metadata types [query] Search the metadata vocabulary (types, subtypes, @attrs) by name or description export Flatten loaded metadata to one canonical JSON artifact docs --out Generate neutral metadata documentation (entity + template pages; --site for HTML site) @@ -40,7 +40,7 @@ EXPORT FLAGS: --out Write output to a file (default: stdout) DOCS FLAGS: - Project root holding metaobjects/ (default: current directory) + Project root to resolve metadata from (default: current directory) --out , -o Output directory for the pages (default: ./docs) --templates Project root to resolve adopter templates/ overrides (default: ) --prompts Extra dir holding prompt .mustache sources for --site (e.g. data/templates/) @@ -91,7 +91,7 @@ ship in later sub-projects. See https://metaobjects.com for docs. /** Focused per-subcommand usage slices shown by ` --help`. */ const COMMAND_HELP: Record = { - gen: `meta gen — codegen TS targets from metaobjects/ entities + gen: `meta gen — codegen TS targets from your declared metadata USAGE: meta gen [...] [flags] @@ -153,7 +153,7 @@ USAGE: meta docs [] [flags] FLAGS: - Project root holding metaobjects/ (default: current directory) + Project root to resolve metadata from (default: current directory) --out , -o Output directory for the pages (default: ./docs) --model Emit the markdown model surface (entity + template pages) --api Emit the markdown api surface (generated SDK reference) @@ -162,7 +162,7 @@ FLAGS: --scaffold-site Copy the site's templates + assets into codegen/docs-site/ to own (theme) them --templates Project root to resolve adopter templates/ overrides (default: ) --prompts Extra dir holding prompt .mustache sources (for --site) when they - live outside metaobjects/ or templates/ (e.g. data/templates/) + live outside the metadata sources or templates/ (e.g. data/templates/) --help, -h Print this help `, init: `meta init — scaffold metaobjects/ + .metaobjects/ in the current repo @@ -274,12 +274,17 @@ export async function run(argv: string[]): Promise { // dumping the full manual (full manual is still available via `meta --help`). // "Is this a MetaObjects project?" routes through resolveCollection — the // single authority on where metadata lives — rather than assuming the - // default `metaobjects/` directory name. - const metaobjectsExists = await resolveCollection(cwd).then(() => true).catch(() => false); - const statusLine = metaobjectsExists - ? `meta — MetaObjects CLI (v${VERSION}) · metaobjects/ found` - : `meta — MetaObjects CLI (v${VERSION}) · no metaobjects/ here`; - const nextSteps = metaobjectsExists + // default `metaobjects/` directory name. The status line must not assert + // it either: a project whose config points `sources` at + // `../shared-model/metadata` would be told "metaobjects/ found", which is + // false, and one that resolves nothing would be told there is no + // `metaobjects/` here when the real problem may be a declared source that + // failed to resolve. + const metadataResolves = await resolveCollection(cwd).then(() => true).catch(() => false); + const statusLine = metadataResolves + ? `meta — MetaObjects CLI (v${VERSION}) · metadata found` + : `meta — MetaObjects CLI (v${VERSION}) · no MetaObjects project here`; + const nextSteps = metadataResolves ? [ " meta gen Run codegen", " meta verify Check for drift", @@ -287,7 +292,7 @@ export async function run(argv: string[]): Promise { " meta --help Full command reference", ] : [ - " meta init Scaffold metaobjects/ in this directory", + " meta init Scaffold a MetaObjects project in this directory", " meta --help Full command reference", ]; log.info(`${statusLine}\n\n${nextSteps.join("\n")}\n`); diff --git a/server/typescript/packages/cli/src/lib/output.ts b/server/typescript/packages/cli/src/lib/output.ts index 3bf9a3b27..c2382e605 100644 --- a/server/typescript/packages/cli/src/lib/output.ts +++ b/server/typescript/packages/cli/src/lib/output.ts @@ -192,7 +192,7 @@ export function genResultToData(result: GenResultShape): { ? `no entities to generate in ${result.outDir}` : parts.join(", "); const help = result.files.length === 0 - ? ["author entities under metaobjects/ then re-run `meta gen`"] + ? ["author entities in this project's metadata sources then re-run `meta gen`"] : ["typecheck the generated code with `npx tsc`", "create your database tables with `meta migrate --from-db --db --dialect --slug init --apply`"]; return { gen: result.files.map((f) => ({ file: f.path, status: f.status })), summary, help }; } diff --git a/server/typescript/packages/cli/test/__snapshots__/cli.test.ts.snap b/server/typescript/packages/cli/test/__snapshots__/cli.test.ts.snap index 6e6c6a66c..f9a8fd923 100644 --- a/server/typescript/packages/cli/test/__snapshots__/cli.test.ts.snap +++ b/server/typescript/packages/cli/test/__snapshots__/cli.test.ts.snap @@ -10,7 +10,7 @@ COMMANDS: init Scaffold metaobjects/ + .metaobjects/ in the current repo init --refresh-docs Refresh .metaobjects/AGENTS.md + CLAUDE.md after CLI upgrades agent-docs Scaffold only the agent-context (.metaobjects/ + .claude/skills/) — canonical redirect target for all language ports - gen [...] Codegen TS targets from metaobjects/ entities + gen [...] Codegen TS targets from your declared metadata types [query] Search the metadata vocabulary (types, subtypes, @attrs) by name or description export Flatten loaded metadata to one canonical JSON artifact docs --out Generate neutral metadata documentation (entity + template pages; --site for HTML site) @@ -33,7 +33,7 @@ EXPORT FLAGS: --out Write output to a file (default: stdout) DOCS FLAGS: - Project root holding metaobjects/ (default: current directory) + Project root to resolve metadata from (default: current directory) --out , -o Output directory for the pages (default: ./docs) --templates Project root to resolve adopter templates/ overrides (default: ) --prompts Extra dir holding prompt .mustache sources for --site (e.g. data/templates/) diff --git a/server/typescript/packages/cli/test/collection-routing.test.ts b/server/typescript/packages/cli/test/collection-routing.test.ts index ff73d82b4..8810f72b5 100644 --- a/server/typescript/packages/cli/test/collection-routing.test.ts +++ b/server/typescript/packages/cli/test/collection-routing.test.ts @@ -1,7 +1,8 @@ -import { describe, test, expect } from "bun:test"; +import { describe, test, expect, spyOn } from "bun:test"; import { mkdtempSync, mkdirSync, rmSync, writeFileSync } from "node:fs"; import { join, resolve } from "node:path"; import { genCommand } from "../src/commands/gen.js"; +import { run } from "../src/index.js"; // Place temp dirs inside the monorepo so metaobjects.config.ts's // `@metaobjectsdev/*` imports resolve the same way the existing @@ -83,3 +84,67 @@ export default defineConfig({ } }); }); + +// The probe in `run()` answers "is this a MetaObjects project?" through +// `resolveCollection`. What it PRINTS has to agree: a project whose config +// points `sources` at a sibling module has no directory of the default name at +// all, so "metaobjects/ found" is simply false — and a directory that resolves +// nothing may be failing on a declared source rather than on a missing default. +// Naming a directory in either message re-asserts the assumption the routing +// removed. +describe("the no-args project probe says what resolved, never a directory name", () => { + /** Run the CLI with no command and return everything it wrote to stdout. */ + async function statusOf(dir: string): Promise { + const lines: string[] = []; + const spy = spyOn(console, "log").mockImplementation((...args: unknown[]) => { + lines.push(args.map(String).join(" ")); + }); + try { + expect(await run(["--cwd", dir])).toBe(0); + } finally { + spy.mockRestore(); + } + return lines.join("\n"); + } + + test("a project whose sources point elsewhere is not told a directory was found", async () => { + mkdirSync(WORKSPACE_TMP, { recursive: true }); + const root = mkdtempSync(join(WORKSPACE_TMP, "probe-declared-")); + try { + mkdirSync(join(root, ".git")); + mkdirSync(join(root, "model"), { recursive: true }); + writeFileSync( + join(root, "model", "meta.a.json"), + JSON.stringify({ + "metadata.root": { package: "acme", children: [{ "object.entity": { name: "Order" } }] }, + }), + ); + mkdirSync(join(root, "apps", "ui", ".metaobjects"), { recursive: true }); + writeFileSync( + join(root, "apps", "ui", ".metaobjects", "config.json"), + JSON.stringify({ schema_version: 1, sources: [{ path: "../../model" }] }), + ); + + const out = await statusOf(join(root, "apps", "ui")); + expect(out).toContain("metadata found"); + expect(out).not.toContain("metaobjects/"); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + test("a directory with no project names no directory either", async () => { + mkdirSync(WORKSPACE_TMP, { recursive: true }); + const root = mkdtempSync(join(WORKSPACE_TMP, "probe-empty-")); + try { + mkdirSync(join(root, ".git")); + const out = await statusOf(root); + expect(out).toContain("no MetaObjects project here"); + // Including the next-step line: `meta init` scaffolds a project, and the + // layout it writes is its own business to describe, not this probe's. + expect(out).not.toContain("metaobjects/"); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +}); From 3c702ec5eb7f38c4fe32a72e5824d29e218c08f5 Mon Sep 17 00:00:00 2001 From: Doug Mealing Date: Tue, 18 Aug 2026 09:28:24 -0400 Subject: [PATCH 39/44] docs: state the rule where the next reader will meet it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `metaobjects/` is the DEFAULT VALUE of `sources` and nothing else. That was implicit in the design and contradicted by three code paths; with those fixed, write it down where someone will hit it before repeating them. - CLAUDE.md names the four sites that may spell the directory — the constant's definition, `DEFAULT_SOURCES`, `resolveCollection` applying it, and `meta init` writing the layout — and points at the test that enforces the list. - `metadata-sources.md` states the rule positively in the body rather than as a footnote: a project that declares `sources` may put metadata anywhere and need not have a directory of that name at all, and every command follows the config together. - Upgrading gains the consequence of the discovery change, phrased as the rule: a project boundary is a `.metaobjects/config.json`, so a subdirectory holding metadata but no config resolves to its nearest ancestor config. If it should own its metadata, give it one. - The retired-workspace-walk entry said "every CLI read path"; `loadMemory` itself now resolves the same way, so it says every read path. Co-Authored-By: Claude Opus 5 (1M context) --- AGENTS.md | 2 +- docs/features/metadata-sources.md | 43 ++++++++++++++++++++++++++----- 2 files changed, 37 insertions(+), 8 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 4e1500dc1..484941f3e 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -200,7 +200,7 @@ import { EntityFetcherProvider, EntityGrid } from "@metaobjectsdev/tanstack"; - **Codegen substrate**: ts-poet for greenfield emit, ts-morph for in-place edits, Biome for format pass, `git merge-file --diff3` for hand-edit-preserving regen. - **Runtime substrate**: Kysely for TS (user-provided connection, async-only). - **Migration substrate**: Postgres + SQLite for TS v0.3. -- **Metadata location**: resolved via `resolveCollection()` (`@metaobjectsdev/sdk`) — the single authority. The directory name lives in ONE literal, `DEFAULT_METADATA_DIR` in `sdk/src/memory.ts`; every other site (`meta init`'s scaffold included) imports it. No code path may write the literal again. See [docs/features/metadata-sources.md](docs/features/metadata-sources.md). +- **Metadata location**: resolved via `resolveCollection()` (`@metaobjectsdev/sdk`) — the single authority. `metaobjects/` is the **default value of `sources`** and nothing else: no other module, command or user-facing message may assert that a directory of that name exists or is where metadata lives. Exactly four sites may name it — `sdk/src/metadata-files.ts` (`DEFAULT_METADATA_DIR`, its single definition), `sdk/src/sources.ts` (`DEFAULT_SOURCES`, **the** default), `sdk/src/collection.ts` (inside `resolveCollection`, *applying* that default), and `cli/src/commands/init.ts` (the scaffolder **writing** the layout). Enforced by `sdk/test/no-hardcoded-metadata-dir.test.ts`, whose allowlist demands a written reason per entry. See [docs/features/metadata-sources.md](docs/features/metadata-sources.md). ## Explicitly out of scope diff --git a/docs/features/metadata-sources.md b/docs/features/metadata-sources.md index 16c40f974..561a46b80 100644 --- a/docs/features/metadata-sources.md +++ b/docs/features/metadata-sources.md @@ -3,8 +3,15 @@ **Where does my metadata come from?** From the `sources` set in `.metaobjects/config.json`. When that key is absent or empty, `sources` takes its default value — the `metaobjects/` directory sitting beside the `.metaobjects/` -folder that holds the config. Nothing else in the toolchain assumes that directory -name. +folder that holds the config. + +**`metaobjects/` is that default value, and nothing more.** It is not a requirement, +and it is not a convention any part of the toolchain is allowed to assume: a project +that declares `sources` may put its metadata anywhere — a sibling module, a shared +model repository, a single file — and need not have a directory of that name at all. +Every command answers "where is the metadata?" by reading the config, so pointing +`sources` elsewhere moves *all* of them together. Nothing greps for the directory, +tests for its existence, or mentions it in a message. **How do I point it somewhere else?** Declare it: @@ -105,6 +112,11 @@ root's. sharing its directory with `.git` — is still reachable from any subdirectory. 4. If nothing is found, the starting directory is used with the default `sources`. +That config file is the **only** thing the walk looks for. A directory holding a +`metaobjects/` directory but no config is not a project root — see +[A project boundary is a `.metaobjects/config.json`](#a-project-boundary-is-a-metaobjectsconfigjson) +under Upgrading. + Collections are **never auto-discovered**. Nothing globs the tree for directories that merely look like metadata homes; a collection exists only where a config names one. @@ -420,9 +432,26 @@ type) surfaces as the config load error and stops the command. ## Upgrading A project with one config at its root, no `sources` and no `scope` resolves the same -files it always did and generates the same code. Six changes are still worth knowing +files it always did and generates the same code. Seven changes are still worth knowing about before you upgrade. +### A project boundary is a `.metaobjects/config.json` + +Discovery walks up for that file and stops at nothing else. A directory holding a +`metaobjects/` directory but no config of its own is **not** a project root, so a +command run inside it resolves the nearest ancestor config — including that config's +`sources`, which may point somewhere neither directory contains. + +This is the rule rather than a caveat: `metaobjects/` is the default *value* of +`sources`, so a directory of that name says nothing about whether a project lives +there. Treating it as a second marker would put a second answer to "where does +metadata live?" back into the toolchain, and would be silently wrong for every +project that declares `sources` elsewhere. + +If a subdirectory should own its metadata, give it a config — `meta init` writes +one, and a `"sources": []` config is enough to claim the directory and take the +default. + ### The workspace `extends:` walk is retired `loadMemory` used to have a second, hidden way of finding metadata: if the project @@ -431,10 +460,10 @@ could be discovered (`pnpm-workspace.yaml`, or `package.json` `workspaces`), it walked that dependency graph and loaded each peer package's `metaobjects/` directory first, in topological order. -Every CLI read path now resolves its files through `sources`, which does no such -walk. Two ways to find metadata, one of them implicit and reachable only from a -particular repository layout, is precisely the divergence this feature exists to -remove — and one of them was undocumented. +Every read path — `loadMemory` itself included — now resolves its files through +`sources`, which does no such walk. Two ways to find metadata, one of them implicit +and reachable only from a particular repository layout, is precisely the divergence +this feature exists to remove — and one of them was undocumented. **It fails loudly, not silently.** A model that depended on a peer package's declarations now fails to load with `ERR_UNRESOLVED_SUPER` naming the `extends:` From 066fded6f7529adc30d53d88044544ad49000c1d Mon Sep 17 00:00:00 2001 From: Doug Mealing Date: Tue, 18 Aug 2026 09:34:08 -0400 Subject: [PATCH 40/44] test(sdk): make "metaobjects/ is a config default" enforceable, not aspirational MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Eight call sites once hardcoded the metadata directory. Routing them through `resolveCollection` fixed those eight and nothing about the ninth. This is the part that lasts: a test walking `sdk/src` and `cli/src` that fails when a file outside an allowlist names the directory in code. Three things keep it from being a gate that passes because it checks nothing: the allowlist is file + REASON, so a new entry costs a written justification; a STALE entry fails, because an allowlisted file that no longer holds the reference is an unwatched exemption the next file inherits; and comments are excluded, proven against `detect-stack.ts`, which mentions the directory only in a comment saying it does not assume it. Both failure modes were demonstrated rather than assumed. A temporary `join("x", "metaobjects")` in `discovery.ts` produced `["sdk/src/discovery.ts:113"]`; a temporary allowlist entry for a file with no reference produced `["sdk/src/scope.ts"]`. Both were then removed. Writing the guard found what a grep did not. Three lines were the PRODUCT name in prose — "the metaobjects ledger", "reach for metaobjects metadata", the `metaobjects:` error prefix — so a reference now requires a following `/` or closing quote. That is the guard's sharpest limit and its limits are recorded in the file header, measured row by row: it catches plain literals, template literals and messages with a trailing slash; it misses any computed spelling and the bare word without a slash. Two consequences beyond the four sanctioned sites: - `cli/src/index.ts`'s two `init` help lines named the directory to everyone, in every project, including one whose sources point elsewhere. Reworded — `meta init`'s OWN output still announces the layout it writes, which is where that belongs. - `sdk/src/agent-docs/body.ts` is allowlisted with its reason: it is the scaffolded documentation prose, reachable by no read path, teaching the default layout a fresh project gets. Flagged as a wording gap for a project that declares `sources` elsewhere, not a resolution one. Co-Authored-By: Claude Opus 5 (1M context) --- server/typescript/packages/cli/src/index.ts | 4 +- .../cli/test/__snapshots__/cli.test.ts.snap | 2 +- .../test/no-hardcoded-metadata-dir.test.ts | 258 ++++++++++++++++++ 3 files changed, 261 insertions(+), 3 deletions(-) create mode 100644 server/typescript/packages/sdk/test/no-hardcoded-metadata-dir.test.ts diff --git a/server/typescript/packages/cli/src/index.ts b/server/typescript/packages/cli/src/index.ts index 8f6bff229..24cda0466 100644 --- a/server/typescript/packages/cli/src/index.ts +++ b/server/typescript/packages/cli/src/index.ts @@ -14,7 +14,7 @@ USAGE: meta [flags] COMMANDS: - init Scaffold metaobjects/ + .metaobjects/ in the current repo + init Scaffold a MetaObjects project in the current repo init --refresh-docs Refresh .metaobjects/AGENTS.md + CLAUDE.md after CLI upgrades agent-docs Scaffold only the agent-context (.metaobjects/ + .claude/skills/) — canonical redirect target for all language ports gen [...] Codegen TS targets from your declared metadata @@ -165,7 +165,7 @@ FLAGS: live outside the metadata sources or templates/ (e.g. data/templates/) --help, -h Print this help `, - init: `meta init — scaffold metaobjects/ + .metaobjects/ in the current repo + init: `meta init — scaffold a MetaObjects project in the current repo USAGE: meta init [flags] diff --git a/server/typescript/packages/cli/test/__snapshots__/cli.test.ts.snap b/server/typescript/packages/cli/test/__snapshots__/cli.test.ts.snap index f9a8fd923..99fa1137c 100644 --- a/server/typescript/packages/cli/test/__snapshots__/cli.test.ts.snap +++ b/server/typescript/packages/cli/test/__snapshots__/cli.test.ts.snap @@ -7,7 +7,7 @@ USAGE: meta [flags] COMMANDS: - init Scaffold metaobjects/ + .metaobjects/ in the current repo + init Scaffold a MetaObjects project in the current repo init --refresh-docs Refresh .metaobjects/AGENTS.md + CLAUDE.md after CLI upgrades agent-docs Scaffold only the agent-context (.metaobjects/ + .claude/skills/) — canonical redirect target for all language ports gen [...] Codegen TS targets from your declared metadata diff --git a/server/typescript/packages/sdk/test/no-hardcoded-metadata-dir.test.ts b/server/typescript/packages/sdk/test/no-hardcoded-metadata-dir.test.ts new file mode 100644 index 000000000..251f3d66d --- /dev/null +++ b/server/typescript/packages/sdk/test/no-hardcoded-metadata-dir.test.ts @@ -0,0 +1,258 @@ +// server/typescript/packages/sdk/test/no-hardcoded-metadata-dir.test.ts +// +// THE enforcer for the rule the whole source-resolution design rests on: +// +// `metaobjects/` is the DEFAULT VALUE of `sources` and nothing else. +// Everywhere else reads the config, through `resolveCollection`. +// +// Eight independent call sites once hardcoded that directory; routing them +// through one authority fixed the eight, and fixed nothing about the ninth +// somebody adds next month. This test is the part that lasts. It walks the +// `sdk` and `cli` source trees and fails when a file outside the allowlist +// names the directory in CODE. +// +// Three properties keep it from becoming a gate that passes because it checks +// nothing: +// +// 1. The allowlist is file + REASON. A fifth entry costs a sentence explaining +// why that file is allowed to know the name. +// 2. A STALE entry fails. An allowlisted file that no longer contains the +// reference silently re-opens the hole it was covering, so the allowlist +// must be exact in both directions. +// 3. Comments are excluded, so the paragraph explaining the rule is not itself +// a violation — and that exclusion is tested against a real file that +// mentions the directory only in a comment, not merely asserted. +// +// WHAT IT DOES NOT CATCH — write these down or the guard becomes a claim rather +// than a check. Measured, not assumed (each row was run): +// +// CAUGHT join(d, "metaobjects") a plain literal, either quote +// CAUGHT `${d}/metaobjects` a template literal, end or mid +// CAUGHT "no metaobjects/ here" a message naming the directory +// MISSED join(d, "meta" + "objects") any computed spelling +// MISSED const N = "meta"; N + "objects" the same, through a variable +// MISSED "author under metaobjects" the word with no trailing `/` +// +// The last is deliberate, not an oversight: `metaobjects` followed by a space +// is the PRODUCT name far more often than a path ("the metaobjects ledger", +// the `metaobjects:` error prefix), and three such lines were the guard's first +// false positives. No lexical rule separates them. The computed-spelling misses +// are the honest ceiling of a source-text check — this catches the way the +// violation is actually written, which is how all eight original ones were +// written, and it will not catch someone evading it on purpose. +// +// It also scans TypeScript SOURCE only: the four other language ports, the +// `docs/` tree, and JSON/YAML fixtures are outside it. +import { describe, test, expect } from "bun:test"; +import { readFileSync } from "node:fs"; +import { readdir } from "node:fs/promises"; +import { join, relative, resolve, sep } from "node:path"; +import { DEFAULT_METADATA_DIR } from "../src/metadata-files.js"; + +/** `packages/`, so both trees are reachable from this one test file. */ +const PACKAGES = resolve(import.meta.dirname, "../.."); +const TREES = [join(PACKAGES, "sdk", "src"), join(PACKAGES, "cli", "src")]; + +/** + * The complete set of files permitted to name the default metadata directory, + * each with the reason it is permitted. Adding an entry is a deliberate act: + * write down why that file needs to know, or route it through + * `resolveCollection` instead. + * + * Paths are relative to `packages/`. + */ +const ALLOWED: ReadonlyMap = new Map([ + [ + "sdk/src/metadata-files.ts", + "the constant's single definition — DEFAULT_METADATA_DIR is declared here and nowhere else", + ], + [ + "sdk/src/sources.ts", + "DEFAULT_SOURCES — THE default, the value config resolution applies when a project declares no sources", + ], + [ + "sdk/src/collection.ts", + "inside resolveCollection, the one authority: it APPLIES that default (the does-it-exist probe and the ERR_COLLECTION_NOT_FOUND text naming it)", + ], + [ + "sdk/src/index.ts", + "package barrel — a bare re-export of the constant, so `meta init` can import it instead of restating the literal. No use.", + ], + [ + "cli/src/commands/init.ts", + "the scaffolder WRITING the default layout — creating that directory, never assuming one exists", + ], + [ + "sdk/src/agent-docs/body.ts", + "the agent-docs PROSE `meta init` scaffolds beside that layout — documentation content, reachable by no read path; it teaches the default a fresh project gets. A project that declares `sources` elsewhere is given docs that name the default, which is a known wording gap, not a resolution one.", + ], +]); + +// --------------------------------------------------------------------------- +// Comment stripping +// --------------------------------------------------------------------------- + +/** + * Blank out `//` and block comments, preserving every newline so reported line + * numbers stay true. + * + * String and template literals are tracked so a `//` inside one is not mistaken + * for a comment (`"https://example.com"` must keep its text). Quote states also + * reset at a newline: an unterminated quote — which the stripper could only + * reach by mis-scanning something exotic — then costs one line rather than the + * rest of the file. + */ +function stripComments(src: string): string { + type State = "code" | "line" | "block" | "sq" | "dq" | "tpl"; + let state: State = "code"; + const out: string[] = []; + for (let i = 0; i < src.length; i++) { + const c = src[i]!; + const next = src[i + 1]; + if (state === "code") { + if (c === "/" && next === "/") { state = "line"; out.push(" ", " "); i++; continue; } + if (c === "/" && next === "*") { state = "block"; out.push(" ", " "); i++; continue; } + if (c === "'") state = "sq"; + else if (c === '"') state = "dq"; + else if (c === "`") state = "tpl"; + out.push(c); + continue; + } + if (state === "line") { + if (c === "\n") { state = "code"; out.push(c); } else out.push(" "); + continue; + } + if (state === "block") { + if (c === "*" && next === "/") { state = "code"; out.push(" ", " "); i++; continue; } + out.push(c === "\n" ? c : " "); + continue; + } + // sq / dq / tpl — literal text is kept verbatim; the opening quote was + // consumed by the `code` branch above, so a matching quote here CLOSES. + out.push(c); + if (c === "\\" && next !== undefined) { out.push(next); i++; continue; } + const closer = state === "sq" ? "'" : state === "dq" ? '"' : "`"; + if (c === closer) { state = "code"; continue; } + if (c === "\n" && state !== "tpl") state = "code"; + } + return out.join(""); +} + +// --------------------------------------------------------------------------- +// Violation detection +// --------------------------------------------------------------------------- + +/** A preceding character meaning the word is part of a LONGER token, or names + * the STATE directory rather than the metadata one: `.metaobjects` (a fixed + * convention with its own constant), `@metaobjectsdev/sdk`. */ +const NOT_A_DIR_BEFORE = /[A-Za-z0-9_.@]/; + +/** A following character meaning this really is a path segment: a separator, or + * the quote that ends the literal (`join(d, "metaobjects")`). + * + * Everything else is the PRODUCT name in prose — "the metaobjects ledger", + * "reach for metaobjects metadata", the `metaobjects:` error prefix — or a + * longer token (`metaobjectsdev`, `metaobjects.config.ts`, + * `metaobjects-authoring`). Requiring this is what keeps the guard from + * convicting the product's own name, and it is also the guard's sharpest + * limit: a message that says "under metaobjects" with no trailing slash is + * indistinguishable, lexically, from prose about the product. */ +const PATH_SEGMENT_AFTER = /[/"'`]/; + +/** Every line of `code` naming the default directory, as a path literal or via + * the constant. `code` must already have its comments stripped. */ +function violationLines(code: string): number[] { + const hits = new Set(); + const lineOf = (index: number): number => code.slice(0, index).split("\n").length; + + for (const m of code.matchAll(/DEFAULT_METADATA_DIR/g)) hits.add(lineOf(m.index)); + + for (const m of code.matchAll(new RegExp(DEFAULT_METADATA_DIR, "g"))) { + const before = m.index === 0 ? "" : code[m.index - 1]!; + const after = code[m.index + DEFAULT_METADATA_DIR.length] ?? ""; + if (NOT_A_DIR_BEFORE.test(before)) continue; + if (!PATH_SEGMENT_AFTER.test(after)) continue; + hits.add(lineOf(m.index)); + } + return [...hits].sort((a, b) => a - b); +} + +async function tsFiles(dir: string): Promise { + const out: string[] = []; + for (const entry of (await readdir(dir, { withFileTypes: true })).sort((a, b) => + a.name < b.name ? -1 : 1, + )) { + const full = join(dir, entry.name); + if (entry.isDirectory()) out.push(...(await tsFiles(full))); + else if (entry.name.endsWith(".ts") || entry.name.endsWith(".tsx")) out.push(full); + } + return out; +} + +/** Every scanned file naming the directory in code, keyed by `packages/`-relative + * path (always `/`-separated, so the allowlist reads the same on any platform). */ +async function scan(): Promise> { + const found = new Map(); + for (const tree of TREES) { + for (const file of await tsFiles(tree)) { + const lines = violationLines(stripComments(readFileSync(file, "utf8"))); + if (lines.length > 0) found.set(relative(PACKAGES, file).split(sep).join("/"), lines); + } + } + return found; +} + +describe("`metaobjects/` is a config default and nothing else", () => { + test("no file outside the allowlist names the default metadata directory", async () => { + const offenders = [...(await scan()).entries()] + .filter(([file]) => !ALLOWED.has(file)) + .map(([file, lines]) => `${file}:${lines.join(",")}`); + expect(offenders).toEqual([]); + }); + + test("every allowlisted file still contains the reference it was allowed for", async () => { + const found = await scan(); + // A stale entry is not cosmetic: it is an allowlisted hole nobody is + // watching, and the next file to take that path inherits the exemption. + expect([...ALLOWED.keys()].filter((f) => !found.has(f))).toEqual([]); + }); + + test("every allowlist entry carries a reason", () => { + expect([...ALLOWED].filter(([, why]) => why.trim().length < 20).map(([f]) => f)).toEqual([]); + }); + + test("a comment-only mention is not a violation — proven against a real file", () => { + // `detect-stack.ts` explains, in a comment, that it reads the resolved + // collection "rather than assuming `metaobjects/`". Saying so is the + // opposite of a violation, and the guard must not convict it. + const file = join(PACKAGES, "cli", "src", "lib", "detect-stack.ts"); + const raw = readFileSync(file, "utf8"); + expect(raw).toContain(`${DEFAULT_METADATA_DIR}/`); // the mention is really there + expect(violationLines(stripComments(raw))).toEqual([]); // and it is in a comment + }); + + test("the stripper removes comments without eating code", () => { + const cases: [string, boolean][] = [ + [`// join(dir, "${DEFAULT_METADATA_DIR}")`, false], + [`/* a ${DEFAULT_METADATA_DIR}/ tree */`, false], + [`/** ${DEFAULT_METADATA_DIR}/ */\nconst a = 1;`, false], + [`const p = join(dir, "${DEFAULT_METADATA_DIR}");`, true], + [`const p = \`\${d}/${DEFAULT_METADATA_DIR}\`;`, true], + [`const msg = "no ${DEFAULT_METADATA_DIR}/ here";`, true], + // A `//` inside a string is not a comment: the literal must survive. + [`const u = "https://x/${DEFAULT_METADATA_DIR}/y";`, true], + // Longer tokens that merely contain the word are never violations. + [`import x from "@${DEFAULT_METADATA_DIR}dev/sdk";`, false], + [`const f = "${DEFAULT_METADATA_DIR}.config.ts";`, false], + [`const d = ".${DEFAULT_METADATA_DIR}/config.json";`, false], + [`const s = ".claude/skills/${DEFAULT_METADATA_DIR}-authoring";`, false], + // The PRODUCT name in prose is not a directory reference. This is the + // guard's deliberate blind spot, pinned so it stays deliberate. + [`log.error("the ${DEFAULT_METADATA_DIR} ledger is absent");`, false], + [`throw new Error("${DEFAULT_METADATA_DIR}: could not resolve");`, false], + ]; + for (const [src, isViolation] of cases) { + expect(violationLines(stripComments(src)).length > 0).toBe(isViolation); + } + }); +}); From d66b4e0134ae1d0d3cedd0a8f1785f939ea7bbee Mon Sep 17 00:00:00 2001 From: Doug Mealing Date: Tue, 18 Aug 2026 09:36:23 -0400 Subject: [PATCH 41/44] fix: two minors from the branch re-review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 1 — `runGen`'s `scope?` JSDoc told callers to adapt a compiled scope with `(fqn) => matchesScope(fqn, collection.scope)`. There is no `collection.scope`: a `Collection` exposes the predicate directly as `inScope`, and the only caller passes it straight through. The paragraph's substantive point — do not turn this seam into a config surface — still stands; only its closing sentence was false. Same stale expression in a comment in `run-gen.test.ts`. 2 — `warnIfLedgerRelocated` compared against `resolvePath(metaRoot, config.outDir)`, but under `--migration-format flyway` with a default outDir the directory the run WRITES to comes from `resolveFormatOutDir`, which redirects to `src/main/resources/db/migration`. In that combination the warning named a directory the invocation would never touch — a message whose entire job is to say which directory is in use. It now takes the resolved format directory. The warning was covered by nothing at all, which is why it could be wrong. A test now runs the exact layout it was written for — a subdirectory holding its own ledger, under a project root that declares the config — and asserts the warning names the flyway directory and not the default one. Confirmed red against the previous call. Co-Authored-By: Claude Opus 5 (1M context) --- .../packages/cli/src/commands/migrate.ts | 13 +++++-- .../cli/test/migrate-format-flyway.test.ts | 38 ++++++++++++++++++- .../packages/codegen-ts/src/runner.ts | 6 +-- .../packages/codegen-ts/test/run-gen.test.ts | 9 +++-- 4 files changed, 54 insertions(+), 12 deletions(-) diff --git a/server/typescript/packages/cli/src/commands/migrate.ts b/server/typescript/packages/cli/src/commands/migrate.ts index b78795dbd..2e4270207 100644 --- a/server/typescript/packages/cli/src/commands/migrate.ts +++ b/server/typescript/packages/cli/src/commands/migrate.ts @@ -143,9 +143,10 @@ function resolveFormatOutDir(config: ResolvedMigrateConfig, metaRoot: string): s * * Conditioned on the local directory EXISTING, so the ordinary case — a run from * anywhere inside a project with one ledger at its root — says nothing. - * `--out-dir` (and a `migrate.outDir` in the config) is honoured: the caller - * passes the RESOLVED directory, so a deliberate redirection is compared, not - * the default that was overridden. + * `--out-dir` (and a `migrate.outDir` in the config) is honoured, as is the + * active output format's own convention: the caller passes the directory this + * run will actually WRITE to (`resolveFormatOutDir`), so a deliberate + * redirection is compared rather than a default that was overridden. */ function warnIfLedgerRelocated(cwd: string, resolvedOutDir: string): void { const local = resolvePath(cwd, MIGRATE_DEFAULT_OUT_DIR); @@ -354,7 +355,11 @@ export async function migrateCommand( // resolves and the directory the metadata comes from cannot diverge. const metaRoot = await resolveConfigDir(cwd); const config = await resolveMigrateConfig(flags, metaRoot); - warnIfLedgerRelocated(cwd, resolvePath(metaRoot, config.outDir)); + // `resolveFormatOutDir`, not `resolvePath(metaRoot, config.outDir)`: under + // `--migration-format flyway` with a default outDir the run writes to + // Flyway's conventional location instead, so the unredirected path names a + // directory this invocation will never touch. + warnIfLedgerRelocated(cwd, resolveFormatOutDir(config, metaRoot)); try { // #192 — Flyway owns apply + history (flyway_schema_history). We generate the diff --git a/server/typescript/packages/cli/test/migrate-format-flyway.test.ts b/server/typescript/packages/cli/test/migrate-format-flyway.test.ts index 36819cc38..388f4d7fd 100644 --- a/server/typescript/packages/cli/test/migrate-format-flyway.test.ts +++ b/server/typescript/packages/cli/test/migrate-format-flyway.test.ts @@ -4,7 +4,7 @@ // generate but never apply), and the emit layout (V__/U__ into Flyway's // conventional dir, with --out-dir overriding it). -import { describe, test, expect, afterAll } from "bun:test"; +import { describe, test, expect, afterAll, spyOn } from "bun:test"; import { mkdtemp, rm, mkdir, writeFile, readdir } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; @@ -195,3 +195,39 @@ describe("migrate --migration-format flyway — emit", () => { expect(entries[0]!.endsWith("-add-note")).toBe(true); }); }); + +describe("the relocated-ledger warning under the flyway layout", () => { + // The warning exists to say "the ledger you can see here is not the one this + // run uses". Under `--migration-format flyway` with a default `outDir` the + // directory the run uses comes from `resolveFormatOutDir`, which redirects to + // Flyway's conventional location — so comparing against the unredirected + // `outDir` named a directory the run would never touch. + test("names the directory the run will actually use, not the default outDir", async () => { + const root = await mkdtemp(join(tmpdir(), "mts-flyway-warn-")); + dirs.push(root); + // The project root declares the config; the subdirectory the command runs + // from holds a ledger of its own but no config — the exact layout the + // warning was written for. + await mkdir(join(root, ".metaobjects"), { recursive: true }); + await writeFile(join(root, ".metaobjects", "config.json"), '{"schema_version":1}', "utf8"); + const sub = join(root, "apps", "api"); + await mkdir(join(sub, ".metaobjects", "migrations"), { recursive: true }); + + const stderr: string[] = []; + const spy = spyOn(console, "error").mockImplementation((...args: unknown[]) => { + stderr.push(args.map(String).join(" ")); + }); + try { + // `--dialect d1` is refused by the flyway adapter immediately AFTER the + // warning, so this exercises the warning without needing a database. + await run(["migrate", "--cwd", sub, "--migration-format", "flyway", "--dialect", "d1"]); + } finally { + spy.mockRestore(); + } + + const warning = stderr.find((l) => l.includes("using the migrations directory")); + expect(warning).toBeDefined(); + expect(warning).toContain(join(root, "src", "main", "resources", "db", "migration")); + expect(warning).not.toContain(join(root, ".metaobjects", "migrations")); + }); +}); diff --git a/server/typescript/packages/codegen-ts/src/runner.ts b/server/typescript/packages/codegen-ts/src/runner.ts index c85a7a12e..7a18f0f37 100644 --- a/server/typescript/packages/codegen-ts/src/runner.ts +++ b/server/typescript/packages/codegen-ts/src/runner.ts @@ -90,9 +90,9 @@ export interface RunGenOpts { * port identically to a `pom.xml` / `metaobjects.config.yaml` in every * language port; it says nothing about internal plumbing between two * TypeScript packages in this one repo. Do not "fix" this into a config - * shape — `cli`'s `gen`/`verify` commands are the only callers, and they - * already hold a compiled `CompiledScope` from `resolveCollection()` and - * adapt it to this predicate with `(fqn) => matchesScope(fqn, collection.scope)`. + * shape — `cli`'s `gen`/`verify` commands are the only callers, and a + * `Collection` already exposes exactly this predicate as `inScope`, which + * they pass straight through. */ scope?: (fqn: string) => boolean; } diff --git a/server/typescript/packages/codegen-ts/test/run-gen.test.ts b/server/typescript/packages/codegen-ts/test/run-gen.test.ts index 6dad7c008..d3731257c 100644 --- a/server/typescript/packages/codegen-ts/test/run-gen.test.ts +++ b/server/typescript/packages/codegen-ts/test/run-gen.test.ts @@ -391,10 +391,11 @@ describe("runGen — scope", () => { }; // The real-world "no scope declared" path (`meta gen`) still ALWAYS passes a - // predicate — `matchesScope(fqn, collection.scope)` with an empty compiled - // scope, which matches everything. So the byte-identical guarantee that - // matters is exactly this: omitting `scope` entirely vs. a predicate that - // matches every entity must produce identical output, not merely "close". + // predicate — `collection.inScope`, which an unconfigured project compiles + // from an empty include/exclude, so it admits everything. The byte-identical + // guarantee that matters is exactly this: omitting `scope` entirely vs. a + // predicate that matches every entity must produce identical output, not + // merely "close". const outA = await runGen({ config: defineConfig({ ...baseConfig, outDir: noScopeDir }), metadata: result.root, From 8d2ffc6ba439ab2851c234fcfb826ca943c771d7 Mon Sep 17 00:00:00 2001 From: Doug Mealing Date: Tue, 18 Aug 2026 09:58:14 -0400 Subject: [PATCH 42/44] fix: close five residuals from the branch re-review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Four accuracy defects (prose disagreeing with the code it describes) and one genuine behaviour fix, per the source-resolution rule: metaobjects/ is a config default, resolved everywhere through resolveCollection. - CLAUDE.md: the rule's own allowlist citation said "four sites" against a six-entry allowlist (sdk/src/index.ts and agent-docs/body.ts were added since); named all six. - docs/features/metadata-sources.md: "nothing tests for its existence or mentions it in a message" was false on both counts; the Upgrading intro miscounted its own subsections ("Seven" over six); loadMemory's no-files rejection silently changed from a plain Error to a ParseError/ERR_COLLECTION_NOT_FOUND and that break was undocumented. - sdk/test/discovery.test.ts: a comment still described the withdrawn "bare metaobjects/ directory is a second stop marker" behaviour that the describe block below it now asserts is NOT the case. - sdk/test/no-hardcoded-metadata-dir.test.ts: recorded one more blind spot in the guard's comment-stripper — a regex literal containing `//` is misread as a line comment, blanking a violation to its right. - cli/src/commands/migrate.ts: the relocated-ledger warning used the Kysely-path directory convention for a plain `--dialect d1` run, so it could name a directory that run never writes to. Fixed rather than just re-documented: D1's own convention is now a shared helper (resolveD1OutDir) called once the wrangler binding resolves, from inside runD1Migrate, instead of the generic pre-dispatch check guessing at it. The `--migration-format flyway` + `--dialect d1` combination (refused before either directory is touched) keeps using the generic check, unchanged. sdk 247/0, cli 580/3 skip/0 (+1 test for the migrate.ts fix); full workspace typecheck green across all 18 packages. Co-Authored-By: Claude Opus 5 (1M context) --- AGENTS.md | 2 +- docs/features/metadata-sources.md | 16 ++++- .../packages/cli/src/commands/migrate.ts | 61 ++++++++++++++----- .../cli/test/migrate-format-flyway.test.ts | 39 ++++++++++++ .../packages/sdk/test/discovery.test.ts | 4 +- .../test/no-hardcoded-metadata-dir.test.ts | 12 +++- 6 files changed, 114 insertions(+), 20 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 484941f3e..96394efb0 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -200,7 +200,7 @@ import { EntityFetcherProvider, EntityGrid } from "@metaobjectsdev/tanstack"; - **Codegen substrate**: ts-poet for greenfield emit, ts-morph for in-place edits, Biome for format pass, `git merge-file --diff3` for hand-edit-preserving regen. - **Runtime substrate**: Kysely for TS (user-provided connection, async-only). - **Migration substrate**: Postgres + SQLite for TS v0.3. -- **Metadata location**: resolved via `resolveCollection()` (`@metaobjectsdev/sdk`) — the single authority. `metaobjects/` is the **default value of `sources`** and nothing else: no other module, command or user-facing message may assert that a directory of that name exists or is where metadata lives. Exactly four sites may name it — `sdk/src/metadata-files.ts` (`DEFAULT_METADATA_DIR`, its single definition), `sdk/src/sources.ts` (`DEFAULT_SOURCES`, **the** default), `sdk/src/collection.ts` (inside `resolveCollection`, *applying* that default), and `cli/src/commands/init.ts` (the scaffolder **writing** the layout). Enforced by `sdk/test/no-hardcoded-metadata-dir.test.ts`, whose allowlist demands a written reason per entry. See [docs/features/metadata-sources.md](docs/features/metadata-sources.md). +- **Metadata location**: resolved via `resolveCollection()` (`@metaobjectsdev/sdk`) — the single authority. `metaobjects/` is the **default value of `sources`** and nothing else: no other module, command or user-facing message may assert that a directory of that name exists or is where metadata lives. Exactly six sites may name it — `sdk/src/metadata-files.ts` (`DEFAULT_METADATA_DIR`, its single definition), `sdk/src/sources.ts` (`DEFAULT_SOURCES`, **the** default), `sdk/src/collection.ts` (inside `resolveCollection`, *applying* that default), `sdk/src/index.ts` (the barrel re-export of the constant, no use), `cli/src/commands/init.ts` (the scaffolder **writing** the layout), and `sdk/src/agent-docs/body.ts` (the agent-docs prose `meta init` scaffolds beside that layout). Enforced by `sdk/test/no-hardcoded-metadata-dir.test.ts`, whose allowlist demands a written reason per entry. See [docs/features/metadata-sources.md](docs/features/metadata-sources.md). ## Explicitly out of scope diff --git a/docs/features/metadata-sources.md b/docs/features/metadata-sources.md index 561a46b80..930a63044 100644 --- a/docs/features/metadata-sources.md +++ b/docs/features/metadata-sources.md @@ -10,8 +10,10 @@ and it is not a convention any part of the toolchain is allowed to assume: a pro that declares `sources` may put its metadata anywhere — a sibling module, a shared model repository, a single file — and need not have a directory of that name at all. Every command answers "where is the metadata?" by reading the config, so pointing -`sources` elsewhere moves *all* of them together. Nothing greps for the directory, -tests for its existence, or mentions it in a message. +`sources` elsewhere moves *all* of them together. Outside `resolveCollection`'s own +default-applies check, nothing greps for the directory or assumes it exists — and the +only place that names it in a message is the `meta init` scaffolded agent-docs prose, +which is documentation content, not a resolution path. **How do I point it somewhere else?** Declare it: @@ -432,7 +434,7 @@ type) surfaces as the config load error and stops the command. ## Upgrading A project with one config at its root, no `sources` and no `scope` resolves the same -files it always did and generates the same code. Seven changes are still worth knowing +files it always did and generates the same code. Six changes are still worth knowing about before you upgrade. ### A project boundary is a `.metaobjects/config.json` @@ -469,6 +471,14 @@ this feature exists to remove — and one of them was undocumented. declarations now fails to load with `ERR_UNRESOLVED_SUPER` naming the `extends:` target it cannot find; nothing generates from a half-resolved model. +**A caller invoking `loadMemory` directly, not through the CLI, changes too.** With no +`options.files`, `loadMemory` now resolves its own file list via `resolveCollection` — +so a project with nothing to resolve (no declared `sources`, no default +`metaobjects/`) used to reject with a plain `Error("cannot read metadata directory +…")` and now rejects with a `ParseError` carrying `code: "ERR_COLLECTION_NOT_FOUND"` +(`sdk/src/collection.ts:158-163`) — the same structured code every other command +reports. A `catch` matching the old message text stops matching, silently. + **The replacement is a declared source**, which is explicit and works in any layout, workspace or not: diff --git a/server/typescript/packages/cli/src/commands/migrate.ts b/server/typescript/packages/cli/src/commands/migrate.ts index 2e4270207..83e525a3e 100644 --- a/server/typescript/packages/cli/src/commands/migrate.ts +++ b/server/typescript/packages/cli/src/commands/migrate.ts @@ -128,6 +128,22 @@ function resolveFormatOutDir(config: ResolvedMigrateConfig, metaRoot: string): s return resolvePath(metaRoot, config.outDir); } +/** + * D1's OWN directory convention — `--out-dir` > `wrangler.toml`'s + * `migrations_dir` > `"migrations"` — kept apart from `resolveFormatOutDir` + * because the middle term is not knowable until a wrangler binding has been + * resolved (`runD1Migrate` step 1), while every other dialect can answer + * immediately from `config` alone. + */ +function resolveD1OutDir( + config: ResolvedMigrateConfig, + metaRoot: string, + migrationsDirHint: string | undefined, +): string { + const isDefaultOutDir = config.outDir === MIGRATE_DEFAULT_OUT_DIR; + return resolvePath(metaRoot, isDefaultOutDir ? (migrationsDirHint ?? "migrations") : config.outDir); +} + /** * Say so when the migrations directory this run will use is NOT the one sitting * in the working directory. @@ -143,10 +159,15 @@ function resolveFormatOutDir(config: ResolvedMigrateConfig, metaRoot: string): s * * Conditioned on the local directory EXISTING, so the ordinary case — a run from * anywhere inside a project with one ledger at its root — says nothing. - * `--out-dir` (and a `migrate.outDir` in the config) is honoured, as is the - * active output format's own convention: the caller passes the directory this - * run will actually WRITE to (`resolveFormatOutDir`), so a deliberate - * redirection is compared rather than a default that was overridden. + * `--out-dir` (and a `migrate.outDir` in the config) is honoured: the caller + * must pass the directory THIS run will actually WRITE to, never a default + * that was overridden — a deliberate redirection is compared, not a stale + * guess. That answer comes from **two** call sites, because there are two + * directory conventions: every dialect but d1 resolves via + * `resolveFormatOutDir` before the format/dialect dispatch below; d1 has its + * own convention (`resolveD1OutDir`, wrangler.toml's `migrations_dir`) that is + * unknowable until its binding resolves, so it calls this again for itself + * from inside `runD1Migrate`, once that binding is in hand. */ function warnIfLedgerRelocated(cwd: string, resolvedOutDir: string): void { const local = resolvePath(cwd, MIGRATE_DEFAULT_OUT_DIR); @@ -359,7 +380,17 @@ export async function migrateCommand( // `--migration-format flyway` with a default outDir the run writes to // Flyway's conventional location instead, so the unredirected path names a // directory this invocation will never touch. - warnIfLedgerRelocated(cwd, resolveFormatOutDir(config, metaRoot)); + // + // Skipped for a plain `--dialect d1` run (format !== flyway): d1 resolves + // its OWN directory from wrangler.toml, unknowable until its binding + // resolves, so it issues this warning for itself from inside + // `runD1Migrate` instead — `resolveFormatOutDir` here would name the + // Kysely-path default, a directory that run never writes to. A d1 + + // `--migration-format flyway` run is refused just below before either + // directory is ever touched, so THAT combination still wants this one. + if (config.dialect !== "d1" || config.format === "flyway") { + warnIfLedgerRelocated(cwd, resolveFormatOutDir(config, metaRoot)); + } try { // #192 — Flyway owns apply + history (flyway_schema_history). We generate the @@ -442,7 +473,7 @@ export async function migrateCommand( ); return 2; } - return await runD1Migrate(config, metaRoot, wranglerRunner ?? defaultWranglerRunner, fmt); + return await runD1Migrate(config, metaRoot, cwd, wranglerRunner ?? defaultWranglerRunner, fmt); } // `migrate baseline` — seed the committed reference snapshot, emit no migration. @@ -1216,6 +1247,7 @@ async function runRollback( async function runD1Migrate( config: ResolvedMigrateConfig, metaRoot: string, + cwd: string, runner: WranglerRunner, fmt: OutputFormat = "text", ): Promise { @@ -1243,6 +1275,12 @@ async function runD1Migrate( binding = { binding: config.d1.binding!, database_name: "", database_id: "", migrations_dir: undefined }; } + // The binding — and with it wrangler.toml's `migrations_dir` — is only now + // known, so this is the earliest point d1 can honestly answer "where will + // this run write?" (the caller skipped its own generic check for exactly + // this reason; see the guard around that call). + warnIfLedgerRelocated(cwd, resolveD1OutDir(config, metaRoot, binding.migrations_dir)); + // 2. Build a D1Runner closure over the wrangler runner. const d1Runner: D1Runner = async (sql) => { const args = buildWranglerExecuteArgs({ @@ -1396,14 +1434,9 @@ async function runD1Migrate( const combinedUp = emitResult.up; const combinedDown = emitResult.down; - // Migration dir resolution: --out-dir > wrangler.toml's migrations_dir > "migrations". - // The default outDir (./.metaobjects/migrations) is the Kysely-path default; for D1 - // we fall back to wrangler conventions when the caller hasn't overridden it. - const isDefaultOutDir = config.outDir === MIGRATE_DEFAULT_OUT_DIR; - const migrationsDir = resolvePath( - metaRoot, - isDefaultOutDir ? (binding.migrations_dir ?? "migrations") : config.outDir, - ); + // Migration dir resolution — same convention `warnIfLedgerRelocated` was + // just given above, so the two cannot drift apart. + const migrationsDir = resolveD1OutDir(config, metaRoot, binding.migrations_dir); if (config.dryRun) { log.info(`-- UP --\n${combinedUp}\n\n-- DOWN --\n${combinedDown}`); diff --git a/server/typescript/packages/cli/test/migrate-format-flyway.test.ts b/server/typescript/packages/cli/test/migrate-format-flyway.test.ts index 388f4d7fd..8007bbbb3 100644 --- a/server/typescript/packages/cli/test/migrate-format-flyway.test.ts +++ b/server/typescript/packages/cli/test/migrate-format-flyway.test.ts @@ -231,3 +231,42 @@ describe("the relocated-ledger warning under the flyway layout", () => { expect(warning).not.toContain(join(root, ".metaobjects", "migrations")); }); }); + +describe("the relocated-ledger warning under a plain d1 run", () => { + // A default (non-flyway) `--dialect d1` run has its OWN directory + // convention — wrangler.toml's `migrations_dir`, falling back to + // `"migrations"` — which the Kysely-path `resolveFormatOutDir` knows + // nothing about. Before the fix, this case named the Kysely-path default + // (`.metaobjects/migrations`), a directory a d1 run never writes to. + test("names d1's own migrations directory, not the Kysely-path default", async () => { + const root = await mkdtemp(join(tmpdir(), "mts-d1-warn-")); + dirs.push(root); + // The project root declares the config; the subdirectory the command runs + // from holds a ledger of its own but no config — the exact layout the + // warning was written for. + await mkdir(join(root, ".metaobjects"), { recursive: true }); + await writeFile(join(root, ".metaobjects", "config.json"), '{"schema_version":1}', "utf8"); + const sub = join(root, "apps", "api"); + await mkdir(join(sub, ".metaobjects", "migrations"), { recursive: true }); + + const stderr: string[] = []; + const spy = spyOn(console, "error").mockImplementation((...args: unknown[]) => { + stderr.push(args.map(String).join(" ")); + }); + try { + // No wrangler.toml and no `metaobjects/` directory: an explicit --d1 + // binding bypasses the wrangler.toml requirement, so the run reaches + // the warning (issued right after the binding resolves) and then fails + // cleanly at metadata resolution (ERR_COLLECTION_NOT_FOUND, exit 2) — + // exercising the warning without needing a real D1 database. + await run(["migrate", "--cwd", sub, "--dialect", "d1", "--d1", "DB"]); + } finally { + spy.mockRestore(); + } + + const warning = stderr.find((l) => l.includes("using the migrations directory")); + expect(warning).toBeDefined(); + expect(warning).toContain(join(root, "migrations")); + expect(warning).not.toContain(join(root, ".metaobjects", "migrations")); + }); +}); diff --git a/server/typescript/packages/sdk/test/discovery.test.ts b/server/typescript/packages/sdk/test/discovery.test.ts index c4ca08778..b6f205d92 100644 --- a/server/typescript/packages/sdk/test/discovery.test.ts +++ b/server/typescript/packages/sdk/test/discovery.test.ts @@ -10,7 +10,9 @@ const cfg = (rel: string) => { mk(join(rel, ".metaobjects")); writeFileSync(join(root, rel, ".metaobjects/config.json"), '{"schema_version":1}', "utf8"); }; -/** The second stop marker: a `metaobjects/` directory, no config. */ +/** A `metaobjects/` directory with no config — used below to prove it is + * NOT a stop marker; the walk stops on `.metaobjects/config.json` and the + * `.git` boundary only. */ const meta = (rel: string) => mk(join(rel, "metaobjects")); beforeEach(() => { root = mkdtempSync(join(tmpdir(), "metaobjects-discovery-")); mk(".git"); }); diff --git a/server/typescript/packages/sdk/test/no-hardcoded-metadata-dir.test.ts b/server/typescript/packages/sdk/test/no-hardcoded-metadata-dir.test.ts index 251f3d66d..37f211bcd 100644 --- a/server/typescript/packages/sdk/test/no-hardcoded-metadata-dir.test.ts +++ b/server/typescript/packages/sdk/test/no-hardcoded-metadata-dir.test.ts @@ -32,8 +32,9 @@ // MISSED join(d, "meta" + "objects") any computed spelling // MISSED const N = "meta"; N + "objects" the same, through a variable // MISSED "author under metaobjects" the word with no trailing `/` +// MISSED /https:\/\//metaobjects/ a regex literal containing `//` // -// The last is deliberate, not an oversight: `metaobjects` followed by a space +// The third is deliberate, not an oversight: `metaobjects` followed by a space // is the PRODUCT name far more often than a path ("the metaobjects ledger", // the `metaobjects:` error prefix), and three such lines were the guard's first // false positives. No lexical rule separates them. The computed-spelling misses @@ -41,6 +42,15 @@ // violation is actually written, which is how all eight original ones were // written, and it will not catch someone evading it on purpose. // +// The fourth is a real blind spot in `stripComments`, not a deliberate +// tradeoff: a regex literal containing `//` (e.g. `/https:\/\//`) drives the +// stripper into line-comment state, same as a real `//`, and blanks the rest +// of that physical line — so a violation sitting to its right on the same +// line is silently missed. The stripper has no regex-literal-vs-division +// disambiguation (that requires knowing the preceding token, which a +// character-at-a-time scan does not track). No such construct exists in +// either scanned tree today. +// // It also scans TypeScript SOURCE only: the four other language ports, the // `docs/` tree, and JSON/YAML fixtures are outside it. import { describe, test, expect } from "bun:test"; From 4baf19a93d226fdf5a6062a0bafd4e94e7870f6f Mon Sep 17 00:00:00 2001 From: Doug Mealing Date: Tue, 18 Aug 2026 21:22:15 -0400 Subject: [PATCH 43/44] no-mistakes(review): address stale changelog bullet and migrate scope provenance refusal --- CHANGELOG.md | 13 +++-- .../packages/cli/src/commands/migrate.ts | 49 ++++++++-------- .../packages/cli/src/commands/verify.ts | 24 +++++--- .../packages/cli/src/lib/migrate-scope.ts | 57 ++++++++++++------- .../test/integration/migrate-db-scope.test.ts | 27 +++++++++ .../test/integration/support/scope-fixture.ts | 31 ++++++++++ .../packages/cli/test/migrate-scope.test.ts | 41 +++++++++++++ 7 files changed, 188 insertions(+), 54 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4e09c812b..75a8ce566 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -135,10 +135,15 @@ the same migrations. Three changes are visible even to that project. Adopter gui resolved directory differs from `/.metaobjects/migrations` and that local directory exists; `--out-dir` overrides, and giving the subdirectory its own `.metaobjects/config.json` makes it a project root. -- **Discovery stops at a `metaobjects/` directory, not only at a config.** A nested - project holding its own `metaobjects/` and no `.metaobjects/config.json` keeps - reading its own metadata, as it always did, rather than adopting an ancestor's - model and `outDir`. +- **A project boundary is a `.metaobjects/config.json` — a bare `metaobjects/` + directory is not one.** Discovery walks up for a config and stops at nothing + else short of the `.git` boundary, so a command run inside a nested directory + that holds metadata but declares no config of its own resolves the nearest + ancestor config — adopting its `sources` and `outDir`. `metaobjects/` is the + default *value* of `sources`, so a directory of that name says nothing about + whether a project lives there. If a subdirectory should own its metadata, give + it a config: `meta init` writes one, and a `"sources": []` config is enough to + claim the directory and take the default. ## [0.23.2] — npm `0.23.2` · PyPI `0.23.2` · NuGet `0.23.2` · Maven `7.23.2` diff --git a/server/typescript/packages/cli/src/commands/migrate.ts b/server/typescript/packages/cli/src/commands/migrate.ts index 83e525a3e..3d6041145 100644 --- a/server/typescript/packages/cli/src/commands/migrate.ts +++ b/server/typescript/packages/cli/src/commands/migrate.ts @@ -12,7 +12,6 @@ import { toonEncode } from "../lib/format.js"; import { buildKyselyFromUrl, redactUrl } from "../lib/kysely.js"; import { log } from "../lib/log.js"; import { loadMemory, resolveCollection, resolveConfigDir, type Collection } from "@metaobjectsdev/sdk"; -import type { MetaRoot } from "@metaobjectsdev/metadata"; import { loadMetaobjectsConfig } from "../lib/load-metaobjects-config.js"; import { migrateScopeMismatch, outOfScopeNote } from "../lib/migrate-scope.js"; import { @@ -47,6 +46,7 @@ import { type D1Binding, type EmitResult, type D1Runner, + type SchemaProvenance, } from "@metaobjectsdev/migrate-ts"; import { buildWranglerExecuteArgs, @@ -217,10 +217,10 @@ function emitStructuredError(error: string, hint: string, fmt: OutputFormat): vo */ function refuseScopeMismatch( collection: Collection, - root: MetaRoot, + provenance: () => SchemaProvenance, fmt: OutputFormat, ): number | undefined { - const mismatch = migrateScopeMismatch(collection, root); + const mismatch = migrateScopeMismatch(collection, provenance); if (mismatch === undefined) return undefined; log.error(`migrate: ${mismatch}`); emitStructuredError(`migrate: ${mismatch}`, "fix or remove migrate.scope in .metaobjects/config.json", fmt); @@ -545,9 +545,6 @@ export async function migrateCommand( return 2; } - const scopeRc = refuseScopeMismatch(collection, metadata, fmt); - if (scopeRc !== undefined) return scopeRc; - let kysely; try { kysely = await buildKyselyFromUrl(config.databaseUrl, config.dialect); @@ -579,18 +576,18 @@ export async function migrateCommand( // view DDL (create/drop/replace + dependency-recreate) and emit() renders it — // there is no separate view-migration emitter. const expectedViews = buildProjectionViews(metadata, { dialect: kysely.dialect, columnNamingStrategy }); + const built = buildExpectedSchemaWithProvenance(metadata, { + dialect: kysely.dialect, + columnNamingStrategy, + views: expectedViews, + }); + const scopeRc = refuseScopeMismatch(collection, () => built.provenance, fmt); + if (scopeRc !== undefined) return scopeRc; // Per-command scope: objects outside `migrate.scope` are another owner's. They // leave the expected schema here and are suppressed on the actual side below — // dropping them from `expected` ALONE would propose DROP TABLE for every one of // them that exists in the database. - const scoped = scopeExpectedSchema( - buildExpectedSchemaWithProvenance(metadata, { - dialect: kysely.dialect, - columnNamingStrategy, - views: expectedViews, - }), - collection.inMigrateScope, - ); + const scoped = scopeExpectedSchema(built, collection.inMigrateScope); const expected = scoped.snapshot; logOutOfScope(scoped.outOfScope, fmt); let actual; @@ -1074,7 +1071,17 @@ export async function runOfflineGenerate( return 2; } - const scopeRc = refuseScopeMismatch(collection, metadata, fmt); + const offlineDialect = config.dialect; + const offlineViews = buildProjectionViews(metadata, { dialect: offlineDialect, columnNamingStrategy: offlineStrategy }); + const scopeRc = refuseScopeMismatch( + collection, + () => buildExpectedSchemaWithProvenance(metadata, { + dialect: offlineDialect, + columnNamingStrategy: offlineStrategy, + views: offlineViews, + }).provenance, + fmt, + ); if (scopeRc !== undefined) return scopeRc; const outDir = resolvePath(metaRoot, config.outDir); @@ -1104,7 +1111,6 @@ export async function runOfflineGenerate( const collectedAmbiguous: AmbiguousChange[] = []; const onAmbiguousResolution = mapOnAmbiguous(config.onAmbiguous); - const offlineViews = buildProjectionViews(metadata, { dialect: config.dialect, columnNamingStrategy: offlineStrategy }); const offlineScope = collection.inMigrateScope; let plan; @@ -1325,9 +1331,6 @@ async function runD1Migrate( return 2; } - const scopeRc = refuseScopeMismatch(collection, metadata, fmt); - if (scopeRc !== undefined) return scopeRc; - // 4. Build expected schema + introspect actual. let columnNamingStrategy: "snake_case" | "literal" | "kebab-case" = "snake_case"; try { @@ -1337,11 +1340,11 @@ async function runD1Migrate( // metaobjects.config.ts absent or invalid — use default snake_case } const expectedViews = buildProjectionViews(metadata, { dialect: "d1", columnNamingStrategy }); + const built = buildExpectedSchemaWithProvenance(metadata, { dialect: "d1", columnNamingStrategy, views: expectedViews }); + const scopeRc = refuseScopeMismatch(collection, () => built.provenance, fmt); + if (scopeRc !== undefined) return scopeRc; // Per-command scope — both-sided, exactly as on the Kysely path above. - const scoped = scopeExpectedSchema( - buildExpectedSchemaWithProvenance(metadata, { dialect: "d1", columnNamingStrategy, views: expectedViews }), - collection.inMigrateScope, - ); + const scoped = scopeExpectedSchema(built, collection.inMigrateScope); const expected = scoped.snapshot; logOutOfScope(scoped.outOfScope, fmt); let actual; diff --git a/server/typescript/packages/cli/src/commands/verify.ts b/server/typescript/packages/cli/src/commands/verify.ts index fbdbfa763..02d6cfe5b 100644 --- a/server/typescript/packages/cli/src/commands/verify.ts +++ b/server/typescript/packages/cli/src/commands/verify.ts @@ -27,7 +27,7 @@ import { } from "../lib/wrangler.js"; import type { MetaobjectsGenConfig } from "@metaobjectsdev/codegen-ts"; import { buildProjectionViews } from "@metaobjectsdev/codegen-ts"; -import { buildKyselyFromUrl, type Dialect } from "../lib/kysely.js"; +import { buildKyselyFromUrl, inferDialect, type Dialect } from "../lib/kysely.js"; import { tokensToAllowOptions, describeChange } from "../lib/allow.js"; import { computeDrift, @@ -35,6 +35,7 @@ import { collectUnmanagedNames, excludeFromSnapshot, scopedDiffInputs, + buildExpectedSchemaWithProvenance, type GovernedScope, introspect, diff, @@ -418,12 +419,21 @@ export async function verifyCommand( const usingD1 = flags.dialect === "d1"; if ((flags.db === undefined && !usingD1) || flags.skipSchema) return 0; - // A `migrate.scope` matching nothing is refused, not tolerated — it would make - // this gate compare zero objects and report "in sync" (see `migrateScopeMismatch`). - // Checked HERE rather than beside the other collection work at the top of - // `verifyCommand`, because `migrate.scope` governs only the schema gate: a stale - // pattern must not fail a `--templates` run that never consults it. - const scopeMismatch = migrateScopeMismatch(collection, root); + // A `migrate.scope` matching nothing it could govern is refused, not tolerated — + // it would make this gate compare zero objects and report "in sync" (see + // `migrateScopeMismatch`). Checked HERE rather than beside the other collection + // work at the top of `verifyCommand`, because `migrate.scope` governs only the + // schema gate: a stale pattern must not fail a `--templates` run that never + // consults it. + const scopeMismatch = migrateScopeMismatch(collection, () => { + const dialect: Dialect = usingD1 ? "d1" : (flags.dialect ?? inferDialect(flags.db as string)); + const viewStrategy = forgeConfig?.columnNamingStrategy ?? "snake_case"; + return buildExpectedSchemaWithProvenance(root, { + dialect, + columnNamingStrategy: viewStrategy, + views: buildProjectionViews(root, { dialect, columnNamingStrategy: viewStrategy }), + }).provenance; + }); if (scopeMismatch !== undefined) { log.error(`verify: ${scopeMismatch}`); return 2; diff --git a/server/typescript/packages/cli/src/lib/migrate-scope.ts b/server/typescript/packages/cli/src/lib/migrate-scope.ts index 568afd9b1..8d2d814b2 100644 --- a/server/typescript/packages/cli/src/lib/migrate-scope.ts +++ b/server/typescript/packages/cli/src/lib/migrate-scope.ts @@ -10,7 +10,7 @@ // or a refusal that drifted between them would be drift the user reads. import type { Collection } from "@metaobjectsdev/sdk"; -import type { MetaRoot } from "@metaobjectsdev/metadata"; +import type { SchemaProvenance } from "@metaobjectsdev/migrate-ts"; /** * Say what a declared `migrate.scope` left out, for `migrate` and `verify --db` @@ -37,13 +37,15 @@ export function outOfScopeNote(command: string, names: readonly string[]): strin const EXAMPLE_FQN_CAP = 3; /** - * The refusal for a `migrate.scope` that matches NOTHING. + * The refusal for a `migrate.scope` that matches NOTHING it could govern. * - * A scope matching zero loaded objects can never be what someone meant — it says - * "every table in this model belongs to somebody else", which is a project with no - * schema to migrate at all, expressed the hard way. In practice it is a typo'd or - * stale package pattern, and it is silent: migrate reports "no changes" while - * having compared nothing. + * A scope matching zero of the objects that declare a table or view can never be + * what someone meant — it says "every table in this model belongs to somebody + * else", which is a project with no schema to migrate at all, expressed the hard + * way. In practice it is a typo'd or stale package pattern, or a scope over a + * package that holds only value objects and abstracts (shapes that can never + * contribute a table or view), and it is silent: migrate reports "no changes" + * while having compared nothing. * * It is also actively dangerous, which is why this is a refusal and not a warning. * An empty expected side is what `diff` reads as "no model, govern the whole @@ -53,22 +55,37 @@ const EXAMPLE_FQN_CAP = 3; * stops the wrong scope going unnoticed in the first place. * * Returns the message to report, or `undefined` when there is nothing to refuse — - * no scope declared, or at least one loaded object inside it. Callers report it and - * exit 2 (a configuration error), rather than this throwing, so it reads like every - * other config failure in these commands. + * no scope declared, or at least one table- or view-declaring object inside it. + * Callers report it and exit 2 (a configuration error), rather than this throwing, + * so it reads like every other config failure in these commands. */ export function migrateScopeMismatch( collection: Collection, - root: MetaRoot, + /** + * The UNSCOPED expected schema's provenance (migrate-ts + * `buildExpectedSchemaWithProvenance`) — qualified table/view name → declaring + * FQN. Supplied lazily because it is consulted only under a declared scope, so + * a project with no `migrate.scope` pays nothing for this check and its runs + * are byte-for-byte what they always were. + */ + provenance: () => SchemaProvenance, ): string | undefined { const { inMigrateScope, migrateScopePatterns } = collection; if (inMigrateScope === undefined) return undefined; - // ADR-0039: `objects()` is the resolving accessor — the loaded object set, which - // is exactly what `migrate.scope` claims to be a subset of. - const fqns = root.objects().map((o) => o.resolutionKey()); - // No objects at all is not a scope error: there is nothing for a pattern to miss, - // and an empty model has its own (much louder) failure modes downstream. + // The declaring FQNs of every table and view the UNSCOPED model contributes — + // the same provenance `scopeExpectedSchema` decides scope on, so the refusal + // asks exactly the question the run answers. NOT the loaded object set: that + // counts value objects and abstracts, which can never declare a table or view + // (persistability derives from a declared/inherited writable source, never + // from a subtype — #248), so a scope over only those objects passed this + // refusal while governing zero tables. And not a fresh walk either: it would + // have to re-implement the builder's skip rules (abstract, TPH subtype, no + // writable source, `@unmanaged`) and would drift from them. + const fqns = [...new Set(provenance().values())]; + // A model that declares no table or view at all is not a scope error: there is + // nothing for a pattern to govern, scoped or not, and an empty schema has its + // own (much louder) failure modes downstream. if (fqns.length === 0) return undefined; if (fqns.some(inMigrateScope)) return undefined; @@ -76,10 +93,10 @@ export function migrateScopeMismatch( const examples = fqns.slice(0, EXAMPLE_FQN_CAP).join(", "); const more = fqns.length > EXAMPLE_FQN_CAP ? `, …and ${fqns.length - EXAMPLE_FQN_CAP} more` : ""; return ( - `migrate.scope matched none of the ${fqns.length} object(s) loaded, so this run would ` + - `treat every one of them as another owner's and compare nothing. ` + - `Patterns: ${patterns}. Loaded: ${examples}${more}. ` + + `migrate.scope matched none of the ${fqns.length} object(s) declaring a table or view, ` + + `so this run would treat every one of them as another owner's and compare nothing. ` + + `Patterns: ${patterns}. Declaring a table or view: ${examples}${more}. ` + `Fix the patterns in .metaobjects/config.json (migrate.scope), or remove the key to ` + - `govern everything loaded.` + `govern everything the model declares.` ); } diff --git a/server/typescript/packages/cli/test/integration/migrate-db-scope.test.ts b/server/typescript/packages/cli/test/integration/migrate-db-scope.test.ts index 612e44b2c..aac031040 100644 --- a/server/typescript/packages/cli/test/integration/migrate-db-scope.test.ts +++ b/server/typescript/packages/cli/test/integration/migrate-db-scope.test.ts @@ -11,6 +11,12 @@ * having compared nothing, while an empty expected side is exactly what the diff * reads as "no model, govern the whole database". Refused, with the patterns and * the loaded FQNs named, so the author can see what missed. + * + * The near-miss variant matters just as much: a scope matching only value + * objects and abstracts matches LOADED objects but none that can declare a + * table or view — the run still compares nothing, so it is refused on the same + * question, answered against the expected schema's provenance rather than the + * loaded object set. */ import { describe, test, expect, beforeEach, afterEach } from "bun:test"; import { rmSync } from "node:fs"; @@ -55,6 +61,27 @@ describe("meta migrate --db — migrate.scope", () => { } }); + test("a scope matching only value objects and abstracts is refused — they declare no table", async () => { + const { repo, dbUrl } = scaffold("metaobjects-migrate-scope-"); + try { + // `acme::shared` (scaffolded by the fixture) holds an abstract base and a + // value object: loaded objects, but none that can contribute a table or + // view. Matching them is not governing anything — the run would compare + // nothing and report "no changes" against a database it was told to check. + declareScope(repo, ["acme::shared::**"]); + expect(await migrateFromDb(repo, dbUrl)).toBe(2); + const all = [...out, ...err].join("\n"); + expect(all).toContain("matched none"); + // The patterns that missed, and the table-declaring objects they could + // have matched — the refusal is decided against those, not against every + // loaded object. + expect(all).toContain("acme::shared::**"); + expect(all).toContain("acme::platform::Job"); + } finally { + rmSync(repo, { recursive: true, force: true }); + } + }); + test("a scope that matches something still runs (the refusal is not a blanket break)", async () => { const { repo, dbUrl } = scaffold("metaobjects-migrate-scope-"); try { diff --git a/server/typescript/packages/cli/test/integration/support/scope-fixture.ts b/server/typescript/packages/cli/test/integration/support/scope-fixture.ts index 2d917fa3d..4966c363e 100644 --- a/server/typescript/packages/cli/test/integration/support/scope-fixture.ts +++ b/server/typescript/packages/cli/test/integration/support/scope-fixture.ts @@ -59,6 +59,36 @@ export const arena = (opts: { venue: boolean } = { venue: false }): string => JS /** Absolute path of the arena metadata file, for a test that rewrites it. */ export const arenaFile = (repo: string): string => join(repo, "metaobjects", "meta.arena.json"); +/** + * A package of shared SHAPES: an abstract base and a value object. Both are + * loaded objects, but neither can declare a table or view — persistability + * needs a writable source — so a `migrate.scope` over only this package + * governs zero tables however well its patterns match. + */ +export const SHARED = JSON.stringify({ + "metadata.root": { + package: "acme::shared", + children: [ + { + "object.entity": { + name: "BaseRecord", + abstract: true, + children: [{ "field.long": { name: "id" } }], + }, + }, + { + "object.value": { + name: "Address", + children: [ + { "field.string": { name: "line1" } }, + { "field.string": { name: "line2" } }, + ], + }, + }, + ], + }, +}); + /** * A throwaway project holding both packages, plus the sqlite URL beside it. * `prefix` names the temp directory so a failing run says which suite made it. @@ -68,6 +98,7 @@ export function scaffold(prefix: string): { repo: string; dbUrl: string } { mkdirSync(join(repo, "metaobjects"), { recursive: true }); writeFileSync(join(repo, "metaobjects", "meta.platform.json"), PLATFORM, "utf8"); writeFileSync(arenaFile(repo), arena(), "utf8"); + writeFileSync(join(repo, "metaobjects", "meta.shared.json"), SHARED, "utf8"); return { repo, dbUrl: `file:${join(repo, "local.db")}` }; } diff --git a/server/typescript/packages/cli/test/migrate-scope.test.ts b/server/typescript/packages/cli/test/migrate-scope.test.ts index d91f9fcf1..4c6f27bb7 100644 --- a/server/typescript/packages/cli/test/migrate-scope.test.ts +++ b/server/typescript/packages/cli/test/migrate-scope.test.ts @@ -137,4 +137,45 @@ describe("meta migrate — migrate.scope", () => { expect(await runOfflineGenerate(cfg(), root)).toBe(1); expect(await migrationDirs(root)).toHaveLength(0); }); + + test("a scope matching only value objects and abstracts is refused before the snapshot gate", async () => { + const root = await project(); + // A package of shapes that can never declare a table: the scope matches + // loaded objects, but none the run could actually govern. + await writeFile( + join(root, "metaobjects", "meta.shared.json"), + JSON.stringify({ + "metadata.root": { + package: "acme::shared", + children: [ + { "object.entity": { name: "BaseRecord", abstract: true, children: [ + { "field.long": { name: "id" } }, + ] } }, + { "object.value": { name: "Address", children: [ + { "field.string": { name: "line1" } }, + { "field.string": { name: "line2" } }, + ] } }, + ], + }, + }), + "utf8", + ); + await declareScope(root, ["acme::shared::**"]); + // No baseline was run, so there is no snapshot: a run that got PAST the + // refusal would report "no schema snapshot" — also exit 2 — so the message + // is what pins that the scope error is the one reported, and that the + // refusal fires before the snapshot is ever read. + const errors: string[] = []; + const origErr = console.error; + console.error = (...a: unknown[]) => { errors.push(a.map(String).join(" ")); }; + try { + expect(await runOfflineGenerate(cfg(), root)).toBe(2); + } finally { + console.error = origErr; + } + const all = errors.join("\n"); + expect(all).toContain("matched none"); + expect(all).toContain("acme::shared::**"); + expect(all).toContain("acme::platform::Job"); + }); }); From 232a1af87e1c27b84dccaaf09b9f9ab0fe0ff3ef Mon Sep 17 00:00:00 2001 From: Doug Mealing Date: Tue, 18 Aug 2026 21:53:50 -0400 Subject: [PATCH 44/44] no-mistakes(document): Updated requirements.md to clarify metaobjects/ is default location --- docs/features/requirements.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/features/requirements.md b/docs/features/requirements.md index 36b9be081..a76c4b3db 100644 --- a/docs/features/requirements.md +++ b/docs/features/requirements.md @@ -27,7 +27,7 @@ disproof to the thing being resurrected, in one line. ## Declaring one -Requirements live in `metaobjects/` beside the entities they describe: +Requirements live beside the entities they describe (by default in `metaobjects/`): ```jsonc { "metadata.root": {