fix(metadata-protocol): os migrate packageless-overlays also moves overlay rows bound to the read-only base itself into its .custom pair

.changeset/29392-packageless-overlays-base-bound.md

@@ -0,0 +1,14 @@
+---
+'@objectstack/metadata-protocol': patch
+'@objectstack/cli': patch
+---
+
+fix(metadata-protocol): `os migrate packageless-overlays` also moves a view, dashboard, report, translation or email template row bound to the read-only package itself into its `.custom` pair
+
+Clause-②: no
+
+Metadata-protocol: none
+
+Before the write naming a read-only code package itself (`?package=<base>`) was closed, such a save stored the customization as a row bound to that package, declaring nothing, served at the package's address in place of its own document. The ceremony's census read rows bound to no package only, so it never listed such a row, and its one exit was a delete, which drops the customization. The census now lists, beside the package-less rows, every row of the five overlay types bound to a read-only package that ships the name (`boundToBase` in `--json`, "bound to PACKAGE" in the listing), classified the same way with that package as its base. `--apply` moves it into the pair exactly as it moves a package-less row: the declared override with `basedOn` (or, for an email template locale the package does not ship, the pair's own member), read back, and only then the bound row deleted. The outcomes are the same `moved` ones.
+
+A row bound to a writable package (the pair included), or to a read-only package that does not ship the name, is not listed. When one item has both a package-less row and a row bound to the package, the package-less row moves and the bound one is kept and reported failed, never merged over it; the next preview lists it `shadowed`, with that disposition's ways out.

packages/cli/src/commands/migrate/packageless-overlays.integration.test.ts

@@ -24,6 +24,9 @@
  * controls, and a FRESH boot of the same database then serves the moved
  * content under the base's name on the package-less read, the read naming the
  * base, the layered read and the base's list; a second `--apply` moves 0.
+ *
+ * [#29392] A second template holds a row bound to the package itself (a save
+ * naming it, before that write was closed): listed and moved the same way.
  */
 
 import { describe, it, expect, beforeAll, afterAll, beforeEach, afterEach, vi } from 'vitest';
@@ -78,7 +81,7 @@ const viewBody = (name: string, columns: readonly string[]) => ({
   config: { type: 'grid', data: { provider: 'object', object: 'm28413_task' }, columns: columns.map((field) => ({ field })) },
 });
 
-interface SeedRow { type: string; name: string; state: string; body: Record<string, unknown> }
+interface SeedRow { type: string; name: string; state: string; body: Record<string, unknown>; packageId?: string }
 const SEED: SeedRow[] = [
   { type: 'view', name: VIEW, state: 'active', body: viewBody(VIEW, STORED_COLUMNS) },
   { type: 'view', name: VIEW, state: 'draft', body: viewBody(VIEW, DRAFT_COLUMNS) },
@@ -89,22 +92,52 @@ const SEED: SeedRow[] = [
   { type: 'permission', name: 'm28413_rep', state: 'draft', body: { name: 'm28413_rep', label: 'Saved in the environment', objects: {} } },
 ];
 
+/**
+ * [#29392] A database whose customization a save naming the package itself
+ * stored before that write was closed: an active row of the package's view
+ * bound to the package, beside the package-less environment row (control).
+ */
+const BOUND_SEED: SeedRow[] = [
+  { type: 'view', name: VIEW, state: 'active', body: viewBody(VIEW, STORED_COLUMNS), packageId: PKG },
+  { type: 'view', name: ENV_VIEW, state: 'active', body: viewBody(ENV_VIEW, STORED_COLUMNS) },
+];
+
 let dir: string;
 let templateDb: string;
+let boundTemplateDb: string;
 const savedCwd = process.cwd();
 const ENV_KEYS = ['OS_ARTIFACT_PATH', 'OS_DATABASE_URL', 'DATABASE_URL', 'OS_DATABASE_DRIVER', 'OS_HOME', 'NODE_ENV',
   'OS_AUTH_SECRET', 'AUTH_SECRET', 'BETTER_AUTH_SECRET', 'OS_ENVIRONMENT_ID'] as const;
 const savedEnv: Record<string, string | undefined> = {};
 
 let caseSeq = 0;
-function caseDb(): string {
+function caseDb(template = templateDb): string {
   caseSeq += 1;
   const file = join(dir, 'cases', `${caseSeq}.db`);
   mkdirSync(dirname(file), { recursive: true });
-  copyFileSync(templateDb, file);
+  copyFileSync(template, file);
   return file;
 }
 
+/** The rows an older release left, written at the driver the way that release stored them. */
+async function insertRows(dbFile: string, rows: readonly SeedRow[]): Promise<void> {
+  const driver = new SqlDriver({ client: 'better-sqlite3', connection: { filename: dbFile }, useNullAsDefault: true });
+  try {
+    const k = (driver as any).knex;
+    const now = new Date().toISOString();
+    let n = 0;
+    for (const row of rows) {
+      n += 1;
+      await k('sys_metadata').insert({
+        id: `seed_${n}`, type: row.type, name: row.name, package_id: row.packageId ?? null, state: row.state,
+        version: 1, checksum: null, created_at: now, updated_at: now, metadata: JSON.stringify(row.body),
+      });
+    }
+  } finally {
+    await driver.disconnect();
+  }
+}
+
 /** Every `sys_metadata` row, `type/name@binding/state`, read on a connection of our own. */
 async function storedRows(dbFile: string): Promise<string[]> {
   const driver = new SqlDriver({ client: 'better-sqlite3', connection: { filename: dbFile }, useNullAsDefault: true });
@@ -215,21 +248,10 @@ beforeAll(async () => {
     extraPlugins: [new PackageServicePlugin()],
   });
   await seed.shutdown();
-  const driver = new SqlDriver({ client: 'better-sqlite3', connection: { filename: templateDb }, useNullAsDefault: true });
-  try {
-    const k = (driver as any).knex;
-    const now = new Date().toISOString();
-    let n = 0;
-    for (const row of SEED) {
-      n += 1;
-      await k('sys_metadata').insert({
-        id: `seed_${n}`, type: row.type, name: row.name, package_id: null, state: row.state,
-        version: 1, checksum: null, created_at: now, updated_at: now, metadata: JSON.stringify(row.body),
-      });
-    }
-  } finally {
-    await driver.disconnect();
-  }
+  boundTemplateDb = join(dir, 'bound-template.db');
+  copyFileSync(templateDb, boundTemplateDb);
+  await insertRows(templateDb, SEED);
+  await insertRows(boundTemplateDb, BOUND_SEED);
 }, CASE_TIMEOUT_MS);
 
 afterAll(() => {
@@ -311,3 +333,54 @@ describe('os migrate packageless-overlays', () => {
     expect(await storedRows(db)).toEqual(before);
   }, CASE_TIMEOUT_MS);
 });
+
+/**
+ * [#29392] A row bound to the read-only package itself, stored by a save
+ * naming it before that write was closed: the census lists it beside the
+ * package-less rows (`boundToBase`), and `--apply` moves it into the pair with
+ * the declaration, the same outcome. Before, the census read package-less
+ * rows only: the preview listed nothing and the row's one exit was a delete.
+ */
+describe('os migrate packageless-overlays: a row bound to the read-only package', () => {
+  it('the preview lists it with its target pair and writes nothing', async () => {
+    const db = caseDb(boundTemplateDb);
+    const before = await storedRows(db);
+    const { payload, exitCode } = await runJson(['--database-url', `file:${db}`, '--json']);
+    expect(payload.error, JSON.stringify(payload).slice(0, 600)).toBeUndefined();
+    expect(exitCode).toBe(0);
+    expect(payload.rows.map((r: any) => `${r.type}/${r.name}/${r.state} ${r.disposition} ${r.base} → ${r.pair} ${r.boundToBase} ${r.outcome}`)).toEqual([
+      `view/${VIEW}/active movable ${PKG} → ${PAIR} true listed`,
+    ]);
+    expect(payload.movable).toBe(1);
+    expect(payload.environmentRows).toBe(1);
+    expect(await storedRows(db)).toEqual(before);
+    expect(await storedPackages(db)).not.toContain(PAIR);
+  }, CASE_TIMEOUT_MS);
+
+  it('--apply moves it into the pair with its `overrides` declaration, keeps the control, serves the same customization over the package\'s document, and a re-run moves 0', async () => {
+    const db = caseDb(boundTemplateDb);
+    const served = await servedColumns(db);
+    expect(served[1]).toEqual(STORED_COLUMNS);
+
+    const { payload, exitCode } = await runJson(['--apply', '--yes', '--database-url', `file:${db}`, '--json']);
+    expect(payload.error, JSON.stringify(payload).slice(0, 600)).toBeUndefined();
+    expect(exitCode).toBe(0);
+    expect(payload.bases).toEqual([{ base: PKG, pair: PAIR, pairCreated: true, moved: 1, declarationsCompleted: 0, failed: 0, shadowed: 0 }]);
+    expect(payload.rows.map((r: any) => `${r.state} ${r.outcome} ${r.packageId} ${r.basedOn} ${r.boundToBase}`)).toEqual([
+      `active moved ${PAIR} 1.4.0 true`,
+    ]);
+    expect(await storedRows(db)).toEqual([`view/${ENV_VIEW}@-/active`, `view/${VIEW}@${PAIR}/active`]);
+    expect((await storedBody(db, { type: 'view', name: VIEW, package_id: PAIR, state: 'active' })).overrides)
+      .toEqual({ package: PKG, basedOn: '1.4.0' });
+    expect(await storedPackages(db)).toContain(PAIR);
+
+    // A fresh boot serves the customization at every read of the package's item.
+    expect(await servedColumns(db)).toEqual([STORED_COLUMNS, STORED_COLUMNS, STORED_COLUMNS, STORED_COLUMNS]);
+
+    const again = await runJson(['--apply', '--yes', '--database-url', `file:${db}`, '--json']);
+    expect(again.payload.error, JSON.stringify(again.payload).slice(0, 600)).toBeUndefined();
+    expect(again.exitCode).toBe(0);
+    expect(again.payload.moved).toBe(0);
+    expect(again.payload.rows).toEqual([]);
+  }, CASE_TIMEOUT_MS * 3);
+});

packages/cli/src/commands/migrate/packageless-overlays.ts

@@ -65,7 +65,11 @@ function reasonOf(row: CensusRow): string {
       return `'${row.base}' has no paired .custom package id, so there is nowhere to move it. Left as it is.`;
     case 'foreign_declaration':
       return `its body already carries an \`overrides\` key that does not name '${row.base}'. Left as it is: `
-        + 'remove the key with a save naming no package, then re-run: the move declares the right override.';
+        + (row.boundToBase
+          // [#29392] A save naming the base is closed, so the key cannot be removed in place.
+          ? `delete it with a delete naming '${row.base}', then save its body into '${row.pair}' declaring `
+            + `overrides: { package: '${row.base}', basedOn }.`
+          : 'remove the key with a save naming no package, then re-run: the move declares the right override.');
     case 'shadowed':
       return (row.shadowDeclares
         ? `'${row.pair}' already overrides this item (a customization made in Studio), and that row is the one `
@@ -94,6 +98,7 @@ function jsonRow(row: CensusRow, outcome?: Outcome): Record<string, unknown> {
     ...(row.pair !== undefined ? { pair: row.pair } : {}),
     ...(row.shadowDeclares !== undefined ? { shadowDeclares: row.shadowDeclares } : {}),
     ...(row.addedMember !== undefined ? { addedMember: row.addedMember } : {}),
+    ...(row.boundToBase ? { boundToBase: true } : {}),
     outcome: outcome?.outcome ?? (row.disposition === 'movable' ? 'listed' : 'left'),
     ...(outcome && outcome.outcome !== 'failed'
       ? { packageId: outcome.packageId, ...(outcome.basedOn !== undefined ? { basedOn: outcome.basedOn } : {}) }
@@ -106,7 +111,9 @@ function jsonRow(row: CensusRow, outcome?: Outcome): Record<string, unknown> {
  * `os migrate packageless-overlays` — list, and with `--apply` move, every
  * package-less overlay row of a name exactly one installed code package ships
  * into that package's paired `<base>.custom`, as a declared override
- * (ADR-0146 D11, the migration half of N-16).
+ * (ADR-0146 D11, the migration half of N-16) — and every overlay row bound to
+ * a read-only code package that ships its name, which a save naming that
+ * package stored before such a save was closed (listed `boundToBase`).
  *
  * ## Why a ceremony
  *
@@ -125,7 +132,7 @@ function jsonRow(row: CensusRow, outcome?: Outcome): Record<string, unknown> {
  * header holds the rules): the pair is ensured through its one creation door,
  * each body is saved into the pair through the save door with the declaration
  * (`basedOn` = the installed base's version now), read back as the served
- * override, and only then is the package-less row deleted. A name no package
+ * override, and only then is the moved row deleted. A name no package
  * ships is environment metadata and is not touched; a name several packages
  * ship, and a row the pair already overrides (a Studio customization made
  * first), are listed and left alone. A pair row declaring an override with no
@@ -158,8 +165,8 @@ function jsonRow(row: CensusRow, outcome?: Outcome): Record<string, unknown> {
 export default class MigratePackagelessOverlays extends Command {
   static override description =
     'List, and with --apply move, the package-less view, dashboard, report, translation and email-template rows '
-    + 'over a name one installed package ships into that package\'s paired .custom package, as declared overrides '
-    + '(ADR-0146 D11). Run once after upgrading.';
+    + 'over a name one installed package ships, and those bound to a read-only package that ships the name, into '
+    + 'that package\'s paired .custom package, as declared overrides (ADR-0146 D11). Run once after upgrading.';
 
   static override examples = [
     '$ os migrate packageless-overlays',
@@ -242,7 +249,7 @@ export default class MigratePackagelessOverlays extends Command {
         }
         printWarning(
           'Apply mode moves the listed sys_metadata rows into each base\'s paired .custom package (created when '
-          + 'absent) and deletes the package-less rows. Re-run with --yes to confirm, or run without --apply to preview.',
+          + 'absent) and deletes the rows moved. Re-run with --yes to confirm, or run without --apply to preview.',
         );
         this.exit(1);
         return;
@@ -381,7 +388,8 @@ export default class MigratePackagelessOverlays extends Command {
         console.log(chalk.bold(`${movable.length} row(s) to move:`));
         for (const row of movable) {
           // [#28904] A member the base does not ship moves in as the pair's own, declaring nothing.
-          console.log(`  • ${itemOf(row)} → ${row.pair} ${row.addedMember !== undefined
+          // [#29392] A row bound to the base (a save naming it, before that write was closed) moves the same way.
+          console.log(`  • ${itemOf(row)}${row.boundToBase ? ` bound to ${row.base}` : ''} → ${row.pair} ${row.addedMember !== undefined
             ? `(its own member; ${row.base} ships none at ${row.addedMember})`
             : `(overrides ${row.base})`}`);
         }
@@ -413,7 +421,7 @@ export default class MigratePackagelessOverlays extends Command {
         + `${line.declarationsCompleted} basedOn recorded, ${line.shadowed} shadowed, ${line.failed} failed`);
     }
     if (failed > 0) {
-      printError(`${failed} row(s) could not be moved; each package-less row that failed is kept as it was.`);
+      printError(`${failed} row(s) could not be moved; each row that failed is kept as it was.`);
     } else {
       printSuccess(`Done ${chalk.dim(`(${timer.display()})`)}. Restart the server so it serves the moved rows.`);
     }