fix(spec): os migrate meta --stored lists a stored object bound external to a stored or registered managed datasource

.changeset/25279-stored-external-binding-known-datasources.md

@@ -0,0 +1,15 @@
+---
+'@objectstack/spec': patch
+'@objectstack/metadata-protocol': patch
+---
+
+fix(spec): `os migrate meta --stored` lists a stored object bound `external` to a stored or registered managed datasource
+
+Clause-②: yes (widening)
+
+Metadata-protocol: none
+
+The stored-shape listing for `object-external-binding-managed-datasource-refused` resolved an object's named datasource only from the stack being converted. A stored object row reaches the conversion chain alone, so the listing judged only a binding on the primary datasource (`'default'`), while the object save door also refuses one on a datasource stored or registered beside the object. The stored-metadata pass therefore did not list a stored object row bound `external` to a managed datasource, the row whose next save is refused with `422 INVALID_METADATA`.
+
+- **Now:** for each stored `object` row carrying `external` and naming a datasource, the pass reads that datasource the way the object save door does (the environment's stored row, else the definition the code layer registers; both go through one read) and hands it to the chain. The row is listed under `rows[].todos` with this entry's id exactly when that door refuses its next save. A datasource declared `schemaMode: 'external'` or `'validate-only'`, and a datasource no layer holds, give no listing, as at the door. Nothing is rewritten.
+- **`@objectstack/spec`:** `ApplyConversionsOptions` (and so `StoredConversionOptions` for `applyConversionsToStoredItem`) gains an optional `knownDatasources`, a map of datasource definitions by name that the caller's door can see beyond the stack. Only the refused-shape listing reads it, so only when an `onTodo` sink is passed. A datasource the stack declares is read from the stack first. Without it, behaviour is unchanged.

packages/metadata-protocol/src/protocol.external-binding-named-datasource-refusal.test.ts

@@ -327,3 +327,76 @@ describe('[#25211] controls — what the doors do not refuse', () => {
         expect((await protocol.saveMetaItem({ type: 'datasource', name: 'wh', item: datasource('wh') })).success).toBe(true);
     });
 });
+
+/**
+ * [#25279] `os migrate meta --stored` lists the object rows the object save
+ * door above refuses: a stored object bound `external` to a stored or
+ * registered managed datasource. The row is planted as stored before that door
+ * existed (straight into the table), then the pass is run in preview, and the
+ * same body is offered to the door, so the listing and the refusal are read
+ * off one environment.
+ */
+describe('[#25279] the stored-migration pass lists the object rows that door refuses', () => {
+    const plantObjectRow = (rows: Map<string, Row>, body: Record<string, unknown>) => {
+        const id = `r_${rows.size + 1}`;
+        rows.set(id, {
+            id, type: 'object', name: String(body.name), state: 'active', package_id: null,
+            metadata: JSON.stringify(body),
+        });
+    };
+    const LISTED_ID = 'object-external-binding-managed-datasource-refused';
+    const listedRows = (report: any) =>
+        report.rows
+            .filter((r: any) => r.todos.some((t: any) => t.conversionId === LISTED_ID))
+            .map((r: any) => `${r.type}/${r.name}|${r.outcome}`);
+
+    for (const [shape, dsExtra] of MANAGED_SHAPES) {
+        it(`a stored object on a stored datasource with ${shape} is listed, and its next save is refused`, async () => {
+            const { protocol, rows } = makeSession();
+            await protocol.saveMetaItem({ type: 'datasource', name: 'wh', item: datasource('wh', dsExtra) });
+            plantObjectRow(rows, object('wh_order'));
+
+            const report = await protocol.migrateStoredMetadata({});
+
+            expect(listedRows(report)).toEqual(['object/wh_order|skipped']);
+            const todo = report.rows.find((r: any) => r.name === 'wh_order').todos.find((t: any) => t.conversionId === LISTED_ID);
+            expect(todo.path).toBe('objects[0].external');
+            expect(todo.reason).toContain(managedFinding('wh_order'));
+            const err = await refusalOf(protocol.saveMetaItem({ type: 'object', name: 'wh_order', item: object('wh_order') }));
+            expectRefusal(err, 'object/wh_order', 'external', ['wh_order']);
+        });
+    }
+
+    it('a stored object on a datasource the code layer registers as managed is listed the same way', async () => {
+        const { protocol, rows } = makeSession({ registryDatasources: { wh: datasource('wh', { schemaMode: 'managed' }) } });
+        plantObjectRow(rows, object('wh_order'));
+
+        expect(listedRows(await protocol.migrateStoredMetadata({}))).toEqual(['object/wh_order|skipped']);
+    });
+
+    it("the environment's stored datasource row is the one judged, over the registered definition", async () => {
+        const { protocol, rows } = makeSession({ registryDatasources: { wh: datasource('wh', { schemaMode: 'managed' }) } });
+        await protocol.saveMetaItem({ type: 'datasource', name: 'wh', item: datasource('wh', FEDERATED) });
+        plantObjectRow(rows, object('wh_order'));
+
+        const report = await protocol.migrateStoredMetadata({});
+
+        expect(listedRows(report)).toEqual([]);
+        expect(report.canonical).toBeGreaterThanOrEqual(1);
+        await expect(protocol.saveMetaItem({ type: 'object', name: 'wh_order', item: object('wh_order') })).resolves.toBeDefined();
+    });
+
+    it.each([
+        ['a federated stored datasource', (p: any) => p.saveMetaItem({ type: 'datasource', name: 'wh', item: datasource('wh', FEDERATED) })],
+        ['a datasource no layer holds', async () => {}],
+    ])('a stored object on %s is not listed', async (_label, seed) => {
+        const { protocol, rows } = makeSession();
+        await seed(protocol);
+        plantObjectRow(rows, object('wh_order'));
+
+        const report = await protocol.migrateStoredMetadata({});
+
+        expect(listedRows(report)).toEqual([]);
+        expect(report.rows.filter((r: any) => r.name === 'wh_order')).toEqual([]);
+    });
+});

packages/metadata-protocol/src/protocol.ts

@@ -6817,11 +6817,15 @@ export class ObjectStackProtocolImplementation implements
      * site as stored and emits NO notice for it, so "no notices" alone means
      * "nothing to persist", never "on protocol": {@link migrateStoredMetadata}
      * reads `todos` to tell the two apart. They change nothing in `item`.
+     *
+     * [#25279] `knownDatasources` is handed to the chain for the refused-shape
+     * listing ({@link storedRowKnownDatasources}); it changes nothing either.
      */
     private convertStoredItemDetailed(
         type: string,
         data: unknown,
         onNotice?: (notice: ConversionNotice) => void,
+        knownDatasources?: ReadonlyMap<string, { readonly schemaMode?: unknown }>,
     ): { item: unknown; notices: ConversionNotice[]; todos: ConversionTodoNotice[] } {
         const singular = PLURAL_TO_SINGULAR[type] ?? type;
         if (singular === 'flow') return { item: data, notices: [], todos: [] };
@@ -6835,10 +6839,37 @@ export class ObjectStackProtocolImplementation implements
             onTodo: (t) => {
                 todos.push(t);
             },
+            ...(knownDatasources ? { knownDatasources } : {}),
         });
         return { item, notices, todos };
     }
 
+    /**
+     * [ADR-0015 §4.2 · #25279] The datasource a stored `object` row's
+     * `external` binding names, as {@link externalBindingSaveRefusal} would
+     * read it at that row's next save ({@link externalBindingDatasourceDefinition}),
+     * for {@link migrateStoredMetadata}'s refused-shape listing.
+     *
+     * A stored object row reaches the conversion chain alone, so the listing's
+     * `external` detector could answer only the primary half (`'default'`)
+     * while the save door also refuses a binding on a stored or registered
+     * managed datasource. Handed down here, the listing names the row exactly
+     * when that door refuses it. `undefined` for any other row, a row with no
+     * `external`, and a datasource no layer holds (no verdict, as at the door).
+     * A read that fails propagates, as it does at the door.
+     */
+    private async storedRowKnownDatasources(
+        singularType: string,
+        body: unknown,
+    ): Promise<ReadonlyMap<string, { readonly schemaMode?: unknown }> | undefined> {
+        if (singularType !== 'object' || !body || typeof body !== 'object' || Array.isArray(body)) return undefined;
+        const { external, datasource } = body as { external?: unknown; datasource?: unknown };
+        if (external === undefined || external === null) return undefined;
+        if (typeof datasource !== 'string' || datasource === '') return undefined;
+        const definition = await this.externalBindingDatasourceDefinition(datasource);
+        return definition ? new Map([[datasource, definition]]) : undefined;
+    }
+
     /**
      * Resolve a flow canonicalizer from the live services registry (#4498).
      *
@@ -22553,11 +22584,8 @@ export class ObjectStackProtocolImplementation implements
             const datasource = typeof body.datasource === 'string' && body.datasource !== '' ? body.datasource : undefined;
             const definition = datasource === undefined
                 ? undefined
-                : (await this.readMetaItem({ type: 'datasource', name: datasource }))?.item;
-            const refusal = externalBindingDatasourceRefusal(
-                name, body.external, datasource,
-                definition && typeof definition === 'object' ? definition as { schemaMode?: unknown } : undefined,
-            );
+                : await this.externalBindingDatasourceDefinition(datasource);
+            const refusal = externalBindingDatasourceRefusal(name, body.external, datasource, definition);
             if (refusal) issues.push({ path: refusal.path, message: refusal.message, code: 'custom' });
         } else if (singularType === 'datasource') {
             for (const bound of await this.objectsBoundExternallyTo(name)) {
@@ -22575,6 +22603,22 @@ export class ObjectStackProtocolImplementation implements
         return err;
     }
 
+    /**
+     * [#25211 · #25279] The definition of the datasource an `external` object
+     * names, as this environment resolves it by name ({@link readMetaItem}: the
+     * stored row, else the definition the code layer registers), or `undefined`
+     * when no layer holds one. The ONE read behind the object half of
+     * {@link externalBindingSaveRefusal} and the stored-migration pass's listing
+     * of the same refusal ({@link storedRowKnownDatasources}), so the two cannot
+     * judge a binding against different datasources.
+     */
+    private async externalBindingDatasourceDefinition(
+        datasource: string,
+    ): Promise<{ readonly schemaMode?: unknown } | undefined> {
+        const definition = (await this.readMetaItem({ type: 'datasource', name: datasource }))?.item;
+        return definition && typeof definition === 'object' ? definition as { schemaMode?: unknown } : undefined;
+    }
+
     /**
      * [#25211] Every object this environment holds that binds `datasource`
      * with `external`, for {@link externalBindingSaveRefusal}'s datasource
@@ -25058,9 +25102,15 @@ export class ObjectStackProtocolImplementation implements
             // flow carrying a `{…}` token at a value slot — a CRUD filter
             // operand among them — parsed, rewrote nothing and was counted
             // `canonical`, while `registerFlow` refused it at boot.
+            //
+            // [#25279] A stored object's `external` binding is judged against the
+            // datasource the save door reads for it, so the listing names the
+            // row exactly when that door refuses its next save.
             const { item, notices, todos } = flowResult
                 ? { item: flowResult.storable, notices: flowResult.notices, todos: flowResult.todos ?? [] }
-                : this.convertStoredItemDetailed(singular, body);
+                : this.convertStoredItemDetailed(
+                    singular, body, undefined, await this.storedRowKnownDatasources(singular, body),
+                );
             const changed = flowResult ? item !== body : notices.length > 0;
             // [#17321] A site the chain left as stored — no lossless rewrite
             // exists for it (ADR-0087 D3: a structured TODO, never silence) —

packages/spec/src/conversions/apply.ts

@@ -80,6 +80,21 @@ export interface ApplyConversionsOptions {
    * nothing of it runs, and in either case nothing is rewritten.
    */
   onTodo?: (todo: ConversionTodoNotice) => void;
+  /**
+   * [#25279] Datasource definitions the caller's door can see beyond the stack
+   * being converted, by name. Read only by the refused-shape listing above
+   * (so only with an `onTodo` sink), and there only by the `external`-binding
+   * detector: a stored object row arrives alone, so without this the listing
+   * could judge only a binding on the primary datasource, while the save door
+   * also judges one on a datasource stored or registered beside it.
+   *
+   * Each value is the named datasource's definition as the caller's door
+   * resolves it (an absent `schemaMode` is the spec default, `'managed'`); a
+   * name absent from the map gets no verdict, exactly as a door that sees no
+   * definition gives none. A datasource the stack itself declares is read from
+   * the stack first. Nothing is converted from it and nothing is rewritten.
+   */
+  knownDatasources?: ReadonlyMap<string, { readonly schemaMode?: unknown }>;
   /**
    * Conversion ids this seam refuses to replay, whatever `includeRetired`
    * says. Empty/absent by default: every seam replays the whole window it
@@ -122,7 +137,7 @@ export function applyConversions(
   stack: Record<string, unknown>,
   options: ApplyConversionsOptions = {},
 ): Record<string, unknown> {
-  const { onNotice, onTodo, includeRetired = false, excludeConversionIds } = options;
+  const { onNotice, onTodo, includeRetired = false, excludeConversionIds, knownDatasources } = options;
   const excluded = excludeConversionIds && excludeConversionIds.length > 0
     ? new Set(excludeConversionIds)
     : null;
@@ -185,7 +200,7 @@ export function applyConversions(
   // judged on the converted stack and listed through the same sink: a row the
   // chain leaves refused must not read as already on protocol.
   if (onTodo) {
-    reportRefusedStoredShapes(current, (refused, detail) =>
+    reportRefusedStoredShapes(current, { knownDatasources }, (refused, detail) =>
       onTodo({
         code: CONVERSION_TODO_CODE,
         conversionId: refused.id,

packages/spec/src/conversions/refused-shapes.test.ts

@@ -36,12 +36,17 @@ function todosOf(stack: Record<string, unknown>): { stack: Record<string, unknow
   return { stack: out, todos };
 }
 
-function storedTodos(type: string, row: Record<string, unknown>) {
+function storedTodos(
+  type: string,
+  row: Record<string, unknown>,
+  knownDatasources?: ReadonlyMap<string, { readonly schemaMode?: unknown }>,
+) {
   const todos: ConversionTodoNotice[] = [];
   const notices: ConversionNotice[] = [];
   const item = applyConversionsToStoredItem(type, row, {
     onTodo: (t) => todos.push(t),
     onNotice: (n) => notices.push(n),
+    ...(knownDatasources ? { knownDatasources } : {}),
   }) as Record<string, any>;
   return { item, todos, notices };
 }
@@ -534,3 +539,63 @@ describe('#25168 — a stored object whose `external` binding is on a managed da
     expect(todosOf(stack).todos.filter((t) => t.conversionId === 'object-external-binding-managed-datasource-refused')).toEqual([]);
   });
 });
+
+describe('#25279 — a stored object bound `external` to a datasource the caller resolves is judged against it', () => {
+  const listed = (todos: ConversionTodoNotice[]) =>
+    todos.filter((t) => t.conversionId === 'object-external-binding-managed-datasource-refused');
+
+  it.each<[string, Record<string, unknown>]>([
+    ["declared `schemaMode: 'managed'`", { name: 'warehouse', driver: 'sqlite', config: {}, schemaMode: 'managed' }],
+    ['declaring no `schemaMode` (managed by the spec default)', { name: 'warehouse', driver: 'sqlite', config: {} }],
+  ])('a stored row on a known datasource %s is listed, once, and not rewritten', (_label, datasource) => {
+    const row = federatedRow({ datasource: 'warehouse' });
+    const { item, todos, notices } = storedTodos('object', row, new Map([['warehouse', datasource]]));
+    expect(item).toEqual(row);
+    expect(notices).toEqual([]);
+    expect(todos).toHaveLength(1);
+    const todo = todos[0]!;
+    expect(todo.conversionId).toBe('object-external-binding-managed-datasource-refused');
+    expect(todo.path).toBe('objects[0].external');
+    expect(todo.reason).toContain("Object 'wh_order' declares `external` on datasource 'warehouse', whose `schemaMode` is 'managed'");
+    expect(JSON.parse(todo.from)).toEqual({ datasource: 'warehouse', external: { remoteName: 'orders' } });
+  });
+
+  it.each([
+    ["a federated datasource (`schemaMode: 'external'`)", 'external'],
+    ["a `schemaMode: 'validate-only'` datasource", 'validate-only'],
+  ])('a stored row on %s is not listed', (_label, schemaMode) => {
+    const known = new Map([['warehouse', { name: 'warehouse', schemaMode }]]);
+    expect(listed(storedTodos('object', federatedRow({ datasource: 'warehouse' }), known).todos)).toEqual([]);
+  });
+
+  it('a known datasource the row does not name gives no verdict', () => {
+    const known = new Map([['other', { name: 'other', schemaMode: 'managed' }]]);
+    expect(listed(storedTodos('object', federatedRow({ datasource: 'warehouse' }), known).todos)).toEqual([]);
+  });
+
+  it('a datasource the stack declares is read before the caller\'s', () => {
+    const stack = {
+      datasources: [{ name: 'warehouse', driver: 'sqlite', config: {}, schemaMode: 'external', external: {} }],
+      objects: [federatedRow({ datasource: 'warehouse' })],
+    };
+    const todos: ConversionTodoNotice[] = [];
+    applyConversions(stack, {
+      includeRetired: true,
+      onTodo: (t) => todos.push(t),
+      knownDatasources: new Map([['warehouse', { schemaMode: 'managed' }]]),
+    });
+    expect(listed(todos)).toEqual([]);
+  });
+
+  it('without an `onTodo` sink nothing reads the known datasources', () => {
+    const known = new Map<string, { schemaMode?: unknown }>();
+    const get = known.get.bind(known);
+    let reads = 0;
+    known.get = (name: string) => {
+      reads++;
+      return get(name);
+    };
+    applyConversionsToStoredItem('object', federatedRow({ datasource: 'warehouse' }), { knownDatasources: known });
+    expect(reads).toBe(0);
+  });
+});