diff --git a/docs/10.joined.md b/docs/10.joined.md index d7b5071..c41df45 100644 --- a/docs/10.joined.md +++ b/docs/10.joined.md @@ -116,6 +116,17 @@ declare userName: string; Joined fields are always sorted without a database index (`@Sortable` is forced to non-indexed for them), since the value comes from a runtime lookup rather than a stored column. +## Deferred Resolution on Lists + +`Query.List` only materializes the joined groups that the request's sort or filters reference before the count. The remaining display-only groups are deferred: the returned stream resolves them itself as soon as an operation can observe them, so destructuring `[query, total]` alone always yields rows with every `@Joined` field resolved. + +- Paging the stream (`slice`, `nth`) appends the lookups after the page boundary, so they only run on the returned rows. +- A bare `count()` — and any field aggregate (`sum`, `min`, `distinct(field)`, ...) on a non-joined field — skips them entirely (left-join lookups never change the row count nor other fields). +- `changes()` attaches to the raw stream, since change feeds cannot carry the join lookups. +- Any other operation (`filter`, `orderBy`, `map`, ...) materializes them first, since it may read the joined values. + +The page-only benefit therefore holds as long as paging is the first thing the caller does. An operation applied to the whole stream before paging — a filter, a sort, or the `lookup` that `Query.Foreign` stages for a `@Foreign` field — materializes the joined groups over the whole matched set instead. The lookups still stay out of the count pipeline, but a caller that wants them strictly on the page must pass `{ exposeDeferredJoined: true }` to `Query.List` and resolve them itself after the slice: the raw stream is then returned unchanged and the third tuple element names the joined fields left to resolve with `Query.Joined`. This is what the default list route does, which also lets it limit the page lookups to the plucked fields. + ## Cross-Schema Joins By default, the remote table is resolved in the same schema as the controller's table. Use the `schema` option to join against a table registered in a different schema; the join is resolved against that schema's default instance. diff --git a/src/components.ts b/src/components.ts index e9b33d8..d32d1b7 100644 --- a/src/components.ts +++ b/src/components.ts @@ -536,6 +536,168 @@ export namespace Query { ); } + interface DeferredJoinContext { + db: SchemaInstance; + meta: DataAPIMeta; + names: Set; + raw: Stream; + } + + /** + * Stream returned by {@link List} when the deferred joined fields are not + * exposed to the caller: it resolves them itself, so consumers that + * destructure `[query, total]` alone still observe every `@Joined` field. + * The lookups are appended after `slice`/`nth`, keeping them on the + * requested page only; bare `count()` calls and field aggregates on + * non-joined fields skip them (left-join lookups never change the row + * count nor other fields); `changes()` attaches to the raw stream, since + * change feeds cannot carry the lookups. Any other operation may observe + * the joined fields, so it materializes them first. + */ + class DeferredJoinedStream extends Stream { + declare deferredJoin: DeferredJoinContext; + } + + type DeferredJoinedOperation = ( + this: DeferredJoinedStream, + ...args: any[] + ) => any; + + function materializeDeferredJoin(stream: DeferredJoinedStream) { + const { db, meta, names, raw } = stream.deferredJoin; + return Joined(db, meta, raw, names); + } + + function pagedDeferredJoinOperation(name: string): DeferredJoinedOperation { + return function (this: DeferredJoinedStream, ...args: any[]) { + const { db, meta, names, raw } = this.deferredJoin; + const paged = (raw as any)[name](...args); + return Joined(db, meta, paged, names); + }; + } + + /** + * Resolves the joined fields through the stream form of {@link Joined} + * rather than its datum form: only the datum form has no way to drop the + * temporary key it stages (neither `Datum` nor `ValueProxy` exposes + * `without`), so it would leave `__joined_orig_*` on the row. + */ + function nthDeferredJoinOperation(): DeferredJoinedOperation { + return function (this: DeferredJoinedStream, n: number) { + const { db, meta, names, raw } = this.deferredJoin; + return Joined(db, meta, raw.slice(n, 1), names).nth(0); + }; + } + + function rawDeferredJoinOperation(name: string): DeferredJoinedOperation { + return function (this: DeferredJoinedStream, ...args: any[]) { + return (this.deferredJoin.raw as any)[name](...args); + }; + } + + function fieldAggregateDeferredJoinOperation( + name: string, + bareCallOnRaw: boolean, + ): DeferredJoinedOperation { + return function (this: DeferredJoinedStream, field?: string) { + const { db, meta, names, raw } = this.deferredJoin; + if (!field) { + const bareTarget = bareCallOnRaw ? raw : materializeDeferredJoin(this); + return (bareTarget as any)[name](); + } + const target = names.has(field) + ? Joined(db, meta, raw, new Set([field])) + : raw; + return (target as any)[name](field); + }; + } + + function forwardingDeferredJoinOperation( + name: string, + ): DeferredJoinedOperation { + return function (this: DeferredJoinedStream, ...args: any[]) { + return (materializeDeferredJoin(this) as any)[name](...args); + }; + } + + const DEFERRED_JOIN_OPERATION_FACTORIES: Record< + string, + (name: string) => DeferredJoinedOperation + > = { + slice: pagedDeferredJoinOperation, + nth: nthDeferredJoinOperation, + changes: rawDeferredJoinOperation, + count: (name) => fieldAggregateDeferredJoinOperation(name, true), + sum: (name) => fieldAggregateDeferredJoinOperation(name, false), + avg: (name) => fieldAggregateDeferredJoinOperation(name, false), + min: (name) => fieldAggregateDeferredJoinOperation(name, false), + max: (name) => fieldAggregateDeferredJoinOperation(name, false), + distinct: (name) => fieldAggregateDeferredJoinOperation(name, false), + }; + + const DEFERRED_JOIN_INHERITED_OPERATIONS = new Set(["constructor", "cast"]); + + function wireDeferredJoinedStream() { + const operationNames = [ + ...Object.getOwnPropertyNames(Stream.prototype), + "run", + "cursor", + "build", + ].filter((name) => !DEFERRED_JOIN_INHERITED_OPERATIONS.has(name)); + for (const name of operationNames) { + const factory = + DEFERRED_JOIN_OPERATION_FACTORIES[name] ?? + forwardingDeferredJoinOperation; + (DeferredJoinedStream.prototype as any)[name] = factory(name); + } + } + wireDeferredJoinedStream(); + + /** + * Wraps `raw` into a {@link DeferredJoinedStream}. The wrapper keeps the + * raw stages as its own (an unobservable fallback, since every operation + * is overridden) while `build()` serializes the materialized pipeline, so + * a wrapper embedded as an argument of another query never leaks rows + * without the joined fields. + */ + function wrapDeferredJoined( + db: SchemaInstance, + meta: DataAPIMeta, + names: Set, + raw: Stream, + ): Stream { + const wrapped: DeferredJoinedStream = Object.create( + DeferredJoinedStream.prototype, + ); + (wrapped as any).stages = raw.build(); + wrapped.deferredJoin = { db, meta, names, raw }; + return wrapped; + } + + function listResultTuple>( + tmpRequest: Stream, + total: Datum, + meta: DataAPIMeta, + joinedSplit: JoinedFieldSplit, + db?: SchemaInstance, + options?: ListOptions, + ): [sorted: Stream, total: Datum, deferredJoined: Set] { + const deferredJoined = deferredJoinedFields(meta, joinedSplit); + if (options?.exposeDeferredJoined || !db || deferredJoined.size === 0) { + return [tmpRequest, total, deferredJoined]; + } + return [ + wrapDeferredJoined( + db, + meta, + deferredJoined, + tmpRequest as Stream, + ) as Stream, + total, + new Set(), + ]; + } + function resolveSchemaDb( sourceDb: SchemaInstance, schemaName: string | undefined, @@ -784,6 +946,26 @@ export namespace Query { : table.get(id as string); } + export interface ListOptions { + /** + * Return the joined field names {@link List} did not materialize as the + * third tuple element instead of a self-resolving stream, leaving their + * page-time resolution to the caller. The default list route uses this + * to restrict the page lookups to the plucked fields. + */ + exposeDeferredJoined?: boolean; + } + + /** + * Builds the filtered and sorted stream of a list request along with its + * total count. Joined fields that no filter or sort references are not + * materialized in either pipeline; by default the returned stream resolves + * them itself once the caller pages it (or otherwise observes rows), so the + * third tuple element is empty. With + * {@link ListOptions.exposeDeferredJoined} — or when no `db` is provided, + * since no lookup can be built without one — the raw stream is returned + * and the third element names the joined fields left for the caller. + */ export function List>( obj: any, meta: DataAPIMeta, @@ -792,6 +974,7 @@ export namespace Query { sorting?: [string, "asc" | "desc" | undefined], filters?: Record, db?: SchemaInstance, + options?: ListOptions, ): [sorted: Stream, total: Datum, deferredJoined: Set] { const filterList = Object.entries(meta.filters).filter( ([name]) => filters && name in filters, @@ -881,7 +1064,7 @@ export namespace Query { if (shouldSort && !shouldSort.indexed && sortField) { tmpRequest = tmpRequest.orderBy(sortField, sorting?.[1] ?? "asc"); } - return [tmpRequest, total, deferredJoinedFields(meta, joinedSplit)]; + return listResultTuple(tmpRequest, total, meta, joinedSplit, db, options); } export function Delete(table: Table, id: string | string[]) { diff --git a/src/index.ts b/src/index.ts index 0c6f469..2f2513a 100644 --- a/src/index.ts +++ b/src/index.ts @@ -221,6 +221,7 @@ export namespace DefaultRoutes { sort, params?.filters, model.database, + { exposeDeferredJoined: true }, ); const pluck: Set | undefined = diff --git a/src/tests/components/deferred_joined.test.ts b/src/tests/components/deferred_joined.test.ts new file mode 100644 index 0000000..e13e702 --- /dev/null +++ b/src/tests/components/deferred_joined.test.ts @@ -0,0 +1,452 @@ +import path from "node:path"; +import type { RequestContext } from "@antelopejs/interface-api"; +import { Controller } from "@antelopejs/interface-api"; +import { + DataController, + DefaultRoutes, + GetDataControllerMeta, + RegisterDataController, +} from "@antelopejs/interface-data-api"; +import { Query } from "@antelopejs/interface-data-api/components"; +import { + Access, + AccessMode, + Filter, + Joined, + Listable, + ModelReference, + Sortable, +} from "@antelopejs/interface-data-api/metadata"; +import { + Schema, + type Stream, + type ValueProxy, +} from "@antelopejs/interface-database"; +import { + BasicDataModel, + Field, + Model, + RegisterSchema, + RegisterTable, + Table, +} from "@antelopejs/interface-database-decorators"; +import { expect } from "chai"; +import { getSchemaInstance } from "../utils"; + +const currentTestName = path + .basename(__filename) + .replace(/\.test\.(ts|js)$/, ""); +const authorTableName = `authors-${currentTestName}`; +const bookTableName = `books-${currentTestName}`; +const schemaName = "default"; + +@RegisterTable(authorTableName, schemaName) +class Author extends Table { + declare _id: string; + + @Field("string") + declare name: string; + + @Field("string") + declare email: string; +} +class AuthorModel extends BasicDataModel(Author, authorTableName) {} + +@RegisterTable(bookTableName, schemaName) +class Book extends Table { + declare _id: string; + + @Field("string") + declare authorId: string; + + @Field("string") + declare title: string; +} +class BookModel extends BasicDataModel(Book, bookTableName) {} + +@RegisterDataController() +class _DeferredJoinedTestAPI extends DataController( + Book, + { + list: DefaultRoutes.List, + }, + Controller(`/${currentTestName}`), +) { + @ModelReference() + @Model(BookModel) + declare bookModel: BookModel; + + @Listable() + @Access(AccessMode.ReadOnly) + declare _id: string; + + @Listable() + @Access(AccessMode.ReadWrite) + declare authorId: string; + + @Listable() + @Access(AccessMode.ReadWrite) + declare title: string; + + @Listable(["authorId"]) + @Joined({ + table: authorTableName, + localKey: "authorId", + remoteField: "name", + }) + @Sortable({ noIndex: true }) + @Filter() + declare name: string; + + @Listable(["authorId"]) + @Joined({ + table: authorTableName, + localKey: "authorId", + remoteField: "email", + }) + declare email: string; +} + +interface BookListed { + _id: string; + authorId: string; + title: string; + name: string | null; + email: string | null; +} + +interface PipelineStage { + stage: string; +} + +interface PipelineStaged { + build(): PipelineStage[]; +} + +interface ListHarness { + thisObj: _DeferredJoinedTestAPI; + model: BookModel; +} + +const authorsDataset: Partial[] = [ + { name: "Alice Carter", email: "alice@example.com" }, + { name: "Bob Stone", email: "bob@example.com" }, + { name: "Carol Wilde", email: "carol@example.com" }, +]; + +const orphanAuthorId = "orphan-author-id"; +const reqCtx = {} as RequestContext; + +describe("Query.List deferred joined self-resolution", () => { + it("resolves joined fields on a paged two-element destructure", async () => + await resolvesJoinedFieldsOnPagedTwoElementDestructure()); + it("keeps page lookups after the slice and off the count pipeline", async () => + await keepsLookupsAfterSliceAndOffCountPipeline()); + it("materializes joined fields for a caller-added filter", async () => + await materializesJoinedFieldsForCallerFilter()); + it("resolves joined fields on a directly awaited stream", async () => + await resolvesJoinedFieldsOnDirectAwait()); + it("resolves joined fields on an nth datum", async () => + await resolvesJoinedFieldsOnNthDatum()); + it("orders by a joined field when the caller sorts the stream", async () => + await ordersByJoinedFieldOnCallerSort()); + it("materializes the whole joined group when sorting through List", async () => + await materializesJoinedGroupOnListSort()); + it("exposes the deferred set and raw stream only on demand", async () => + await exposesDeferredSetOnDemand()); + it("keeps the change feed on the raw pipeline", async () => + await keepsChangeFeedOnRawPipeline()); + it("scopes field aggregates to the joined fields they read", async () => + await scopesFieldAggregatesToJoinedFields()); + it("still pages the lookups after a cast", async () => + await stillPagesLookupsAfterCast()); + it("materializes joined fields when embedded in another query", async () => + await materializesJoinedFieldsWhenEmbedded()); + it("resolves joined fields on async iteration", async () => + await resolvesJoinedFieldsOnAsyncIteration()); + it("returns the unresolved names when no database is provided", async () => + await returnsUnresolvedNamesWithoutDatabase()); +}); + +async function _seedTables(): Promise { + await RegisterSchema(schemaName); + const schema = Schema.get(schemaName); + if (schema) { + await schema.instance().table(bookTableName).delete(); + await schema.instance().table(authorTableName).delete(); + } + + const schemaInstance = getSchemaInstance(schemaName); + const authorModel = new AuthorModel(schemaInstance); + const bookModel = new BookModel(schemaInstance); + + const authorIdsRecord = await authorModel.insert(authorsDataset); + const authorIds = Object.values(authorIdsRecord); + + const booksDataset: Partial[] = [ + { authorId: authorIds[0], title: "Alpha Rising" }, + { authorId: authorIds[1], title: "Beta Stories" }, + { authorId: authorIds[2], title: "Gamma Tales" }, + { authorId: authorIds[0], title: "Alpha Returns" }, + { authorId: orphanAuthorId, title: "Lost Chapter" }, + ]; + await bookModel.insert(booksDataset); + + const thisObj: _DeferredJoinedTestAPI = Object.create( + _DeferredJoinedTestAPI.prototype, + ); + return { thisObj, model: bookModel }; +} + +function listAsConsumer(harness: ListHarness) { + const meta = GetDataControllerMeta(harness.thisObj); + return Query.List( + harness.thisObj, + meta, + harness.model.table, + reqCtx, + undefined, + undefined, + harness.model.database, + ); +} + +function stageNames(staged: PipelineStaged): string[] { + return staged.build().map((entry) => entry.stage); +} + +async function resolvesJoinedFieldsOnPagedTwoElementDestructure() { + const harness = await _seedTables(); + const [query, queryTotal] = listAsConsumer(harness); + + const page = (await query + .slice(0, 10) + .pluck("_internal", "title", "name", "email")) as BookListed[]; + expect(await queryTotal).to.equal(5); + expect(page).to.have.lengthOf(5); + + const alphaRising = page.find((book) => book.title === "Alpha Rising"); + expect(alphaRising?.name).to.equal("Alice Carter"); + expect(alphaRising?.email).to.equal("alice@example.com"); + + const lostChapter = page.find((book) => book.title === "Lost Chapter"); + expect(lostChapter?.name).to.equal(null); + expect(lostChapter?.email).to.equal(null); +} + +async function keepsLookupsAfterSliceAndOffCountPipeline() { + const harness = await _seedTables(); + const [query, queryTotal] = listAsConsumer(harness); + + const countStages = stageNames(queryTotal); + expect(countStages).to.include("count"); + expect(countStages).to.not.include("lookup"); + + const directCountStages = stageNames(query.count()); + expect(directCountStages).to.include("count"); + expect(directCountStages).to.not.include("lookup"); + + const pageStages = stageNames(query.slice(0, 2)); + expect(pageStages.indexOf("lookup")).to.be.greaterThan( + pageStages.indexOf("slice"), + ); +} + +async function materializesJoinedFieldsForCallerFilter() { + const harness = await _seedTables(); + let [query] = listAsConsumer(harness); + + query = query.filter((row) => + (row as unknown as ValueProxy).key("name").eq("Alice Carter"), + ) as Stream; + + const matches = (await query) as BookListed[]; + expect(matches).to.have.lengthOf(2); + for (const match of matches) { + expect(match.name).to.equal("Alice Carter"); + expect(match.email).to.equal("alice@example.com"); + } + expect(await query.count()).to.equal(2); +} + +async function resolvesJoinedFieldsOnDirectAwait() { + const harness = await _seedTables(); + const [query] = listAsConsumer(harness); + + const rows = (await query) as BookListed[]; + expect(rows).to.have.lengthOf(5); + const betaStories = rows.find((book) => book.title === "Beta Stories"); + expect(betaStories?.name).to.equal("Bob Stone"); + expect(betaStories?.email).to.equal("bob@example.com"); +} + +async function resolvesJoinedFieldsOnNthDatum() { + const harness = await _seedTables(); + const [query] = listAsConsumer(harness); + + const first = (await query.nth(0)) as BookListed; + expect(first.title).to.be.a("string"); + expect(first).to.have.property("name"); + expect(first).to.have.property("email"); + + const internalKeys = Object.keys(first).filter((key) => + key.startsWith("__joined_orig_"), + ); + expect(internalKeys).to.deep.equal([]); + expect(first.authorId).to.be.a("string"); + + const second = (await query.nth(1)) as BookListed; + expect(second.title).to.not.equal(first.title); +} + +async function ordersByJoinedFieldOnCallerSort() { + const harness = await _seedTables(); + const [query] = listAsConsumer(harness); + + const rows = (await (query as unknown as Stream).orderBy( + "name", + "desc", + )) as BookListed[]; + expect(rows).to.have.lengthOf(5); + const names = rows.map((book) => book.name).filter((name) => name !== null); + expect(names).to.deep.equal([...names].sort().reverse()); + expect(names[0]).to.equal("Carol Wilde"); +} + +async function materializesJoinedGroupOnListSort() { + const harness = await _seedTables(); + const meta = GetDataControllerMeta(harness.thisObj); + const [query, , deferredJoined] = Query.List( + harness.thisObj, + meta, + harness.model.table, + reqCtx, + ["name", "asc"], + undefined, + harness.model.database, + ); + + expect(deferredJoined.size).to.equal(0); + const rows = (await query.slice(0, 5)) as BookListed[]; + const gammaTales = rows.find((book) => book.title === "Gamma Tales"); + expect(gammaTales?.name).to.equal("Carol Wilde"); + expect(gammaTales?.email).to.equal("carol@example.com"); +} + +async function keepsChangeFeedOnRawPipeline() { + const harness = await _seedTables(); + const [query] = listAsConsumer(harness); + + const feedStages = stageNames(query.changes()); + expect(feedStages).to.include("changes"); + expect(feedStages).to.not.include("lookup"); +} + +async function scopesFieldAggregatesToJoinedFields() { + const harness = await _seedTables(); + const [query] = listAsConsumer(harness); + + const listedQuery = query as unknown as Stream; + const joinedFieldStages = stageNames(listedQuery.count("name")); + expect(joinedFieldStages).to.include("count"); + expect(joinedFieldStages).to.include("lookup"); + + const plainFieldStages = stageNames(query.count("title")); + expect(plainFieldStages).to.include("count"); + expect(plainFieldStages).to.not.include("lookup"); + + const distinctRowStages = stageNames( + (query as Stream).distinct() as unknown as PipelineStaged, + ); + expect(distinctRowStages).to.include("distinct"); + expect(distinctRowStages).to.include("lookup"); +} + +async function stillPagesLookupsAfterCast() { + const harness = await _seedTables(); + const [query] = listAsConsumer(harness); + + const pageStages = stageNames(query.cast().slice(0, 2)); + expect(pageStages.indexOf("lookup")).to.be.greaterThan( + pageStages.indexOf("slice"), + ); + + const page = (await query.cast().slice(0, 10)) as BookListed[]; + const alphaRising = page.find((book) => book.title === "Alpha Rising"); + expect(alphaRising?.name).to.equal("Alice Carter"); +} + +async function materializesJoinedFieldsWhenEmbedded() { + const harness = await _seedTables(); + const [query] = listAsConsumer(harness); + + const unioned = (await (harness.model.table as unknown as Stream).union( + query, + )) as BookListed[]; + expect(unioned).to.have.lengthOf(10); + const withJoined = unioned.filter((row) => "name" in row); + expect(withJoined).to.have.lengthOf(5); + expect( + withJoined.filter((row) => row.name === "Alice Carter"), + ).to.have.lengthOf(2); +} + +async function resolvesJoinedFieldsOnAsyncIteration() { + const harness = await _seedTables(); + const [query] = listAsConsumer(harness); + + const names = new Set(); + for await (const row of query as unknown as Stream) { + names.add(row.name); + if (names.has("Alice Carter") && names.has("Bob Stone")) { + break; + } + } + expect(names.has("Alice Carter")).to.equal(true); + expect(names.has("Bob Stone")).to.equal(true); +} + +async function returnsUnresolvedNamesWithoutDatabase() { + const harness = await _seedTables(); + const meta = GetDataControllerMeta(harness.thisObj); + + const [query, , deferredJoined] = Query.List( + harness.thisObj, + meta, + harness.model.table, + reqCtx, + ); + + expect(Array.from(deferredJoined).sort()).to.deep.equal(["email", "name"]); + const pageStages = stageNames(query.slice(0, 5)); + expect(pageStages).to.not.include("lookup"); +} + +async function exposesDeferredSetOnDemand() { + const harness = await _seedTables(); + const meta = GetDataControllerMeta(harness.thisObj); + + const [, , resolvedByLayer] = listAsConsumer(harness); + expect(resolvedByLayer.size).to.equal(0); + + const [rawQuery, rawTotal, deferredJoined] = Query.List( + harness.thisObj, + meta, + harness.model.table, + reqCtx, + undefined, + undefined, + harness.model.database, + { exposeDeferredJoined: true }, + ); + + expect(Array.from(deferredJoined).sort()).to.deep.equal(["email", "name"]); + expect(await rawTotal).to.equal(5); + + const rawRows = (await rawQuery.slice(0, 5)) as BookListed[]; + expect(rawRows).to.have.lengthOf(5); + for (const row of rawRows) { + expect(row).to.not.have.property("name"); + expect(row).to.not.have.property("email"); + } +}