From 77a59ce69ce7b1562151f1761c5b26813a5823d4 Mon Sep 17 00:00:00 2001 From: MrSociety404 Date: Fri, 10 Jul 2026 10:40:05 +0200 Subject: [PATCH 1/3] perf: defer unused joined-field lookups to the paginated result set Query.List applied every @Joined lookup to the full stream before the filter, the count() and the list route's slice(), so list endpoints paid for all joined groups across the entire matched set (twice, counting the count aggregation) even when the request neither sorted nor filtered by them. Split joined groups the same way computed fields already are: groups referenced by filters (or forced in by a filtered computed field, whose expression may read joined fields) stay before the filter and count; groups needed only for sorting (or whose local key a later Foreign lookup replaces) are applied after count() and before orderBy, so their lookups never enter the count pipeline; the remaining display-only groups are resolved by the list route after slice(), so their lookups only run on the returned page. Joined() gains an optional `only` set and the new Query.DeferredJoinedFields() exposes the deferred names to the route. Left-join lookups never change the row count, so totals and responses are unchanged. Refs #17 --- src/components.ts | 100 +++++++++++++++++++++++++++- src/index.ts | 22 ++++++ src/tests/components/joined.test.ts | 31 +++++++++ 3 files changed, 150 insertions(+), 3 deletions(-) diff --git a/src/components.ts b/src/components.ts index 6c97d2c..40fc9ef 100644 --- a/src/components.ts +++ b/src/components.ts @@ -463,6 +463,81 @@ export namespace Query { return Array.from(groups.values()); } + interface JoinedFieldSplit { + filter: Set; + sort: Set; + } + + function addAll(target: Set, names: string[]) { + for (const name of names) { + target.add(name); + } + } + + /** + * Splits the joined fields of `meta` between those {@link List} must + * materialize before the filter and count (`filter`) and those it only + * needs after the count (`sort`). Joined fields referenced by neither are + * left out so the list route can defer their lookups to the returned page. + * + * Computed expressions may read joined fields, so materializing a computed + * field pulls every remaining joined group in with it. Groups whose local + * key is also a foreign field must resolve before `Foreign` replaces the + * key with the looked-up record. + */ + function splitJoinedFields( + meta: DataAPIMeta, + sorting?: [string, "asc" | "desc" | undefined], + filters?: Record, + ): JoinedFieldSplit { + const split: JoinedFieldSplit = { filter: new Set(), sort: new Set() }; + const filterNames = Object.keys(meta.filters).filter( + (name) => filters && name in filters, + ); + const hasComputedFilter = filterNames.some( + (name) => meta.fields[name]?.computed, + ); + const sortField = sorting?.[0]; + const hasComputedSort = !!sortField && !!meta.fields[sortField]?.computed; + for (const group of collectJoinedGroups(meta)) { + const names = group.fields.map((field) => field.name); + if ( + hasComputedFilter || + names.some((name) => filterNames.includes(name)) + ) { + addAll(split.filter, names); + } else if ( + hasComputedSort || + (sortField && names.includes(sortField)) || + meta.fields[group.localKey]?.foreign + ) { + addAll(split.sort, names); + } + } + return split; + } + + /** + * Joined field names that {@link List} does not materialize for the given + * sort/filter parameters. The default list route resolves these after + * pagination so their lookups only run on the returned page. + */ + export function DeferredJoinedFields( + meta: DataAPIMeta, + sorting?: [string, "asc" | "desc" | undefined], + filters?: Record, + ): Set { + const split = splitJoinedFields(meta, sorting, filters); + return new Set( + Object.entries(meta.fields) + .filter( + ([name, field]) => + field.joined && !split.filter.has(name) && !split.sort.has(name), + ) + .map(([name]) => name), + ); + } + function resolveSchemaDb( sourceDb: SchemaInstance, schemaName: string | undefined, @@ -477,18 +552,23 @@ export namespace Query { db: SchemaInstance, meta: DataAPIMeta, query: Stream, + only?: Set, ): Stream; export function Joined( db: SchemaInstance, meta: DataAPIMeta, query: Datum, + only?: Set, ): Datum; export function Joined( db: SchemaInstance, meta: DataAPIMeta, query: Stream | Datum, + only?: Set, ): Stream | Datum { - const groups = collectJoinedGroups(meta); + const groups = collectJoinedGroups(meta).filter( + (group) => !only || group.fields.some((field) => only.has(field.name)), + ); if (groups.length === 0) { return query as any; } @@ -730,8 +810,14 @@ export namespace Query { ? request.getAll(indexedFilter?.[0] ?? "", index) : request; - if (db) { - tmpRequest = Joined(db, meta, tmpRequest as Stream) as Stream; + const joinedSplit = splitJoinedFields(meta, sorting, filters); + if (db && joinedSplit.filter.size > 0) { + tmpRequest = Joined( + db, + meta, + tmpRequest as Stream, + joinedSplit.filter, + ) as Stream; } const filteredComputed = new Set( @@ -773,6 +859,14 @@ export namespace Query { }, tmpRequest); } const total = tmpRequest.count(); + if (db && joinedSplit.sort.size > 0) { + tmpRequest = Joined( + db, + meta, + tmpRequest as Stream, + joinedSplit.sort, + ) as Stream; + } if ( db && sortField && diff --git a/src/index.ts b/src/index.ts index c6cc1f1..f38187e 100644 --- a/src/index.ts +++ b/src/index.ts @@ -162,6 +162,18 @@ function displayOnlyComputedFields( return new Set(names); } +function displayOnlyJoinedFields( + meta: DataAPIMeta, + params: Parameters.ListParameters, + pluck: Set | undefined, + sort: [string, "asc" | "desc" | undefined] | undefined, +): Set { + const names = Array.from( + Query.DeferredJoinedFields(meta, sort, params.filters), + ).filter((name) => params.noPluck || pluck?.has(name)); + return new Set(names); +} + export namespace DefaultRoutes { class Methods { async get(_reqCtx: RequestContext, params: Parameters.GetParameters) { @@ -229,6 +241,16 @@ export namespace DefaultRoutes { const limit = params.limit || 10; let queryPaged = query.slice(offset, limit); + const displayJoined = displayOnlyJoinedFields(meta, params, pluck, sort); + if (displayJoined.size > 0) { + queryPaged = Query.Joined( + model.database, + meta, + queryPaged, + displayJoined, + ); + } + const displayComputed = displayOnlyComputedFields(meta, params, pluck); if (displayComputed.size > 0) { queryPaged = Query.Computed( diff --git a/src/tests/components/joined.test.ts b/src/tests/components/joined.test.ts index cb7a5d8..ea6ef9e 100644 --- a/src/tests/components/joined.test.ts +++ b/src/tests/components/joined.test.ts @@ -95,6 +95,8 @@ describe("Field Joined", () => { it("sorts by joined field descending", async () => await sortsByJoinedFieldDescending()); it("filters by joined field", async () => await filtersByJoinedField()); + it("keeps joined fields on paginated rows when unused by sort and filter", async () => + await keepsJoinedFieldsOnPaginatedRows()); it("returns null for orphan foreign key", async () => await returnsNullForOrphanForeignKey()); it("ignores joined field on edit body", async () => @@ -268,6 +270,35 @@ async function filtersByJoinedField() { } } +async function keepsJoinedFieldsOnPaginatedRows() { + await _createDataController(getFunctionName()); + + const response = await listRequest(getFunctionName(), { + sortKey: "title", + sortDirection: "asc", + offset: "1", + limit: "2", + }); + expect(response.status).to.equal(200); + const data = (await response.json()) as { + results: BookListed[]; + total: number; + }; + expect(data.total).to.equal(5); + expect(data.results.map((b) => b.title)).to.deep.equal([ + "Alpha Rising", + "Beta Stories", + ]); + expect(data.results.map((b) => b.name)).to.deep.equal([ + "Alice Carter", + "Bob Stone", + ]); + expect(data.results.map((b) => b.email)).to.deep.equal([ + "alice@example.com", + "bob@example.com", + ]); +} + async function returnsNullForOrphanForeignKey() { await _createDataController(getFunctionName()); From 4ef8c3b7179289c95e4f7083f39725b2e0e4ffb0 Mon Sep 17 00:00:00 2001 From: MrSociety404 Date: Fri, 10 Jul 2026 10:57:27 +0200 Subject: [PATCH 2/3] perf: address pre-PR review feedback Materialize all deferred joined fields on the page whenever a display-only computed field is resolved post-slice, since its expression may read a joined field that is not itself plucked. Mirrors the conservative rule Query.List already applies for filter- and sort-materialized computed fields; the final pluck() strips any fields the response did not request. Adds a regression test: a listable computed field whose expression reads a non-listable joined field previously evaluated against a missing field and returned null. --- src/index.ts | 26 ++++++++--- src/tests/components/joined.test.ts | 70 ++++++++++++++++++++++++++++- 2 files changed, 90 insertions(+), 6 deletions(-) diff --git a/src/index.ts b/src/index.ts index f38187e..d6f7bc5 100644 --- a/src/index.ts +++ b/src/index.ts @@ -167,10 +167,15 @@ function displayOnlyJoinedFields( params: Parameters.ListParameters, pluck: Set | undefined, sort: [string, "asc" | "desc" | undefined] | undefined, + materializeAll: boolean, ): Set { - const names = Array.from( - Query.DeferredJoinedFields(meta, sort, params.filters), - ).filter((name) => params.noPluck || pluck?.has(name)); + const deferred = Query.DeferredJoinedFields(meta, sort, params.filters); + if (materializeAll) { + return deferred; + } + const names = Array.from(deferred).filter( + (name) => params.noPluck || pluck?.has(name), + ); return new Set(names); } @@ -241,7 +246,19 @@ export namespace DefaultRoutes { const limit = params.limit || 10; let queryPaged = query.slice(offset, limit); - const displayJoined = displayOnlyJoinedFields(meta, params, pluck, sort); + const displayComputed = displayOnlyComputedFields(meta, params, pluck); + // A computed expression may read any joined field, so when a computed + // field is materialized after the slice, every deferred joined group + // must be merged first regardless of pluck (mirroring the conservative + // rule Query.List applies pre-count). The extra lookups only run on the + // page and the final pluck() strips fields the response did not ask for. + const displayJoined = displayOnlyJoinedFields( + meta, + params, + pluck, + sort, + displayComputed.size > 0, + ); if (displayJoined.size > 0) { queryPaged = Query.Joined( model.database, @@ -251,7 +268,6 @@ export namespace DefaultRoutes { ); } - const displayComputed = displayOnlyComputedFields(meta, params, pluck); if (displayComputed.size > 0) { queryPaged = Query.Computed( model.database, diff --git a/src/tests/components/joined.test.ts b/src/tests/components/joined.test.ts index ea6ef9e..9235212 100644 --- a/src/tests/components/joined.test.ts +++ b/src/tests/components/joined.test.ts @@ -8,13 +8,14 @@ import { import { Access, AccessMode, + Computed, Filter, Joined, Listable, ModelReference, Sortable, } from "@antelopejs/interface-data-api/metadata"; -import { Schema } from "@antelopejs/interface-database"; +import { Schema, type ValueProxy } from "@antelopejs/interface-database"; import { BasicDataModel, Field, @@ -97,6 +98,8 @@ describe("Field Joined", () => { it("filters by joined field", async () => await filtersByJoinedField()); it("keeps joined fields on paginated rows when unused by sort and filter", async () => await keepsJoinedFieldsOnPaginatedRows()); + it("resolves computed fields reading a non-plucked joined field", async () => + await resolvesComputedFieldReadingNonPluckedJoinedField()); it("returns null for orphan foreign key", async () => await returnsNullForOrphanForeignKey()); it("ignores joined field on edit body", async () => @@ -155,6 +158,52 @@ async function _createDataController(testName: string) { declare email: string; } + return _seedTables(); +} + +// Regression fixture for the lazy joined-field split: `name` is a joined +// field that is neither listable, filtered, nor sorted, while `display` is a +// listable computed field whose expression reads it. The deferred joined +// lookup must still run before the computed expression on the paginated rows. +async function _createComputedJoinedController(testName: string) { + @RegisterDataController() + class _JoinedComputedTestAPI extends DataController( + Book, + { + list: DefaultRoutes.List, + }, + Controller(`/${testName}`), + ) { + @ModelReference() + @Model(BookModel) + declare bookModel: BookModel; + + @Listable() + @Access(AccessMode.ReadOnly) + declare _id: string; + + @Listable() + @Access(AccessMode.ReadWrite) + declare title: string; + + @Joined({ + table: authorTableName, + localKey: "authorId", + remoteField: "name", + }) + declare name: string; + + @Listable() + @Computed((row) => + (row.key("name") as ValueProxy).concat(" (author)"), + ) + declare display: string; + } + + return _seedTables(); +} + +async function _seedTables() { await RegisterSchema(schemaName); await _dropTables(); @@ -299,6 +348,25 @@ async function keepsJoinedFieldsOnPaginatedRows() { ]); } +async function resolvesComputedFieldReadingNonPluckedJoinedField() { + await _createComputedJoinedController(getFunctionName()); + + const response = await listRequest(getFunctionName(), {}); + expect(response.status).to.equal(200); + const data = (await response.json()) as { + results: { title: string; display: string | null; name?: string }[]; + total: number; + }; + expect(data.total).to.equal(5); + expect(data.results).to.have.lengthOf(5); + + const alphaRising = data.results.find((b) => b.title === "Alpha Rising"); + expect(alphaRising).to.not.equal(undefined); + expect(alphaRising?.display).to.equal("Alice Carter (author)"); + // the joined field itself is not listable, so it must stay out of the response + expect(alphaRising?.name).to.equal(undefined); +} + async function returnsNullForOrphanForeignKey() { await _createDataController(getFunctionName()); From 4ca8a448d07678de661ffb7d0a8d97f9fd8957b8 Mon Sep 17 00:00:00 2001 From: MrSociety404 Date: Fri, 10 Jul 2026 11:15:40 +0200 Subject: [PATCH 3/3] address greptile review feedback (greploop iteration 1) --- src/components.ts | 14 ++++++-------- src/index.ts | 9 +++------ 2 files changed, 9 insertions(+), 14 deletions(-) diff --git a/src/components.ts b/src/components.ts index 40fc9ef..e9b33d8 100644 --- a/src/components.ts +++ b/src/components.ts @@ -519,15 +519,13 @@ export namespace Query { /** * Joined field names that {@link List} does not materialize for the given - * sort/filter parameters. The default list route resolves these after - * pagination so their lookups only run on the returned page. + * split. The default list route resolves these after pagination so their + * lookups only run on the returned page. */ - export function DeferredJoinedFields( + function deferredJoinedFields( meta: DataAPIMeta, - sorting?: [string, "asc" | "desc" | undefined], - filters?: Record, + split: JoinedFieldSplit, ): Set { - const split = splitJoinedFields(meta, sorting, filters); return new Set( Object.entries(meta.fields) .filter( @@ -794,7 +792,7 @@ export namespace Query { sorting?: [string, "asc" | "desc" | undefined], filters?: Record, db?: SchemaInstance, - ): [sorted: Stream, total: Datum] { + ): [sorted: Stream, total: Datum, deferredJoined: Set] { const filterList = Object.entries(meta.filters).filter( ([name]) => filters && name in filters, ); @@ -883,7 +881,7 @@ export namespace Query { if (shouldSort && !shouldSort.indexed && sortField) { tmpRequest = tmpRequest.orderBy(sortField, sorting?.[1] ?? "asc"); } - return [tmpRequest, total]; + return [tmpRequest, total, deferredJoinedFields(meta, joinedSplit)]; } export function Delete(table: Table, id: string | string[]) { diff --git a/src/index.ts b/src/index.ts index d6f7bc5..0c6f469 100644 --- a/src/index.ts +++ b/src/index.ts @@ -163,13 +163,11 @@ function displayOnlyComputedFields( } function displayOnlyJoinedFields( - meta: DataAPIMeta, + deferred: Set, params: Parameters.ListParameters, pluck: Set | undefined, - sort: [string, "asc" | "desc" | undefined] | undefined, materializeAll: boolean, ): Set { - const deferred = Query.DeferredJoinedFields(meta, sort, params.filters); if (materializeAll) { return deferred; } @@ -215,7 +213,7 @@ export namespace DefaultRoutes { "asc" | "desc" | undefined, ]) : undefined; - let [query, queryTotal] = Query.List( + let [query, queryTotal, deferredJoined] = Query.List( this, meta, model.table, @@ -253,10 +251,9 @@ export namespace DefaultRoutes { // rule Query.List applies pre-count). The extra lookups only run on the // page and the final pluck() strips fields the response did not ask for. const displayJoined = displayOnlyJoinedFields( - meta, + deferredJoined, params, pluck, - sort, displayComputed.size > 0, ); if (displayJoined.size > 0) {