From 4af04518f6f06b27b74919f089cc4d3a64e0eb35 Mon Sep 17 00:00:00 2001 From: Taylor Date: Sat, 18 Jul 2026 11:30:05 -0400 Subject: [PATCH] =?UTF-8?q?feat(query):=20existence-test=20error=20surface?= =?UTF-8?q?s=20and=20docs=20=E2=80=94=20the=20feature's=20edges?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every remaining surface where a reverse or M2M relation can be named now answers with the supported spelling, and the docs teach existence tests end to end. Error surfaces beyond the proxy's own (those shipped in the tracer bullet): - join()/left_join() on a reverse/M2M edge stays rejected — the pinned "a join never multiplies root rows" property is preserved by rejection, not implicit DISTINCT — with the error naming .exists() (previously an opaque "must return a relation path" TypeError). - in_() with a query RHS stays a TypeError whose message names the existence test when the RHS is a query (the #307 repro's second guess); a non-query RHS keeps the plain message. - include() rejection verified unchanged for M2M edges too — population remains a separate future mechanism. Docs: - New guide section "Existence Tests on Reverse & Many-to-Many Relations": real models (Assignment + Annotated tabs), lambda-style predicates throughout, rendered SQL for the #307/#308 shapes, the explicit grouping contrast, negation, traversal-inside-the-test, nesting, M2M, the loud dead ends, and the exists-test vs query-terminal disambiguation. Backed by two new runnable example scripts exercised by the docs harness. - Operators table gains the .exists() row; the forward existence/absence section and the reverse-relation filtering section cross-link the reverse direction. - Reference (api/queries.md) states the one-verb rule and its rejections. - Roadmap and "Not Yet Supported" now distinguish reverse predicates (shipped — the existence test) from reverse population (still future; the include() rejection stays), and track cross-scope correlation as #309. Refs #311, #317 --- docs/examples/existence_tests.py | 195 +++++++++++++++++++++ docs/examples/existence_tests_annotated.py | 101 +++++++++++ docs/pages/api/queries.md | 2 + docs/pages/guide/queries.md | 126 ++++++++++++- docs/pages/roadmap.md | 3 +- src/ferro/query/builder.py | 12 ++ src/ferro/query/nodes.py | 17 ++ tests/test_query_exists.py | 51 ++++++ 8 files changed, 504 insertions(+), 3 deletions(-) create mode 100644 docs/examples/existence_tests.py create mode 100644 docs/examples/existence_tests_annotated.py diff --git a/docs/examples/existence_tests.py b/docs/examples/existence_tests.py new file mode 100644 index 0000000..b5f4ed9 --- /dev/null +++ b/docs/examples/existence_tests.py @@ -0,0 +1,195 @@ +"""Runnable companion to the Existence Tests guide (docs/pages/guide/queries.md). + +Seeds the transfer/split-line domain the guide walks through (one-to-one and +to-many BackRefs plus a category both layers point at) and a tagged-user M2M +pair, then runs every existence test the guide shows and asserts the exact +rows that come back. +""" + +import asyncio +from typing import Annotated + +from ferro import BackRef, Field, ForeignKey, ManyToMany, Model, Relation, connect, engines + + +# --8<-- [start:schema] +class Category(Model): + id: int | None = Field(default=None, primary_key=True) + name: str + transactions: Relation[list["Transaction"]] = BackRef() + lines: Relation[list["SplitLine"]] = BackRef() + + +class Transaction(Model): + id: int | None = Field(default=None, primary_key=True) + amount: int + category: Annotated[ + Category | None, ForeignKey(related_name="transactions", on_delete="SET NULL") + ] = None + # One-to-one BackRefs: a transfer references a transaction via a unique FK + transfer_out: "Transfer" = BackRef() + transfer_in: "Transfer" = BackRef() + # To-many BackRef: a split transaction carries its lines + lines: Relation[list["SplitLine"]] = BackRef() + + +class Transfer(Model): + id: int | None = Field(default=None, primary_key=True) + outflow_transaction: Annotated[ + Transaction | None, ForeignKey(related_name="transfer_out", unique=True) + ] = None + inflow_transaction: Annotated[ + Transaction | None, ForeignKey(related_name="transfer_in", unique=True) + ] = None + + +class SplitLine(Model): + id: int | None = Field(default=None, primary_key=True) + txn: Annotated[Transaction, ForeignKey(related_name="lines", on_delete="CASCADE")] + category: Annotated[ + Category | None, ForeignKey(related_name="lines", on_delete="SET NULL") + ] = None + amount: int = 0 +# --8<-- [end:schema] + + +# --8<-- [start:m2m-schema] +class Tag(Model): + id: int | None = Field(default=None, primary_key=True) + name: str + users: Relation[list["User"]] = ManyToMany(related_name="tags") + + +class User(Model): + id: int | None = Field(default=None, primary_key=True) + username: str + tags: Relation[list["Tag"]] = BackRef() +# --8<-- [end:m2m-schema] + + +async def main() -> None: + await connect("sqlite::memory:", auto_migrate=True) + + async with engines.session(): + outflow = await Transaction.create(amount=-100) + inflow = await Transaction.create(amount=100) + plain = await Transaction.create(amount=-20) + assert plain.amount == -20 + await Transfer.create(outflow_transaction=outflow) + await Transfer.create(inflow_transaction=inflow) + + # --8<-- [start:bare] + # "Is this transaction part of any transfer?" — membership via either + # FK column, one EXISTS per side, each matching row exactly once. + in_transfer = await Transaction.where( + lambda t: t.transfer_out.exists() | t.transfer_in.exists() + ).all() + + # The negated branch: ~ renders NOT EXISTS + not_in_transfer = await Transaction.where( + lambda t: ~t.transfer_out.exists() & ~t.transfer_in.exists() + ).all() + # --8<-- [end:bare] + assert {t.amount for t in in_transfer} == {-100, 100} + assert {t.amount for t in not_in_transfer} == {-20} + + groceries = await Category.create(name="Groceries") + split = await Transaction.create(amount=-70) # category vacated while split + await SplitLine.create(txn=split, category=groceries, amount=-30) + await SplitLine.create(txn=split, category=groceries, amount=-40) + await Transaction.create(amount=-10, category=groceries) + ids = [groceries.id] + + # --8<-- [start:scoped] + # The inner lambda is a full ferro predicate over the related model: + # "transactions carrying the category at the root OR on any line". + # A line-less transaction survives through the OR's root branch, and + # a transaction with several matching lines comes back exactly once. + matching = await Transaction.where( + lambda t: t.category_id.in_(ids) + | t.lines.exists(lambda line: line.category_id.in_(ids)) + ).all() + # --8<-- [end:scoped] + assert {t.amount for t in matching} == {-70, -10} + + # --8<-- [start:composes] + # Root-shaped results: existence tests compose with every other + # predicate, ordering, and paging — nothing about the query changes. + page = ( + await Transaction.where( + lambda t: t.category_id.in_(ids) + | t.lines.exists(lambda line: line.category_id.in_(ids)) + ) + .order_by("amount", "desc") + .limit(1) + .all() + ) + # --8<-- [end:composes] + assert [t.amount for t in page] == [-10] + + fuel_run = await Transaction.create(amount=-55) + await SplitLine.create(txn=fuel_run, category=groceries, amount=-55) + spread = await Transaction.create(amount=-60) + await SplitLine.create(txn=spread, category=groceries, amount=-5) + await SplitLine.create(txn=spread, amount=-55) + + # --8<-- [start:grouping] + # Grouping is YOUR choice, spelled explicitly — the two shapes below + # are different questions with different answers. + + # One line matches BOTH conditions: + one_line_both = await Transaction.where( + lambda t: t.lines.exists( + lambda line: line.category_id.in_(ids) & (line.amount <= -50) + ) + ).all() + + # SOME line matches each condition (possibly different lines): + some_line_each = await Transaction.where( + lambda t: t.lines.exists(lambda line: line.category_id.in_(ids)) + & t.lines.exists(lambda line: line.amount <= -50) + ).all() + # --8<-- [end:grouping] + assert {t.amount for t in one_line_both} == {-55} + assert {t.amount for t in some_line_each} == {-55, -60} + + # --8<-- [start:traversal-inside] + # Forward traversal works inside the test (joins render INSIDE the + # EXISTS subquery, ADR-0006 semantics unchanged), and tests nest. + by_name = await Transaction.where( + lambda t: t.lines.exists(lambda line: line.category.name == "Groceries") + ).all() + + active_categories = await Category.where( + lambda c: c.lines.exists(lambda line: line.txn.amount < -50) + ).all() + # --8<-- [end:traversal-inside] + assert {t.amount for t in by_name} == {-70, -55, -60} + assert {c.name for c in active_categories} == {"Groceries"} + + admin = await Tag.create(name="admin") + beta = await Tag.create(name="beta") + alice = await User.create(username="alice") + bob = await User.create(username="bob") + await User.create(username="carol") + await admin.users.add(alice) + await beta.users.add(alice, bob) + + # --8<-- [start:m2m] + # Many-to-many spells identically — the test correlates through the + # join table, and the inner lambda scopes over the target model. + admins = await User.where( + lambda u: u.tags.exists(lambda tag: tag.name == "admin") + ).all() + tagged = await User.where(lambda u: u.tags.exists()).all() + untagged = await User.where(lambda u: ~u.tags.exists()).all() + # --8<-- [end:m2m] + assert {u.username for u in admins} == {"alice"} + assert {u.username for u in tagged} == {"alice", "bob"} + assert {u.username for u in untagged} == {"carol"} + + print("existence_tests example ran successfully") + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/docs/examples/existence_tests_annotated.py b/docs/examples/existence_tests_annotated.py new file mode 100644 index 0000000..f274c40 --- /dev/null +++ b/docs/examples/existence_tests_annotated.py @@ -0,0 +1,101 @@ +"""Annotated-style companion to existence_tests.py (AGENTS.md I-7).""" + +import asyncio +from typing import Annotated + +from ferro import BackRef, Field, ForeignKey, ManyToMany, Model, Relation, connect, engines + + +# --8<-- [start:schema] +class Category(Model): + id: Annotated[int | None, Field(default=None, primary_key=True)] + name: str + transactions: Relation[list["Transaction"]] = BackRef() + lines: Relation[list["SplitLine"]] = BackRef() + + +class Transaction(Model): + id: Annotated[int | None, Field(default=None, primary_key=True)] + amount: int + category: Annotated[ + Category | None, ForeignKey(related_name="transactions", on_delete="SET NULL") + ] = None + # One-to-one BackRefs: a transfer references a transaction via a unique FK + transfer_out: "Transfer" = BackRef() + transfer_in: "Transfer" = BackRef() + # To-many BackRef: a split transaction carries its lines + lines: Relation[list["SplitLine"]] = BackRef() + + +class Transfer(Model): + id: Annotated[int | None, Field(default=None, primary_key=True)] + outflow_transaction: Annotated[ + Transaction | None, ForeignKey(related_name="transfer_out", unique=True) + ] = None + inflow_transaction: Annotated[ + Transaction | None, ForeignKey(related_name="transfer_in", unique=True) + ] = None + + +class SplitLine(Model): + id: Annotated[int | None, Field(default=None, primary_key=True)] + txn: Annotated[Transaction, ForeignKey(related_name="lines", on_delete="CASCADE")] + category: Annotated[ + Category | None, ForeignKey(related_name="lines", on_delete="SET NULL") + ] = None + amount: int = 0 +# --8<-- [end:schema] + + +# --8<-- [start:m2m-schema] +class Tag(Model): + id: Annotated[int | None, Field(default=None, primary_key=True)] + name: str + users: Relation[list["User"]] = ManyToMany(related_name="tags") + + +class User(Model): + id: Annotated[int | None, Field(default=None, primary_key=True)] + username: str + tags: Relation[list["Tag"]] = BackRef() +# --8<-- [end:m2m-schema] + + +async def main() -> None: + await connect("sqlite::memory:", auto_migrate=True) + + async with engines.session(): + outflow = await Transaction.create(amount=-100) + await Transaction.create(amount=-20) + await Transfer.create(outflow_transaction=outflow) + + in_transfer = await Transaction.where( + lambda t: t.transfer_out.exists() | t.transfer_in.exists() + ).all() + assert {t.amount for t in in_transfer} == {-100} + + groceries = await Category.create(name="Groceries") + split = await Transaction.create(amount=-70) + await SplitLine.create(txn=split, category=groceries, amount=-70) + + line_aware = await Transaction.where( + lambda t: t.category_id.in_([groceries.id]) + | t.lines.exists(lambda line: line.category_id.in_([groceries.id])) + ).all() + assert {t.amount for t in line_aware} == {-70} + + admin = await Tag.create(name="admin") + alice = await User.create(username="alice") + await User.create(username="bob") + await admin.users.add(alice) + + admins = await User.where( + lambda u: u.tags.exists(lambda tag: tag.name == "admin") + ).all() + assert {u.username for u in admins} == {"alice"} + + print("existence_tests_annotated example ran successfully") + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/docs/pages/api/queries.md b/docs/pages/api/queries.md index 615d893..ca749b0 100644 --- a/docs/pages/api/queries.md +++ b/docs/pages/api/queries.md @@ -6,6 +6,8 @@ Prefix `~` negates **any** predicate — leaf comparison or `&`/`|` compound — `where()` and `order_by()` lambdas may **traverse** a forward-FK relation (`lambda t: t.account.ledger_id == 1`): each hop renders one INNER join, deduplicated by relation path (ADR-0006). `join()` forces a join on a relation path (a bare `join()` is an existence filter on a nullable relation), and `left_join()` marks the whole path LEFT to keep relation-less rows. See the [Querying Across Relationships](../guide/queries.md#querying-across-relationships) guide for worked examples. +A reverse (`BackRef`) or many-to-many relation in a predicate supports exactly one verb — the **existence test** `t.rel.exists(inner_lambda=None)` (ADR-0007). It renders as a correlated `EXISTS` at every cardinality (never a join, so the result stays root-shaped and each matching root returns once), negates with `~`, and the optional inner lambda is a full ferro predicate over the related model (operators, `&`/`|`/`~`, forward traversal rendered inside the subquery, nested tests). Everything else on a reverse edge — column access, comparisons (including `!= None`), `in_` (including a query RHS), `join()`/`left_join()` — raises at build time naming `.exists()`; an inner lambda referencing any scope but its own parameter is likewise rejected ([#309](https://github.com/syn54x/ferro-orm/issues/309)). See [Existence Tests](../guide/queries.md#existence-tests-on-reverse-many-to-many-relations) for worked examples. + ## `include()` and populated relations `include(lambda t: t.account)` delivers each result with the relation **populated** (ADR-0008): access becomes a plain attribute holding the complete related instance — no await, no query — while unpopulated relations keep the awaitable contract. Include is the third orthogonal query axis (joins decide membership, projection decides shape, include decides attached data): it never changes which rows come back, `.all()` still returns `list[Model]`, and `count()`/`exists()` are unaffected. Paths populate whole (`include(lambda t: t.account.owner)` populates both hops); includes are cumulative, order-free, and idempotent; populated instances run the full session identity-map protocol, and a refresh keeps a population only while the row's FK still points at it. diff --git a/docs/pages/guide/queries.md b/docs/pages/guide/queries.md index dfa4115..029c259 100644 --- a/docs/pages/guide/queries.md +++ b/docs/pages/guide/queries.md @@ -101,6 +101,7 @@ Lambda predicates keep the call site fully type-checked: the proxy's attributes | `== None` | `IS NULL` | `lambda user: user.deleted_at == None` | | `!= None` | `IS NOT NULL` | `lambda user: user.deleted_at != None` | | `~` | `NOT` | `lambda user: ~user.role.in_(["admin", "moderator"])` | +| `.exists(...)` | `EXISTS (SELECT 1 …)` | `lambda user: user.posts.exists(lambda post: post.published == True)` — reverse/M2M relations only; see [Existence Tests](#existence-tests-on-reverse-many-to-many-relations) | ```python --8<-- "docs/examples/predicates.py:operators" @@ -329,6 +330,8 @@ For the opposite question — "has *no* related row" — compare the relation to --8<-- "docs/examples/traversal.py:is-null" ``` +Both spellings here are the **forward** direction — the FK column lives on the queried table. Asking the same question in the *reverse* direction ("has at least one / no related child row") is an [existence test](#existence-tests-on-reverse-many-to-many-relations): `t.lines.exists()` / `~t.lines.exists()`. + ### Keeping rows that have no relation When you want the relation-less rows *kept* rather than filtered out, opt into a `left_join`. It marks **every edge of its path** LEFT (the whole-path rule), so a left-marked two-hop path retains rows missing the relation at either hop: @@ -412,6 +415,8 @@ latest = await author.posts.order_by(lambda post: post.created_at, "desc").limit n = await author.posts.count() ``` +That is a query *from* an instance. To filter the **root query** on membership in a reverse relation — "authors who have at least one published post" — use an [existence test](#existence-tests-on-reverse-many-to-many-relations): `Author.where(lambda a: a.posts.exists(lambda post: post.published == True))`. + ### Results are plain root instances A traversed query is **shape-preserving**: filtering `Transaction` through `transaction.account.ledger_id` still returns `Transaction` instances, no matter how deep the predicate reaches. Traversal does *not* pre-load the related rows onto the results — `await transaction.account` still issues its own query, exactly as it does without any traversal. Attaching related data is a separate, explicit request: [`include()`](#populating-relations-with-include). @@ -438,6 +443,122 @@ Do the two-step: fetch the primary keys with the joined query, then mutate by th See [Relationships](relationships.md) for the schema-declaration side of foreign keys and reverse relations. +## Existence Tests on Reverse & Many-to-Many Relations + +Traversal reaches *forward* along a foreign key. The reverse question — "which transactions have at least one split line?", "which transactions appear in any transfer?" — is a different shape: the related rows live on the **other** table, keyed back at you. In a predicate, a reverse (`BackRef`) or many-to-many relation supports exactly one verb for that question, the **existence test**: + +```python +matches = await Transaction.where(lambda t: t.lines.exists()).all() +``` + +`.exists()` renders as a correlated `EXISTS` subquery — never a join — so the result stays **root-shaped**: each matching row comes back exactly once (a transaction with three lines is one result, no `DISTINCT` bookkeeping), rows are never multiplied, and the test composes with every other predicate, ordering, and paging. One verb covers every cardinality: a one-to-one `BackRef`, a to-many `BackRef`, and an M2M edge all spell the same, so a schema cardinality change never breaks a call site. + +The examples below use this schema — a transfer links two transactions through unique FKs (one-to-one BackRefs), split lines hang off a transaction (to-many), and a category is referenced by both layers: + +=== "Assignment" + + ```python + --8<-- "docs/examples/existence_tests.py:schema" + ``` + +=== "Annotated" + + ```python + --8<-- "docs/examples/existence_tests_annotated.py:schema" + ``` + +### Bare tests and negation + +A bare `.exists()` asks "is any related row there?". Negation is the uniform `~` — NOT EXISTS is not a separate spelling: + +```python +--8<-- "docs/examples/existence_tests.py:bare" +``` + +The first query ships to the database as: + +```sql +SELECT ... FROM transaction t +WHERE EXISTS (SELECT 1 FROM transfer WHERE transfer.outflow_transaction_id = t.id) + OR EXISTS (SELECT 1 FROM transfer WHERE transfer.inflow_transaction_id = t.id) +``` + +### Scoping with an inner predicate + +Pass a lambda to filter *which* related rows count. The inner lambda is a **full ferro predicate over the related model** — every operator, `&`/`|`/`~`, forward traversal, even nested existence tests — not a sub-language: + +```python +--8<-- "docs/examples/existence_tests.py:scoped" +``` + +Rendered SQL — the root branch keeps line-less transactions, the `EXISTS` branch finds categories on any line: + +```sql +SELECT ... FROM transaction t +WHERE t.category_id IN (...) + OR EXISTS (SELECT 1 FROM split_line l + WHERE l.txn_id = t.id AND l.category_id IN (...)) +``` + +Because the result is root-shaped, keyset ordering and paging compose unchanged: + +```python +--8<-- "docs/examples/existence_tests.py:composes" +``` + +### Grouping is explicit + +With conditions on the *same* relation, there are two different questions: does **one** related row match all conditions, or does **some** related row match each? The lambda scope makes the choice visible — one test with a compound inner predicate, or two tests combined outside: + +```python +--8<-- "docs/examples/existence_tests.py:grouping" +``` + +This is why reverse relations have an explicit combinator rather than implicit path traversal (`t.lines.category_id == x` raises): with implicit traversal, nothing on the page says which of those two questions `(t.lines.a == 1) & (t.lines.b == 2)` asks (ADR-0007). + +### Traversal inside the test, and nesting + +Forward-FK traversal works inside the inner lambda — its joins render *inside* the `EXISTS` subquery with the same [INNER semantics](#every-traversed-hop-is-an-inner-join) as everywhere else — and existence tests nest to any depth: + +```python +--8<-- "docs/examples/existence_tests.py:traversal-inside" +``` + +### Many-to-many + +An M2M relation spells identically, from either side — the test correlates through the association table and the inner lambda scopes over the target model: + +=== "Assignment" + + ```python + --8<-- "docs/examples/existence_tests.py:m2m-schema" + ``` + +=== "Annotated" + + ```python + --8<-- "docs/examples/existence_tests_annotated.py:m2m-schema" + ``` + +```python +--8<-- "docs/examples/existence_tests.py:m2m" +``` + +### One verb, loud dead ends + +Reverse relations are **tested, never traversed** (ADR-0007). Every other way of naming a reverse or M2M relation in a query fails at build time with the supported spelling in the message: + +- **Column access** (`t.lines.category_id`) raises `AttributeError` — scope the columns with the inner lambda instead: `t.lines.exists(lambda line: line.category_id == ...)`. +- **Comparisons**, including the tempting `t.transfer_out != None`, raise `TypeError` — a reverse relation has no root-side column to be `NULL`; the spelling is `t.transfer_out.exists()` (and `~t.transfer_out.exists()` for absence). +- **`in_()` with a query** (`t.id.in_(subquery)`) raises `TypeError` — the workloads an `IN (subquery)` serves are existence-test workloads, without hand-correlating on id columns. +- **`join()` / `left_join()` on a reverse edge** raise `TypeError` — a join there would multiply root rows; membership is the existence test's job. +- **The inner lambda sees only its own parameter.** Referencing the outer lambda's parameter (or comparing column-to-column) is a build-time error — cross-scope correlation is a tracked future capability ([#309](https://github.com/syn54x/ferro-orm/issues/309)), never a silent misrender. + +Existence tests answer **membership** only. *Populating* a reverse collection onto results (the data axis) is a separate future mechanism — see [Not Yet Supported](#not-yet-supported); until it lands, fetch collections through the relation itself (`await txn.lines.all()`). + +!!! note "Two `exists`, two levels" + `t.lines.exists(...)` inside a predicate is the existence *test* on a relation. `await query.exists()` is the query *terminal* asking whether the whole query matches any row. Same word, deliberately — both ask "is there at least one?" — at different levels. + ## Populating Relations with include() Every relation access is a query. A list view that renders 100 transactions with their account labels awaits `transaction.account` 100 times — 101 statements for one screen. `include()` cures that N+1: ask the query to bring the related rows along, and each result's relation arrives **populated**: @@ -536,7 +657,7 @@ Every fetch refreshes instances the session already holds. A refresh keeps each Misuse raises at build time, before any SQL: -- **Forward foreign keys only.** Including a `BackRef` or `ManyToMany` relation raises `TypeError`: reverse and M2M population will be a separate mechanism (a batched second query stitched onto the results), not `include()`. Until it lands, fetch collections through the relation itself (`await author.posts.all()`). +- **Forward foreign keys only.** Including a `BackRef` or `ManyToMany` relation raises `TypeError`: reverse and M2M population will be a separate mechanism (a batched second query stitched onto the results), not `include()`. Until it lands, fetch collections through the relation itself (`await author.posts.all()`). *Filtering* on reverse membership is a different axis and already works — the [existence test](#existence-tests-on-reverse-many-to-many-relations). - **Lambda selectors only.** `include("account")` raises pointing at the lambda form — strings never traverse. Selecting a *column* (`include(lambda t: t.account.label)`) raises too: every populated hop is a complete row, so there is nothing to select per column. - **No include × projection.** A query carries exactly one materialization plan — populated instances or projected records, never both — and record results are flat, permanently. Either order raises `ValueError` pointing at [traversed projection](#reaching-across-a-relation), the record-shaped way across a relation. - **No mutations.** `update()`/`delete()` on an included query raise — a mutation returns no instances to populate. @@ -675,7 +796,8 @@ Aggregation builds directly on this machinery — `select(lambda t: {"total": t. The following query features are **not yet implemented** — see the [Roadmap](../roadmap.md): - `having()` — post-aggregation filtering; `where()` rejects aggregate predicates pointing at it ([#291](https://github.com/syn54x/ferro-orm/issues/291)) - - Reverse (`BackRef`) and many-to-many population — [`include()`](#populating-relations-with-include) covers forward FKs; collection population is a separate future mechanism + - Reverse (`BackRef`) and many-to-many **population** — [`include()`](#populating-relations-with-include) covers forward FKs; collection population is a separate future mechanism. (*Filtering* on reverse/M2M membership is supported — that's the [existence test](#existence-tests-on-reverse-many-to-many-relations).) + - Cross-scope correlation inside an existence test — comparing an inner-lambda column to an outer column ([#309](https://github.com/syn54x/ferro-orm/issues/309)); rejected loudly at build time today - Case-insensitive `ilike()` ## See Also diff --git a/docs/pages/roadmap.md b/docs/pages/roadmap.md index 45db79d..1a6d9de 100644 --- a/docs/pages/roadmap.md +++ b/docs/pages/roadmap.md @@ -6,7 +6,8 @@ Ferro is pre-1.0 and under active development. The items below are known gaps we - **`having()`** — post-aggregation filtering for [grouped queries](guide/aggregations.md); `where()` rejects aggregate predicates pointing at it ([#291](https://github.com/syn54x/ferro-orm/issues/291)). Workaround: filter groups in Python after `all()`. - **Typed record fields** — a projected `Row`'s fields type as `Any` on access today; inferring per-field types from the selector (so `row.total` checks as `int | None`) is [#290](https://github.com/syn54x/ferro-orm/issues/290). -- **Reverse and many-to-many population** — [`include()`](guide/queries.md#populating-relations-with-include) populates forward-FK paths in one statement; populating `BackRef` collections and M2M sets is a separate future mechanism (a batched second query stitched onto the results). Today each awaited collection is its own query. +- **Reverse and many-to-many population** — [`include()`](guide/queries.md#populating-relations-with-include) populates forward-FK paths in one statement; populating `BackRef` collections and M2M sets is a separate future mechanism (a batched second query stitched onto the results). Today each awaited collection is its own query. (*Filtering* on reverse/M2M membership already works — the [existence test](guide/queries.md#existence-tests-on-reverse-many-to-many-relations), `t.lines.exists(...)`.) +- **Cross-scope correlation in existence tests** — comparing an inner-lambda column against the outer scope (`t.lines.exists(lambda line: line.category_id == t.category_id)`) is rejected at build time today; correlated column-to-column comparison is [#309](https://github.com/syn54x/ferro-orm/issues/309). - **`ilike()`** — case-insensitive pattern matching. Workaround: `like()` with normalized case. - **Atomic update expressions** — database-side expressions in batch updates, e.g. `update(view_count=Post.view_count + 1)`, avoiding the read-modify-write race. Workaround today: load, mutate, `save()` (or raw SQL). diff --git a/src/ferro/query/builder.py b/src/ferro/query/builder.py index 47a64e5..a7c1aa1 100644 --- a/src/ferro/query/builder.py +++ b/src/ferro/query/builder.py @@ -158,6 +158,18 @@ def _resolve_join_selector( "(e.g. `lambda t: t.account.name`); a join selector names a relation " "path (e.g. `lambda t: t.account`)." ) + if isinstance(result, ReverseRelationProxy): + # Joining a reverse/M2M edge would multiply root rows — the pinned + # "a join never multiplies root rows" property is preserved by + # rejection (ADR-0007); membership is the existence test's job. + relation = result._name + raise TypeError( + f"join()/left_join() cannot join the reverse relation " + f"{relation!r}: a join on a reverse or many-to-many edge would " + "multiply root rows. Test membership with a predicate instead — " + f"where(lambda t: t.{relation}.exists(...)), negated with ~ — " + "reverse relations are tested, not traversed (ADR-0007)." + ) if not isinstance(result, RelationProxy): raise TypeError( "join()/left_join() selector must return a relation path " diff --git a/src/ferro/query/nodes.py b/src/ferro/query/nodes.py index 75f3e3d..3808f5f 100644 --- a/src/ferro/query/nodes.py +++ b/src/ferro/query/nodes.py @@ -466,6 +466,23 @@ def in_( 'IN' """ if not isinstance(other, (list, tuple, set)): + # Late import: builder imports this module at load time. + # ProjectedQuery and Relation subclass Query, so one check + # covers every query shape. + from .builder import Query + + if isinstance(other, Query): + # in_(subquery) is declined, not deferred (ADR-0007): the + # workloads it serves are existence-test workloads, and + # hand-correlating on id columns leaks the join column into + # every call site. + raise TypeError( + f"The 'in_' operator expects a list, tuple, or set, got " + f"{type(other).__name__}. To filter on membership in " + "related rows, use the existence test on the relation — " + "where(lambda t: t..exists(lambda r: ...)) — " + "instead of an in_(subquery) (ADR-0007)." + ) raise TypeError( f"The 'in_' operator expects a list, tuple, or set, got {type(other).__name__}" ) diff --git a/tests/test_query_exists.py b/tests/test_query_exists.py index bdfdf3a..b67c6a7 100644 --- a/tests/test_query_exists.py +++ b/tests/test_query_exists.py @@ -644,6 +644,57 @@ async def test_m2m_exists_composes_with_root_predicates(db_url): assert n == 2 +# --------------------------------------------------------------------------- +# Remaining error surfaces (#317): every place a reverse or M2M relation can +# be named now answers with the supported spelling. +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_left_join_on_reverse_edge_names_exists(db_url): + """``left_join()`` on a reverse edge stays rejected — the pinned "a join + never multiplies root rows" property is preserved by rejection — and the + error now names ``.exists()``.""" + await connect(db_url, auto_migrate=True) + with pytest.raises(TypeError, match=r"\.exists\("): + ExTxn.select().left_join(lambda t: t.transfer_out) + with pytest.raises(TypeError, match=r"\.exists\("): + ExTxn.select().join(lambda t: t.lines) + + +@pytest.mark.asyncio +async def test_left_join_on_m2m_edge_names_exists(db_url): + await connect(db_url, auto_migrate=True) + with pytest.raises(TypeError, match=r"\.exists\("): + ExUser.select().left_join(lambda u: u.tags) + + +@pytest.mark.asyncio +async def test_in_with_query_rhs_names_exists(db_url): + """``in_()`` with a query RHS stays a TypeError, and the message names the + existence test when the RHS is a query (the #307 repro's second guess).""" + await connect(db_url, auto_migrate=True) + sub = ExLine.select(lambda line: line.txn_id) + with pytest.raises(TypeError, match=r"\.exists\("): + ExTxn.where(lambda t: t.id.in_(sub)) + with pytest.raises(TypeError, match=r"\.exists\("): + ExTxn.where(lambda t: t.id.in_(ExLine.select())) + # A non-query, non-collection RHS keeps the plain message — no exists + # hint where none applies. + with pytest.raises(TypeError, match="expects a list, tuple, or set") as exc_info: + ExTxn.where(lambda t: t.id.in_(42)) + assert ".exists(" not in str(exc_info.value) + + +@pytest.mark.asyncio +async def test_include_on_m2m_edge_unchanged(db_url): + """``include()`` population rejection is unchanged for M2M too — reverse + population stays a separate future mechanism.""" + await connect(db_url, auto_migrate=True) + with pytest.raises(TypeError, match="reverse .BackRef. or many-to-many"): + ExUser.select().include(lambda u: u.tags) + + @pytest.mark.asyncio async def test_m2m_proxy_rejects_everything_but_exists(db_url): """The M2M reverse proxy has the same single verb as the reverse-FK one."""