diff --git a/docs/tutorials/pydantic-guide.md b/docs/tutorials/pydantic-guide.md new file mode 100644 index 00000000..680d1c20 --- /dev/null +++ b/docs/tutorials/pydantic-guide.md @@ -0,0 +1,565 @@ +# Pydantic models for TerminusDB — user guide + +This guide covers the pydantic-native schema interface for the TerminusDB +Python client: declare your domain model once as pydantic models, and let +TerminusDB hold the schema. Documents validate against the same models on the +way in and out — no second declaration, no drift. + +The pydantic interface is the primary way to manage schemas from Python. The +older `DocumentTemplate`/`WOQLSchema` API remains supported; see +[Migrating from DocumentTemplate](#migrating-from-documenttemplate). + +## Why pydantic + TerminusDB + +A TerminusDB schema is a graph of class documents. A pydantic model is a +typed Python class. They describe the same thing — this package makes them +interchangeable: + +- Declare models once. The schema in the database is generated from them, or + the models are generated from the schema — your choice, per project. +- Documents coming out of the database validate into your models. A document + that fails validation means schema drift, and it fails loudly. +- Everything pydantic gives you applies: FastAPI request/response models, + `model_dump`/`model_validate`, JSON Schema export, mypy-friendly typing. + +Three supported workflows: + +- **Code-first** — author models in Python, `schema.commit(client)` pushes + the schema. Best for greenfield. +- **Schema-first** — evolve the schema in TerminusDB, `tdbpy sync` regenerates + `schema.py`. The generated file is a check-in artefact: schema changes and + model diffs land in the same PR. +- **Dynamic** — `PydanticSchema.from_db(client)` builds models from the + database at runtime. Drift is impossible by construction; the trade-off is + no static types. For tools, notebooks and agents. + +`tdbpy check` is the guard rail: it diffs your local models against the +schema graph and exits non-zero when they disagree. + +## Installation + +```bash +pip install "terminusdb[pydantic]" +``` + +Requires pydantic >= 2 and Python >= 3.9. The core client keeps working +without the extra; importing `terminusdb_client.schema.pydantic` without +pydantic installed raises a single `ImportError` telling you to install the +extra. + +## Declaring models + +Subclass `TerminusModel` and annotate fields exactly as you would in any +pydantic model: + +```python +from typing import List, Optional +from terminusdb_client.schema.pydantic import TerminusModel +from terminusdb_client.schema import LexicalKey + +class Person(TerminusModel): + """A person tracked by the system. + + Attributes + ---------- + name : str + Full display name. + """ + + _key = LexicalKey("name") + + name: str + age: Optional[int] = None + tags: List[str] = [] +``` + +This class emits the TerminusDB schema document: + +```python +{"@type": "Class", "@id": "Person", + "@documentation": {"@comment": "A person tracked by the system.", + "@properties": {"name": "Full display name."}}, + "@key": {"@type": "Lexical", "@fields": ["name"]}, + "name": "xsd:string", + "age": {"@type": "Optional", "@class": "xsd:integer"}, + "tags": {"@type": "List", "@class": "xsd:string"}} +``` + +Everything `DocumentTemplate` can declare, a `TerminusModel` can declare: + +### Class attributes + +| Attribute | Effect on the schema | +|---|---| +| `_key` | `@key` — document id scheme. All key classes: `HashKey`, `LexicalKey`, `ValueHashKey`, `RandomKey` (default), `UuidV7Key` | +| `_subdocument = []` | `@subdocument` — embedded document, no independent id | +| `_shared = True` | `"@shared": []` — reference-counted document, cascade-deleted on last unreference (see Shared documents) | +| `_unfoldable = True` | `"@unfoldable": []` — always expanded on read (see links) | +| `_abstract = []` | `@abstract` — no instances | +| `_foreign = True` | `{"@type": "Foreign"}` — opaque reference to another data product (see Foreign references) | +| `_base = "prefix/"` | `@base` — id minting prefix for this class | +| `_metadata = {...}` | `@metadata` — arbitrary class metadata (`render_as`, `order_by`, embedding config, …) | +| `_schema = my_schema` | registers the class into a `Schema` at definition time | + +Underscore-prefixed names are pydantic-private by convention, never +fields. `UuidV7Key` accepts any supplied IRI as `@id` (no `@base` enforcement) — +useful when loading data that already carries external identifiers. + +### Field types + +The annotation alone determines the schema entry — validators, defaults and +constraints (`Field(gt=)`, `pattern=`) are client-side concerns and never +appear in the schema document. `Optional` is the only way to say a field may +be absent. + +| Annotation | Schema entry | +|---|---| +| `str` / `int` / `float` / `bool` | `xsd:string` / `xsd:integer` / `xsd:double` / `xsd:boolean` | +| `datetime`, `date`, `time`, `timedelta` | `xsd:dateTime`, `xsd:date`, `xsd:time`, `xsd:duration` | +| `decimal.Decimal` | `xsd:decimal` | +| `dict` | `sys:JSON` (arbitrary JSON blob) | +| `woql_type` NewTypes (`gYear`, `anyURI`, `token`, `long`, `dayTimeDuration`, …) | their `xsd:` counterparts | +| `Optional[X]` / `X \| None` | `{"@type": "Optional", "@class": X}` | + +Write `Optional[X] = None` when the field should be truly optional at +construction time — `Optional[X]` alone is still a *required* field in +pydantic. `Optional[Set[X]]` and `Optional[List[X]]` are invalid +(`TypeError`): a collection already permits zero elements. +| `List[X]` / `list[X]` | `{"@type": "List", "@class": X}` | +| `Set[X]` / `set[X]` | `{"@type": "Set", "@class": X}` | +| `Set[X]` with `Field(min_length=, max_length=)` | adds `@min_cardinality`/`@max_cardinality` (equal values collapse to `@cardinality`) | +| `Annotated[list[X], Array]` | `{"@type": "Array", "@class": X}` — ordered, gaps allowed; `@dimensions` from list nesting (`list[list[float]]` → `2`) | +| `None` (inside `TerminusUnion`) | `sys:Unit` — a tagged-union member carrying no value | +| another `TerminusModel` | object property — the class name; embedded when the target is a subdocument | +| `Enum` subclass | the enum's `@id`; the enum itself emits `{"@type": "Enum", "@value": [...]}` | +| forward reference `"Person"` | the class name (self- and mutual references) | +| `Literal`, `Union[A, B]` (non-Optional), `tuple`, `frozenset`, `Callable` | `TypeError` — not representable; see `TerminusUnion` below | + +### Enums + +Enums register with `TerminusEnum` (a `str, Enum` subclass); member values +are the stored values: + +```python +from terminusdb_client.schema.pydantic import TerminusEnum + +class Colour(TerminusEnum): + red = "red" + blue = "blue" + information_technology = "Information Technology" + +class Widget(TerminusModel): + shade: Colour +``` + +`Colour` emits `{"@type": "Enum", "@id": "Colour", "@value": ["red", "blue", +"Information Technology"]}` and registers into the same `_schema`. Values +serialise as `member.value`. + +### Subdocuments + +A subdocument is embedded inside its parent — it mints no id of its own: + +```python +class Address(TerminusModel): + _subdocument = [] + street: str + city: str + +class Person(TerminusModel): + address: Optional[Address] = None +``` + +`address` emits `{"@type": "Optional", "@class": "Address"}` and the stored +document embeds the `Address` object inline. Engine rule: a subdocument's +key must be `RandomKey`, `UuidV7Key` or `ValueHashKey`. + +### Shared documents — `_shared` + +A shared document is a *regular* document — own id, any key type, linkable +from many parents — whose lifecycle is reference-counted: when its last +inbound reference is removed (parent deleted, field updated, triple +removed), the engine cascade-deletes it at commit, recursively, including +circular islands of shared documents that reference only each other. + +```python +class Footnote(TerminusModel): + _shared = True + label: str + content: str + +class Article(TerminusModel): + notes: Set[Footnote] # several articles may share one Footnote +``` + +Constraints: `_shared` is mutually exclusive with `_subdocument` (a class +definition carrying both raises `ValueError` at declaration time), is +inherited, and unlike a subdocument it does **not** auto-unfold — add +`_unfoldable` on the class or `Unfold` on the property for that. A shared +document with zero references may be created standalone; the cascade only +fires when a reference is *removed*. + +### Foreign references — `_foreign` + +`Foreign` types are opaque references to documents living in another data +product: no fields, no referential integrity — just an identifier. + +```python +class Person(TerminusModel): + _foreign = True + +class Event(TerminusModel): + person: Optional[Person] = None # stores the IRI only +``` + +`Person` emits `{"@type": "Foreign", "@id": "Person"}` (no properties). +Assign a plain string id/IRI on write; on read it materialises as a stub +carrying `_backend_id` — fields never unfold because nothing local exists +to unfold. + +### Tagged unions — `TerminusUnion` + +A tagged union says "the document carries the fields of exactly one member". +Declare members as fields on a `TerminusUnion` subclass: + +```python +from terminusdb_client.schema.pydantic import TerminusUnion + +class Contact(TerminusUnion): + email: str # "xsd:string" member + phone: Optional[str] = None +``` + +emits `{"@type": "TaggedUnion", "@id": "Contact", "@oneOf": {"email": ..., +"phone": ...}}`. Non-Optional `Union[A, B]` annotations are *not* the same +thing — use `TerminusUnion`. + +A member with no value is `sys:Unit`, written as a `None`-annotated field — +the canonical binary-tree example: + +```python +class BinaryTree(TerminusUnion): + leaf: None # "leaf": "sys:Unit" + node: "Node" +``` + +(TerminusDB also supports the general `@oneOf` list on ordinary `Class` +documents — multiple disjoint property groups. `TerminusUnion` covers the +`TaggedUnion` sugar only; the general form is a documented limitation, see +the reference chapter.) + +### Inheritance and abstract classes + +```python +class Party(TerminusModel): + _abstract = [] + +class Person(Party): + name: str +``` + +`Person` emits `"@inherits": "Party"` — fields resolve through `__mro__` +exactly as pydantic resolves them. + +### Documentation + +The class docstring is parsed with `numpydoc` (`ClassDoc`) and becomes +`@documentation`; `Field(description=...)` adds or overrides a property +entry. Per-language (`@language`) documentation is handled by the engine — +author the default text in code. + +## Documents and links: the unfold contract + +Links to other documents are the one place where TerminusDB storage and +pydantic models differ in shape. TerminusDB may return a link as an id +string (`"Person/bob"`), as a `{"@type", "@id"}` reference, or as a full +nested document — depending on schema markers and the request flag. The +client maps each mechanism to exactly one pydantic construct: + +| TerminusDB mechanism | Pydantic equivalent | +|---|---| +| `@subdocument` class | Nested `TerminusModel` — always a full embedded object | +| `"@unfoldable": []` on the class | `_unfoldable = True` on the model — always expanded | +| `"@unfold": true` on a property | `Annotated[Sub, Unfold]` marker — this property always expands | +| `unfold` flag on a document request | `client.get_document(id, unfold=True)` — expands reachable links server-side | +| folded link (`"Class/id"` string or `{"@type","@id"}`) | **stub** — an id-carrying model with no populated fields | + +```python +from typing import Annotated +from terminusdb_client.schema.pydantic import TerminusModel, Unfold + +class Audit(TerminusModel): + _subdocument = [] + _unfoldable = True # class marker: this type always expands + +class Ticket(TerminusModel): + title: str + audit: Annotated[Optional[Audit], Unfold] = None # property marker + assignee: Optional[Person] = None # ordinary link — may be a stub +``` + +`Unfold` is a metadata marker for `Annotated`; it emits `"@unfold": true` +on the property — `{"@class": "Audit", "@unfold": true}` for a required +field, `{"@type": "Optional", "@class": "Audit", "@unfold": true}` inside +complex specs (`Optional`, `Set`, `List`, `Array`). `_unfoldable` emits +`"@unfoldable": []` on the class document. Unfoldability is transitive +through inheritance, matching the engine. + +Unfolding is bounded: the engine detects cycles (a document that unfolds +into itself comes back as an IRI at the repeated leaf) and caps expansion +at a configurable default of 500,000 documents. + +### Stubs + +A folded link loads as a **stub**: a model instance carrying only its +`_backend_id`, with no fields populated. Accessing a field on a stub raises +`AttributeError`; serialising a stub writes its id back — a document with +unresolved links round-trips losslessly. + +```python +ticket = schema.parse_document(client.get_document("Ticket/1")) +ticket.assignee._backend_id # "Person/bob" +ticket.assignee.name # AttributeError — it's a stub + +bob = schema.parse_document(client.get_document(ticket.assignee._backend_id)) +``` + +Stubs are the same compromise `DocumentTemplate` always made — the +alternative (fetching every linked document eagerly) is not offered. When +you want real objects, fetch with `unfold=True` and every link arrives as a +full model instead: + +```python +ticket = schema.parse_document(client.get_document("Ticket/1", unfold=True)) +ticket.assignee.name # works — unfolded document +``` + +## The schema lifecycle + +### Model-first: commit + +Register models into a `Schema` and push: + +```python +from terminusdb_client.schema import Schema +from terminusdb_client.client import Client + +schema = Schema(title="Ticketing", description="Support tickets") +for cls in (Person, Ticket, Audit, Colour): + schema.add_obj(cls.__name__, cls) + +client = Client("http://localhost:6363") +client.connect(db="tickets", team="admin", account="admin", key="root") +schema.commit(client, commit_msg="ticketing schema v1") +``` + +Setting `cls._schema = schema` at class-definition time registers +automatically — the same idiom as `DocumentTemplate`. `Schema` also accepts +`metadata={...}`: arbitrary JSON written verbatim into the context object's +`@metadata` — this is where data-product options such as +`terminusdb.options` (`store_indices`, `store_clustering`) live. + +### Schema-first: `tdbpy sync` + +Inside a `tdbpy` project (`tdbpy startproject` creates `config.json` + +`.TDB`): + +```bash +tdbpy sync --format pydantic # writes schema.py with TerminusModel classes +tdbpy sync # writes schema.py with DocumentTemplate classes (default) +``` + +`sync --format pydantic` also records `"model_backend": "pydantic"` in +`config.json`; subsequent `sync` runs keep the pydantic emitter, and +`commit`/`check` know which module shape to expect. The generated file is +meant to be committed to version control — review schema and code together. + +### Drift gate: `tdbpy check` + +```bash +tdbpy check # exit 0: models match the schema graph +tdbpy check --ignore-docs # ignore @documentation differences +``` + +On drift it prints which classes are missing on each side and a unified +diff of differing classes, then exits 1. Wire it into CI and "the schema +changed without regenerating models" becomes impossible to merge. +`Schema.diff(client, ignore_docs=False)` exposes the same comparison +(returning a list of diff lines) for tests and tooling. + +## Typed documents + +`parse_document` is the single entry point — it finds the model by `@type` +and validates: + +```python +schema = PydanticSchema().from_db(client) # or a Schema with models added via add_obj/_schema + +ticket = schema.parse_document(client.get_document("Ticket/1")) +ticket.title # str, validated +ticket._backend_id # "Ticket/1" +``` + +Writing uses the ordinary document API — models serialise through +`_obj_to_dict`, which the client already understands: + +```python +new_ticket = Ticket(title="Login broken", audit=Audit(by="alice")) +client.insert_document(new_ticket) # mints + stamps id +new_ticket._backend_id # "Ticket/" + +client.update_document(new_ticket) # writes changed fields +client.replace_document(new_ticket) # full replace +client.delete_document(new_ticket) # or an id string +``` + +After a successful insert, `client` stamps the minted id onto the object +(`_backend_id`), exactly as it does for `DocumentTemplate` instances. + +Multi-object writes use the same `@capture`/`@ref` graph the engine +expects: linking a not-yet-inserted object emits a `@ref`, and +`insert_document` writes the whole reachable graph in one call: + +```python +bob = Person(name="bob") +ticket = Ticket(title="x", assignee=bob) +client.insert_document(ticket) # writes bob + ticket in one request +``` + +`to_document()` returns `(doc, references)` — the dict your model becomes +on the wire. `from_document(doc)` / `schema.parse_document(doc)` is the +inverse, for when you fetch raw dicts yourself (e.g. via WOQL). + +## Evolving the schema + +Two loops, one source of truth. + +**Database-first (recommended for production).** Evolve the schema in +TerminusDB — via the migration API or a schema push — then regenerate: + +```bash +tdbpy sync --format pydantic # models follow the database +git diff schema.py # review what changed +tdbpy check # 0 — local matches server +``` + +The regenerated file *is* the reviewable artefact of the migration. + +**Model-first (local development).** Edit the model, check the diff, push: + +```bash +tdbpy check # shows the pending schema diff +tdbpy commit # pushes schema.py, full_replace +``` + +When a model-first push would lose data (dropped fields, narrowed types) +the engine's migration machinery decides what happens — see the schema +migration reference. `--ignore-docs` on `check` separates cosmetic doc +edits from structural drift. + +## Dynamic loading + +```python +from terminusdb_client.schema.pydantic import PydanticSchema + +schema = PydanticSchema().from_db(client, select=["Person", "Ticket"]) +Person = schema.object["Person"] +person = schema.parse_document(client.get_document("Person/bob")) +``` + +`from_db` rebuilds the full class graph — inheritance, enums, tagged +unions, subdocuments, keys, forward and self references — as real pydantic +models. Use it for tools, notebooks and agents that must match whatever the +database currently says; use codegen (`sync`) for application code, where +static types and reviewable diffs matter. + +## Migrating from DocumentTemplate + +The declaration syntaxes coexist in one `Schema` — migrate a class at a +time, and the emitted schema document (hence the database) does not change: + +| `DocumentTemplate` | `TerminusModel` | +|---|---| +| `class P(DocumentTemplate):` | `class P(TerminusModel):` | +| `name: str` | `name: str` — unchanged | +| `_key = LexicalKey("name")` | unchanged | +| `_subdocument`, `_abstract`, `_base` | unchanged | +| `EnumTemplate` + `transform_enum_dict` | `TerminusEnum` (`str, Enum`) with explicit values | +| `TaggedUnion` | `TerminusUnion` (fields are the union members) | +| `typeguard` on `__setattr__` | pydantic validators | +| `obj._id` | `obj._backend_id` for loaded ids; `_id` still computes custom ids | +| — (new constructs) | `_shared`, `_foreign`, `_metadata`, `Unfold`, `Array`, `Field` cardinality — pydantic-only additions | + +`DocumentTemplate` stays supported — existing code keeps working. New code +should use `TerminusModel`: it is the documented interface going forward, +and the only one that gains the codegen/check tooling. + +## Recipes + +**FastAPI.** The same model is the API contract and the database schema: + +```python +@app.post("/tickets") +async def create_ticket(ticket: Ticket): # pydantic validates the request + client.insert_document(ticket) # and the document + return {"id": ticket._backend_id} +``` + +**Notebooks.** Pull the live schema, explore: + +```python +schema = PydanticSchema().from_db(client) +for name, cls in schema.object.items(): + print(name, list(cls.model_fields)) +``` + +**CI drift gate.** + +```yaml +- run: tdbpy sync --format pydantic && tdbpy check +``` + +## Reference + +### Error catalogue + +- `TypeError: use TerminusUnion for non-Optional unions` — `Union[A, B]` + or `Literal` is not a schema type; declare `TerminusUnion`/`Enum`. +- `AttributeError` on a model field — the value is a **stub**, not a + populated object; fetch with `unfold=True` or resolve `_backend_id`. +- `ValidationError` from `parse_document` — the stored document does not + satisfy the model. That is drift: sync the models or fix the data. +- `ImportError: install terminusdb[pydantic]` — missing the extra. +- `ValueError` from `client` on insert — "Subdocument cannot be added + directly" (subdocuments only exist embedded) or a non-model object. +- `ValueError` at class definition — `_shared` and `_subdocument` on one + class (mutually exclusive), or `_foreign` with declared fields. + +### Limitations + +Not yet mapped to pydantic constructs — `from_db` raises a precise error +rather than round-tripping these incorrectly: + +- the general `@oneOf` list on `Class` documents (only the `TaggedUnion` + sugar is covered by `TerminusUnion`) +- the `Table` type family (engine surface without reference-guide docs) +- `@language` multilingual `@documentation` (author the default language; + per-language docs are engine-managed) +- custom context prefixes and prefixed names (`layer:Layer`, + `layer:identifier`) — `_base`/`schema_ref` cover the common cases +- per-member enum docs (`@values` on `Enum` `@documentation`) +- `sys:JSONDocument` — unstructured JSON storage is an insert-flag feature + (`client.insert_document(raw_dict)` / CLI `-j`), not a schema construct + +### FAQ + +- *Do field defaults go into the schema?* No — defaults and constraints + are client-side. Only `Optional[]` changes the schema. +- *Can a project mix `DocumentTemplate` and `TerminusModel`?* Yes — one + `Schema`, both kinds of class, one commit. +- *Which file does `sync` write?* Always `schema.py`; `model_backend` in + `config.json` decides whether it contains `TerminusModel` or + `DocumentTemplate` classes. +- *What about `Literal` fields?* Not schema types; use an `Enum`. +- *Is `pydantic` required for the rest of the client?* No — it is an + optional extra; everything else works as before. diff --git a/pyproject.toml b/pyproject.toml index 5fd9c8c8..be61df56 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,6 +20,10 @@ typeguard = "^4.0.0" tqdm = "*" click = ">=8.0" shed = "*" +pydantic = { version = ">=2.7,<3", optional = true } + +[tool.poetry.extras] +pydantic = ["pydantic"] [tool.poetry.group.dev.dependencies] pytest = ">= 3" diff --git a/terminusdb_client/client/Client.py b/terminusdb_client/client/Client.py index 45214fd4..4a8ab219 100644 --- a/terminusdb_client/client/Client.py +++ b/terminusdb_client/client/Client.py @@ -1422,7 +1422,7 @@ def _conv_to_dict(self, obj): return obj.to_dict() elif hasattr(obj, "_to_dict"): if hasattr(obj, "_isinstance") and obj._isinstance: - if hasattr(obj.__class__, "_subdocument"): + if getattr(obj.__class__, "_subdocument", None) is not None: raise ValueError("Subdocument cannot be added directly") (d, refs) = obj._obj_to_dict() # merge all refs @@ -1453,7 +1453,7 @@ def _convert_document(self, document, graph_type): "Inserting Schema object into non-schema graph." ) item_dict = self._conv_to_dict(item) - if hasattr(item, "_capture"): + if getattr(item, "_capture", None) is not None: seen[item._capture] = item_dict else: if isinstance(item_dict, list): @@ -1582,7 +1582,9 @@ def insert_document( result = json.loads(_finish_response(result)) if isinstance(all_docs, list): for idx, item in enumerate(all_docs): - if hasattr(item, "_obj_to_dict") and not hasattr(item, "_backend_id"): + if hasattr(item, "_obj_to_dict") and getattr( + item, "_backend_id", None + ) is None: item._backend_id = result[idx] return result @@ -1668,7 +1670,9 @@ def replace_document( result = json.loads(_finish_response(result)) if isinstance(all_docs, list): for idx, item in enumerate(all_docs): - if hasattr(item, "_obj_to_dict") and not hasattr(item, "_backend_id"): + if hasattr(item, "_obj_to_dict") and getattr( + item, "_backend_id", None + ) is None: item._backend_id = result[idx].removeprefix( "terminusdb:///data/" ) diff --git a/terminusdb_client/schema/pydantic.py b/terminusdb_client/schema/pydantic.py new file mode 100644 index 00000000..5ca3268c --- /dev/null +++ b/terminusdb_client/schema/pydantic.py @@ -0,0 +1,739 @@ +"""Pydantic-native schema layer for TerminusDB. + +TerminusModel subclasses declare the same schema DocumentTemplate declares, +and emit byte-identical class documents. Instances serialise through +``_obj_to_dict`` so the client document API works unchanged. +""" +import decimal +import types as _types +from enum import Enum +from typing import ( + Annotated, + ClassVar, + ForwardRef, + List, + Optional, + Set, + Union, + get_args, + get_origin, +) + +from numpydoc.docscrape import ClassDoc + +try: + from pydantic import BaseModel, ConfigDict, Field, PrivateAttr, create_model + from pydantic.fields import FieldInfo +except ImportError as e: # pragma: no cover + raise ImportError( + "install terminusdb[pydantic] to use pydantic models" + ) from e + +from .. import woql_type as wt +from ..client.Client import Client +from .schema import ( + HashKey, + LexicalKey, + RandomKey, + Schema, + UuidV7Key, + ValueHashKey, + _check_and_fix_custom_id, +) + +_PY_CONVERT_TYPE = dict(wt.CONVERT_TYPE) +_PY_CONVERT_TYPE[decimal.Decimal] = "xsd:decimal" + +_KEY_TYPES = { + k.at_type: k + for k in (HashKey, LexicalKey, ValueHashKey, RandomKey, UuidV7Key) +} +_SUBDOC_KEYS = {"Random", "ValueHash", "UuidV7"} + + +class Unfold: + """Annotated marker: emit ``"@unfold": true`` on this property.""" + + +class Array: + """Annotated marker: emit ``"@type": "Array"`` for this property. + + ``@dimensions`` is inferred from list nesting depth. + """ + + +def _is_subdoc(cls): + return getattr(cls, "_subdocument", None) is not None + + +def _unwrap_optional(ann): + """Return (is_optional, inner) for Optional[...] / X | None.""" + origin = get_origin(ann) + if origin is Union or origin is _types.UnionType: + args = get_args(ann) + if len(args) == 2 and type(None) in args: + inner = args[0] if args[1] is type(None) else args[1] + return True, inner + raise TypeError( + f"use TerminusUnion for non-Optional unions: {ann}" + ) + return False, ann + + +def _annotation_to_woql(ann): + """Map a pydantic field annotation to a TerminusDB property spec.""" + if ann is None or ann is type(None): + return "sys:Unit" + _, ann = _unwrap_optional_pair(ann) + + origin = get_origin(ann) + if origin is not None and (origin is Union or origin is _types.UnionType): + # only reachable for unions whose optional-ness was already consumed + raise TypeError(f"use TerminusUnion for non-Optional unions: {ann}") + if origin in (list, List): + return {"@type": "List", "@class": _annotation_to_woql(get_args(ann)[0])} + if origin is frozenset: + raise TypeError( + "frozenset is not a TerminusDB type; use Set[X] / set[X]" + ) + if origin in (set, Set): + return {"@type": "Set", "@class": _annotation_to_woql(get_args(ann)[0])} + if isinstance(ann, str): + return ann # class-name reference (generated code, mutual refs) + if isinstance(ann, ForwardRef): + return ann.__forward_arg__ + if ann in _PY_CONVERT_TYPE: + return _PY_CONVERT_TYPE[ann] + if isinstance(ann, type): + if issubclass(ann, (TerminusModel, Enum)): + return ann.__name__ + raise TypeError( + f"annotation {ann!r} cannot be converted to a TerminusDB type; " + "declare an Enum or TerminusUnion instead" + ) + + +def _unwrap_optional_pair(ann): + """Return (optional_wrapped_spec_or_None, inner_ann).""" + is_opt, inner = _unwrap_optional(ann) + return is_opt, inner + + +def _field_to_woql(field): + """Map a pydantic FieldInfo to a TerminusDB property spec, + applying Unfold/Array markers and Set cardinality.""" + ann = field.annotation + metadata = list(field.metadata) + # peel Annotated if it survived on .annotation + while get_origin(ann) is Annotated: + inner_args = get_args(ann) + metadata = list(inner_args[1:]) + metadata + ann = inner_args[0] + + is_opt, inner = _unwrap_optional(ann) + while get_origin(inner) is Annotated: + inner_args = get_args(inner) + metadata = list(inner_args[1:]) + metadata + inner = inner_args[0] + spec = _annotation_to_woql(inner) + if is_opt: + if isinstance(spec, dict) and spec.get("@type") in ("Set", "List"): + raise TypeError( + "Optional cannot wrap Set/List — a collection already " + "permits zero elements" + ) + spec = {"@type": "Optional", "@class": spec} + + if any(m is Array or isinstance(m, Array) for m in metadata): + spec = _to_array_spec(inner) + + if any(m is Unfold or isinstance(m, Unfold) for m in metadata): + if isinstance(spec, dict): + spec = {**spec, "@unfold": True} + else: + spec = {"@class": spec, "@unfold": True} + + # Set cardinality from Field(min_length=, max_length=) + if isinstance(spec, dict) and spec.get("@type") == "Set": + lo = hi = None + for m in metadata: + if getattr(m, "min_length", None) is not None: + lo = m.min_length + if getattr(m, "max_length", None) is not None: + hi = m.max_length + if lo is not None or hi is not None: + if lo == hi: + spec["@cardinality"] = lo + else: + if lo is not None: + spec["@min_cardinality"] = lo + if hi is not None: + spec["@max_cardinality"] = hi + return spec + + +def _to_array_spec(ann): + """Build an Array spec; @dimensions from list nesting depth.""" + dims = 0 + inner = ann + while get_origin(inner) in (list, List): + dims += 1 + inner = get_args(inner)[0] + spec = {"@type": "Array", "@class": _annotation_to_woql(inner)} + if dims > 1: + spec["@dimensions"] = dims + return spec + + +def _enum_to_dict(cls): + """Schema document for an Enum subclass.""" + return { + "@type": "Enum", + "@id": cls.__name__, + "@value": [m.value for m in cls], + } + + +class TerminusEnum(str, Enum): + """``str, Enum`` with TerminusDB schema emission and optional + ``_schema`` registration. Values are the stored wire values.""" + + def __init_subclass__(cls, **kwargs): + super().__init_subclass__(**kwargs) + schema = cls.__dict__.get("_schema") + if schema is not None: + if not hasattr(schema, "object"): + schema.object = {} + schema.add_obj(cls.__name__, cls) + + def __str__(self): + return self.value + + @classmethod + def _to_dict(cls): + return _enum_to_dict(cls) + + +class TerminusModel(BaseModel): + """A TerminusDB document class declared as a pydantic model.""" + + model_config = ConfigDict(extra="ignore") + + _schema: ClassVar = None + _key: ClassVar = RandomKey() + _subdocument: ClassVar = None + _abstract: ClassVar = None + _shared: ClassVar = None + _unfoldable: ClassVar = None + _foreign: ClassVar = None + _metadata: ClassVar = None + _base: ClassVar = None + _backend_id = PrivateAttr(default=None) + _custom_id = PrivateAttr(default=None) + _isinstance = PrivateAttr(default=True) + _capture = PrivateAttr(default=None) + + def __hash__(self): + return id(self) + + def __init__(self, _id=None, _backend_id=None, **data): + super().__init__(**data) + if _backend_id is not None: + self._backend_id = _backend_id + if _id is not None: + if not self._allows_custom_id(): + raise ValueError( + f"Customized id is not allowed. {self.__class__} is a" + " subdocument or has set id key scheme." + ) + self._custom_id = _id + cls = self.__class__ + self._capture = f"{cls.__name__}{id(cls)}/{cls._capture_order}" + cls._capture_order += 1 + + @classmethod + def __pydantic_init_subclass__(cls, **kwargs): + super().__pydantic_init_subclass__(**kwargs) + cls._capture_order = 0 + # declaration-time validations + if _is_subdoc(cls) and getattr(cls, "_shared", None): + raise ValueError( + f"{cls.__name__}: @shared and @subdocument are mutually exclusive" + ) + if getattr(cls, "_foreign", None) and cls.model_fields: + raise ValueError( + f"{cls.__name__}: a Foreign class declares no fields" + ) + declared_key = cls.__dict__.get("_key") + if ( + _is_subdoc(cls) + and declared_key is not None + and declared_key.__class__.at_type not in _SUBDOC_KEYS + ): + raise ValueError( + f"{cls.__name__}: a subdocument key must be Random, " + "ValueHash or UuidV7" + ) + schema = cls.__dict__.get("_schema", cls._schema) + if schema is not None: + if not hasattr(schema, "object"): + schema.object = {} + schema.add_obj(cls.__name__, cls) + + @classmethod + def _allows_custom_id(cls): + if _is_subdoc(cls): + return False + declared = cls.__dict__.get("_key") + if declared is None: + return True + return isinstance(declared, (RandomKey, UuidV7Key)) + + @property + def _id(self): + if self._backend_id: + return self._backend_id + if self._custom_id: + if isinstance(self.__class__._key, UuidV7Key): + return self._custom_id + return _check_and_fix_custom_id( + self.__class__.__name__, self._custom_id + ) + return None + + @_id.setter + def _id(self, custom_id): + if self._allows_custom_id(): + self._custom_id = custom_id + else: + raise ValueError( + f"Customized id is not allowed. {self.__class__} is a" + " subdocument or has set id key scheme." + ) + + def _embedded_rep(self): + """Representation when embedded as an object property.""" + if _is_subdoc(self.__class__): + return self._obj_to_dict() + if self._id: + return {"@id": self._id, "@type": "@id"} + return {"@ref": self._capture} + + @classmethod + def _to_dict(cls, skip_checking=False): + """Emit the TerminusDB class document.""" + if not skip_checking: + _check_cycling(cls) + if getattr(cls, "_foreign", None): + return {"@type": "Foreign", "@id": cls.__name__} + is_union = issubclass(cls, TerminusUnion) and cls is not TerminusUnion + result = { + "@type": "TaggedUnion" if is_union else "Class", + "@id": cls.__name__, + } + stop = TerminusUnion if is_union else TerminusModel + parents = [x.__name__ for x in cls.__mro__] + inherited = parents[1 : parents.index(stop.__name__)] + if inherited: + result["@inherits"] = inherited + + if cls.__doc__: + doc_obj = ClassDoc(cls) + prop_doc = {} + for thing in doc_obj["Attributes"]: + if thing.desc and thing.name in cls.model_fields: + prop_doc[thing.name] = "\n".join(thing.desc) + for fname, field in cls.model_fields.items(): + if field.description: + prop_doc[fname] = field.description + result["@documentation"] = { + "@comment": "\n".join( + doc_obj["Summary"] + doc_obj["Extended Summary"] + ), + "@properties": prop_doc, + } + + if getattr(cls, "_base", None) is not None: + result["@base"] = cls._base + if _is_subdoc(cls): + result["@subdocument"] = cls._subdocument + key_type = "Random" + key = getattr(cls, "_key", None) + if key is not None and key.__class__.at_type in _SUBDOC_KEYS: + key_type = key.__class__.at_type + result["@key"] = {"@type": key_type} + if getattr(cls, "_abstract", None) is not None: + result["@abstract"] = cls._abstract + if getattr(cls, "_unfoldable", None): + result["@unfoldable"] = [] + if getattr(cls, "_shared", None): + result["@shared"] = [] + if getattr(cls, "_metadata", None) is not None: + result["@metadata"] = cls._metadata + if not _is_subdoc(cls) and getattr(cls, "_key", None) is not None: + if hasattr(cls._key, "_keys"): + result["@key"] = { + "@type": cls._key.__class__.at_type, + "@fields": cls._key._keys, + } + else: + result["@key"] = {"@type": cls._key.__class__.at_type} + for attr, field in cls.model_fields.items(): + result[attr] = _field_to_woql(field) + return result + + def _obj_to_dict(self, skip_checking=False): + """Emit the instance document; returns (doc, references).""" + result = {"@type": self.__class__.__name__} + if self._id: + result["@id"] = self._id + elif not _is_subdoc(self.__class__): + result["@capture"] = self._capture + + references = {} + for item, field in self.__class__.model_fields.items(): + if not hasattr(self, item): + continue + the_item = getattr(self, item) + if the_item is None: + if ( + field.annotation is type(None) + and item in self.model_fields_set + ): + result[item] = [] + continue + if hasattr(the_item, "_embedded_rep"): + ref_obj = the_item._embedded_rep() + if "@ref" in ref_obj: + references[ref_obj["@ref"]] = the_item + elif "@id" in ref_obj: + pass + else: + (sub_item, refs) = ref_obj + references = {**references, **refs} + ref_obj = sub_item + result[item] = ref_obj + elif isinstance(the_item, (list, set, frozenset)): + new_item = [] + for sub_item in the_item: + if hasattr(sub_item, "_embedded_rep"): + ref_obj = sub_item._embedded_rep() + if "@ref" in ref_obj: + references[ref_obj["@ref"]] = sub_item + elif "@id" in ref_obj: + pass + else: + (sub_item2, refs) = ref_obj + references = {**references, **refs} + ref_obj = sub_item2 + new_item.append(ref_obj) + elif isinstance(sub_item, Enum): + new_item.append(sub_item.value) + else: + new_item.append(sub_item) + result[item] = new_item + else: + if isinstance(the_item, Enum): + result[item] = the_item.value + else: + result[item] = wt.datetime_to_woql(the_item) + return (result, references) + + def to_document(self): + """Public alias for :meth:`_obj_to_dict`.""" + return self._obj_to_dict() + + @classmethod + def _stub(cls, doc_id): + """An id-carrying instance with no fields — a folded link.""" + obj = cls.model_construct() + obj._backend_id = doc_id + return obj + + @classmethod + def from_document(cls, doc, registry=None): + """Build a validated instance from a TerminusDB document dict. + + ``registry`` maps class names to classes for resolving linked + documents (stubs) — pass ``schema.object``. + """ + data = {} + for key, value in doc.items(): + if key.startswith("@"): + continue + field = cls.model_fields.get(key) + if field is not None: + value = _coerce_value(field.annotation, value, registry) + data[key] = value + obj = cls.model_validate(data) + if doc.get("@id"): + obj._backend_id = doc["@id"] + return obj + + +class TerminusUnion(TerminusModel): + """Declares ``"@type": "TaggedUnion"`` — fields are the members. + + A ``None``-annotated field emits ``"sys:Unit"`` (a member carrying no + value). + """ + + +def _check_cycling(cls): + """A subdocument may not embed (a superclass of) itself.""" + if getattr(cls, "_subdocument", None) is None: + return + mro_names = {c.__name__ for c in cls.__mro__} + for field in cls.model_fields.values(): + ann = field.annotation + _, inner = _unwrap_optional(ann) + while get_origin(inner) in (list, List, set, Set): + inner = get_args(inner)[0] + if isinstance(inner, ForwardRef): + name = inner.__forward_arg__.split(".")[-1].strip("'\"") + elif isinstance(inner, str): + name = inner + else: + name = getattr(inner, "__name__", str(inner)) + if name in mro_names: + raise RecursionError(f"Embbding {inner} cause recursions.") + + +def _coerce_value(ann, value, registry): + """Convert a wire value into what pydantic can validate for ``ann``.""" + if value is None: + return value + is_opt, inner = _unwrap_optional(ann) + origin = get_origin(inner) + if origin in (list, List): + items = value if isinstance(value, list) else [value] + return [_coerce_value(get_args(inner)[0], v, registry) for v in items] + if origin in (set, frozenset, Set): + items = value if isinstance(value, (list, set)) else [value] + return { + _coerce_value(get_args(inner)[0], v, registry) for v in items + } + target = inner + if isinstance(target, (str, ForwardRef)): + name = target if isinstance(target, str) else target.__forward_arg__ + name = name.split(".")[-1].strip("'\"") + target = registry.get(name) if registry else None + if isinstance(target, type) and issubclass(target, TerminusModel): + if isinstance(value, str): + return target._stub(value) + if isinstance(value, dict): + keys = [k for k in value if not k.startswith("@")] + if not keys or value.get("@type") == "@id": + return target._stub(value.get("@id")) + return target.from_document(value, registry) + return value + if inner is type(None) and value == []: + return None + return value + + +def _spec_to_annotation(spec): + """Convert a TerminusDB property spec to (annotation, default) for + ``create_model``.""" + if isinstance(spec, str): + if spec == "sys:Unit": + return (Optional[type(None)], None) + if spec[:4] == "xsd:" or spec[:4] == "sys:": + return (wt.from_woql_type(spec, skip_convert_error=True), ...) + return (spec, ...) # class-name reference + if not isinstance(spec, dict): + raise RuntimeError(f"{spec} is not a TerminusDB property spec") + spec = dict(spec) + unfold = spec.pop("@unfold", None) + cardinality = {} + for key in ("@cardinality", "@min_cardinality", "@max_cardinality"): + if key in spec: + cardinality[key] = spec.pop(key) + ann, default = _spec_inner(spec) + if cardinality: + lo = cardinality.get("@min_cardinality", cardinality.get("@cardinality")) + hi = cardinality.get("@max_cardinality", cardinality.get("@cardinality")) + default = Field(min_length=lo, max_length=hi) + if lo is None and hi is None: + default = Field() + if unfold: + ann = Annotated[ann, Unfold] + return (ann, default) + + +def _spec_inner(spec): + spec_type = spec.get("@type") + inner_spec = spec.get("@class") + if inner_spec is None: + raise RuntimeError(f"{spec} is not a TerminusDB property spec") + if spec_type is None: + # bare {"@class": X} — a plain link property + ann, default = _spec_to_annotation(inner_spec) + return (ann, ...) + if spec_type == "Optional": + ann, _ = _spec_to_annotation(inner_spec) + return (Optional[ann], None) + if spec_type == "Set": + ann, _ = _spec_to_annotation(inner_spec) + return (Set[ann], Field(default_factory=set)) + if spec_type == "List": + ann, _ = _spec_to_annotation(inner_spec) + return (List[ann], Field(default_factory=list)) + if spec_type == "Array": + dims = spec.get("@dimensions", 1) + ann, _ = _spec_to_annotation(inner_spec) + for _ in range(dims): + ann = list[ann] + return (Annotated[ann, Array], ...) + if spec_type == "Cardinality": + # deprecated family: treat as Set; caller adds cardinality props + ann, _ = _spec_to_annotation(inner_spec) + return (Set[ann], ...) + raise RuntimeError(f"{spec} is not a TerminusDB property spec") + + +class PydanticSchema(Schema): + """A Schema whose ``from_db`` rebuilds pydantic models.""" + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self._building = set() + + def _construct_context(self, context_dict): + super()._construct_context(context_dict) + if context_dict.get("@metadata") is not None: + self.metadata = context_dict["@metadata"] + + def _construct_class(self, class_obj_dict): + class_id = class_obj_dict.get("@id") + existing = self.object.get(class_id) + if existing is not None and not isinstance(existing, str): + return existing + if class_id in self._building: + raise RuntimeError(f"inheritance cycle involving {class_id}") + obj_type = class_obj_dict.get("@type") + + if obj_type == "Enum": + members = class_obj_dict.get("@value") + if not members: + raise RuntimeError(f"{class_obj_dict} not exist in database schema") + enum_dict = { + v.lower().replace(" ", "_"): v for v in members + } + new_class = TerminusEnum(class_id, enum_dict) + self.add_obj(class_id, new_class) + return new_class + + self._building.add(class_id) + try: + if obj_type == "TaggedUnion": + base = TerminusUnion + superclasses = [base] + elif obj_type == "Foreign": + superclasses = [TerminusModel] + elif obj_type == "Class": + inherits = class_obj_dict.get("@inherits") + if inherits: + if isinstance(inherits, str): + inherits = [inherits] + superclasses = [] + for parent in inherits: + parent_obj = self.object.get(parent) + if parent_obj is None or isinstance(parent_obj, str): + parent_doc = self._all_existing_classes.get(parent) + if parent_doc is None: + raise RuntimeError( + f"{parent} not exist in database schema" + ) + parent_obj = self._construct_class(parent_doc) + superclasses.append(parent_obj) + else: + superclasses = [TerminusModel] + else: + raise RuntimeError(f"{class_obj_dict} is not a schema class") + + fields = {} + attrs = {} + docstring = None + prop_docs = {} + for key, value in class_obj_dict.items(): + if key[0] != "@": + fields[key] = _spec_to_annotation(value) + elif key == "@subdocument": + attrs["_subdocument"] = value + elif key == "@abstract": + attrs["_abstract"] = value + elif key == "@unfoldable": + attrs["_unfoldable"] = True + elif key == "@shared": + attrs["_shared"] = True + elif key == "@metadata": + attrs["_metadata"] = value + elif key == "@base": + attrs["_base"] = value + elif key == "@key": + key_type = _KEY_TYPES.get(value.get("@type")) + if key_type is None: + raise RuntimeError( + f"{value} is not a TerminusDB key type" + ) + fields_key = value.get("@fields") + attrs["_key"] = ( + key_type(fields_key) if fields_key else key_type() + ) + elif key == "@documentation": + doc = value[0] if isinstance(value, list) else value + docstring = doc.get("@comment") or None + prop_docs = doc.get("@properties") or {} + for prop, descr in prop_docs.items(): + if prop in fields: + ann, default = fields[prop] + if isinstance(default, FieldInfo): + default.description = descr + else: + default = Field(default=default, description=descr) + fields[prop] = (ann, default) + + new_class = create_model( + class_id, + __base__=tuple(superclasses), + __doc__=docstring, + __config__=ConfigDict(extra="ignore"), + **fields, + ) + for attr, val in attrs.items(): + setattr(new_class, attr, val) + if obj_type == "Foreign": + new_class._foreign = True + self.add_obj(class_id, new_class) + return new_class + finally: + self._building.discard(class_id) + + def from_db(self, client: Client, select: Optional[List[str]] = None): + super().from_db(client, select=select) + namespace = self.object + for cls in list(namespace.values()): + if isinstance(cls, type) and issubclass(cls, TerminusModel): + try: + cls.model_rebuild(_types_namespace=namespace) + except Exception: + pass # unresolved refs stay ForwardRef; _to_dict still works + return self + + def parse_document(self, doc): + """Materialise a document dict into the registered class.""" + obj_type = doc.get("@type") + cls = self.object.get(obj_type) + if cls is None or isinstance(cls, str): + raise ValueError( + f"{obj_type} is not in current schema. (Received {doc})" + ) + if isinstance(cls, type) and issubclass(cls, TerminusModel): + return cls.from_document(doc, self.object) + return self._construct_object(doc) + + def import_objects(self, obj_dict): + if isinstance(obj_dict, dict): + return self.parse_document(obj_dict) + return [self.parse_document(d) for d in obj_dict] diff --git a/terminusdb_client/schema/schema.py b/terminusdb_client/schema/schema.py index e97fe8ca..18f5c97d 100644 --- a/terminusdb_client/schema/schema.py +++ b/terminusdb_client/schema/schema.py @@ -82,7 +82,6 @@ class RandomKey(TerminusKey): class UuidV7Key(TerminusKey): """Generating ID with UUID v7 (RFC 9562); accepts any IRI as @id""" - at_type = "UuidV7" @@ -496,6 +495,7 @@ def __init__( authors: Optional[List[str]] = None, schema_ref=None, base_ref=None, + metadata: Optional[dict] = None, ): self.object = {} self._all_existing_classes = {} @@ -504,6 +504,7 @@ def __init__( self.authors = authors self.schema_ref = schema_ref self.base_ref = base_ref + self.metadata = metadata @property def context(self): @@ -518,12 +519,15 @@ def context(self): documentation = {"@title": title, "@description": description} if self.authors is not None: documentation["@authors"] = self.authors - return { + context = { "@type": "@context", "@documentation": documentation, "@schema": self.schema_ref, "@base": self.base_ref, } + if getattr(self, "metadata", None) is not None: + context["@metadata"] = self.metadata + return context @context.setter def context(self, value): @@ -652,6 +656,8 @@ def _construct_context(self, context_dict): self.authors = documentation["@authors"] self.base_ref = context_dict.get("@base") self.schema_ref = context_dict.get("@schema") + if context_dict.get("@metadata") is not None: + self.metadata = context_dict["@metadata"] def _construct_object(self, obj_dict): obj_type = obj_dict.get("@type") @@ -943,6 +949,67 @@ def to_dict(self): all_obj.sort(key=lambda item: item.get("@id")) return [self.context] + all_obj + def diff(self, client: Client, ignore_docs=False): + """Compare this schema with the database schema graph. + + Returns a list of diff lines; empty means the schemas match. + """ + import difflib + + server_docs = client.get_all_documents( + graph_type=GraphType.SCHEMA, as_list=True + ) + local_docs = self.to_dict() + + def norm(obj): + if isinstance(obj, dict): + out = { + k: norm(v) + for k, v in sorted(obj.items()) + if not (ignore_docs and k == "@documentation") + } + # Random is the engine default key; an explicit "@key": + # {"@type": "Random"} and its absence are equivalent + if out.get("@key") == {"@type": "Random"}: + del out["@key"] + # the engine emits a lone parent as a string, the client + # always emits a list + if isinstance(out.get("@inherits"), str): + out["@inherits"] = [out["@inherits"]] + return out + if isinstance(obj, list): + return [norm(v) for v in obj] + return obj + + def index(docs): + out = {} + for doc in docs: + out[doc.get("@id") or "__context__"] = doc + return out + + local = index(local_docs) + server = index(server_docs) + lines = [] + for item in sorted(set(server) - set(local)): + lines.append(f"missing locally: {item}") + for item in sorted(set(local) - set(server)): + lines.append(f"missing on server: {item}") + for item in sorted(set(local) & set(server)): + a = norm(local[item]) + b = norm(server[item]) + if a != b: + lines.append(f"{item} differs:") + lines += list( + difflib.unified_diff( + json.dumps(b, indent=2, sort_keys=True).splitlines(), + json.dumps(a, indent=2, sort_keys=True).splitlines(), + fromfile=f"server:{item}", + tofile=f"local:{item}", + lineterm="", + ) + ) + return lines + def to_json_schema(self, class_object: Union[str, dict]): """Return the schema in the json schema (http://json-schema.org/) format as a dictionary for the class object. diff --git a/terminusdb_client/scripts/scripts.py b/terminusdb_client/scripts/scripts.py index ec275af6..1199e926 100644 --- a/terminusdb_client/scripts/scripts.py +++ b/terminusdb_client/scripts/scripts.py @@ -336,14 +336,236 @@ def add_docstring(self, obj_dict): return print_script -def _sync(client): +_PY_KEY_CLASSES = { + "Lexical": "LexicalKey", + "Hash": "HashKey", + "ValueHash": "ValueHashKey", + "Random": "RandomKey", + "UuidV7": "UuidV7Key", +} + +_PY_BASES = {"DocumentTemplate": "TerminusModel", "TaggedUnion": "TerminusUnion"} + + +def _enum_member_name(value): + name = "".join(c if c.isalnum() or c == "_" else "_" for c in value) + if not name.isidentifier() or (name and name[0].isdigit()): + name = "value_" + name + return name + + +def _pydantic_spec_src(spec, used_keys, used_types): + """Render a TerminusDB property spec as a pydantic field annotation. + + Returns (annotation_source, default_source_or_None). + """ + if isinstance(spec, str): + if spec == "sys:Unit": + return ("Optional[None]", "None") + src = wt.from_woql_type(spec, skip_convert_error=True, as_str=True) + if src.startswith("'"): + return (src, None) # class-name forward reference + if src == "dict": + return ("dict", None) + if hasattr(wt, src): + used_types.add(src) + return (src, None) + if not isinstance(spec, dict): + raise RuntimeError(f"{spec} is not a TerminusDB property spec") + + spec = dict(spec) + unfold = spec.pop("@unfold", None) + cardinality = {} + for key in ("@cardinality", "@min_cardinality", "@max_cardinality"): + if key in spec: + cardinality[key] = spec.pop(key) + + spec_type = spec.get("@type") + inner = spec.get("@class") + if inner is None: + raise RuntimeError(f"{spec} is not a TerminusDB property spec") + inner_src, _ = _pydantic_spec_src(inner, used_keys, used_types) + + if spec_type == "Optional": + src, default = f"Optional[{inner_src}]", "None" + elif spec_type == "Set": + src, default = f"Set[{inner_src}]", None + elif spec_type in ("Cardinality",): + src, default = f"Set[{inner_src}]", None + cardinality = {**spec, **cardinality} + elif spec_type == "List": + src, default = f"List[{inner_src}]", None + elif spec_type == "Array": + dims = spec.get("@dimensions", 1) + wrapped = inner_src + for _ in range(dims): + wrapped = f"list[{wrapped}]" + src, default = f"Annotated[{wrapped}, Array]", None + else: + raise RuntimeError(f"{spec} is not a TerminusDB property spec") + + if unfold: + src = f"Annotated[{src}, Unfold]" + if cardinality: + lo = cardinality.get("@min_cardinality", cardinality.get("@cardinality")) + hi = cardinality.get("@max_cardinality", cardinality.get("@cardinality")) + params = [] + if lo is not None: + params.append(f"min_length={lo}") + if hi is not None: + params.append(f"max_length={hi}") + default = f"Field({', '.join(params)})" + return (src, default) + + +def _create_pydantic_script(obj_list): + """Emit a pydantic-model schema.py from schema documents.""" + obj_list = list(obj_list) + context = obj_list.pop(0) if obj_list and obj_list[0].get("@type") == "@context" else None + + used_keys, used_types, result_list = set(), set(), [] + print_script = ( + "####\n" + "# This is the script for storing the schema of your TerminusDB\n" + "# database for your project.\n" + "# Use 'terminusdb commit' to commit changes to the database and\n" + "# use 'terminusdb sync' to change this file according to\n" + "# the exsisting database schema\n" + "####\n" + ) + if context is not None: + documentation = context.get("@documentation") or {} + print_script += '"""\n' + if documentation.get("@title"): + print_script += "Title: " + documentation["@title"] + "\n" + if documentation.get("@description"): + print_script += "Description: " + documentation["@description"] + "\n" + if documentation.get("@authors"): + print_script += "Authors: " + ", ".join(documentation["@authors"]) + "\n" + print_script += '"""\n' + + for obj in obj_list: + if not obj.get("@id"): + continue + obj_id, obj_type = obj["@id"], obj.get("@type") + script = "" + if obj_type == "Enum": + script = f"class {obj_id}(TerminusEnum):\n" + for value in obj.get("@value", []): + script += f' {_enum_member_name(value)} = "{value}"\n' + elif obj_type == "Foreign": + script = f"class {obj_id}(TerminusModel):\n _foreign = True\n" + else: + if obj_type == "TaggedUnion": + bases = ["TerminusUnion"] + else: + parents = obj.get("@inherits") + if parents: + if isinstance(parents, str): + parents = [parents] + bases = [_PY_BASES.get(p, p) for p in parents] + else: + bases = ["TerminusModel"] + script = f"class {obj_id}({', '.join(bases)}):\n" + prop_docs = {} + if obj.get("@documentation"): + doc = obj["@documentation"] + if doc.get("@comment"): + script += f' """{doc["@comment"]}"""\n' + prop_docs = doc.get("@properties") or {} + if obj.get("@subdocument") is not None: + script += " _subdocument = []\n" + elif obj.get("@key"): + key = obj["@key"] + key_name = _PY_KEY_CLASSES.get(key.get("@type")) + if key_name is None: + raise RuntimeError(f"{key} is not a TerminusDB key type") + used_keys.add(key_name) + if key.get("@fields"): + script += f" _key = {key_name}({key['@fields']!r})\n" + else: + script += f" _key = {key_name}()\n" + if obj.get("@abstract") is not None: + script += " _abstract = []\n" + if obj.get("@shared") is not None: + script += " _shared = True\n" + if obj.get("@unfoldable") is not None: + script += " _unfoldable = True\n" + if obj.get("@base") is not None: + script += f' _base = "{obj["@base"]}"\n' + if obj.get("@metadata") is not None: + script += f" _metadata = {obj['@metadata']!r}\n" + field_lines = "" + for key, value in obj.items(): + if key[0] == "@": + continue + src, default = _pydantic_spec_src(value, used_keys, used_types) + line = f" {key}: {src}" + descr = prop_docs.get(key) + if descr is not None: + if default == "None": + default = f'Field(default=None, description={descr!r})' + elif default is None: + default = f'Field(description={descr!r})' + else: # Field(...) for cardinality: append description + default = default[:-1] + f', description={descr!r})' + if default is not None: + line += f" = {default}" + field_lines += line + "\n" + if field_lines: + script += field_lines + elif obj_type == "Class": + script += " pass\n" + # ordering bookkeeping: parents must print before children + entry = {"name": obj_id, "script": script} + if obj_type == "Class" and obj.get("@inherits"): + parents = obj["@inherits"] + entry["parents"] = [parents] if isinstance(parents, str) else list(parents) + else: + entry["parents"] = [] + result_list.append(entry) + + body = "" + printed = [] + while result_list: + entry = result_list.pop(0) + if all(p in printed or p not in {e["name"] for e in result_list + [entry]} + for p in entry["parents"]): + body += entry["script"] + "\n" + printed.append(entry["name"]) + else: + result_list.append(entry) + + imports = "from typing import Annotated, List, Optional, Set\n" + if "Field(" in body: + imports += "from pydantic import Field\n" + imports += ( + "from terminusdb_client.schema.pydantic import " + "Array, TerminusEnum, TerminusModel, TerminusUnion, Unfold\n" + ) + if used_keys: + imports += ( + f"from terminusdb_client.schema import {', '.join(sorted(used_keys))}\n" + ) + if used_types: + imports += ( + f"from terminusdb_client.woql_type import {', '.join(sorted(used_types))}\n" + ) + print_script += imports + body + return print_script + + +def _sync(client, model_backend="template"): # all_existing_obj = client.get_all_documents(graph_type="schema") # all_obj_list = [] # for obj in all_existing_obj: # all_obj_list.append(obj) all_obj_list = client.get_all_documents(graph_type="schema", as_list=True) if len(all_obj_list) > 1: - print_script = _create_script(all_obj_list) + if model_backend == "pydantic": + print_script = _create_pydantic_script(all_obj_list) + else: + print_script = _create_script(all_obj_list) print_script = shed(source_code=print_script) file = open("schema.py", "w") file.write(print_script) @@ -354,15 +576,28 @@ def _sync(client): @click.command() -def sync(): +@click.option( + "--format", + "fmt", + type=click.Choice(["template", "pydantic"]), + default=None, + help="Model backend for the generated schema.py", +) +def sync(fmt): """Pull the current schema plan in database to schema.py""" settings = _load_settings() status = _load_settings(".TDB", check=[]) settings.update(status) - settings["database"] + backend = fmt or settings.get("model_backend") or "template" + if fmt is not None and fmt != settings.get("model_backend"): + with open("config.json") as fp: + stored = json.load(fp) + stored["model_backend"] = fmt + with open("config.json", "w") as outfile: + json.dump(stored, outfile, sort_keys=True, indent=4) client, msg = _connect(settings, new_db=False) click.echo(msg) - click.echo(_sync(client)) + click.echo(_sync(client, backend)) @click.command() @@ -410,12 +645,67 @@ def commit(message): if isinstance(obj, (woqlschema.TerminusClass, enum.EnumMeta)): if obj_str not in ["DocumentTemplate", "EnumTemplate", "TaggedUnion"]: schema_obj.add_obj(obj.__name__, obj) + elif _is_terminus_model(obj): + schema_obj.add_obj(obj.__name__, obj) if message is None: message = "Schema updated by Python client." schema_obj.commit(client, commit_msg=message, full_replace=True) click.echo(f"{database} schema updated.") +def _is_terminus_model(obj): + try: + from ..schema.pydantic import TerminusModel + except ImportError: + return False + return ( + isinstance(obj, type) + and issubclass(obj, TerminusModel) + and obj not in (TerminusModel,) + ) + + +@click.command() +@click.option( + "--ignore-docs", + is_flag=True, + help="Ignore @documentation differences when comparing schemas", +) +def check(ignore_docs): + """Diff schema.py against the database schema. Exit 1 on drift.""" + settings = _load_settings() + status = _load_settings(".TDB", check=[]) + settings.update(status) + client, msg = _connect(settings, new_db=False) + click.echo(msg) + sys.path.append(os.getcwd()) + try: + schema_plan = __import__("schema", globals(), locals(), [], 0) + except ModuleNotFoundError: + click.echo("No schema.py found in the current directory.") + raise SystemExit(2) + schema_obj = WOQLSchema() + found = False + for obj_str in dir(schema_plan): + obj = eval(f"schema_plan.{obj_str}") # noqa: S307 + if isinstance(obj, (woqlschema.TerminusClass, enum.EnumMeta)): + if obj_str not in ["DocumentTemplate", "EnumTemplate", "TaggedUnion"]: + schema_obj.add_obj(obj.__name__, obj) + found = True + elif _is_terminus_model(obj): + schema_obj.add_obj(obj.__name__, obj) + found = True + if not found: + click.echo("No schema classes found in schema.py.") + raise SystemExit(2) + lines = schema_obj.diff(client, ignore_docs=ignore_docs) + if lines: + for line in lines: + click.echo(line) + raise SystemExit(1) + click.echo("schema.py is in sync with the database schema.") + + @click.command() def deletedb(): """Delete the database in this project.""" @@ -988,6 +1278,7 @@ def try_parsing(value): tdbpy.add_command(startproject) tdbpy.add_command(sync) tdbpy.add_command(commit) +tdbpy.add_command(check) tdbpy.add_command(deletedb) tdbpy.add_command(importcsv) tdbpy.add_command(exportcsv) diff --git a/terminusdb_client/tests/integration_tests/test_client.py b/terminusdb_client/tests/integration_tests/test_client.py index e87caa0e..dae9fcea 100644 --- a/terminusdb_client/tests/integration_tests/test_client.py +++ b/terminusdb_client/tests/integration_tests/test_client.py @@ -222,6 +222,53 @@ def test_woql_substr(docker_url): assert result["bindings"][0]["Substring"]["@value"] == "Soc" +def test_woql_triple_relation(docker_url): + client = Client(docker_url, user_agent=test_user_agent) + client.connect() + db_name = "relations" + str(random()) + client.create_database(db_name) + client.connect(db=db_name) + schema = { + "@type": "Class", + "@id": "Event", + "interval": "xdd:dateTimeInterval", + } + client.insert_document(schema, graph_type="schema") + client.insert_document( + [ + { + "@type": "Event", + "@id": "Event/ev-2025-Q1", + "interval": { + "@type": "xdd:dateTimeInterval", + "@value": "2025-01-01T00:00:00Z/2025-04-01T00:00:00Z", + }, + }, + { + "@type": "Event", + "@id": "Event/ev-2025-Q2", + "interval": { + "@type": "xdd:dateTimeInterval", + "@value": "2025-04-01T00:00:00Z/2025-07-01T00:00:00Z", + }, + }, + ] + ) + result = client.query( + WOQLQuery().triple_relation( + "v:doc", + "@schema:interval", + "v:iv", + "meets", + { + "@type": "xdd:dateTimeInterval", + "@value": "2025-04-01T00:00:00Z/2025-07-01T00:00:00Z", + }, + ) + ) + assert [b["doc"] for b in result["bindings"]] == ["Event/ev-2025-Q1"] + + def test_diff_apply_version(docker_url): client = Client(docker_url, user_agent=test_user_agent) client.connect() diff --git a/terminusdb_client/tests/integration_tests/test_pydantic_integration.py b/terminusdb_client/tests/integration_tests/test_pydantic_integration.py new file mode 100644 index 00000000..14e69f7a --- /dev/null +++ b/terminusdb_client/tests/integration_tests/test_pydantic_integration.py @@ -0,0 +1,208 @@ +import datetime as dt + +import pytest + +pytest.importorskip("pydantic", reason="requires terminusdb[pydantic]") + +from terminusdb_client.client.Client import Client +from terminusdb_client.schema.pydantic import ( + PydanticSchema, + TerminusEnum, + TerminusModel, + TerminusUnion, + Unfold, +) +from terminusdb_client.schema.schema import ( + DocumentTemplate, + LexicalKey, + Schema, + WOQLSchema, +) +from typing import Annotated, Optional, Set + +test_user_agent = "terminusdb-client-python-tests" +_DB = "test_pydantic_integration" + + +class Genre(TerminusEnum): + fiction = "fiction" + non_fiction = "non_fiction" + + +class Address(TerminusModel): + _subdocument = [] + street: str + city: str + + +class Person(TerminusModel): + _key = LexicalKey("name") + name: str + age: Optional[int] = None + home: Optional[Address] = None + friend_of: Set["Person"] = set() + + +class Employee(Person): + staff_since: Optional[dt.date] = None + contact: Optional["Either"] = None + + +class Either(TerminusUnion): + local_number: Optional[int] = None + international: Optional[str] = None + + +class Shelf(TerminusModel): + _shared = True + _unfoldable = True + genre: Optional[Genre] = None + label: str + + +class Library(TerminusModel): + name: str + shelves: Annotated[Set[Shelf], Unfold] = set() + + +@pytest.fixture(scope="module", autouse=True) +def cleanup_databases(docker_url): + client = Client(docker_url, user_agent=test_user_agent) + client.connect() + for db in (_DB, _DB + "_mixed"): + try: + client.delete_database(db) + except Exception: + pass + client.create_database(db) + yield + for db in (_DB, _DB + "_mixed"): + try: + client.delete_database(db) + except Exception: + pass + + +def test_push_pydantic_schema_and_load_legacy(docker_url): + client = Client(docker_url, user_agent=test_user_agent) + client.connect(db=_DB) + schema = PydanticSchema( + title="Pydantic test schema", + description="Schema pushed from pydantic models", + authors=["integration test"], + ) + for cls in (Genre, Address, Person, Employee, Either, Shelf, Library): + schema.add_obj(cls.__name__, cls) + schema.commit(client) + + # the same schema loads as legacy DocumentTemplate classes + legacy = WOQLSchema() + legacy.from_db(client) + assert issubclass(legacy.object["Employee"], legacy.object["Person"]) + legacy_docs = legacy._all_existing_classes + for cls in (Address, Person, Employee, Shelf, Library): + doc = cls._to_dict() + ref = legacy_docs[cls.__name__] + for key, value in doc.items(): + if key == "@documentation": + continue + if key == "@inherits" and isinstance(ref.get(key), str): + value = value[0] + assert ref.get(key) == value, f"{cls.__name__}.{key}" + + # and reloads back as pydantic models + pyd = PydanticSchema().from_db(client) + assert issubclass(pyd.object["Employee"], pyd.object["Person"]) + assert pyd.object["Shelf"]._shared is True + assert pyd.object["Shelf"]._unfoldable is True + + +def test_insert_and_parse_document(docker_url): + client = Client(docker_url, user_agent=test_user_agent) + client.connect(db=_DB) + schema = PydanticSchema().from_db(client) + + jane = schema.object["Person"]( + name="Jane", + age=36, + home=schema.object["Address"](street="Main", city="Dublin"), + ) + client.insert_document(jane) + + doc = client.get_document("Person/Jane") + back = schema.parse_document(doc) + assert back.name == "Jane" + assert back.age == 36 + assert back.home.street == "Main" + assert back._backend_id == "Person/Jane" + + # a folded link materialises as a stub + john = schema.object["Person"](name="John", friend_of={back}) + client.insert_document(john) + johndoc = client.get_document("Person/John") + jback = schema.parse_document(johndoc) + friend = next(iter(jback.friend_of)) + assert friend._backend_id == "Person/Jane" + + # replacing through a stub-free path keeps working + back.age = 37 + client.replace_document(back) + doc = client.get_document("Person/Jane") + assert doc["age"] == 37 + + +def test_shared_and_unfold(docker_url): + client = Client(docker_url, user_agent=test_user_agent) + client.connect(db=_DB) + schema = PydanticSchema().from_db(client) + + shelf = schema.object["Shelf"](label="A1", genre=schema.object["Genre"].fiction) + lib = schema.object["Library"](name="Central", shelves={shelf}) + client.insert_document(lib) + + # Shelf is @unfoldable so get_document returns JSONL; list docs instead + docs = client.get_all_documents(as_list=True) + lib_doc = next(d for d in docs if d.get("@type") == "Library") + back = schema.parse_document(lib_doc) + s = next(iter(back.shelves)) + assert s.label == "A1" or s._backend_id.startswith("Shelf/") + + +def test_schema_diff_and_check(docker_url): + client = Client(docker_url, user_agent=test_user_agent) + client.connect(db=_DB) + schema = PydanticSchema().from_db(client) + assert schema.diff(client) == [] + + # perturb: same @id, extra field the db does not have + class Person(TerminusModel): + _key = LexicalKey("name") + name: str + nickname: Optional[str] = None + + schema2 = PydanticSchema() + schema2.add_obj("Person", Person) + lines = schema2.diff(client) + assert any("nickname" in line for line in lines) + + +def test_mixed_template_and_pydantic_commit(docker_url): + client = Client(docker_url, user_agent=test_user_agent) + client.connect(db=_DB + "_mixed") + + class LegPerson(DocumentTemplate): + _key = LexicalKey("name") + name: str + + class PyDoc(TerminusModel): + label: str + + schema = Schema() + schema.add_obj("LegPerson", LegPerson) + schema.add_obj("PyDoc", PyDoc) + schema.commit(client) + + loaded = PydanticSchema().from_db(client) + assert "LegPerson" in loaded.object + assert "PyDoc" in loaded.object + assert issubclass(loaded.object["PyDoc"], TerminusModel) diff --git a/terminusdb_client/tests/test_pydantic_from_db.py b/terminusdb_client/tests/test_pydantic_from_db.py new file mode 100644 index 00000000..55e1bf92 --- /dev/null +++ b/terminusdb_client/tests/test_pydantic_from_db.py @@ -0,0 +1,189 @@ +"""Tests for PydanticSchema.from_db — rebuilding models from schema docs.""" + +from unittest.mock import Mock +from typing import Optional + +import pytest + +pytest.importorskip("pydantic", reason="requires terminusdb[pydantic]") + +from terminusdb_client.schema.pydantic import ( + PydanticSchema, + TerminusModel, + TerminusUnion, +) +from terminusdb_client.schema.schema import LexicalKey, RandomKey, ValueHashKey + +SCHEMA_DOCS = [ + {"@type": "@context", "@schema": "http://s#", "@base": "http://i/"}, + { + "@type": "Class", + "@id": "Person", + "@key": {"@type": "Lexical", "@fields": ["name"]}, + "@documentation": { + "@comment": "A person.", + "@properties": {"name": "The name."}, + }, + "name": "xsd:string", + "age": {"@type": "Optional", "@class": "xsd:integer"}, + "friend_of": {"@type": "Set", "@class": "Person"}, + }, + { + "@type": "Class", + "@id": "Employee", + "@inherits": "Person", + "@key": {"@type": "Lexical", "@fields": ["name"]}, + "managed_by": "Employee", + }, + { + "@type": "Class", + "@id": "Address", + "@subdocument": [], + "@key": {"@type": "Random"}, + "street": "xsd:string", + }, + {"@type": "Enum", "@id": "Team", "@value": ["IT", "Information Technology"]}, + { + "@type": "TaggedUnion", + "@id": "Contact", + "local_number": "xsd:integer", + "international": "xsd:string", + }, + { + "@type": "Class", + "@id": "Unfoldable", + "@unfoldable": [], + "name": "xsd:string", + }, + { + "@type": "Class", + "@id": "SharedDoc", + "@shared": [], + "@key": {"@type": "ValueHash"}, + "name": "xsd:string", + }, + {"@type": "Foreign", "@id": "ExternalThing"}, + { + "@type": "Class", + "@id": "Meta", + "@metadata": {"order_by": ["name"]}, + "name": "xsd:string", + }, + { + "@type": "Class", + "@id": "Holder", + "maybe": { + "@type": "Optional", + "@class": "Unfoldable", + "@unfold": True, + }, + "always": {"@class": "Unfoldable", "@unfold": True}, + "grid": {"@type": "Array", "@class": "xsd:decimal", "@dimensions": 2}, + "three": { + "@type": "Set", + "@class": "Person", + "@cardinality": 3, + }, + "unit_field": "sys:Unit", + }, +] + + +def _client(docs=SCHEMA_DOCS): + client = Mock() + client.get_all_documents.return_value = docs + return client + + +def test_from_db_builds_models(): + schema = PydanticSchema().from_db(_client()) + for name in ["Person", "Employee", "Address", "Contact", "Holder"]: + assert name in schema.object + Person = schema.object["Person"] + assert issubclass(Person, TerminusModel) + assert Person._key.__class__ is LexicalKey + assert Person._key._keys == ["name"] + Employee = schema.object["Employee"] + assert issubclass(Employee, Person) + Address = schema.object["Address"] + assert hasattr(Address, "_subdocument") + Team = schema.object["Team"] + assert list(Team.__members__) == ["it", "information_technology"] + assert issubclass(schema.object["Contact"], TerminusUnion) + + +def test_from_db_annotations_resolve(): + schema = PydanticSchema().from_db(_client()) + Person = schema.object["Person"] + from typing import Set as TSet + + assert Person.model_fields["friend_of"].annotation == TSet[Person] + assert Person.model_fields["age"].annotation == Optional[int] + Employee = schema.object["Employee"] + assert Employee.model_fields["managed_by"].annotation is Employee + + +def test_from_db_reemits_identical_docs(): + schema = PydanticSchema().from_db(_client()) + emitted = {d["@id"]: d for d in schema.to_dict() if "@id" in d} + for doc in SCHEMA_DOCS: + doc_id = doc.get("@id") + if doc_id is None: + continue + got = emitted[doc_id] + for key, value in doc.items(): + if key == "@inherits" and isinstance(value, str): + value = [value] + assert got.get(key) == value, f"{doc_id}.{key}: {got.get(key)!r} != {value!r}" + + +def test_from_db_docstring(): + schema = PydanticSchema().from_db(_client()) + assert "A person." in schema.object["Person"].__doc__ + + +def test_from_db_markers(): + schema = PydanticSchema().from_db(_client()) + assert schema.object["Unfoldable"]._unfoldable is True + assert schema.object["SharedDoc"]._shared is True + assert schema.object["SharedDoc"]._key.__class__ is ValueHashKey + assert schema.object["ExternalThing"]._foreign is True + assert schema.object["Meta"]._metadata == {"order_by": ["name"]} + + +def test_from_db_field_markers(): + schema = PydanticSchema().from_db(_client()) + Holder = schema.object["Holder"] + doc = Holder._to_dict() + assert doc["maybe"] == { + "@type": "Optional", + "@class": "Unfoldable", + "@unfold": True, + } + assert doc["always"] == {"@class": "Unfoldable", "@unfold": True} + assert doc["grid"] == { + "@type": "Array", + "@class": "xsd:decimal", + "@dimensions": 2, + } + assert doc["three"] == {"@type": "Set", "@class": "Person", "@cardinality": 3} + assert doc["unit_field"] == "sys:Unit" + + +def test_from_db_select(): + schema = PydanticSchema().from_db(_client(), select=["Address"]) + assert "Address" in schema.object + assert "Person" not in schema.object + + +def test_from_db_context(): + schema = PydanticSchema().from_db(_client()) + assert schema.context["@schema"] == "http://s#" + assert schema.context["@base"] == "http://i/" + + +def test_from_db_random_key_default(): + schema = PydanticSchema().from_db(_client()) + # Person has Lexical; Address Random (explicit), Holder no @key + assert schema.object["Address"]._key.__class__ is RandomKey + assert schema.object["Holder"]._key.__class__ is RandomKey diff --git a/terminusdb_client/tests/test_pydantic_model.py b/terminusdb_client/tests/test_pydantic_model.py new file mode 100644 index 00000000..5d05fa5e --- /dev/null +++ b/terminusdb_client/tests/test_pydantic_model.py @@ -0,0 +1,469 @@ +"""Tests for TerminusModel declaration -> schema document emission. + +Golden parity: pydantic twins of the conftest DocumentTemplate classes must +emit byte-identical class documents. +""" + +import datetime as dt +from enum import Enum +from typing import Annotated, List, Optional, Set, Union + +import pytest + +pytest.importorskip("pydantic", reason="requires terminusdb[pydantic]") + +from terminusdb_client.schema.pydantic import ( + Array, + TerminusModel, + TerminusUnion, + Unfold, +) +from terminusdb_client.schema.schema import ( + LexicalKey, + Schema, + UuidV7Key, + ValueHashKey, +) +from terminusdb_client.woql_type import gYear + +from .conftest import ( + Address, + Contact, + Coordinate, + Country, + Employee, + Person, +) + +pydantic_schema = Schema() + + +class PCoordinate(TerminusModel): + _schema = pydantic_schema + x: float + y: float + + +class PCountry(TerminusModel): + _schema = pydantic_schema + _key = LexicalKey(["name"]) + name: str + perimeter: List[PCoordinate] + + +class PAddress(TerminusModel): + """This is address""" + + _subdocument = [] + _schema = pydantic_schema + street: str + postal_code: str + country: "PCountry" + + +class PPerson(TerminusModel): + """This is a person + + Attributes + ---------- + name : str + Name of the person. + age : int + Age of the person. + """ + + _schema = pydantic_schema + name: str + age: int + friend_of: Set["PPerson"] + + +class PEmployee(PPerson): + _schema = pydantic_schema + address_of: PAddress + contact_number: Optional[str] + managed_by: "PEmployee" + member_of: "PTeam" + permisstion: Set["PRole"] + + +class PTeam(str, Enum): + IT = "Information Technology" + Marketing = "Marketing" + + +class PRole(str, Enum): + "Test Enum in a set" + + Admin = "Admin" + Read = "Read" + Write = "Write" + + +class PContact(TerminusUnion): + _schema = pydantic_schema + local_number: int + international: str + + +def _expected(docs): + return {d["@id"]: d for d in docs} + + +# --- declaration parity vs DocumentTemplate --------------------------------- + + +def _rename(value, mapping): + if isinstance(value, str): + return mapping.get(value, value) + if isinstance(value, list): + return [_rename(v, mapping) for v in value] + if isinstance(value, dict): + return {k: _rename(v, mapping) for k, v in value.items()} + return value + + +def test_class_doc_parity(): + """Each pydantic twin emits the same class doc as its DocumentTemplate + twin, modulo class names.""" + pairs = [ + (PCoordinate, Coordinate), + (PCountry, Country), + (PAddress, Address), + (PPerson, Person), + (PEmployee, Employee), + (PContact, Contact), + ] + mapping = {p.__name__: t.__name__ for p, t in pairs} + mapping.update({"PTeam": "Team", "PRole": "Role"}) + for pcls, tcls in pairs: + got = _rename(pcls._to_dict(), mapping) + assert got == tcls._to_dict() + + +def test_enum_parity(): + assert PTeam.__members__["IT"].value == "Information Technology" + from terminusdb_client.schema.pydantic import _enum_to_dict + + assert _enum_to_dict(PTeam) == { + "@type": "Enum", + "@id": "PTeam", + "@value": ["Information Technology", "Marketing"], + } + + +def test_schema_registration(): + """_schema on a TerminusModel subclass registers like DocumentTemplate.""" + names = set(pydantic_schema.object) + assert { + "PCoordinate", + "PCountry", + "PAddress", + "PPerson", + "PEmployee", + "PContact", + } <= names + + +def test_no_globals_leak(): + """TerminusModel subclasses do not leak into module globals.""" + import terminusdb_client.schema.pydantic as pyd_mod + + assert "PPerson" not in vars(pyd_mod) + + +# --- annotation -> schema mapping ------------------------------------------- + + +def test_scalar_mappings(): + import decimal + + class Scalars(TerminusModel): + s: str + i: int + f: float + b: bool + d: dt.date + t: dt.time + ts: dt.datetime + dur: dt.timedelta + blob: dict + dec: decimal.Decimal + y: gYear + + doc = Scalars._to_dict() + assert doc["s"] == "xsd:string" + assert doc["i"] == "xsd:integer" + assert doc["f"] == "xsd:double" + assert doc["b"] == "xsd:boolean" + assert doc["d"] == "xsd:date" + assert doc["t"] == "xsd:time" + assert doc["ts"] == "xsd:dateTime" + assert doc["dur"] == "xsd:duration" + assert doc["blob"] == "sys:JSON" + assert doc["dec"] == "xsd:decimal" + assert doc["y"] == "xsd:gYear" + + +def test_container_mappings(): + class Containers(TerminusModel): + opt: Optional[int] + union_none: Union[int, None] + lst: List[str] + st: Set[str] + builtin_lst: "list[str]" + builtin_opt: "str | None" + doc = Containers._to_dict() + assert doc["opt"] == {"@type": "Optional", "@class": "xsd:integer"} + assert doc["union_none"] == {"@type": "Optional", "@class": "xsd:integer"} + assert doc["lst"] == {"@type": "List", "@class": "xsd:string"} + assert doc["st"] == {"@type": "Set", "@class": "xsd:string"} + assert doc["builtin_lst"] == {"@type": "List", "@class": "xsd:string"} + assert doc["builtin_opt"] == {"@type": "Optional", "@class": "xsd:string"} + + +def test_optional_cannot_wrap_collections(): + """The engine rejects Optional over Set/List — collections already + permit zero elements.""" + class Bad(TerminusModel): + opt_lst: Optional[List[int]] + + with pytest.raises(TypeError, match="Optional cannot wrap"): + Bad._to_dict() + + class BadSet(TerminusModel): + opt_st: Optional[Set[str]] + + with pytest.raises(TypeError, match="Optional cannot wrap"): + BadSet._to_dict() + + +def test_unsupported_annotations_raise(): + class Bad(TerminusModel): + u: Union[int, str] + + with pytest.raises(TypeError, match="TerminusUnion"): + Bad._to_dict() + + class Bad2(TerminusModel): + t: tuple + + with pytest.raises(TypeError): + Bad2._to_dict() + + class Bad3(TerminusModel): + f: frozenset + + with pytest.raises(TypeError): + Bad3._to_dict() + + +def test_field_description_merges_into_documentation(): + from pydantic import Field + + class Documented(TerminusModel): + """A documented class. + + Attributes + ---------- + name : str + Docstring says this. + """ + name: str = Field(description="Field description wins") + age: Optional[int] = Field(description="Only the field text") + + doc = Documented._to_dict() + props = doc["@documentation"]["@properties"] + assert props["name"] == "Field description wins" + assert props["age"] == "Only the field text" + assert doc["@documentation"]["@comment"] == "A documented class." + + +def test_default_does_not_make_optional(): + class WithDefault(TerminusModel): + name: str = "x" + + assert WithDefault._to_dict()["name"] == "xsd:string" + + +# --- class attributes -------------------------------------------------------- + + +def test_class_markers(): + class Marked(TerminusModel): + _abstract = [] + _base = "marked/" + _unfoldable = True + _metadata = {"render_as": {"desc": "markdown"}} + name: str + + doc = Marked._to_dict() + assert doc["@abstract"] == [] + assert doc["@base"] == "marked/" + assert doc["@unfoldable"] == [] + assert doc["@metadata"] == {"render_as": {"desc": "markdown"}} + + +def test_key_emissions(): + class ValueHashDoc(TerminusModel): + _key = ValueHashKey() + name: str + + assert ValueHashDoc._to_dict()["@key"] == {"@type": "ValueHash"} + + class UuidDoc(TerminusModel): + _key = UuidV7Key() + name: str + + assert UuidDoc._to_dict()["@key"] == {"@type": "UuidV7"} + + class RandomDoc(TerminusModel): + name: str + + assert RandomDoc._to_dict()["@key"] == {"@type": "Random"} + + +def test_subdocument_key_rule(): + class Sub(TerminusModel): + _subdocument = [] + name: str + + assert Sub._to_dict()["@key"] == {"@type": "Random"} + + class SubVH(TerminusModel): + _subdocument = [] + _key = ValueHashKey() + name: str + + assert SubVH._to_dict()["@key"] == {"@type": "ValueHash"} + + with pytest.raises(ValueError, match="subdocument"): + + class BadSub(TerminusModel): + _subdocument = [] + _key = LexicalKey("name") + name: str + + +def test_shared_mutual_exclusion(): + class Shared(TerminusModel): + _shared = True + name: str + + assert Shared._to_dict()["@shared"] == [] + + with pytest.raises(ValueError, match="subdocument"): + + class BadShared(TerminusModel): + _shared = True + _subdocument = [] + name: str + + +def test_foreign_class(): + class ForeignPerson(TerminusModel): + _foreign = True + + assert ForeignPerson._to_dict() == { + "@type": "Foreign", + "@id": "ForeignPerson", + } + + with pytest.raises(ValueError, match="Foreign"): + + class BadForeign(TerminusModel): + _foreign = True + name: str + + +# --- unfold markers ---------------------------------------------------------- + + +def test_unfold_property_marker(): + class Sub(TerminusModel): + _subdocument = [] + name: str + + class Host(TerminusModel): + req: Annotated[Sub, Unfold] + opt: Annotated[Optional[Sub], Unfold] + plain: Optional[Sub] + + doc = Host._to_dict() + assert doc["req"] == {"@class": "Sub", "@unfold": True} + assert doc["opt"] == { + "@type": "Optional", + "@class": "Sub", + "@unfold": True, + } + assert doc["plain"] == {"@type": "Optional", "@class": "Sub"} + + +# --- array + cardinality ----------------------------------------------------- + + +def test_array_marker(): + class Grid(TerminusModel): + flat: Annotated[list[float], Array] + grid2d: Annotated[list[list[float]], Array] + + doc = Grid._to_dict() + assert doc["flat"] == {"@type": "Array", "@class": "xsd:double"} + assert doc["grid2d"] == { + "@type": "Array", + "@class": "xsd:double", + "@dimensions": 2, + } + + +def test_set_cardinality(): + from pydantic import Field + + class Capped(TerminusModel): + friends: Set[str] = Field(min_length=3, max_length=3) + tags: Set[str] = Field(min_length=1) + opts: Set[str] = Field(max_length=2, default_factory=set) + plain: Set[str] + + doc = Capped._to_dict() + assert doc["friends"] == { + "@type": "Set", + "@class": "xsd:string", + "@cardinality": 3, + } + assert doc["tags"] == { + "@type": "Set", + "@class": "xsd:string", + "@min_cardinality": 1, + } + assert doc["opts"] == { + "@type": "Set", + "@class": "xsd:string", + "@max_cardinality": 2, + } + assert doc["plain"] == {"@type": "Set", "@class": "xsd:string"} + + +# --- tagged unions ------------------------------------------------------------ + + +def test_union_sys_unit(): + class Node(TerminusModel): + value: int + + class BinaryTree(TerminusUnion): + leaf: None + node: Node + + doc = BinaryTree._to_dict() + assert doc["@type"] == "TaggedUnion" + assert doc["leaf"] == "sys:Unit" + assert doc["node"] == "Node" + + +def test_forward_refs(): + doc = PEmployee._to_dict() + assert doc["managed_by"] == "PEmployee" + assert doc["member_of"] == "PTeam" + assert doc["permisstion"] == {"@type": "Set", "@class": "PRole"} + assert doc["friend_of"] == {"@type": "Set", "@class": "PPerson"} diff --git a/terminusdb_client/tests/test_pydantic_parse.py b/terminusdb_client/tests/test_pydantic_parse.py new file mode 100644 index 00000000..037b486a --- /dev/null +++ b/terminusdb_client/tests/test_pydantic_parse.py @@ -0,0 +1,216 @@ +"""Tests for TerminusModel instance serialisation (to_document/_obj_to_dict) +and reconstruction (from_document/stubs).""" + +import datetime as dt +from enum import Enum +from typing import List, Optional, Set + +import pytest + +pytest.importorskip("pydantic", reason="requires terminusdb[pydantic]") + +from terminusdb_client.schema.pydantic import PydanticSchema, TerminusModel +from terminusdb_client.schema.schema import LexicalKey + + +class Coord(TerminusModel): + _subdocument = [] + x: float + y: float + + +class Nation(TerminusModel): + _key = LexicalKey("name") + name: str + perimeter: List[Coord] + + +class Colour(str, Enum): + red = "red" + blue = "blue" + + +class Widget(TerminusModel): + name: str + shade: Optional[Colour] + parts: Set["Widget"] + + +class CheckDt(TerminusModel): + when: dt.datetime + span: dt.timedelta + + +class Address(TerminusModel): + _subdocument = [] + street: str + city: str + + +class Human(TerminusModel): + name: str + home: Optional[Address] = None + best_friend: Optional["Human"] = None + + +# --- to_document / _obj_to_dict --------------------------------------------- + + +def test_basic_serialise(): + w = Widget(name="bolt", shade=Colour.red, parts=set()) + doc, refs = w._obj_to_dict() + assert doc["@type"] == "Widget" + assert doc["name"] == "bolt" + assert doc["shade"] == "red" # member.value, not "Colour.red" + assert doc["parts"] == [] + assert "@capture" in doc # no id yet -> capture minted + assert refs == {} + + +def test_capture_and_ref_graph(): + """Un-ided linked objects emit @capture/@ref and land in references.""" + katy = Human(name="katy") + bob = Human(name="bob", best_friend=katy) + doc, refs = bob._obj_to_dict() + assert doc["best_friend"] == {"@ref": katy._capture} + assert katy._capture in refs + # bob has no id either + assert "@capture" in doc + + +def test_id_link_form(): + """Keyed objects carry no client-side @id — the engine mints it.""" + ireland = Nation(name="Republic of Ireland", perimeter=[]) + doc, _ = ireland._obj_to_dict() + assert "@id" not in doc + assert doc["@capture"] == ireland._capture + assert doc["perimeter"] == [] + + +def test_subdocument_embeds(): + h = Human(name="bob", home=Address(street="s", city="c")) + doc, _ = h._obj_to_dict() + assert doc["home"] == { + "@type": "Address", + "street": "s", + "city": "c", + } + + +def test_custom_id_and_backend_id(): + w = Widget(name="w", shade=None, parts=set(), _id="custom") + doc, _ = w._obj_to_dict() + assert doc["@id"] == "Widget/custom" + loaded = Widget(name="w", shade=None, parts=set(), _backend_id="Widget/db1") + doc2, _ = loaded._obj_to_dict() + assert doc2["@id"] == "Widget/db1" + + +def test_custom_id_rejected_on_key_scheme(): + with pytest.raises(ValueError, match="Customized id"): + Nation(name="x", perimeter=[], _id="bad") + + +def test_datetime_serialise(): + obj = CheckDt( + when=dt.datetime(2019, 5, 18, 15, 17, 8, 132263), + span=dt.timedelta(days=64, hours=8, minutes=5, seconds=56.00001), + ) + doc, _ = obj._obj_to_dict() + assert doc["when"] == "2019-05-18T15:17:08.132263" + assert doc["span"].startswith("PT") + + +def test_to_document_alias(): + w = Widget(name="w", shade=None, parts=set()) + assert w.to_document() == w._obj_to_dict() + + +# --- stubs / from_document --------------------------------------------------- + + +def test_stub_from_id_string(): + schema = PydanticSchema() + schema.add_obj("Human", Human) + schema.add_obj("Address", Address) + doc = {"@type": "Human", "@id": "Human/bob", "name": "bob", + "best_friend": "Human/katy"} + obj = schema.parse_document(doc) + assert obj._backend_id == "Human/bob" + assert obj.name == "bob" + friend = obj.best_friend + assert friend._backend_id == "Human/katy" + with pytest.raises(AttributeError): + friend.name + + +def test_stub_from_id_dict(): + schema = PydanticSchema() + schema.add_obj("Human", Human) + schema.add_obj("Address", Address) + doc = {"@type": "Human", "@id": "Human/bob", "name": "bob", + "best_friend": {"@type": "@id", "@id": "Human/katy"}} + obj = schema.parse_document(doc) + assert obj.best_friend._backend_id == "Human/katy" + + +def test_unfolded_subdocument_parses(): + schema = PydanticSchema() + schema.add_obj("Human", Human) + schema.add_obj("Address", Address) + doc = {"@type": "Human", "@id": "Human/bob", "name": "bob", + "home": {"@type": "Address", "street": "s", "city": "c"}} + obj = schema.parse_document(doc) + assert obj.home.street == "s" + assert obj.home.city == "c" + + +def test_unfolded_document_parses_to_model(): + schema = PydanticSchema() + schema.add_obj("Human", Human) + schema.add_obj("Address", Address) + doc = {"@type": "Human", "@id": "Human/bob", "name": "bob", + "best_friend": {"@type": "Human", "@id": "Human/katy", + "name": "katy"}} + obj = schema.parse_document(doc) + assert obj.best_friend.name == "katy" + assert obj.best_friend._backend_id == "Human/katy" + + +def test_round_trip(): + schema = PydanticSchema() + schema.add_obj("Nation", Nation) + schema.add_obj("Coord", Coord) + n = Nation(name="Ireland", perimeter=[Coord(x=1.0, y=2.0)]) + doc, _ = n._obj_to_dict() + back = schema.parse_document(doc) + assert back.name == "Ireland" + assert back.perimeter[0].x == 1.0 + assert back._backend_id == n._id + + +def test_scalar_into_set_wraps(): + schema = PydanticSchema() + schema.add_obj("Widget", Widget) + schema.add_obj("Colour", Colour) + doc = {"@type": "Widget", "name": "w", "shade": "red", + "parts": "Widget/other"} + obj = schema.parse_document(doc) + assert isinstance(obj.parts, set) + assert obj.parts.pop()._backend_id == "Widget/other" + assert obj.shade is Colour.red + + +def test_import_objects_list(): + schema = PydanticSchema() + schema.add_obj("Widget", Widget) + docs = [{"@type": "Widget", "name": "a", "shade": None, "parts": []}, + {"@type": "Widget", "name": "b", "shade": None, "parts": []}] + out = schema.import_objects(docs) + assert [o.name for o in out] == ["a", "b"] + + +def test_unknown_type_raises(): + schema = PydanticSchema() + with pytest.raises(ValueError, match="not in current schema"): + schema.parse_document({"@type": "Nope", "@id": "x"}) diff --git a/terminusdb_client/tests/test_pydantic_scripts.py b/terminusdb_client/tests/test_pydantic_scripts.py new file mode 100644 index 00000000..afd37efc --- /dev/null +++ b/terminusdb_client/tests/test_pydantic_scripts.py @@ -0,0 +1,212 @@ +"""Tests for pydantic codegen (_create_pydantic_script), Schema.diff and the +tdbpy check/sync/commit wiring.""" + +import types +from unittest.mock import Mock + +import pytest + +pytest.importorskip("pydantic", reason="requires terminusdb[pydantic]") + +from terminusdb_client.schema.pydantic import ( + PydanticSchema, + TerminusModel, + TerminusUnion, +) +from terminusdb_client.schema.schema import LexicalKey +from terminusdb_client.scripts.scripts import _create_pydantic_script + +SCHEMA_DOCS = [ + { + "@type": "@context", + "@schema": "http://s#", + "@base": "http://i/", + "@documentation": { + "@title": "T", + "@description": "D", + "@authors": ["A U Thor"], + }, + }, + { + "@type": "Class", + "@id": "Person", + "@key": {"@type": "Lexical", "@fields": ["name"]}, + "@documentation": {"@comment": "A person.", + "@properties": {"name": "The name."}}, + "name": "xsd:string", + "age": {"@type": "Optional", "@class": "xsd:integer"}, + "friend_of": {"@type": "Set", "@class": "Person"}, + }, + { + "@type": "Class", + "@id": "Employee", + "@inherits": "Person", + "managed_by": "Employee", + }, + { + "@type": "Class", + "@id": "Address", + "@subdocument": [], + "street": "xsd:string", + }, + {"@type": "Enum", "@id": "Team", "@value": ["IT", "Information Technology"]}, + { + "@type": "TaggedUnion", + "@id": "BinaryTree", + "leaf": "sys:Unit", + "node": "Node", + }, + { + "@type": "Class", + "@id": "Node", + "value": "xsd:integer", + }, + { + "@type": "Class", + "@id": "Shared", + "@shared": [], + "@unfoldable": [], + "name": "xsd:string", + }, + { + "@type": "Class", + "@id": "Holder", + "maybe": { + "@type": "Optional", + "@class": "Shared", + "@unfold": True, + }, + "grid": {"@type": "Array", "@class": "xsd:decimal", "@dimensions": 2}, + "three": {"@type": "Set", "@class": "Person", "@cardinality": 3}, + }, + {"@type": "Foreign", "@id": "Ext"}, +] + + +def _exec_script(source): + mod = types.ModuleType("generated_schema") + exec(compile(source, "schema.py", "exec"), mod.__dict__) + return mod + + +def test_codegen_emits_pydantic_classes(): + source = _create_pydantic_script(list(SCHEMA_DOCS)) + mod = _exec_script(source) + Person = mod.Person + assert issubclass(Person, TerminusModel) + assert Person._key.__class__ is LexicalKey + assert issubclass(mod.Employee, mod.Person) + assert issubclass(mod.BinaryTree, TerminusUnion) + assert issubclass(mod.Team, str) and issubclass(mod.Team, __import__("enum").Enum) + assert mod.Ext._foreign is True + assert mod.Shared._shared is True + assert mod.Shared._unfoldable is True + + +def test_codegen_reemits_identical_docs(): + source = _create_pydantic_script(list(SCHEMA_DOCS)) + mod = _exec_script(source) + emitted = {} + for name in dir(mod): + obj = getattr(mod, name) + if isinstance(obj, type) and issubclass(obj, TerminusModel) \ + and name not in ("TerminusModel", "TerminusUnion"): + emitted[name] = obj._to_dict() + elif isinstance(obj, type) and issubclass(obj, __import__("enum").Enum) \ + and name != "Enum": + from terminusdb_client.schema.pydantic import _enum_to_dict + + emitted[name] = _enum_to_dict(obj) + for doc in SCHEMA_DOCS: + doc_id = doc.get("@id") + if doc_id is None: + continue + assert doc_id in emitted, f"{doc_id} missing from generated module" + got = emitted[doc_id] + for key, value in doc.items(): + if key == "@inherits" and isinstance(value, str): + value = [value] + assert got.get(key) == value, f"{doc_id}.{key}: {got.get(key)!r} != {value!r}" + + +def test_codegen_field_doc(): + source = _create_pydantic_script(list(SCHEMA_DOCS)) + mod = _exec_script(source) + assert "A person." in mod.Person.__doc__ + + +# --- Schema.diff / tdbpy check ------------------------------------------------- + + +def _normalised(docs): + return docs + + +def test_diff_in_sync(): + schema = PydanticSchema() + for cls_doc in SCHEMA_DOCS: + if cls_doc.get("@type") == "@context": + schema.title = cls_doc["@documentation"]["@title"] + schema.description = cls_doc["@documentation"]["@description"] + schema.authors = cls_doc["@documentation"]["@authors"] + schema.schema_ref = cls_doc["@schema"] + schema.base_ref = cls_doc["@base"] + + class Person(TerminusModel): + _schema = schema + _key = LexicalKey("name") + name: str + + client = Mock() + local = schema.to_dict() + client.get_all_documents.return_value = local + assert schema.diff(client) == [] + + +def test_diff_reports_changes(): + schema = PydanticSchema() + + class Person(TerminusModel): + _schema = schema + _key = LexicalKey("name") + name: str + age: int + + schema.schema_ref = "http://s#" + schema.base_ref = "http://i/" + server_docs = [ + {"@type": "@context", "@schema": "http://s#", "@base": "http://i/"}, + {"@type": "Class", "@id": "Person", + "@key": {"@type": "Lexical", "@fields": ["name"]}, + "name": "xsd:string"}, + {"@type": "Class", "@id": "Ghost", "x": "xsd:string"}, + ] + client = Mock() + client.get_all_documents.return_value = server_docs + diff = schema.diff(client) + out = "\n".join(diff) + assert any("missing" in line.lower() or "-" in line for line in diff) + # Ghost is on server only, age locally only + assert "Ghost" in out + assert "age" in out + + +def test_diff_ignore_docs(): + schema = PydanticSchema() + schema.schema_ref = "http://s#" + schema.base_ref = "http://i/" + + class Person(TerminusModel): + """Different comment.""" + _schema = schema + name: str + + client = Mock() + client.get_all_documents.return_value = [ + {"@type": "@context", "@schema": "http://s#", "@base": "http://i/"}, + {"@type": "Class", "@id": "Person", + "@documentation": {"@comment": "Old comment", "@properties": {}}, + "name": "xsd:string"}, + ] + assert schema.diff(client) != [] + assert schema.diff(client, ignore_docs=True) == [] diff --git a/terminusdb_client/tests/test_woqlQuery.py b/terminusdb_client/tests/test_woqlQuery.py index 7c6594d1..65cc0602 100644 --- a/terminusdb_client/tests/test_woqlQuery.py +++ b/terminusdb_client/tests/test_woqlQuery.py @@ -67,6 +67,33 @@ def test_woql_not_method(self): assert woql_object.to_dict() == json_obj assert woql_object_chain.to_dict() == json_obj + def test_triple_relation_method(self): + woql_object = WOQLQuery().triple_relation( + "v:doc", + "scm:interval", + "v:iv", + "meets", + {"@type": "xdd:dateTimeInterval", "@value": "2025-01-01/2025-04-01"}, + ) + json_obj = { + "@type": "TripleRelation", + "subject": {"@type": "NodeValue", "variable": "doc"}, + "predicate": {"@type": "NodeValue", "node": "scm:interval"}, + "object": {"@type": "Value", "variable": "iv"}, + "relation": { + "@type": "DataValue", + "data": {"@type": "xsd:string", "@value": "meets"}, + }, + "value": { + "@type": "DataValue", + "data": { + "@type": "xdd:dateTimeInterval", + "@value": "2025-01-01/2025-04-01", + }, + }, + } + assert woql_object.to_dict() == json_obj + def test_woql_and_method(self): woql_object = WOQLQuery().woql_and( WOQLQuery().triple("a", "b", "c"), WOQLQuery().triple("1", "2", "3") @@ -237,6 +264,36 @@ def test_cast_method(self): woql_object = WOQLQuery().cast("v:Duration", "xsd:integer", "v:Duration_Cast") assert woql_object.to_dict() == WOQL_CAST_JSON + def test_cast_method_prefixes_dict(self): + woql_object = WOQLQuery().cast( + "v:QName_str", + "xsd:QName", + "v:QName_val", + prefixes={"eg": "http://example.org/ns#"}, + ) + json_obj = woql_object.to_dict() + assert json_obj["prefixes"] == { + "@type": "Value", + "data": {"eg": "http://example.org/ns#"}, + } + + def test_cast_method_prefixes_var(self): + woql_object = WOQLQuery().cast( + "v:QName_str", + "xsd:QName", + "v:QName_val", + prefixes="v:Namespaces", + ) + json_obj = woql_object.to_dict() + assert json_obj["prefixes"] == { + "@type": "Value", + "variable": "Namespaces", + } + + def test_cast_method_no_prefixes(self): + woql_object = WOQLQuery().cast("v:Duration", "xsd:integer", "v:Duration_Cast") + assert "prefixes" not in woql_object.to_dict() + def test_cast_method_literal(self): woql_object1 = WOQLQuery().cast( "1", "xsd:integer", "v:Duration_Cast", literal_type="integer" diff --git a/terminusdb_client/woqlquery/woql_query.py b/terminusdb_client/woqlquery/woql_query.py index d9b93bc7..1a8c8116 100644 --- a/terminusdb_client/woqlquery/woql_query.py +++ b/terminusdb_client/woqlquery/woql_query.py @@ -2715,6 +2715,45 @@ def interval_relation_typed(self, relation, x, y): self._cursor["y"] = self._clean_object(y) return self + def triple_relation(self, sub, pred, obj, relation, value): + """Finds triples whose object stands in a typed relation to a + reference value. + + Supports the thirteen Allen interval relations on + xdd:dateTimeInterval objects; instant objects (xsd:dateTime) + compare as zero-duration points. An instant at a meeting boundary + belongs to the interval it starts. + + Parameters + ---------- + sub : str + Subject, has to be a node (URI) or variable + pred : str + Predicate, can be variable (prefix with "v:") or node + obj : str or dict + The interval or instant object or a variable + relation : str or dict + relation name (e.g. "during", "meets") + value : str or dict + the reference interval or instant literal + + Returns + ------- + WOQLQuery object + query object that can be chained and/or execute + """ + if relation is None or value is None: + raise ValueError("TripleRelation takes five parameters") + if self._cursor.get("@type"): + self._wrap_cursor_with_and() + self._cursor["@type"] = "TripleRelation" + self._cursor["subject"] = self._clean_subject(sub) + self._cursor["predicate"] = self._clean_predicate(pred) + self._cursor["object"] = self._clean_object(obj) + self._cursor["relation"] = self._clean_data_value(relation) + self._cursor["value"] = self._clean_data_value(value) + return self + def range_min(self, input_list, result): """Find the minimum value in a list using the standard ordering. @@ -3800,7 +3839,7 @@ def count(self, countvar, query=None): self._cursor["count"] = self._clean_data_value(countvar) return self._add_sub_query(query) - def cast(self, val, user_type, result, literal_type=None): + def cast(self, val, user_type, result, literal_type=None, prefixes=None): """Changes the type of va to type and saves the return in vb Parameters @@ -3814,6 +3853,11 @@ def cast(self, val, user_type, result, literal_type=None): literal_type: str, optional literal type of`val`, can be used to treat `val` as a literal rather than an object or variable in the WOQL query. If literal type is "owl:Thing" or "node", `val` will be treated as object in the graph + prefixes : dict or str or Var, optional + prefix map used to expand or contract expanded names + (xsd:QName, xbrl:SQName) during the cast. A plain dict is sent + as a literal map; a "v:" string or Var is a variable bound + elsewhere (e.g. via dot on a stored namespaces dict). Returns ------- @@ -3824,9 +3868,13 @@ def cast(self, val, user_type, result, literal_type=None): if (literal_type == "owl:Thing" or literal_type == "node") and isinstance( val, str ): - return self.cast({"@type": "Value", "node": val}, user_type, result) + return self.cast( + {"@type": "Value", "node": val}, user_type, result, prefixes=prefixes + ) else: - return self.cast(self.literal(val, literal_type), user_type, result) + return self.cast( + self.literal(val, literal_type), user_type, result, prefixes=prefixes + ) if val and val == "args": return ["value", "type", "result"] if self._cursor.get("@type"): @@ -3835,6 +3883,11 @@ def cast(self, val, user_type, result, literal_type=None): self._cursor["value"] = self._clean_object(val) self._cursor["type"] = self._clean_node_value(user_type) self._cursor["result"] = self._clean_object(result) + if prefixes is not None: + if type(prefixes) is dict and "@type" not in prefixes: + self._cursor["prefixes"] = {"@type": "Value", "data": prefixes} + else: + self._cursor["prefixes"] = self._clean_object(prefixes) return self def type_of(self, value, vtype): diff --git a/terminusdb_client/woqlschema/__init__.py b/terminusdb_client/woqlschema/__init__.py index 1410a51b..fc69cae4 100644 --- a/terminusdb_client/woqlschema/__init__.py +++ b/terminusdb_client/woqlschema/__init__.py @@ -3,3 +3,4 @@ WOQLSchema = Schema # noqa sys.modules["terminusdb_client.woqlschema.woql_schema"] = schema # noqa +del schema # noqa keep the terminusdb_client.schema package importable diff --git a/tox.ini b/tox.ini index 8139d4a3..dfcd4860 100644 --- a/tox.ini +++ b/tox.ini @@ -18,7 +18,7 @@ deps = pytest-mock pytest-xdist commands = - pip install -e . + pip install -e .[pydantic] python -m pytest --tb=native --cov=terminusdb_client terminusdb_client/tests/{posargs} --cov-report xml:cov.xml passenv = TERMINUSX_TOKEN @@ -28,3 +28,7 @@ passenv = TERMINUSX_TOKEN # S501 is turned off for now, check if the verify=false for request is ok ignore = E501,W503,S101,S310,A003,E203,S607,S603,S404,W391,N814,N804,S106,F722,S105,E266,S311,N806 exclude = .*/,__pycache__,docs/,build/,venv/,.tox/ +per-file-ignores = + # pytest.importorskip guards defer imports below module top (E402) + terminusdb_client/tests/test_pydantic_*.py: E402 + terminusdb_client/tests/integration_tests/test_pydantic_*.py: E402