From a8972a843dd3303b8fd0b1ff037aca6c3798e3c5 Mon Sep 17 00:00:00 2001 From: Joseph Mearman Date: Mon, 17 Aug 2026 15:01:35 +0100 Subject: [PATCH 1/3] feat: add the semantic math value schemas (rationals, units, symbol table, expression grammar) The semantic half of the formula model. Exact rationals are canonical decimal-integer strings, because Number loses integer exactness above 2^53 and exactness is the whole point -- unit-conversion chains and quantity equality stay bit-exact over canonical strings. Dimensions are integer-exponent vectors over the seven SI bases. The unit registry carries exact rational factor/offset conversions to coherent SI plus domain normalisation contexts (per-unit systems), and is document-carried data rather than a table shipped in this package. The symbol table keys curation entries by (glyph, scope) mapping to id, quantity kind, preferred unit, and definition source. MathExpression is a closed eight-variant grammar -- num, qty with uncertainty, sym, app over a namespaced operator registry, sum/prod binders with bounds, matrix, and unparsed as a first-class fallback so lowering coverage gaps stay visible data rather than parse failures. The recursive union uses the package's z.custom structural-guard pattern (MathMlNode's), since z.lazy collapses to unknown in the pinned Zod. --- src/index.ts | 1 + src/math.test.ts | 302 +++++++++++++++++++++++++++++++++++++++++++++++ src/math.ts | 287 ++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 590 insertions(+) create mode 100644 src/math.test.ts create mode 100644 src/math.ts diff --git a/src/index.ts b/src/index.ts index 6c77a56..6d837e0 100644 --- a/src/index.ts +++ b/src/index.ts @@ -3,6 +3,7 @@ export * from './geometry'; export * from './style'; export * from './metadata'; export * from './mathml'; +export * from './math'; export * from './content'; export * from './layout'; export * from './package'; diff --git a/src/math.test.ts b/src/math.test.ts new file mode 100644 index 0000000..e2e4c66 --- /dev/null +++ b/src/math.test.ts @@ -0,0 +1,302 @@ +import { describe, expect, it } from 'vitest'; +import { + DimensionVectorSchema, + ExactRationalSchema, + isMathExpression, + MathExpressionSchema, + MathMatrixSchema, + MathNormalisationContextSchema, + MathNumSchema, + MathPresentationSchema, + MathProvenanceSchema, + MathQtySchema, + MathSumSchema, + MathSymbolEntrySchema, + MathSymSchema, + MathUncertaintySchema, + MathUnitSchema, + MathUnparsedSchema, + type MathExpression, + SymbolTableSchema, + type SymbolTable, +} from './math'; + +describe('ExactRationalSchema', () => { + it('accepts canonical integer-string halves, including the integer-as-over-one and exact-negative cases', () => { + expect(ExactRationalSchema.safeParse({ numerator: '0', denominator: '1' }).success).toBe(true); + expect(ExactRationalSchema.safeParse({ numerator: '-381', denominator: '1250' }).success).toBe(true); + expect(ExactRationalSchema.safeParse({ numerator: '196133', denominator: '20000' }).success).toBe(true); + }); + + it('rejects every non-canonical spelling: float text, leading zeros, negative zero, and zero or negative denominators', () => { + for (const numerator of ['1.5', '007', '-0', '1e3', '']) { + expect(ExactRationalSchema.safeParse({ numerator, denominator: '2' }).success).toBe(false); + } + for (const denominator of ['0', '-2', '00', '2.5', '']) { + expect(ExactRationalSchema.safeParse({ numerator: '1', denominator }).success).toBe(false); + } + }); +}); + +describe('DimensionVectorSchema', () => { + it('accepts a sparse vector over any subset of the SI bases, exponents negative included', () => { + // Speed and force in their standard SI decompositions, plus the empty dimensionless vector. + expect(DimensionVectorSchema.safeParse({ length: 1, time: -1 }).success).toBe(true); + expect(DimensionVectorSchema.safeParse({ length: 1, mass: 1, time: -2 }).success).toBe(true); + expect(DimensionVectorSchema.safeParse({}).success).toBe(true); + }); + + it('rejects dimensions outside the seven SI bases and non-integer exponents', () => { + expect(DimensionVectorSchema.safeParse({ information: 1 }).success).toBe(false); + expect(DimensionVectorSchema.safeParse({ length: 1.5 }).success).toBe(false); + expect(DimensionVectorSchema.safeParse({ length: '1' }).success).toBe(false); + }); +}); + +describe('MathUnitSchema', () => { + it('accepts a linear unit with its exact SI conversion (the foot is exactly 381/1250 m)', () => { + expect( + MathUnitSchema.safeParse({ + id: 'imperial:foot', + symbol: 'ft', + name: 'foot', + dimension: { length: 1 }, + factorToSi: { numerator: '381', denominator: '1250' }, + }).success, + ).toBe(true); + }); + + it('accepts an affine unit whose zero differs from SI (degree Celsius: factor 1, offset 5463/20 K)', () => { + expect( + MathUnitSchema.safeParse({ + id: 'si:degree-celsius', + symbol: '°C', + dimension: { thermodynamicTemperature: 1 }, + factorToSi: { numerator: '1', denominator: '1' }, + offsetToSi: { numerator: '5463', denominator: '20' }, + }).success, + ).toBe(true); + }); + + it('rejects a unit with no dimension or no exact conversion factor', () => { + expect(MathUnitSchema.safeParse({ id: 'si:metre', symbol: 'm', factorToSi: { numerator: '1', denominator: '1' } }).success).toBe(false); + expect(MathUnitSchema.safeParse({ id: 'si:metre', symbol: 'm', dimension: { length: 1 } }).success).toBe(false); + }); +}); + +describe('SymbolTableSchema', () => { + it('accepts a table with symbols, their unit registry, and a per-unit normalisation context', () => { + const table: SymbolTable = { + symbols: [ + { + glyph: 'U', + scope: 'document', + id: 'symbols:voltage', + quantityKind: 'si:voltage', + preferredUnit: 'si:volt', + definitionSource: 'prose:sections/2/paragraph-1', + }, + ], + units: [ + { + id: 'si:volt', + symbol: 'V', + dimension: { length: 2, mass: 1, time: -3, electricCurrent: -1 }, + factorToSi: { numerator: '1', denominator: '1' }, + }, + { + id: 'psu:pu-power', + symbol: 'p.u.', + dimension: {}, + factorToSi: { numerator: '1', denominator: '1' }, + context: 'psu:100mva-11kv', + }, + ], + contexts: [ + { + id: 'psu:100mva-11kv', + bases: [ + { unit: 'si:volt-ampere', value: { numerator: '100000000', denominator: '1' } }, + { unit: 'si:volt', value: { numerator: '11000', denominator: '1' } }, + ], + }, + ], + }; + const parsed = SymbolTableSchema.parse(table); + expect(parsed.symbols[0]?.id).toBe('symbols:voltage'); + expect(parsed.contexts?.[0]?.bases).toHaveLength(2); + }); + + it('requires symbols and units outright, with contexts the only optional array', () => { + expect(SymbolTableSchema.safeParse({ symbols: [], units: [] }).success).toBe(true); + expect(SymbolTableSchema.safeParse({ symbols: [] }).success).toBe(false); + expect(SymbolTableSchema.safeParse({ units: [] }).success).toBe(false); + }); + + it('rejects a symbol entry missing the (glyph, scope, id) key it is looked up by', () => { + expect(MathSymbolEntrySchema.safeParse({ glyph: 'U', id: 'symbols:voltage' }).success).toBe(false); + expect(MathSymbolEntrySchema.safeParse({ glyph: 'U', scope: 'document', id: 'symbols:voltage' }).success).toBe(true); + }); + + it('accepts a normalisation context with exact rational bases', () => { + expect( + MathNormalisationContextSchema.safeParse({ + id: 'psu:100mva-11kv', + bases: [{ unit: 'si:volt', value: { numerator: '11000', denominator: '1' } }], + }).success, + ).toBe(true); + expect(MathNormalisationContextSchema.safeParse({ id: 'psu:100mva-11kv', bases: [{ unit: 'si:volt' }] }).success).toBe(false); + }); +}); + +describe('MathPresentationSchema and MathProvenanceSchema', () => { + it('carries the verbatim LaTeX as the presentation layer in full', () => { + expect(MathPresentationSchema.parse({ latex: '\\frac{-b \\pm \\sqrt{b^2 - 4ac}}{2a}' }).latex).toBe( + '\\frac{-b \\pm \\sqrt{b^2 - 4ac}}{2a}', + ); + }); + + it('requires the provenance source and edit trail, with the page reference optional', () => { + expect(MathProvenanceSchema.safeParse({ source: 'odf:content.xml#Object1', editTrail: [] }).success).toBe(true); + expect( + MathProvenanceSchema.safeParse({ source: 'lowered:latex', pageRef: 'page 4', editTrail: ['value corrected against the ledger'] }).success, + ).toBe(true); + expect(MathProvenanceSchema.safeParse({ source: 'odf:content.xml#Object1' }).success).toBe(false); + }); +}); + +describe('MathUncertaintySchema', () => { + it('accepts an exact magnitude with optional unit override and coverage factor', () => { + expect(MathUncertaintySchema.safeParse({ magnitude: { numerator: '1', denominator: '4' } }).success).toBe(true); + expect( + MathUncertaintySchema.safeParse({ + magnitude: { numerator: '196', denominator: '100' }, + unit: 'si:percent', + coverageFactor: 2, + }).success, + ).toBe(true); + }); + + it('rejects a non-positive coverage factor and a float-text magnitude', () => { + expect(MathUncertaintySchema.safeParse({ magnitude: { numerator: '1', denominator: '4' }, coverageFactor: 0 }).success).toBe(false); + expect(MathUncertaintySchema.safeParse({ magnitude: { numerator: '0.25', denominator: '1' } }).success).toBe(false); + }); +}); + +// Pythagoras as the shared deep example: c = sqrt(a^2 + b^2), every recursive kind exercised on the way down. +const pythagoras: MathExpression = { + kind: 'app', + operator: 'math:sqrt', + args: [ + { + kind: 'app', + operator: 'math:add', + args: [ + { kind: 'app', operator: 'math:pow', args: [{ kind: 'sym', id: 'a' }, { kind: 'num', numerator: '2', denominator: '1' }] }, + { kind: 'app', operator: 'math:pow', args: [{ kind: 'sym', id: 'b' }, { kind: 'num', numerator: '2', denominator: '1' }] }, + ], + }, + ], +}; + +describe('the MathExpression grammar', () => { + it('validates each non-recursive variant through its own named schema', () => { + expect(MathNumSchema.safeParse({ kind: 'num', numerator: '-1', denominator: '6' }).success).toBe(true); + expect( + MathQtySchema.safeParse({ + kind: 'qty', + value: { numerator: '196133', denominator: '20000' }, + unit: 'si:metre-per-square-second', + uncertainty: { magnitude: { numerator: '1', denominator: '20000' } }, + }).success, + ).toBe(true); + expect(MathSymSchema.safeParse({ kind: 'sym', id: 'symbols:voltage' }).success).toBe(true); + expect(MathUnparsedSchema.safeParse({ kind: 'unparsed', latex: '\\oint_C \\mathbf{B} \\cdot d\\mathbf{l}' }).success).toBe(true); + }); + + it('validates a deep recursive expression through the z.custom union', () => { + expect(isMathExpression(pythagoras)).toBe(true); + expect(MathExpressionSchema.safeParse(pythagoras).success).toBe(true); + }); + + it('validates the sum and prod binders with their bounds and lexically scoped binder symbol', () => { + const sumOfSquares = { + kind: 'sum', + binder: 'i', + lower: { kind: 'app', operator: 'math:equals', args: [{ kind: 'sym', id: 'i' }, { kind: 'num', numerator: '1', denominator: '1' }] }, + upper: { kind: 'sym', id: 'N' }, + body: { kind: 'app', operator: 'math:pow', args: [{ kind: 'sym', id: 'i' }, { kind: 'num', numerator: '2', denominator: '1' }] }, + } satisfies MathExpression; + expect(MathSumSchema.safeParse(sumOfSquares).success).toBe(true); + expect(MathExpressionSchema.safeParse(sumOfSquares).success).toBe(true); + expect( + MathExpressionSchema.safeParse({ + kind: 'prod', + binder: 'k', + lower: { kind: 'sym', id: 'k' }, + upper: { kind: 'sym', id: 'n' }, + body: { kind: 'sym', id: 'k' }, + }).success, + ).toBe(true); + }); + + it('validates a matrix as rows of expressions and rejects a ragged one on both validation paths', () => { + const identity = { + kind: 'matrix', + rows: [ + [ + { kind: 'num', numerator: '1', denominator: '1' }, + { kind: 'num', numerator: '0', denominator: '1' }, + ], + [ + { kind: 'num', numerator: '0', denominator: '1' }, + { kind: 'num', numerator: '1', denominator: '1' }, + ], + ], + } satisfies MathExpression; + expect(MathMatrixSchema.safeParse(identity).success).toBe(true); + expect(MathExpressionSchema.safeParse(identity).success).toBe(true); + + const ragged = { + kind: 'matrix', + rows: [ + [{ kind: 'num', numerator: '1', denominator: '1' }, { kind: 'num', numerator: '0', denominator: '1' }], + [{ kind: 'num', numerator: '0', denominator: '1' }], + ], + }; + expect(MathMatrixSchema.safeParse(ragged).success).toBe(false); + expect(MathExpressionSchema.safeParse(ragged).success).toBe(false); + }); + + it('keeps unparsed a first-class fallback rather than a parse failure', () => { + const partiallyLowered: MathExpression = { + kind: 'app', + operator: 'math:equals', + args: [ + { kind: 'sym', id: 'c' }, + { kind: 'unparsed', latex: '\\int_0^\\infty e^{-x^2}\\,dx' }, + ], + }; + expect(MathExpressionSchema.safeParse(partiallyLowered).success).toBe(true); + }); + + it('enforces the canonical-integer patterns through the recursive guard, at any depth', () => { + expect( + MathExpressionSchema.safeParse({ kind: 'app', operator: 'math:negate', args: [{ kind: 'num', numerator: '1', denominator: '0' }] }) + .success, + ).toBe(false); + expect(MathExpressionSchema.safeParse({ kind: 'num', numerator: '2.5', denominator: '1' }).success).toBe(false); + }); + + it('rejects unknown kinds and non-record inputs outright', () => { + expect(isMathExpression(null)).toBe(false); + expect(isMathExpression('num')).toBe(false); + expect(isMathExpression(undefined)).toBe(false); + expect(MathExpressionSchema.safeParse({ kind: 'integral', latex: '\\int x\\,dx' }).success).toBe(false); + }); + + it('survives a JSON round trip unchanged, exact rationals included', () => { + const roundTripped: unknown = JSON.parse(JSON.stringify(pythagoras)); + expect(MathExpressionSchema.parse(roundTripped)).toEqual(pythagoras); + }); +}); diff --git a/src/math.ts b/src/math.ts new file mode 100644 index 0000000..65e57bd --- /dev/null +++ b/src/math.ts @@ -0,0 +1,287 @@ +import { z } from 'zod'; + +// The semantic half of this package's two-layer math model. A formula is stored as two co-equal authoritative layers, joined in ContentFormulaSchema (src/content.ts): presentation -- a verbatim LaTeX string a renderer serialises exactly as stored, never re-derived from semantics -- and content -- a MathExpression tree from this module, which a computer can evaluate. Neither layer is stored derived from the other: string-to-tree lowering is total (any input at least degrades to an `unparsed` node), tree-to-string rendering is partial (some trees have no conventional linear form), and storage takes the recoverable side of that asymmetry by carrying both verbatim. The atomic pair-edit rule the whole design serves: editing one layer must never silently mutate the other, and any canonical/normalised form used to match or diff the two is a derived view computed at comparison time, never written back into either layer. This module holds only schemas and structural type guards, no lowering or rendering logic -- those live in the packages that produce and consume formulas. + +// -- Exact rationals -- + +// Canonical encodings for the two halves of an exact rational, as decimal-integer strings rather than JS numbers: Number loses integer exactness above 2^53, and exactness is the entire point of carrying rationals here (unit-conversion chains and quantity equality stay bit-exact when every step is integer arithmetic over canonical strings). The patterns enforce the canonical spellings directly -- numerator '0' or a signed integer with no leading zeros and no '-0', denominator a positive integer with no leading zeros -- so every value that validates has exactly one spelling and string equality is value equality within each half. Lowest terms (2/4 reduced to 1/2) is a producer convention the shape cannot check; cross-multiplying two rationals is the exact comparison for producers that skip the reduction step. +const CANONICAL_SIGNED_INTEGER = /^(0|-?[1-9]\d*)$/; +const CANONICAL_POSITIVE_INTEGER = /^[1-9]\d*$/; + +const EXACT_RATIONAL_FIELDS = { + numerator: z.string().regex(CANONICAL_SIGNED_INTEGER), // carries the rational's sign; '0' is the unique zero + denominator: z.string().regex(CANONICAL_POSITIVE_INTEGER), // strictly positive, so the sign lives on the numerator alone and each rational has one valid shape +}; + +// An exact rational number. Plain integers are the denominator-'1' case; there is deliberately no separate integer node, since one numeric leaf keeps the grammar closed and every consumer's arithmetic uniform. +export const ExactRationalSchema = z.object(EXACT_RATIONAL_FIELDS); +export type ExactRational = z.infer; + +// -- Dimensions -- + +// The seven SI base quantities (SI Brochure order), the axes every derived quantity's dimension is expressed over. Closed on purpose: adding a base dimension is a schema-level change, not something a document may do locally, and the current seven cover every SI-coherent quantity. +export const SI_BASE_DIMENSIONS = [ + 'length', + 'mass', + 'time', + 'electricCurrent', + 'thermodynamicTemperature', + 'amountOfSubstance', + 'luminousIntensity', +] as const; +export type SiBaseDimension = (typeof SI_BASE_DIMENSIONS)[number]; + +// A dimension as exponents over the SI bases, one entry per base with a non-zero exponent -- speed is { length: 1, time: -1 }, force { length: 1, mass: 1, time: -2 }, and an omitted key means exponent zero. Exponents are integers because every SI-coherent derived quantity is an integer product of base quantities. {} is the dimensionless vector (radian, count, a per-unit ratio). +export const DimensionVectorSchema = z.partialRecord(z.enum(SI_BASE_DIMENSIONS), z.number().int()); +export type DimensionVector = z.infer; + +// -- Units -- + +// One entry in a document's unit registry (SymbolTableSchema.units below). Units are referenced everywhere else by their id only, so this entry is the single place a unit's meaning is carried. ids are namespaced ('si:metre', 'imperial:foot', 'psu:pu-power') with the prefix before the first ':' naming the registry the definition belongs to; symbols are the unit's own short written form ('m', 'ft'). The registry is document-carried data, not a table shipped inside this package: this package defines the shapes, and each producer registers the units its quantities actually use, SI ones included, under the 'si:' namespace. Conversion is exact by construction: factorToSi/offsetToSi are exact rationals relating this unit to the coherent SI unit of the same dimension via si_value = value * factorToSi + offsetToSi (linear units such as the foot carry factor 381/1250 and no offset; affine scales such as degree Celsius carry factor 1 and offset 5463/20; compound dimensions convert against the coherent SI product, e.g. the foot-per-second against m/s). Compound quantities reference one registered unit id ('si:metre-per-second') rather than composing units inside an expression -- unit algebra (multiplying dimensions, chaining conversions) happens over these entries' dimension vectors and exact factors, which is where the exact-rational representation pays off. +export const MathUnitSchema = z.object({ + id: z.string(), // namespaced registry id, e.g. 'si:metre', 'imperial:foot' + symbol: z.string(), // the unit's own short written form, e.g. 'm', 'ft' + name: z.string().optional(), // full human name, e.g. 'metre' + dimension: DimensionVectorSchema, // exponents over the SI bases; {} means dimensionless + factorToSi: ExactRationalSchema, // exact scale to the coherent SI unit of this unit's own dimension + offsetToSi: ExactRationalSchema.optional(), // exact affine shift for scales whose zero differs from SI's (temperature scales); absent means zero + context: z.string().optional(), // id of the normalisation context (below) this unit exists inside, set only on domain-normalised units +}); +export type MathUnit = z.infer; + +// A domain normalisation context: the declared base a family of quantities has been divided by, per-unit systems being the canonical example (a power study normalised against 100 MVA and 11 kV bases). A normalised quantity is dimensionless in SI terms, so its unit entry carries this context's id and the base definition lives here -- the thing a consumer needs to de-normalise back to SI units. bases lists the normalisation quantities as unit ids with their exact base values; ids are namespaced like unit ids ('psu:100mva-11kv'). +export const MathNormalisationContextSchema = z.object({ + id: z.string(), + bases: z.array( + z.object({ + unit: z.string(), // registry id of the base quantity's unit, e.g. 'si:volt-ampere' + value: ExactRationalSchema, // the exact base value in that unit, e.g. 100 (MVA) + }), + ), +}); +export type MathNormalisationContext = z.infer; + +// -- The symbol table -- + +// One curation entry: what a single written symbol means. The key is the pair (glyph, scope) -- the written form plus the region of the document it is distinct in -- because the same glyph legitimately names different quantities in different scopes ('m' for mass in one section, metres in another). The payload is the reference side: id is what a MathExpression 'sym' node points at, quantityKind links the symbol into a quantity vocabulary ('si:mass'), preferredUnit is a unit-registry id, and definitionSource records where the definition came from (document prose, a citation, a standard). (glyph, scope) uniqueness is a producer convention the array shape cannot enforce; consumers building a lookup should treat duplicates as a curatorial error, not silently pick one. Entries are presentation-inert by construction -- they describe what a symbol means, never how any formula renders. +export const MathSymbolEntrySchema = z.object({ + glyph: z.string(), // the written form as it appears in presentation, e.g. 'U', 'm_e' + scope: z.string(), // the disambiguating scope path, e.g. 'document', 'sections/2' + id: z.string(), // canonical symbol id, what MathExpression 'sym' nodes reference, e.g. 'symbols:voltage' + quantityKind: z.string().optional(), // quantity vocabulary id, e.g. 'si:mass' + preferredUnit: z.string().optional(), // unit-registry id this symbol's quantities are most naturally expressed in + definitionSource: z.string().optional(), // where the definition came from: a prose anchor, citation, or standard +}); +export type MathSymbolEntry = z.infer; + +// The document-level symbol table: the curation layer that makes a document's equations computable, and the unit registry those equations resolve against. Carried on every ContentDocument arm as the optional `symbolTable` field (src/content.ts) so a formula's expressions stay small -- they reference symbols and units by id, and the definitions live here once per document. A standalone value on purpose: one document's table is importable into another, which is how curated meaning travels without dragging the formulas along. +export const SymbolTableSchema = z.object({ + symbols: z.array(MathSymbolEntrySchema), + units: z.array(MathUnitSchema), // the unit registry: every unit any 'qty' in this document references, SI ones included + contexts: z.array(MathNormalisationContextSchema).optional(), // domain normalisation contexts, present only when the document uses normalised units +}); +export type SymbolTable = z.infer; + +// -- The two layers' carrying shapes -- + +// The rendering-authoritative layer's whole content: the formula's LaTeX, stored verbatim. An object around one field rather than a bare string so the layer can grow sibling fields (alternative notations, rendering hints) without another format change; a renderer serialises this string exactly as it stands and never re-emits it from the semantic layer. +export const MathPresentationSchema = z.object({ + latex: z.string(), +}); +export type MathPresentation = z.infer; + +// Where a formula came from and what has touched it since: source names the origin (a format part path such as 'odf:content.xml#Object1', or a pipeline stage such as 'lowered:latex'), pageRef locates it in the source document when the source is paginated, and editTrail is the append-only audit log in producer-defined detail (who edited, when, what changed) -- free-form strings because this is annotation for humans and provenance tooling, not an input to computation. +export const MathProvenanceSchema = z.object({ + source: z.string(), + pageRef: z.string().optional(), + editTrail: z.array(z.string()), +}); +export type MathProvenance = z.infer; + +// A measured quantity's uncertainty, GUM-style: magnitude is the +/- half-width as an exact rational, expressed in the quantity's own unit unless unit overrides it (rare, but an uncertainty quoted in a percentage unit while the value is absolute is a real convention). coverageFactor is the k the magnitude was expanded by (k = 2 for the usual approximate 95 % interval) -- a stated convention factor, so a plain number rather than an exact rational. +export const MathUncertaintySchema = z.object({ + magnitude: ExactRationalSchema, + unit: z.string().optional(), // registry id; absent means the quantity's own unit + coverageFactor: z.number().positive().optional(), // k factor the magnitude was expanded by; absent means k = 1 (standard uncertainty) +}); +export type MathUncertainty = z.infer; + +// -- The expression grammar: non-recursive variants -- + +// The grammar is closed -- these variants plus the recursive ones below are the whole node vocabulary -- and extensible: domain semantics enter through the namespaced operator registry ('app' below) and new node kinds are schema versions, not local extensions. + +export const MathNumSchema = z.object({ + kind: z.literal('num'), + ...EXACT_RATIONAL_FIELDS, +}); +export type MathNum = z.infer; + +export const MathQtySchema = z.object({ + kind: z.literal('qty'), + value: ExactRationalSchema, // the measured or exact value, itself rational so unit chains over it stay exact + unit: z.string(), // unit-registry id this quantity is expressed in, e.g. 'si:metre' + uncertainty: MathUncertaintySchema.optional(), +}); +export type MathQty = z.infer; + +// A named symbol, referenced by table id rather than written form: the symbol table owns glyph-to-meaning curation, and an expression that embedded glyphs would duplicate it. id resolves lexically -- a 'sum'/'prod' binder whose binder name matches shadows the table entry inside that binder's body (the bound variable is local), and otherwise the id is looked up in the document's symbolTable. An id matching neither is a dangling reference, detectable by a consumer walking the expression against the table, not by this grammar alone. +export const MathSymSchema = z.object({ + kind: z.literal('sym'), + id: z.string(), // symbol-table id, or a binder-local name within the binder that introduced it +}); +export type MathSym = z.infer; + +// The first-class fallback: source LaTeX this grammar could not cover, so a coverage gap stays visible data instead of becoming a parse failure. Anything can degrade to this node; nothing below it is interpreted. +export const MathUnparsedSchema = z.object({ + kind: z.literal('unparsed'), + latex: z.string(), // the verbatim source construct that resisted lowering +}); +export type MathUnparsed = z.infer; + +// -- The expression grammar: recursive variants -- + +// Operator application. operator is a namespaced registry id -- 'math:divide', 'math:sqrt' for the core arithmetic registry every reference consumer of this grammar implements, a domain prefix for a domain registry ('physics:planck', room for later registries) -- with everything about the operator (its arity, argument order, semantics) owned by the registry its prefix names, not restated here. args is variadic for the same reason: the registry defines how many arguments mean what. +export interface MathApp { + kind: 'app'; + operator: string; + args: MathExpression[]; +} + +// Shared shape of the two binder variants (MathSum/MathProd below): binder is the bound variable's own name, lexically scoped to this binder's body (it shadows same-id symbol-table entries there); lower/upper are the bounds as full expressions -- a set-membership lower bound such as i element-of S is itself an 'app', and bounds referencing a table symbol (an infinity entry, say) are 'sym' nodes; body is the summand/product the binder ranges over. +export interface MathSum { + kind: 'sum'; + binder: string; + lower: MathExpression; + upper: MathExpression; + body: MathExpression; +} + +export interface MathProd { + kind: 'prod'; + binder: string; + lower: MathExpression; + upper: MathExpression; + body: MathExpression; +} + +// A matrix as rows of expressions -- nested arrays rather than a flat array plus row/column counts, so the shape is its own dimension statement and the only invariant left to check is that every row has the same number of columns (enforced on both validation paths below). Entry order is row-major, the universal convention. +export interface MathMatrix { + kind: 'matrix'; + rows: MathExpression[][]; +} + +// MathExpression is recursive through app args, binder bounds/bodies, and matrix rows -- hand-written with a structural z.custom() guard below, mirroring ContentBlock's and MathMlNode's identical treatment, since z.lazy() collapses to `unknown` for recursive children in the pinned Zod version. +export type MathExpression = MathNum | MathQty | MathSym | MathApp | MathSum | MathProd | MathMatrix | MathUnparsed; + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +function isExactRational(value: unknown): value is ExactRational { + return ( + isRecord(value) && + typeof value.numerator === 'string' && + CANONICAL_SIGNED_INTEGER.test(value.numerator) && + typeof value.denominator === 'string' && + CANONICAL_POSITIVE_INTEGER.test(value.denominator) + ); +} + +function isMathUncertainty(value: unknown): value is MathUncertainty { + return ( + isRecord(value) && + isExactRational(value.magnitude) && + (value.unit === undefined || typeof value.unit === 'string') && + (value.coverageFactor === undefined || (typeof value.coverageFactor === 'number' && value.coverageFactor > 0)) + ); +} + +// Recursive structural guard, mirroring the per-variant Zod schemas' checks by hand (including the canonical-integer patterns and the equal-width matrix rule) so the z.custom() node validates exactly what the named schemas validate. Used via z.custom so recursive children validate without a recursive Zod schema -- the same treatment as ContentBlockSchema/ContentEmbeddedObjectSchema (src/content.ts) and MathMlNodeSchema (src/mathml.ts). +export function isMathExpression(value: unknown): value is MathExpression { + if (!isRecord(value)) { + return false; + } + const kind = value.kind; + if (kind === 'num') { + return ( + typeof value.numerator === 'string' && + CANONICAL_SIGNED_INTEGER.test(value.numerator) && + typeof value.denominator === 'string' && + CANONICAL_POSITIVE_INTEGER.test(value.denominator) + ); + } + if (kind === 'qty') { + return ( + isExactRational(value.value) && + typeof value.unit === 'string' && + (value.uncertainty === undefined || isMathUncertainty(value.uncertainty)) + ); + } + if (kind === 'sym') { + return typeof value.id === 'string'; + } + if (kind === 'app') { + return typeof value.operator === 'string' && Array.isArray(value.args) && value.args.every(isMathExpression); + } + if (kind === 'sum' || kind === 'prod') { + return ( + typeof value.binder === 'string' && + isMathExpression(value.lower) && + isMathExpression(value.upper) && + isMathExpression(value.body) + ); + } + if (kind === 'matrix') { + if (!Array.isArray(value.rows)) { + return false; + } + const widths = new Set(); + for (const row of value.rows) { + if (!Array.isArray(row) || !row.every(isMathExpression)) { + return false; + } + widths.add(row.length); + } + return widths.size <= 1; + } + if (kind === 'unparsed') { + return typeof value.latex === 'string'; + } + return false; +} + +export const MathExpressionSchema = z.custom(isMathExpression); + +// The per-variant schemas for the recursive kinds, defined after MathExpressionSchema because their child fields go through it -- matching MathMlElementSchema's own placement after MathMlNodeSchema (src/mathml.ts). The non-recursive variants' schemas sit with the leaves above. + +export const MathAppSchema = z.object({ + kind: z.literal('app'), + operator: z.string(), // namespaced operator-registry id, e.g. 'math:divide' + args: z.array(MathExpressionSchema), +}); + +const BINDER_SCHEMA_FIELDS = { + binder: z.string(), + lower: MathExpressionSchema, + upper: MathExpressionSchema, + body: MathExpressionSchema, +}; + +// sum and prod are two variants of one shape rather than one 'binder' variant with an op field, so a consumer's exhaustive switch over the grammar distinguishes them at the discriminant like every other kind. +export const MathSumSchema = z.object({ + kind: z.literal('sum'), + ...BINDER_SCHEMA_FIELDS, +}); + +export const MathProdSchema = z.object({ + kind: z.literal('prod'), + ...BINDER_SCHEMA_FIELDS, +}); + +// The equal-row-width refinement is the one matrix invariant the nested-array shape cannot state structurally; it mirrors the identical check in isMathExpression's 'matrix' branch so both validation paths accept and reject the same values. +export const MathMatrixSchema = z.object({ + kind: z.literal('matrix'), + rows: z.array(z.array(MathExpressionSchema)), +}).refine((matrix) => new Set(matrix.rows.map((row) => row.length)).size <= 1, { + message: 'matrix rows must all have the same number of columns', +}); From e25cd70a79c1b1c49c80efcd410ba81cebe26dcf Mon Sep 17 00:00:00 2001 From: Joseph Mearman Date: Mon, 17 Aug 2026 15:01:50 +0100 Subject: [PATCH 2/3] feat: carry the two-layer math model on ContentFormula and a symbol table on every document arm ContentFormula gains presentation (verbatim LaTeX, authoritative for rendering, serialised exactly as stored and never re-emitted from semantics), content (the computation-authoritative MathExpression), and provenance. Neither layer is stored derived from the other -- string-to-tree lowering is total, tree-to-string rendering is partial, so storage takes the recoverable side by carrying both verbatim -- and the atomic pair-edit rule holds that editing one layer never silently mutates the other, with canonical forms used for matching kept as derived views rather than written back in place. mathml stays required (an empty array for a LaTeX-authored formula) so every existing constructor remains valid and CONTENT_FORMAT_VERSION stays 3. The document-level symbolTable joins all five ContentDocument arms through one shared field spread -- on the envelope rather than inside LayoutMetadataSchema, which LayoutDocument shares and which carries no formulas of its own. --- src/content.test.ts | 105 ++++++++++++++++++++++++++++++++++++++++++++ src/content.ts | 23 +++++++++- 2 files changed, 126 insertions(+), 2 deletions(-) diff --git a/src/content.test.ts b/src/content.test.ts index b29e46a..a09982f 100644 --- a/src/content.test.ts +++ b/src/content.test.ts @@ -417,6 +417,111 @@ describe('ContentDocument formula variant', () => { }); }); +// The two-layer design (src/math.ts's own top comment): the same Pythagoras formula as above, carrying its verbatim LaTeX alongside an equivalent semantic tree, neither derived from the other at rest. An empty mathml array is the LaTeX-authored case -- a formula whose source offered no MathML tree keeps the required field while all its meaning lives in the two layers. +function layeredFormulaDocument(): ContentDocument { + return { + kind: 'formula', + formatVersion: CONTENT_FORMAT_VERSION, + metadata: { title: 'Pythagoras, both layers' }, + formula: { + mathml: [], + presentation: { latex: 'c = \\sqrt{a^2 + b^2}' }, + content: { + kind: 'app', + operator: 'math:equals', + args: [ + { kind: 'sym', id: 'c' }, + { + kind: 'app', + operator: 'math:sqrt', + args: [ + { + kind: 'app', + operator: 'math:add', + args: [ + { kind: 'app', operator: 'math:pow', args: [{ kind: 'sym', id: 'a' }, { kind: 'num', numerator: '2', denominator: '1' }] }, + { kind: 'app', operator: 'math:pow', args: [{ kind: 'sym', id: 'b' }, { kind: 'num', numerator: '2', denominator: '1' }] }, + ], + }, + ], + }, + ], + }, + provenance: { source: 'lowered:latex', editTrail: ['lowered from presentation on ingest'] }, + }, + }; +} + +describe('ContentFormula two-layer model', () => { + it('carries presentation, content, and provenance alongside the MathML tree', () => { + const parsed = ContentDocumentSchema.parse(layeredFormulaDocument()); + if (parsed.kind !== 'formula') { + throw new Error('expected a formula document'); + } + expect(parsed.formula.presentation?.latex).toBe('c = \\sqrt{a^2 + b^2}'); + expect(parsed.formula.content?.kind).toBe('app'); + expect(parsed.formula.provenance?.source).toBe('lowered:latex'); + }); + + it('still validates the pre-existing shape with every new layer absent', () => { + expect(ContentDocumentSchema.safeParse(formulaDocument()).success).toBe(true); + }); + + it('rejects a malformed semantic tree rather than degrading it, keeping coverage gaps the job of explicit unparsed nodes', () => { + const malformed: unknown = { + ...layeredFormulaDocument(), + formula: { + mathml: [], + presentation: { latex: 'x' }, + content: { kind: 'app', operator: 'math:divide', args: [{ kind: 'num', numerator: '1', denominator: '0' }] }, + }, + }; + expect(ContentDocumentSchema.safeParse(malformed).success).toBe(false); + }); +}); + +describe('the document-level symbol table', () => { + const symbolTable = { + symbols: [ + { glyph: 'a', scope: 'document', id: 'leg-a', preferredUnit: 'si:metre' }, + { glyph: 'b', scope: 'document', id: 'leg-b', preferredUnit: 'si:metre' }, + ], + units: [ + { id: 'si:metre', symbol: 'm', dimension: { length: 1 }, factorToSi: { numerator: '1', denominator: '1' } }, + ], + }; + + it('is accepted on every one of the five ContentDocument arms', () => { + for (const document of [ + wordprocessingDocument(), + presentationDocument(), + spreadsheetDocument(), + drawingDocument(), + formulaDocument(), + ]) { + const withTable = { ...document, symbolTable }; + expect(ContentDocumentSchema.safeParse(withTable).success).toBe(true); + } + }); + + it('parses back off the envelope with its entries intact', () => { + const parsed = ContentDocumentSchema.parse({ ...formulaDocument(), symbolTable }); + if (parsed.kind !== 'formula') { + throw new Error('expected a formula document'); + } + expect(parsed.symbolTable?.symbols).toHaveLength(2); + expect(parsed.symbolTable?.units[0]?.id).toBe('si:metre'); + }); + + it('stays absent and optional on documents that carry no math curation', () => { + const parsed = ContentDocumentSchema.parse(wordprocessingDocument()); + if (parsed.kind !== 'wordprocessing') { + throw new Error('expected a wordprocessing document'); + } + expect(parsed.symbolTable).toBeUndefined(); + }); +}); + // The formula ContentDocument kind slots straight into the pre-existing ContentEmbeddedObjectKind 'formula' mechanism -- an embedded equation now carries genuine MathML instead of a wordprocessing document standing in for one. describe('an embedded formula object carrying a real formula document', () => { it('validates as a ContentBlock and inside a whole document', () => { diff --git a/src/content.ts b/src/content.ts index d4c85ec..69b28a9 100644 --- a/src/content.ts +++ b/src/content.ts @@ -3,6 +3,7 @@ import { ColorSchema } from './color'; import type { Color } from './color'; import { BoxSchema, LayoutFrameSchema, MarginsSchema, PageSizeSchema } from './geometry'; import type { Box, LayoutFrame } from './geometry'; +import { MathExpressionSchema, MathPresentationSchema, MathProvenanceSchema, SymbolTableSchema } from './math'; import { MathMlNodeSchema } from './mathml'; import { LayoutMetadataSchema } from './metadata'; import { AlignmentSchema } from './style'; @@ -501,47 +502,65 @@ export const ContentDrawPageSchema = z.object({ }); export type ContentDrawPage = z.infer; -// Formula content model: a standalone equation document (an ODF .odf formula document, or the equation an embedded 'formula' object carries). Unlike the other four kinds this has no page/slide/sheet structure at all -- a formula is one expression, positioned by whatever embeds it, so there is nothing here to paginate or place. +// Formula content model: a standalone equation document (an ODF .odf formula document, or the equation an embedded 'formula' object carries). Unlike the other four kinds this has no page/slide/sheet structure at all -- a formula is one expression, positioned by whatever embeds it, so there is nothing here to paginate or place. Whatever embeds it supplies position through its own frames field (a ContentEmbeddedObjectBlock's frames describe the rendered box the equation occupies); the semantic payload here is position-independent by nature, and the fusion invariant deliberately reaches no further into a formula than that box. +// +// A formula's meaning is carried as two co-equal authoritative layers, joined in this one shape: `presentation` (rendering-authoritative) and `content` (computation-authoritative), alongside the source formats' own trees and strings. Neither layer is stored derived from the other -- string-to-tree lowering is total (worst case an `unparsed` node), tree-to-string rendering is partial, and storage takes the recoverable side by carrying both verbatim. The atomic pair-edit rule: editing one layer must never silently mutate the other -- an editor changing `content` leaves `presentation` byte-identical unless it explicitly rewrites both -- and any canonical form used to match or diff the two layers is a derived view computed at comparison time, never written back in place. That rule is documentation-stated rather than Zod-enforced on purpose: the schema's job is to carry both layers losslessly, and a producer that mutates one layer in place is misbehaving in a way no input shape can prevent -- consumers comparing layers recompute their own canonical views rather than trusting either layer to be normalised. export const ContentFormulaSchema = z.object({ - // The formula's own MathML presentation-layer tree, carried as raw XML nodes (see src/mathml.ts for why that, rather than a MathML-specific element vocabulary). An array rather than a single root because a real formula part's content is a node list -- an XML declaration and/or whitespace text nodes commonly precede the element itself, and dropping them on the way in would make this model lossy for no gain. + // The formula's own MathML presentation-layer tree, carried as raw XML nodes (see src/mathml.ts for why that, rather than a MathML-specific element vocabulary). An array rather than a single root because a real formula part's content is a node list -- an XML declaration and/or whitespace text nodes commonly precede the element itself, and dropping them on the way in would make this model lossy for no gain. Required even when the source carried no MathML of its own (a LaTeX-authored equation lowered on the way in): such a formula carries an empty array, which keeps every existing constructor of this shape valid. mathml: z.array(MathMlNodeSchema), // The equivalent StarMath source, when the producing format carried one alongside the MathML (ODF stores it as the formula's own annotation). Purely informational: MathML is the authoritative content, and a consumer that renders from starMath instead is rendering a secondary encoding of the same expression. starMath: z.string().optional(), + // The rendering-authoritative layer: the formula's LaTeX, stored verbatim (src/math.ts's MathPresentationSchema). A renderer serialises this string exactly as it stands and never re-emits it from the semantic layer below. Absent on a formula whose source offered nothing LaTeX-shaped, in which case rendering falls back to the MathML tree above. + presentation: MathPresentationSchema.optional(), + // The computation-authoritative layer: a MathExpression tree (src/math.ts). Absent means nobody has lowered this formula to semantics yet; an `unparsed` node inside it means somebody tried and hit a construct the grammar does not cover -- coverage gaps stay visible data, never parse failures. The symbol and unit references inside resolve against the embedding document's own symbolTable field (below). + content: MathExpressionSchema.optional(), + // Where this formula came from and what has touched it since (src/math.ts's MathProvenanceSchema). + provenance: MathProvenanceSchema.optional(), }); export type ContentFormula = z.infer; // Bumped whenever ContentDocumentSchema's shape changes incompatibly. 2 added the 'formula' variant below, renamed ContentSheetPrintSettings.scale to scalePercent, made ContentSheetColumn.widthPt/ContentSheetRow.heightPt optional-positive rather than required-nonnegative, and added the 'dateTime' ContentCellValue kind. 3 added the canonical, format-agnostic `headingLevel` field to ContentParagraphSchema (alongside the existing round-trip-only `styleId`), and fused DocumentPackage's own layout half directly onto the content tree: every content-kind leaf that previously carried only a `sourcePath` correlation string (ContentRun, ContentParagraph, ContentImageBlock, ContentPageBreak, ContentTable, ContentTableCell, ContentEmbeddedObjectBlock, ContentShape, every ContentVector variant, ContentSheetCell) now additionally carries an optional `frames: LayoutFrame[]` field of its own rendered page position(s) -- see FusedNode above and DOCUMENT_PACKAGE_FORMAT_VERSION in package.ts, bumped in step. export const CONTENT_FORMAT_VERSION = 3; +// Fields every one of the five ContentDocument arms below carries in addition to its own kind, formatVersion, and metadata -- currently the document-level math symbol table (SymbolTableSchema, src/math.ts): the curation layer mapping each written symbol glyph to its quantity kind, preferred unit, and definition, alongside the unit registry a formula's expressions resolve their symbol and unit references against. Spliced into each arm via spread rather than factored through a base schema the arms extend, because z.discriminatedUnion() needs each member as a plain z.object carrying its own literal `kind` field in place. Optional on every arm: a document with no lowered math content (most of them) simply omits it, and the table is presentation-inert by construction -- it curates what symbols mean, never how any formula renders -- so its presence or absence changes no rendering. It lives on the envelope, not inside LayoutMetadataSchema, because that schema is shared with LayoutDocument (src/metadata.ts) and a math curation layer there would leak onto every layout document, which carries no formulas of its own. +const contentDocumentSharedFields = { + symbolTable: SymbolTableSchema.optional(), +}; + export const ContentDocumentSchema = z.discriminatedUnion('kind', [ z.object({ kind: z.literal('wordprocessing'), formatVersion: z.literal(CONTENT_FORMAT_VERSION), metadata: LayoutMetadataSchema, + ...contentDocumentSharedFields, sections: z.array(ContentSectionSchema), }), z.object({ kind: z.literal('presentation'), formatVersion: z.literal(CONTENT_FORMAT_VERSION), metadata: LayoutMetadataSchema, + ...contentDocumentSharedFields, slides: z.array(ContentSlideSchema), }), z.object({ kind: z.literal('spreadsheet'), formatVersion: z.literal(CONTENT_FORMAT_VERSION), metadata: LayoutMetadataSchema, + ...contentDocumentSharedFields, sheets: z.array(ContentSheetSchema), }), z.object({ kind: z.literal('drawing'), formatVersion: z.literal(CONTENT_FORMAT_VERSION), metadata: LayoutMetadataSchema, + ...contentDocumentSharedFields, pages: z.array(ContentDrawPageSchema), }), z.object({ kind: z.literal('formula'), formatVersion: z.literal(CONTENT_FORMAT_VERSION), metadata: LayoutMetadataSchema, + ...contentDocumentSharedFields, formula: ContentFormulaSchema, }), ]); From f281d73e862a6c6ebe1f85eb3fe88de7f78d5499 Mon Sep 17 00:00:00 2001 From: Joseph Mearman Date: Mon, 17 Aug 2026 15:02:04 +0100 Subject: [PATCH 3/3] build: hand-transcribe the math schemas into the JSON Schema $defs ContentFormulaSchema joins the hand-transcribed set: its mathml and content fields reach the opaque MathMlNodeSchema and MathExpressionSchema custom nodes, so the generator now replaces every occurrence with a $ref to a new $defs.ContentFormula fragment, clearing the auto-generated body first (a real z.object's jsonSchema is already populated at finalize time, unlike a custom node's empty {}). SymbolTableSchema is transcribed too, keeping each arm's symbolTable field one named reference instead of five inlined copies of the whole unit-registry subtree. The regression test's transcribed-vs-live partition moves accordingly -- thirteen non-recursive math leaves join the live-compared set, while ContentFormula, MathExpression, and the recursive variants join the hand-verified set downstream of the MathExpressionSchema custom node. The smoke test's formula-variant assertions follow the new $ref path, and the README's z.custom inventory gains MathExpressionSchema. --- README.md | 6 +- scripts/generate-json-schemas.mjs | 24 ++- src/content-json-schema-defs.test.ts | 30 +++- src/content-json-schema-defs.ts | 215 ++++++++++++++++++++++++++- src/mathml.ts | 2 +- test/smoke.test.mjs | 22 ++- 6 files changed, 286 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 68da675..42d1d3d 100644 --- a/README.md +++ b/README.md @@ -48,7 +48,7 @@ graph TD style schema fill:#f9a825,stroke:#333,stroke-width:3px ``` -`ContentDocument` (the semantic pivot) is a discriminated union of five kinds: `wordprocessing` (docx/odt sections of paragraphs/runs/tables/images), `presentation` (pptx/odp slides of shapes), `spreadsheet` (xlsx/ods sheets of cells, columns, rows, print settings), `drawing` (odg pages of shapes plus vector primitives — rect/ellipse/line/path), and `formula` (an equation carrying its own MathML node tree plus StarMath source when the producing format had one). `ContentEmbeddedObjectSchema` lets any of the five embed another whole `ContentDocument`. Every paragraph/run/image/table/shape/vector/spreadsheet-cell leaf also carries its own canonical `headingLevel`-or-position fields directly: a `ContentParagraph`'s optional `headingLevel` (1 = the outermost heading, independent of the round-trip-only `styleId`), and every such leaf's optional `frames: LayoutFrame[]` — that node's own rendered page position(s) (`pageIndex` plus PDF user-space `xPt`/`yPt`/`widthPt`/`heightPt`), fused directly onto the content tree once a layout pass has run. `LayoutDocument` (the PDF-rendering pivot pdf-codec's `readPdf`/`writePdf` operate on directly, independent of any `ContentDocument`) is pages of positioned `LayoutItem`s (`text`/`image`/`rect`/`line`/`ellipse`/`path`/`link`) in PDF user-space coordinates. `DocumentPackageSchema` wraps `content` (required) with `pages` (optional, derived: each rendered page's own size, indexed to match every node's own `frames[].pageIndex`) — a single fused tree rather than a second, independent `LayoutDocument` correlated back to `content` only by matching `sourcePath` strings; the schema does not keep `content`'s populated `frames` fields and `pages` in sync or detect staleness. +`ContentDocument` (the semantic pivot) is a discriminated union of five kinds: `wordprocessing` (docx/odt sections of paragraphs/runs/tables/images), `presentation` (pptx/odp slides of shapes), `spreadsheet` (xlsx/ods sheets of cells, columns, rows, print settings), `drawing` (odg pages of shapes plus vector primitives — rect/ellipse/line/path), and `formula` (an equation carrying its own MathML node tree plus StarMath source when the producing format had one, extended with the two-layer math model: an optional verbatim-LaTeX `presentation` authoritative for rendering, an optional semantic `content: MathExpression` tree authoritative for computation, and provenance — neither layer stored derived from the other, so editing one never silently mutates the other). `ContentEmbeddedObjectSchema` lets any of the five embed another whole `ContentDocument`. Every paragraph/run/image/table/shape/vector/spreadsheet-cell leaf also carries its own canonical `headingLevel`-or-position fields directly: a `ContentParagraph`'s optional `headingLevel` (1 = the outermost heading, independent of the round-trip-only `styleId`), and every such leaf's optional `frames: LayoutFrame[]` — that node's own rendered page position(s) (`pageIndex` plus PDF user-space `xPt`/`yPt`/`widthPt`/`heightPt`), fused directly onto the content tree once a layout pass has run. `LayoutDocument` (the PDF-rendering pivot pdf-codec's `readPdf`/`writePdf` operate on directly, independent of any `ContentDocument`) is pages of positioned `LayoutItem`s (`text`/`image`/`rect`/`line`/`ellipse`/`path`/`link`) in PDF user-space coordinates. `DocumentPackageSchema` wraps `content` (required) with `pages` (optional, derived: each rendered page's own size, indexed to match every node's own `frames[].pageIndex`) — a single fused tree rather than a second, independent `LayoutDocument` correlated back to `content` only by matching `sourcePath` strings; the schema does not keep `content`'s populated `frames` fields and `pages` in sync or detect staleness. Every one of the five kinds also accepts an optional document-level `symbolTable` — the math curation layer mapping each written symbol glyph (within a scope) to its id, quantity kind, preferred unit, and definition source, alongside the unit registry (SI dimension-exponent vectors, exact rational conversions, per-unit-system normalisation contexts) that the `qty` nodes of lowered formulas resolve against. The package contains only [Zod](https://zod.dev) schemas, their inferred types, trivial schema-attached helpers (hex-colour conversion, recursive structural type guards), and two small structural interfaces (`ContentCodec`/`LayoutCodec`, see [Codecs](#codecs)). No XML, ZIP, PDF, or binary handling; the sole dependency is `zod`. @@ -114,11 +114,11 @@ node_modules/document-schema.js/schemas/content-document.schema.json node_modules/document-schema.js/schemas/layout-document.schema.json ``` -Each file's `$id` is a jsdelivr URL pinned to the exact npm version — immutable and live on publish. The three files cross-reference via `$ref`s, so a validator resolving refs over HTTP can validate a whole `DocumentPackage`. `content-document.schema.json` carries a hand-authored `$defs` block for the recursive paragraph/table/embedded-object and MathML node models (Zod's converter cannot express these directly); `content-json-schema-defs.ts` holds those fragments, and a regression test compares each against a live `z.toJSONSchema()` of its real Zod counterpart so a field changed without updating its fragment fails a test. Fragments downstream of a `z.custom()` node (`ContentBlock`, `ContentTable`/`Cell`/`Row`, `ContentEmbeddedObjectBlock`, `MathMlNode`/`Element`/`Attribute`) still need hand re-verification against `src/content.ts`/`src/mathml.ts` — see below. +Each file's `$id` is a jsdelivr URL pinned to the exact npm version — immutable and live on publish. The three files cross-reference via `$ref`s, so a validator resolving refs over HTTP can validate a whole `DocumentPackage`. `content-document.schema.json` carries a hand-authored `$defs` block for the recursive paragraph/table/embedded-object and MathML node models (Zod's converter cannot express these directly); `content-json-schema-defs.ts` holds those fragments, and a regression test compares each against a live `z.toJSONSchema()` of its real Zod counterpart so a field changed without updating its fragment fails a test. Fragments downstream of a `z.custom()` node (`ContentBlock`, `ContentTable`/`Cell`/`Row`, `ContentEmbeddedObjectBlock`, `MathMlNode`/`Element`/`Attribute`, `ContentFormula`, `MathExpression` and its recursive variants) still need hand re-verification against `src/content.ts`/`src/mathml.ts`/`src/math.ts` — see below. ### `z.custom()` vs `z.lazy()` for recursive schemas -`ContentBlockSchema`, `ContentEmbeddedObjectSchema`, and `MathMlNodeSchema` are `z.custom()` type-guard predicates rather than real Zod schemas, because `z.lazy()` was believed to collapse to `unknown` for recursive children. A throwaway spike (reverted) re-tested `MathMlNodeSchema` (the simplest case) against `zod@4.4.3`. +`ContentBlockSchema`, `ContentEmbeddedObjectSchema`, `MathMlNodeSchema`, and `MathExpressionSchema` are `z.custom()` type-guard predicates rather than real Zod schemas, because `z.lazy()` was believed to collapse to `unknown` for recursive children. A throwaway spike (reverted) re-tested `MathMlNodeSchema` (the simplest case) against `zod@4.4.3`. **Finding: `z.lazy()` works now, with one constructional gotcha.** The naive rewrite — diff --git a/scripts/generate-json-schemas.mjs b/scripts/generate-json-schemas.mjs index fcc3b9f..7da73e5 100644 --- a/scripts/generate-json-schemas.mjs +++ b/scripts/generate-json-schemas.mjs @@ -5,7 +5,7 @@ // // This script is deliberately outside tsconfig.json's "include" and eslint.config.ts's linted set (see the "scripts" entry in both), matching the existing precedent for test/smoke.test.mjs: a standalone build step, not part of the shipped src/ program. // -// -- The z.custom() opacity problem -- ContentDocumentSchema's tree contains three schemas built from a hand-written type-guard predicate (z.custom()) rather than real Zod primitives -- ContentBlockSchema, ContentEmbeddedObjectSchema, and MathMlNodeSchema -- because the recursive block/table/embedded-object and MathML-element structures they represent can't be expressed via z.lazy() in the pinned Zod version (see src/content.ts's own comments on isContentBlock/isContentEmbeddedObject, and src/mathml.ts's on isMathMlNode). z.toJSONSchema() cannot introspect a z.custom() node at all: with `unrepresentable: 'any'` it silently emits an empty `{}` for that node (confirmed by reading node_modules/zod/v4/core/json-schema-processors.js's customProcessor, which does nothing to its `json` argument once `unrepresentable !== 'throw'`); without that option it throws immediately, before override() ever runs (override only patches at finalize() time, strictly after the pass that would otherwise throw). So these three nodes need hand-authored JSON Schema fragments, spliced in via the override() callback below -- the fragments themselves (CONTENT_DEFS, MAX_SAFE_INTEGER, EMBEDDED_OBJECT_KINDS, CONTENT_DOCUMENT_URI) now live in src/content-json-schema-defs.ts rather than inline here, so this script and that module's own regression test (content-json-schema-defs.test.ts) share exactly one copy -- see that src module's own top comment for why it had to move out of this script. +// -- The z.custom() opacity problem -- ContentDocumentSchema's tree contains four schemas built from a hand-written type-guard predicate (z.custom()) rather than real Zod primitives -- ContentBlockSchema, ContentEmbeddedObjectSchema, MathMlNodeSchema, and MathExpressionSchema -- because the recursive block/table/embedded-object, MathML-element, and math-expression structures they represent can't be expressed via z.lazy() in the pinned Zod version (see src/content.ts's own comments on isContentBlock/isContentEmbeddedObject, src/mathml.ts's on isMathMlNode, and src/math.ts's on isMathExpression). z.toJSONSchema() cannot introspect a z.custom() node at all: with `unrepresentable: 'any'` it silently emits an empty `{}` for that node (confirmed by reading node_modules/zod/v4/core/json-schema-processors.js's customProcessor, which does nothing to its `json` argument once `unrepresentable !== 'throw'`); without that option it throws immediately, before override() ever runs (override only patches at finalize() time, strictly after the pass that would otherwise throw). So everything downstream of those four nodes needs hand-authored JSON Schema fragments, spliced in via the override() callback below -- the fragments themselves (CONTENT_DEFS, MAX_SAFE_INTEGER, EMBEDDED_OBJECT_KINDS, CONTENT_DOCUMENT_URI) now live in src/content-json-schema-defs.ts rather than inline here, so this script and that module's own regression test (content-json-schema-defs.test.ts) share exactly one copy -- see that src module's own top comment for why it had to move out of this script. Two real z.objects are replaced with a $ref to their hand-authored fragments too: ContentFormulaSchema (a real z.object, but its mathml/content fields drag in the opaque MathMlNodeSchema/MathExpressionSchema, so its auto-generated body would be hollowed out around them -- the ContentTableCellSchema situation) and SymbolTableSchema (fully generatable, but $ref-ing it keeps each ContentDocument arm's symbolTable field one named reference instead of five inlined copies of the whole unit-registry subtree). // // -- Cross-file references -- Rather than each of the three .schema.json files being a fully independent, self-contained document (duplicating ContentDocument's entire body inside document-package.schema.json), this uses Zod's registry-based multi-schema generation: a dedicated z.registry() (not z.globalRegistry, so a one-shot build step never pollutes shared process-wide state), registry.add(schema, {id}) for all three schemas, then one z.toJSONSchema(registry, {uri, override, unrepresentable: 'any'}) call. Zod automatically produces real $ref-based cross-references between the three output files for any registered schema encountered while generating another (confirmed empirically: see the experiment behind this script's own review) -- DocumentPackageSchema's own `content`/`layout` fields come out as `{ $ref: }` rather than inlining ContentDocument/LayoutDocument's entire bodies. // @@ -22,6 +22,7 @@ import { ContentBlockSchema, ContentDocumentSchema, ContentEmbeddedObjectSchema, + ContentFormulaSchema, CONTENT_DOCUMENT_URI, DocumentPackageSchema, EMBEDDED_OBJECT_KINDS, @@ -30,6 +31,7 @@ import { MAX_SAFE_INTEGER, SCHEMA_FILE_NAMES, schemaUriFor, + SymbolTableSchema, } from '../dist/index.js'; const here = dirname(fileURLToPath(import.meta.url)); @@ -44,13 +46,29 @@ registry.add(DocumentPackageSchema, { id: 'DocumentPackage' }); registry.add(ContentDocumentSchema, { id: 'ContentDocument' }); registry.add(LayoutDocumentSchema, { id: 'LayoutDocument' }); -// Four branches, keyed on reference equality against the exported consts (each z.custom() call has distinct object identity, confirmed empirically). override() fires exactly once per unique Zod schema instance encountered anywhere across the whole registry-processing session, regardless of how many field sites reference it or which of the three output files happens to reach it first -- mutating ctx.jsonSchema in place is what makes one override call apply everywhere that exact schema object is used (e.g. ContentBlockSchema appears in ContentSectionSchema, ContentShapeSchema, and ContentTableCellSchema all at once). +// Six branches, keyed on reference equality against the exported consts (each z.custom() call has distinct object identity, confirmed empirically). override() fires exactly once per unique Zod schema instance encountered anywhere across the whole registry-processing session, regardless of how many field sites reference it or which of the three output files happens to reach it first -- mutating ctx.jsonSchema in place is what makes one override call apply everywhere that exact schema object is used (e.g. ContentBlockSchema appears in ContentSectionSchema, ContentShapeSchema, and ContentTableCellSchema all at once). function override(ctx) { if (ctx.zodSchema === ContentDocumentSchema) { - // Zod's own discriminated-union conversion already produced a correct `oneOf` of the five kind variants on ctx.jsonSchema (each is a real z.object; the 'formula' variant reaches the custom MathMlNodeSchema through ContentFormulaSchema.mathml, patched by its own branch below) -- this only adds the hand-authored $defs block alongside it. + // Zod's own discriminated-union conversion already produced a correct `oneOf` of the five kind variants on ctx.jsonSchema (each is a real z.object; the 'formula' variant reaches the custom nodes through ContentFormulaSchema, replaced wholesale by its own branch below) -- this only adds the hand-authored $defs block alongside it. ctx.jsonSchema.$defs = CONTENT_DEFS; return; } + if (ctx.zodSchema === ContentFormulaSchema) { + // A real z.object, but two of its fields reach opaque custom nodes (mathml -> MathMlNodeSchema, content -> MathExpressionSchema), so the whole thing is transcribed as $defs.ContentFormula and every occurrence replaced -- same treatment as ContentBlockSchema below, except the auto-generated body must be cleared first: unlike a custom node (which starts as `{}`), a real object schema's jsonSchema already carries type/properties/required by the time finalize() runs. + for (const key of Object.keys(ctx.jsonSchema)) { + delete ctx.jsonSchema[key]; + } + ctx.jsonSchema.$ref = '#/$defs/ContentFormula'; + return; + } + if (ctx.zodSchema === SymbolTableSchema) { + // Fully generatable, but $ref-ing the hand-authored $defs.SymbolTable keeps each of the five ContentDocument arms' symbolTable field one named reference instead of five inlined copies of the whole symbol/unit subtree. Same clear-then-set as ContentFormulaSchema above. + for (const key of Object.keys(ctx.jsonSchema)) { + delete ctx.jsonSchema[key]; + } + ctx.jsonSchema.$ref = '#/$defs/SymbolTable'; + return; + } if (ctx.zodSchema === MathMlNodeSchema) { // Recursive -- an element's children may themselves be elements -- so every occurrence, including inside $defs.MathMlElement above, points at one shared definition rather than inlining, exactly as ContentBlockSchema does below. ctx.jsonSchema starts as `{}` here, so this assignment alone is sufficient. ctx.jsonSchema.$ref = '#/$defs/MathMlNode'; diff --git a/src/content-json-schema-defs.test.ts b/src/content-json-schema-defs.test.ts index b822222..460f868 100644 --- a/src/content-json-schema-defs.test.ts +++ b/src/content-json-schema-defs.test.ts @@ -13,9 +13,24 @@ import { } from './content'; import { CONTENT_DEFS } from './content-json-schema-defs'; import { BoxSchema, LayoutFrameSchema } from './geometry'; +import { + DimensionVectorSchema, + ExactRationalSchema, + MathNormalisationContextSchema, + MathNumSchema, + MathPresentationSchema, + MathProvenanceSchema, + MathQtySchema, + MathSymbolEntrySchema, + MathSymSchema, + MathUncertaintySchema, + MathUnitSchema, + MathUnparsedSchema, + SymbolTableSchema, +} from './math'; import { AlignmentSchema } from './style'; -// This is the regression test scripts/generate-json-schemas.mjs's own top comment calls for: the only structural defence that generator has against silently drifting away from src/content.ts/src/color.ts/src/geometry.ts/src/style.ts, since CONTENT_DEFS (content-json-schema-defs.ts) is transcribed by hand rather than generated. Not every entry in CONTENT_DEFS can be checked this way -- ContentBlock/ContentTable/ContentTableRow/ContentTableCell/ContentEmbeddedObjectBlock/MathMlNode/MathMlElement/MathMlAttribute all sit downstream of one of the three genuinely un-representable z.custom() nodes (ContentBlockSchema, ContentEmbeddedObjectSchema, MathMlNodeSchema -- see that module's own top comment), so a bare z.toJSONSchema() call over their real schema counterpart either throws or degrades to `{}` for the recursive/custom part, which is exactly the problem CONTENT_DEFS exists to work around in the first place. What CAN be checked -- because a real, non-recursive, non-custom exported Zod schema exists for it -- is every leaf and near-leaf fragment: Color, Box, LayoutFrame, Alignment, ContentStrokeStyle, ContentBorder, ContentCellBorders, ContentListMembership, ContentRun, ContentParagraph, ContentImageBlock, ContentPageBreak. None of these reaches ContentBlockSchema, ContentEmbeddedObjectSchema, or MathMlNodeSchema from anywhere in their own field tree, so they can be generated live and compared directly. +// This is the regression test scripts/generate-json-schemas.mjs's own top comment calls for: the only structural defence that generator has against silently drifting away from src/content.ts/src/color.ts/src/geometry.ts/src/style.ts/src/math.ts, since CONTENT_DEFS (content-json-schema-defs.ts) is transcribed by hand rather than generated. Not every entry in CONTENT_DEFS can be checked this way -- ContentBlock/ContentTable/ContentTableRow/ContentTableCell/ContentEmbeddedObjectBlock/MathMlNode/MathMlElement/MathMlAttribute all sit downstream of one of the genuinely un-representable z.custom() nodes (ContentBlockSchema, ContentEmbeddedObjectSchema, MathMlNodeSchema), and ContentFormula/MathExpression/MathApp/MathSum/MathProd/MathMatrix sit downstream of the fourth (MathExpressionSchema, reached through ContentFormulaSchema.content for the first and through the grammar's own recursion for the rest) -- see that module's own top comment -- so a bare z.toJSONSchema() call over their real schema counterpart either throws or degrades to `{}` for the recursive/custom part, which is exactly the problem CONTENT_DEFS exists to work around in the first place. What CAN be checked -- because a real, non-recursive, non-custom exported Zod schema exists for it -- is every leaf and near-leaf fragment: Color, Box, LayoutFrame, Alignment, ContentStrokeStyle, ContentBorder, ContentCellBorders, ContentListMembership, ContentRun, ContentParagraph, ContentImageBlock, ContentPageBreak, ExactRational, DimensionVector, MathPresentation, MathProvenance, MathUncertainty, MathNum, MathQty, MathSym, MathUnparsed, MathSymbolEntry, MathUnit, MathNormalisationContext, SymbolTable. None of these reaches ContentBlockSchema, ContentEmbeddedObjectSchema, MathMlNodeSchema, or MathExpressionSchema from anywhere in its own field tree, so each can be generated live and compared directly. // // Comparison strategy: a bare `z.toJSONSchema(SomeSchema)` call, run in isolation, would INLINE every nested schema it encounters (ColorSchema inside ContentRunSchema, AlignmentSchema inside ContentParagraphSchema, etc.) rather than emit the `{ $ref: '#/$defs/X' }` pointers CONTENT_DEFS itself uses -- because those nested schemas aren't registered anywhere. To reproduce the exact cross-reference shape CONTENT_DEFS hand-authors, this test registers the identical set of real schemas under the identical id strings CONTENT_DEFS uses as its own $defs keys, with a `uri` callback matching the `#/$defs/` convention CONTENT_DEFS was written against -- confirmed empirically (see this file's own construction) to make Zod's registry-based multi-schema generation emit exactly that $ref shape for every registered schema referenced from within another. Each per-schema result still carries its own top-level `$schema`/`$id` (since z.toJSONSchema(registry, ...) treats every registered schema as its own standalone root), which CONTENT_DEFS's own nested fragments never have -- those two keys are stripped before comparison, since they're an artefact of testing each fragment as a registry root rather than a real structural difference. @@ -32,6 +47,19 @@ const REGISTERED_SCHEMAS = { ContentParagraph: ContentParagraphSchema, ContentImageBlock: ContentImageBlockSchema, ContentPageBreak: ContentPageBreakSchema, + ExactRational: ExactRationalSchema, + DimensionVector: DimensionVectorSchema, + MathPresentation: MathPresentationSchema, + MathProvenance: MathProvenanceSchema, + MathUncertainty: MathUncertaintySchema, + MathNum: MathNumSchema, + MathQty: MathQtySchema, + MathSym: MathSymSchema, + MathUnparsed: MathUnparsedSchema, + MathSymbolEntry: MathSymbolEntrySchema, + MathUnit: MathUnitSchema, + MathNormalisationContext: MathNormalisationContextSchema, + SymbolTable: SymbolTableSchema, }; const registry = z.registry<{ id: string }>(); diff --git a/src/content-json-schema-defs.ts b/src/content-json-schema-defs.ts index ad72785..15ec6fb 100644 --- a/src/content-json-schema-defs.ts +++ b/src/content-json-schema-defs.ts @@ -1,14 +1,15 @@ import type { z } from 'zod'; +import { SI_BASE_DIMENSIONS } from './math'; import { schemaUriFor } from './schema-io'; // The hand-authored JSON Schema $defs fragments spliced into content-document.schema.json's `override()` callback (scripts/generate-json-schemas.mjs), lifted out into their own src module rather than staying inline in that script. The reason is single-sourcing, not tidiness: this exact object needs to be reachable from two places that cannot share an import graph -- // // 1. scripts/generate-json-schemas.mjs itself, which only ever runs against the freshly-built ../dist/ (it imports every other schema it needs the same way), so it imports CONTENT_DEFS from '../dist/content-json-schema-defs.js', the file tsdown emits for this module (entry: 'src/**/*.ts', one dist file per src file -- see tsdown.config.ts). -// 2. content-json-schema-defs.test.ts (src/, run directly by vitest's "unit" project against source, never against dist), which imports this exact same CONTENT_DEFS value straight from here and asserts it stays byte-for-byte in step with a live z.toJSONSchema() call over each fragment's real exported Zod schema counterpart (ContentParagraphSchema, ContentRunSchema, ContentListMembershipSchema, ContentImageBlockSchema, ContentPageBreakSchema, ColorSchema, BoxSchema, AlignmentSchema, ContentStrokeStyleSchema, ContentBorderSchema, ContentCellBordersSchema) -- see that test file's own top comment for why this is the only structural defence this generator has against silently drifting away from the schemas it's meant to describe. +// 2. content-json-schema-defs.test.ts (src/, run directly by vitest's "unit" project against source, never against dist), which imports this exact same CONTENT_DEFS value straight from here and asserts it stays byte-for-byte in step with a live z.toJSONSchema() call over each fragment's real exported Zod schema counterpart (ContentParagraphSchema, ContentRunSchema, ContentListMembershipSchema, ContentImageBlockSchema, ContentPageBreakSchema, ColorSchema, BoxSchema, AlignmentSchema, ContentStrokeStyleSchema, ContentBorderSchema, ContentCellBordersSchema, plus the non-recursive math leaves from src/math.ts: ExactRationalSchema, DimensionVectorSchema, MathPresentationSchema, MathProvenanceSchema, MathUncertaintySchema, MathNumSchema, MathQtySchema, MathSymSchema, MathUnparsedSchema, MathSymbolEntrySchema, MathUnitSchema, MathNormalisationContextSchema, SymbolTableSchema) -- see that test file's own top comment for why this is the only structural defence this generator has against silently drifting away from the schemas it's meant to describe. // // If CONTENT_DEFS stayed inline in the .mjs script, only path 1 above would work: the script imports Zod schemas exclusively from '../dist/index.js' (a build artefact that may not exist, and per eslint.config.ts/tsconfig.json is deliberately excluded from both linting and typechecking, matching test/smoke.test.mjs's own precedent) -- a test that has to import through that path would only ever run after a build, which `pnpm test` (the "unit" vitest project, run standalone in CI's own "test" job, with no build step beforehand) never guarantees. Living here instead, this is an ordinary, fully typechecked and linted src module like any other -- CONTENT_DEFS just happens to be consumed by a script as well as by the package's own test suite. // -// The fragments below still cover exactly what scripts/generate-json-schemas.mjs's own top-of-file comment already explains: ContentBlockSchema, ContentEmbeddedObjectSchema, and MathMlNodeSchema are z.custom() predicates z.toJSONSchema() cannot introspect at all (recursion the pinned Zod version's z.lazy() can't express -- see src/content.ts's isContentBlock/isContentEmbeddedObject and src/mathml.ts's isMathMlNode), so every schema reachable only through one of those three is transcribed by hand here, field-for-field, from the real Zod object definitions. Anything transcribed here that DOES have a real, non-custom, exported Zod schema counterpart is exactly what content-json-schema-defs.test.ts holds to a live z.toJSONSchema() comparison; re-verify the rest (ContentTableCell/ContentTableRow/ContentTable, ContentEmbeddedObjectBlock, MathMlElement/MathMlNode) against src/content.ts/src/mathml.ts by hand whenever those files' field shapes change, exactly as before. +// The fragments below still cover exactly what scripts/generate-json-schemas.mjs's own top-of-file comment already explains: ContentBlockSchema, ContentEmbeddedObjectSchema, MathMlNodeSchema, and MathExpressionSchema are z.custom() predicates z.toJSONSchema() cannot introspect at all (recursion the pinned Zod version's z.lazy() can't express -- see src/content.ts's isContentBlock/isContentEmbeddedObject, src/mathml.ts's isMathMlNode, and src/math.ts's isMathExpression), so every schema reachable only through one of those four is transcribed by hand here, field-for-field, from the real Zod object definitions. Two further schemas are transcribed despite being real z.objects themselves: ContentFormulaSchema (its mathml/content fields reach the opaque MathMlNodeSchema/MathExpressionSchema nodes, exactly like ContentTableCellSchema's blocks) and SymbolTableSchema (transcribed so each ContentDocument arm's symbolTable field is one named $ref rather than five inlined copies of the whole unit-registry subtree) -- the generator's override() replaces both with a $ref to their fragments here. Anything transcribed here that DOES have a real, non-custom, exported Zod schema counterpart is exactly what content-json-schema-defs.test.ts holds to a live z.toJSONSchema() comparison; re-verify the rest (ContentTableCell/ContentTableRow/ContentTable, ContentEmbeddedObjectBlock, MathMlElement/MathMlNode, MathApp/MathSum/MathProd/MathMatrix/MathExpression, ContentFormula) against src/content.ts/src/mathml.ts/src/math.ts by hand whenever those files' field shapes change, exactly as before. type JsonSchema = z.core.JSONSchema.JSONSchema; @@ -20,9 +21,25 @@ export const EMBEDDED_OBJECT_KINDS = ['formula', 'wordprocessing', 'presentation // The genuine cycle back to a whole ContentDocument: ContentEmbeddedObject(Block)'s own `document` field. Resolved once here since both the ContentEmbeddedObjectBlock fragment below and scripts/generate-json-schemas.mjs's own override() branch for the standalone ContentEmbeddedObjectSchema need the identical URI. export const CONTENT_DOCUMENT_URI = schemaUriFor('ContentDocument'); +// The two binder variants (MathSum/MathProd) differ only in their kind discriminant -- one builder rather than two copies of the same twelve-line fragment, so a binder-field change lands in both or fails the hand re-verification visibly in the diff. +function mathBinderDef(kind: 'sum' | 'prod'): JsonSchema { + return { + type: 'object', + properties: { + kind: { type: 'string', const: kind }, + binder: { type: 'string' }, + lower: { $ref: '#/$defs/MathExpression' }, + upper: { $ref: '#/$defs/MathExpression' }, + body: { $ref: '#/$defs/MathExpression' }, + }, + required: ['kind', 'binder', 'lower', 'upper', 'body'], + additionalProperties: false, + }; +} + // -- Hand-authored $defs, spliced into content-document.schema.json only (via scripts/generate-json-schemas.mjs's own ContentDocumentSchema override branch) -- // -// The fragments below are transcribed by hand, field-for-field, from src/content.ts's real Zod object definitions (ContentParagraphSchema, ContentTableSchema/ContentTableRowSchema/ContentTableCellSchema, ContentImageBlockSchema, ContentPageBreakSchema, ContentRunSchema, ContentListMembershipSchema, ColorSchema, BoxSchema, LayoutFrameSchema, AlignmentSchema, ContentStrokeStyleSchema, ContentBorderSchema, ContentCellBordersSchema -- each cross-checked directly against a real z.toJSONSchema() call over that exact exported schema, and the ones with a real, non-recursive, non-custom counterpart are held to that comparison as a running test by content-json-schema-defs.test.ts) plus the ContentEmbeddedObject/ContentEmbeddedObjectBlock TS interfaces, which have no exported z.object() counterpart at all (both are validated only via the isContentEmbeddedObject*() z.custom() guards). Re-verify this block against src/content.ts whenever that file's field shapes change -- nothing here is generated or checked against the real schemas at build time, other than the twelve leaf/near-leaf fragments the regression test below does cover. +// The fragments below are transcribed by hand, field-for-field, from src/content.ts's real Zod object definitions (ContentParagraphSchema, ContentTableSchema/ContentTableRowSchema/ContentTableCellSchema, ContentImageBlockSchema, ContentPageBreakSchema, ContentRunSchema, ContentListMembershipSchema, ColorSchema, BoxSchema, LayoutFrameSchema, AlignmentSchema, ContentStrokeStyleSchema, ContentBorderSchema, ContentCellBordersSchema -- each cross-checked directly against a real z.toJSONSchema() call over that exact exported schema, and the ones with a real, non-recursive, non-custom counterpart are held to that comparison as a running test by content-json-schema-defs.test.ts) plus the ContentEmbeddedObject/ContentEmbeddedObjectBlock TS interfaces, which have no exported z.object() counterpart at all (both are validated only via the isContentEmbeddedObject*() z.custom() guards), plus the math value schemas of src/math.ts (the semantic half of the two-layer formula model -- see that file's own top comment for how the layers divide). Re-verify this block against src/content.ts/src/math.ts whenever those files' field shapes change -- nothing here is generated or checked against the real schemas at build time, other than the leaf/near-leaf fragments the regression test below does cover. export const CONTENT_DEFS: Record = { Color: { type: 'object', @@ -289,4 +306,196 @@ export const CONTENT_DEFS: Record = { { $ref: '#/$defs/MathMlElement' }, ], }, + // -- The math value schemas (src/math.ts) -- + // + // ContentFormula itself (src/content.ts): its `mathml` field reaches the opaque MathMlNodeSchema and its `content` field the opaque MathExpressionSchema, so the generator's override() replaces every occurrence with a $ref to this fragment (the ContentTableCell precedent -- a real z.object dragged opaque by one field). presentation/provenance/starMath are transcribed alongside rather than left to inline, so the whole fragment tree under `formula` lives here where the regression test can see the leaves. + ContentFormula: { + type: 'object', + properties: { + mathml: { type: 'array', items: { $ref: '#/$defs/MathMlNode' } }, + starMath: { type: 'string' }, + presentation: { $ref: '#/$defs/MathPresentation' }, + content: { $ref: '#/$defs/MathExpression' }, + provenance: { $ref: '#/$defs/MathProvenance' }, + }, + required: ['mathml'], + additionalProperties: false, + }, + // An exact rational's two halves as canonical decimal-integer strings (src/math.ts's CANONICAL_SIGNED_INTEGER/CANONICAL_POSITIVE_INTEGER -- the patterns ARE the canonicalisation: no leading zeros, no '-0', denominator strictly positive). + ExactRational: { + type: 'object', + properties: { + numerator: { type: 'string', pattern: '^(0|-?[1-9]\\d*)$' }, + denominator: { type: 'string', pattern: '^[1-9]\\d*$' }, + }, + required: ['numerator', 'denominator'], + additionalProperties: false, + }, + // A dimension as exponents over the SI bases (DimensionVectorSchema = z.partialRecord(z.enum(SI_BASE_DIMENSIONS), z.number().int())) -- the enum below is spread from that same const so the two cannot drift. + DimensionVector: { + type: 'object', + propertyNames: { type: 'string', enum: [...SI_BASE_DIMENSIONS] }, + additionalProperties: { type: 'integer', minimum: -MAX_SAFE_INTEGER, maximum: MAX_SAFE_INTEGER }, + }, + MathPresentation: { + type: 'object', + properties: { + latex: { type: 'string' }, + }, + required: ['latex'], + additionalProperties: false, + }, + MathProvenance: { + type: 'object', + properties: { + source: { type: 'string' }, + pageRef: { type: 'string' }, + editTrail: { type: 'array', items: { type: 'string' } }, + }, + required: ['source', 'editTrail'], + additionalProperties: false, + }, + MathUncertainty: { + type: 'object', + properties: { + magnitude: { $ref: '#/$defs/ExactRational' }, + unit: { type: 'string' }, + coverageFactor: { type: 'number', exclusiveMinimum: 0 }, + }, + required: ['magnitude'], + additionalProperties: false, + }, + MathSymbolEntry: { + type: 'object', + properties: { + glyph: { type: 'string' }, + scope: { type: 'string' }, + id: { type: 'string' }, + quantityKind: { type: 'string' }, + preferredUnit: { type: 'string' }, + definitionSource: { type: 'string' }, + }, + required: ['glyph', 'scope', 'id'], + additionalProperties: false, + }, + MathUnit: { + type: 'object', + properties: { + id: { type: 'string' }, + symbol: { type: 'string' }, + name: { type: 'string' }, + dimension: { $ref: '#/$defs/DimensionVector' }, + factorToSi: { $ref: '#/$defs/ExactRational' }, + offsetToSi: { $ref: '#/$defs/ExactRational' }, + context: { type: 'string' }, + }, + required: ['id', 'symbol', 'dimension', 'factorToSi'], + additionalProperties: false, + }, + // The bases array's entry object is inlined rather than given its own $def -- MathMlNode's five non-element variants set the precedent for inlining definitions nothing else references. + MathNormalisationContext: { + type: 'object', + properties: { + id: { type: 'string' }, + bases: { + type: 'array', + items: { + type: 'object', + properties: { + unit: { type: 'string' }, + value: { $ref: '#/$defs/ExactRational' }, + }, + required: ['unit', 'value'], + additionalProperties: false, + }, + }, + }, + required: ['id', 'bases'], + additionalProperties: false, + }, + // SymbolTableSchema is a real z.object with no custom node anywhere under it, so z.toJSONSchema() could convert it inline -- it is transcribed here (and the generator $refs to it) so each ContentDocument arm's symbolTable field stays one named reference instead of five duplicated copies of this whole subtree. + SymbolTable: { + type: 'object', + properties: { + symbols: { type: 'array', items: { $ref: '#/$defs/MathSymbolEntry' } }, + units: { type: 'array', items: { $ref: '#/$defs/MathUnit' } }, + contexts: { type: 'array', items: { $ref: '#/$defs/MathNormalisationContext' } }, + }, + required: ['symbols', 'units'], + additionalProperties: false, + }, + // MathExpression and the recursive variants below it sit downstream of the fourth opaque z.custom() node (MathExpressionSchema) -- transcribed from src/math.ts's per-variant Zod definitions and interfaces, re-verified by hand when those shapes change. + MathNum: { + type: 'object', + properties: { + kind: { type: 'string', const: 'num' }, + numerator: { type: 'string', pattern: '^(0|-?[1-9]\\d*)$' }, + denominator: { type: 'string', pattern: '^[1-9]\\d*$' }, + }, + required: ['kind', 'numerator', 'denominator'], + additionalProperties: false, + }, + MathQty: { + type: 'object', + properties: { + kind: { type: 'string', const: 'qty' }, + value: { $ref: '#/$defs/ExactRational' }, + unit: { type: 'string' }, + uncertainty: { $ref: '#/$defs/MathUncertainty' }, + }, + required: ['kind', 'value', 'unit'], + additionalProperties: false, + }, + MathSym: { + type: 'object', + properties: { + kind: { type: 'string', const: 'sym' }, + id: { type: 'string' }, + }, + required: ['kind', 'id'], + additionalProperties: false, + }, + MathApp: { + type: 'object', + properties: { + kind: { type: 'string', const: 'app' }, + operator: { type: 'string' }, + args: { type: 'array', items: { $ref: '#/$defs/MathExpression' } }, + }, + required: ['kind', 'operator', 'args'], + additionalProperties: false, + }, + MathSum: mathBinderDef('sum'), + MathProd: mathBinderDef('prod'), + MathMatrix: { + type: 'object', + properties: { + kind: { type: 'string', const: 'matrix' }, + rows: { type: 'array', items: { type: 'array', items: { $ref: '#/$defs/MathExpression' } } }, + }, + required: ['kind', 'rows'], + additionalProperties: false, + }, + MathUnparsed: { + type: 'object', + properties: { + kind: { type: 'string', const: 'unparsed' }, + latex: { type: 'string' }, + }, + required: ['kind', 'latex'], + additionalProperties: false, + }, + // MathExpression itself (src/math.ts): `MathNum | MathQty | MathSym | MathApp | MathSum | MathProd | MathMatrix | MathUnparsed`, in that exact declared order. + MathExpression: { + oneOf: [ + { $ref: '#/$defs/MathNum' }, + { $ref: '#/$defs/MathQty' }, + { $ref: '#/$defs/MathSym' }, + { $ref: '#/$defs/MathApp' }, + { $ref: '#/$defs/MathSum' }, + { $ref: '#/$defs/MathProd' }, + { $ref: '#/$defs/MathMatrix' }, + { $ref: '#/$defs/MathUnparsed' }, + ], + }, }; diff --git a/src/mathml.ts b/src/mathml.ts index 5d5882e..7614c68 100644 --- a/src/mathml.ts +++ b/src/mathml.ts @@ -65,7 +65,7 @@ function isMathMlAttribute(value: unknown): value is MathMlAttribute { return isRecord(value) && typeof value.name === 'string' && typeof value.value === 'string'; } -// Recursive structural guard. Used via z.custom so element children validate without a recursive Zod schema (which collapses to `unknown` under z.lazy in this Zod version) -- the third such node in this package, alongside ContentBlockSchema and ContentEmbeddedObjectSchema (src/content.ts). +// Recursive structural guard. Used via z.custom so element children validate without a recursive Zod schema (which collapses to `unknown` under z.lazy in this Zod version) -- one of this package's z.custom() recursion nodes, alongside ContentBlockSchema and ContentEmbeddedObjectSchema (src/content.ts) and MathExpressionSchema (src/math.ts). export function isMathMlNode(value: unknown): value is MathMlNode { if (!isRecord(value)) { return false; diff --git a/test/smoke.test.mjs b/test/smoke.test.mjs index 2a7441f..a2fecba 100644 --- a/test/smoke.test.mjs +++ b/test/smoke.test.mjs @@ -69,15 +69,33 @@ describe('smoke: generated JSON Schema files', () => { expect(contentDocument.$defs.ContentBlock.oneOf).toHaveLength(5); }); - it("content-document.schema.json's formula variant refs the hand-authored recursive MathMlNode definition", () => { + it("content-document.schema.json's formula variant refs the hand-authored ContentFormula and MathMlNode definitions", () => { const contentDocument = readSchema('content-document.schema.json'); const formula = contentDocument.oneOf.find((variant) => variant.properties.kind.const === 'formula'); - expect(formula.properties.formula.properties.mathml.items.$ref).toBe('#/$defs/MathMlNode'); + expect(formula.properties.formula.$ref).toBe('#/$defs/ContentFormula'); + expect(contentDocument.$defs.ContentFormula.properties.mathml.items.$ref).toBe('#/$defs/MathMlNode'); expect(contentDocument.$defs.MathMlNode.oneOf).toHaveLength(6); // The recursion itself: an element's children point back at the shared MathMlNode definition rather than inlining. expect(contentDocument.$defs.MathMlElement.properties.children.items.$ref).toBe('#/$defs/MathMlNode'); }); + it("content-document.schema.json's formula and symbol-table definitions carry the two-layer math model", () => { + const contentDocument = readSchema('content-document.schema.json'); + // The two layers join in one hand-authored fragment: verbatim LaTeX presentation, semantic MathExpression content, and provenance alongside the MathML tree. + expect(contentDocument.$defs.ContentFormula.properties.presentation.$ref).toBe('#/$defs/MathPresentation'); + expect(contentDocument.$defs.ContentFormula.properties.content.$ref).toBe('#/$defs/MathExpression'); + expect(contentDocument.$defs.ContentFormula.properties.provenance.$ref).toBe('#/$defs/MathProvenance'); + expect(contentDocument.$defs.ContentFormula.required).toEqual(['mathml']); + // The closed grammar's eight variants, and the unparsed fallback inside them. + expect(contentDocument.$defs.MathExpression.oneOf).toHaveLength(8); + expect(contentDocument.$defs.MathUnparsed.required).toEqual(['kind', 'latex']); + // The document-level symbol table is one named reference from every ContentDocument arm, not five inlined copies. + for (const variant of contentDocument.oneOf) { + expect(variant.properties.symbolTable.$ref).toBe('#/$defs/SymbolTable'); + } + expect(contentDocument.$defs.SymbolTable.required).toEqual(['symbols', 'units']); + }); + it('layout-document.schema.json has the expected pages/images shape', () => { const layoutDocument = readSchema('layout-document.schema.json'); expect(layoutDocument.type).toBe('object');